Change a model's replica count without editing a file on the control plane
The count is scaling.min, and until now the only way to change it was to edit server.yaml on the host and restart — or, for a model added at runtime, to remove it and add it back, which means having its whole definition to hand. `llmgw ctl scale <model> --min N` instead. Only the scaling block moves: everything else is read back from what was stored and written out untouched, so changing a number cannot quietly become a redefinition. The catalog is rebuilt and validated before anything is written, as in Add. A max below a min is refused with the stored definition untouched — persisting a rejected change would come back on the next restart as a catalog that does not build. The answer reports the scaling in force rather than the request. The catalog fills in a max that was left out, and a caller echoing its own request would print a number the cluster is not using. A model declared in configuration is refused, for the reason Remove already refuses one: the YAML is read on every start, so the new count would hold until the next restart and then silently revert. A cluster disagreeing with its own config file is worse than an error saying where to edit. The help says what --max does, because it is not what it looks like: a model with a minimum runs exactly that many and nothing scales it up under load, and an on-demand model loads one replica however high its max. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
10e323132c
commit
707d774c76
6 files changed
+388
No files matched your search
@@ -164,6 +164,73 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Scale changes how many replicas a model runs, without a restart.
|
||||
//
|
||||
// Only the scaling block moves. Everything else about the model — its source,
|
||||
// its profiles, the engine arguments somebody tuned — is read back from what
|
||||
// was stored and written out again untouched, so changing a number cannot
|
||||
// quietly become a redefinition.
|
||||
//
|
||||
// A model declared in configuration is refused, for the same reason Remove
|
||||
// refuses one: the YAML is read on every start, so the new number would hold
|
||||
// until the next restart and then silently revert. A cluster that disagrees
|
||||
// with its own config file is worse than an error message saying where to edit.
|
||||
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.Declared(name) {
|
||||
return catalog.Model{}, fmt.Errorf(
|
||||
"model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name)
|
||||
}
|
||||
|
||||
stored, err := s.storedModels(ctx)
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
e.Scaling = sc
|
||||
found = true
|
||||
}
|
||||
next = append(next, e)
|
||||
}
|
||||
if !found {
|
||||
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
|
||||
}
|
||||
|
||||
// Built before anything is written, like Add: catalog.New is what applies
|
||||
// the defaults (max follows min when it is left out) and what refuses a
|
||||
// max below a min, so the model that comes back is the one that will run
|
||||
// rather than the one that was asked for.
|
||||
built, err := catalog.New(next)
|
||||
if err != nil {
|
||||
return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err)
|
||||
}
|
||||
final, ok := built.Resolve(name)
|
||||
if !ok {
|
||||
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
|
||||
}
|
||||
|
||||
def, err := json.Marshal(final)
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
if err := s.store.PutModel(ctx, store.StoredModel{
|
||||
Name: name, Definition: def, CreatedAt: time.Now(), CreatedBy: by,
|
||||
}); err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
|
||||
s.apply.ReplaceCatalog(ctx, built)
|
||||
s.log.Info("model scaled", "model", name,
|
||||
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
||||
return final, nil
|
||||
}
|
||||
|
||||
// Load builds the catalog at startup: configuration plus what was stored.
|
||||
//
|
||||
// A stored model that no longer validates — its engine left the build, a
|
||||
|
||||
@@ -194,3 +194,146 @@ func TestAStoredModelThatNoLongerValidatesIsSkippedAlone(t *testing.T) {
|
||||
t.Error("a valid stored model was dropped because another was invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalingChangesTheCountAndSurvivesARestart(t *testing.T) {
|
||||
s, st, rec := newService(t)
|
||||
if err := s.Add(t.Context(), model("added"), "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.Scale(t.Context(), "added", catalog.Scaling{Min: 3}, "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Max follows min when it is left out, and the answer reports what the
|
||||
// catalog settled on rather than what was asked for — a caller echoing its
|
||||
// own request would print a max of zero.
|
||||
if got.Scaling.Min != 3 || got.Scaling.Max != 3 {
|
||||
t.Errorf("expected 3/3 in force, got %d/%d", got.Scaling.Min, got.Scaling.Max)
|
||||
}
|
||||
if m, _ := rec.last.Resolve("added"); m.Scaling.Min != 3 {
|
||||
t.Errorf("the applied catalog still says %d", m.Scaling.Min)
|
||||
}
|
||||
|
||||
s2, _, _ := newService(t)
|
||||
s2.store = st
|
||||
built, err := s2.Load(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := built.Resolve("added"); m.Scaling.Min != 3 {
|
||||
t.Errorf("the new count did not survive a restart; got %d", m.Scaling.Min)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalingChangesNothingButTheCount(t *testing.T) {
|
||||
s, _, rec := newService(t)
|
||||
m := model("added")
|
||||
m.Aliases = []string{"a"}
|
||||
m.Profiles[0].Args = []string{"--ctx-size", "16384"}
|
||||
if err := s.Add(t.Context(), m, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := s.Scale(t.Context(), "added", catalog.Scaling{Min: 2}, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Changing a number must not quietly become a redefinition: the source and
|
||||
// the engine arguments somebody tuned are read back and written out again.
|
||||
got, _ := rec.last.Resolve("added")
|
||||
if got.Source != "/models/added" {
|
||||
t.Errorf("the source changed to %q", got.Source)
|
||||
}
|
||||
if len(got.Profiles) != 1 || strings.Join(got.Profiles[0].Args, " ") != "--ctx-size 16384" {
|
||||
t.Errorf("the profile arguments were lost: %+v", got.Profiles)
|
||||
}
|
||||
if len(got.Aliases) != 1 || got.Aliases[0] != "a" {
|
||||
t.Errorf("the aliases were lost: %v", got.Aliases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeclaredModelCannotBeScaledHere(t *testing.T) {
|
||||
s, _, _ := newService(t, model("declared"))
|
||||
|
||||
_, err := s.Scale(t.Context(), "declared", catalog.Scaling{Min: 5}, "ilker")
|
||||
|
||||
// The YAML is read on every start, so the number would hold until the next
|
||||
// restart and then silently revert. Refusing and saying where to edit is
|
||||
// the smaller surprise — the same call Remove makes.
|
||||
if err == nil {
|
||||
t.Fatal("expected scaling a declared model to be refused")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "configuration") {
|
||||
t.Errorf("the error should say where to change it; got %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalingAModelThatWasNeverAddedIsAnError(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
if _, err := s.Scale(t.Context(), "ghost", catalog.Scaling{Min: 1}, "ilker"); err == nil {
|
||||
t.Error("expected an error for a model that is not there")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMaxBelowAMinIsRefusedBeforeAnythingIsWritten(t *testing.T) {
|
||||
s, st, _ := newService(t)
|
||||
if err := s.Add(t.Context(), model("added"), "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := s.Scale(t.Context(), "added", catalog.Scaling{Min: 3, Max: 1}, "ilker"); err == nil {
|
||||
t.Fatal("expected a max below a min to be refused")
|
||||
}
|
||||
|
||||
// Refused before the write, so the stored definition is untouched: a
|
||||
// rejected change that had already been persisted would come back on the
|
||||
// next restart as a catalog that does not build.
|
||||
var stored catalog.Model
|
||||
if err := json.Unmarshal(st.rows["added"].Definition, &stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.Scaling.Min != 1 {
|
||||
t.Errorf("the stored definition was changed to %d despite the refusal", stored.Scaling.Min)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalingToZeroKeepsTheModelInTheCatalog(t *testing.T) {
|
||||
s, _, rec := newService(t)
|
||||
if err := s.Add(t.Context(), model("added"), "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.Scale(t.Context(), "added", catalog.Scaling{Min: 0, Max: 0}, "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Configured and deliberately not running is a different and equally
|
||||
// useful thing from removed: the entry stays so it can be brought back
|
||||
// with one number.
|
||||
if got.Scaling.OnDemand() {
|
||||
t.Error("min 0 max 0 is not on-demand; it is off")
|
||||
}
|
||||
if _, ok := rec.last.Resolve("added"); !ok {
|
||||
t.Error("scaling to zero must not remove the model")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinZeroWithAMaxIsOnDemand(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
if err := s.Add(t.Context(), model("added"), "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.Scale(t.Context(), "added",
|
||||
catalog.Scaling{Min: 0, Max: 2, IdleTimeout: 10 * time.Minute}, "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Scaling.OnDemand() {
|
||||
t.Error("min 0 with a max above it should be on-demand")
|
||||
}
|
||||
if got.Scaling.IdleTimeout != 10*time.Minute {
|
||||
t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user