Change a model's replica count without editing a file on the control plane
The count is scaling.min, and until now the only way to change it was to edit server.yaml on the host and restart — or, for a model added at runtime, to remove it and add it back, which means having its whole definition to hand. `llmgw ctl scale <model> --min N` instead. Only the scaling block moves: everything else is read back from what was stored and written out untouched, so changing a number cannot quietly become a redefinition. The catalog is rebuilt and validated before anything is written, as in Add. A max below a min is refused with the stored definition untouched — persisting a rejected change would come back on the next restart as a catalog that does not build. The answer reports the scaling in force rather than the request. The catalog fills in a max that was left out, and a caller echoing its own request would print a number the cluster is not using. A model declared in configuration is refused, for the reason Remove already refuses one: the YAML is read on every start, so the new count would hold until the next restart and then silently revert. A cluster disagreeing with its own config file is worse than an error saying where to edit. The help says what --max does, because it is not what it looks like: a model with a minimum runs exactly that many and nothing scales it up under load, and an on-demand model loads one replica however high its max. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
10e323132c
commit
707d774c76
6 files changed
+388
No files matched your search
@@ -252,9 +252,24 @@ type fakeModels struct {
|
||||
added []catalog.Model
|
||||
actors []string
|
||||
removed []string
|
||||
scaled []catalog.Scaling
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) {
|
||||
if f.err != nil {
|
||||
return catalog.Model{}, f.err
|
||||
}
|
||||
f.scaled = append(f.scaled, s)
|
||||
f.actors = append(f.actors, by)
|
||||
// Max following min is what the catalog does; the fake has to do it too or
|
||||
// the handler test would pass while the real answer reported a zero.
|
||||
if s.Max == 0 && s.Min > 0 {
|
||||
s.Max = s.Min
|
||||
}
|
||||
return catalog.Model{Name: name, Scaling: s}, nil
|
||||
}
|
||||
|
||||
func (f *fakeModels) Add(_ context.Context, m catalog.Model, by string) error {
|
||||
if f.err != nil {
|
||||
return f.err
|
||||
@@ -353,3 +368,53 @@ func TestRemovingAModelOverTheAPI(t *testing.T) {
|
||||
t.Errorf("removed = %v", fm.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScalingAModelOverTheAPI(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
fm := &fakeModels{}
|
||||
a.SetModels(fm)
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/models/m1/scale",
|
||||
strings.NewReader(`{"min":3}`))
|
||||
req.Header.Set("X-Llmgw-Actor", "ilker")
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body)
|
||||
}
|
||||
var res ScaleResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The scaling in force, not the request echoed back: max follows min when
|
||||
// it is left out, and a caller printing its own request would show 0.
|
||||
if res.Min != 3 || res.Max != 3 {
|
||||
t.Errorf("expected 3/3 in force, got %d/%d", res.Min, res.Max)
|
||||
}
|
||||
if len(fm.actors) != 1 || fm.actors[0] != "ilker" {
|
||||
t.Errorf("the change should be attributed to whoever made it; got %v", fm.actors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANegativeReplicaCountIsRefused(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
fm := &fakeModels{}
|
||||
a.SetModels(fm)
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/models/m1/scale",
|
||||
strings.NewReader(`{"min":-1}`))
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400, got %d", rec.Code)
|
||||
}
|
||||
if len(fm.scaled) != 0 {
|
||||
t.Error("nothing should have reached the service")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user