Change a model's replica count without editing a file on the control plane

The count is scaling.min, and until now the only way to change it was to edit
server.yaml on the host and restart — or, for a model added at runtime, to
remove it and add it back, which means having its whole definition to hand.

`llmgw ctl scale <model> --min N` instead. Only the scaling block moves:
everything else is read back from what was stored and written out untouched, so
changing a number cannot quietly become a redefinition.

The catalog is rebuilt and validated before anything is written, as in Add. A
max below a min is refused with the stored definition untouched — persisting a
rejected change would come back on the next restart as a catalog that does not
build.

The answer reports the scaling in force rather than the request. The catalog
fills in a max that was left out, and a caller echoing its own request would
print a number the cluster is not using.

A model declared in configuration is refused, for the reason Remove already
refuses one: the YAML is read on every start, so the new count would hold until
the next restart and then silently revert. A cluster disagreeing with its own
config file is worse than an error saying where to edit.

The help says what --max does, because it is not what it looks like: a model
with a minimum runs exactly that many and nothing scales it up under load, and
an on-demand model loads one replica however high its max.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 11:25:05 +02:00
1 parent 10e323132c
commit 707d774c76
6 files changed
+388

No files matched your search

+55
View File
@@ -37,6 +37,7 @@ func (a *API) Register(mux *http.ServeMux) {
// it does: reaching this port already means holding the control plane.
mux.HandleFunc("POST /api/v1/models", a.addModel)
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("GET /api/v1/nodes", a.listNodes)
mux.HandleFunc("GET /api/v1/models", a.listModels)
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
@@ -304,6 +305,7 @@ func (a *API) Replicas(model string) []ReplicaView {
type Models interface {
Add(ctx context.Context, m catalog.Model, by string) error
Remove(ctx context.Context, name string) error
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
}
// SetModels supplies the catalog-changing service.
@@ -330,6 +332,59 @@ func (a *API) addModel(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"added": m.Name})
}
// ScaleRequest is how many replicas a model should run.
type ScaleRequest struct {
Min int `json:"min"`
// Max omitted follows Min, which is what the catalog does for a model that
// declares only a minimum.
Max int `json:"max,omitempty"`
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
}
// ScaleResponse reports the scaling that is now in force.
//
// What is in force rather than what was asked for: the catalog fills in a max
// that was left out, and a caller that echoed its own request would print a
// number the cluster is not using.
type ScaleResponse struct {
Model string `json:"model"`
Min int `json:"min"`
Max int `json:"max"`
IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
OnDemand bool `json:"on_demand"`
}
func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req ScaleRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the scaling request is not valid JSON: "+err.Error())
return
}
if req.Min < 0 || req.Max < 0 {
writeError(w, http.StatusBadRequest, "a replica count cannot be negative")
return
}
name := r.PathValue("name")
m, err := a.models.Scale(r.Context(), name,
catalog.Scaling{Min: req.Min, Max: req.Max, IdleTimeout: req.IdleTimeout},
r.Header.Get("X-Llmgw-Actor"))
if err != nil {
// Like addModel: every refusal here is the request being wrong — a
// model declared in configuration, one that was never added, a max
// below a min. None are server faults.
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, ScaleResponse{
Model: m.Name, Min: m.Scaling.Min, Max: m.Scaling.Max,
IdleTimeout: m.Scaling.IdleTimeout, OnDemand: m.Scaling.OnDemand(),
})
}
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")