Scale and place by what a replica was measured to serve
Release / build (push) Successful in 1m8s
CI / test (push) Successful in 4m24s
Release / notes (push) Successful in 3s

Stored ctl bench results reach the reconciler: requests per replica past
the measured SLO concurrency scale a model up, a replica is taken back only
if the rest stay under 70% of theirs, and among profiles of equal priority
the most measured tokens per GPU or core is tried first. ctl model target
sizes a model for a throughput against the room there is.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 18:21:08 +02:00
1 parent 9dc1b62fdc
commit 78e41460f9
16 files changed
+503 -15

No files matched your search

+4 -1
View File
@@ -60,6 +60,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
a.registerQuotas(mux)
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
@@ -399,6 +400,8 @@ type HeadroomView struct {
Limited bool `json:"limited,omitempty"`
// Blocked says why one more does not fit, a line per node considered.
Blocked []string `json:"blocked,omitempty"`
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
TokPerSec float64 `json:"tok_per_sec,omitempty"`
}
// RejectionView is one rejected candidate.
@@ -958,7 +961,7 @@ func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
return HeadroomView{}, false
}
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
Limited: h.Limited, Blocked: h.Blocked}, true
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec}, true
}
func (a *API) Explain(model string) ExplainView {
+63
View File
@@ -0,0 +1,63 @@
package adminapi
import (
"fmt"
"math"
"net/http"
"strconv"
)
// TargetView is how many replicas a model needs to serve a throughput, from
// what one replica was measured to serve, and whether the cluster has room.
type TargetView struct {
Model string `json:"model"`
// TokPerSec is what was asked for; PerReplica what one replica was
// measured to serve at its SLO, on Profile.
TokPerSec float64 `json:"tok_per_sec"`
PerReplica float64 `json:"per_replica_tok_per_sec"`
Profile string `json:"profile"`
// Needed replicas in all; Current hold capacity now; More is how many of
// Needed are not running, and Room how many more fit.
Needed int `json:"needed"`
Current int `json:"current"`
More int `json:"more"`
Room int `json:"room"`
Fits bool `json:"fits"`
Blocked []string `json:"blocked,omitempty"`
}
// Target sizes a model for a throughput.
func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model)
if !ok || meas.TokPerSec <= 0 {
return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model)
}
v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile,
Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)}
h, ok := a.Headroom(model, v.Needed)
if !ok {
return TargetView{}, fmt.Errorf("no model %q", model)
}
v.Current = h.Current
v.More = max(v.Needed-h.Current, 0)
v.Room = h.More
v.Fits = v.More <= h.More || h.Limited
if !v.Fits {
v.Blocked = h.Blocked
}
return v, nil
}
func (a *API) target(w http.ResponseWriter, r *http.Request) {
tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64)
if err != nil || tps <= 0 {
writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number")
return
}
v, err := a.Target(r.PathValue("name"), tps)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, v)
}
+30
View File
@@ -0,0 +1,30 @@
package adminapi
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
func TestATargetIsSizedFromTheMeasurementAndTheRoom(t *testing.T) {
// Three 8-GPU nodes; one tp8 replica was measured at 500 tokens/s.
a, ctl := setup(t, testModels(), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
if _, err := a.Target("llama", 1000); err == nil || !strings.Contains(err.Error(), "ctl bench llama --save") {
t.Errorf("an unmeasured model should say how to measure it: %v", err)
}
ctl.Reconciler().SetMeasurements(func() map[string]reconciler.Measured {
return map[string]reconciler.Measured{"llama/tp8": {MaxConcurrency: 16, TokPerSec: 500}}
})
v, err := a.Target("llama", 1200)
if err != nil {
t.Fatal(err)
}
if v.Needed != 3 || v.PerReplica != 500 || v.Profile != "tp8" || v.More != 3 || !v.Fits {
t.Errorf("1200 tok/s at 500 each is 3 replicas on 3 empty nodes: %+v", v)
}
v, _ = a.Target("llama", 2000)
if v.Needed != 4 || v.Fits || v.Room != 3 || len(v.Blocked) == 0 {
t.Errorf("a fourth does not fit on three nodes, and should say why: %+v", v)
}
}