Scale and place by what a replica was measured to serve
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9dc1b62fdc
commit
78e41460f9
16 files changed
+503
-15
No files matched your search
@@ -60,6 +60,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -399,6 +400,8 @@ type HeadroomView struct {
|
||||
Limited bool `json:"limited,omitempty"`
|
||||
// Blocked says why one more does not fit, a line per node considered.
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
|
||||
TokPerSec float64 `json:"tok_per_sec,omitempty"`
|
||||
}
|
||||
|
||||
// RejectionView is one rejected candidate.
|
||||
@@ -958,7 +961,7 @@ func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||
return HeadroomView{}, false
|
||||
}
|
||||
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||
Limited: h.Limited, Blocked: h.Blocked}, true
|
||||
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec}, true
|
||||
}
|
||||
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// TargetView is how many replicas a model needs to serve a throughput, from
|
||||
// what one replica was measured to serve, and whether the cluster has room.
|
||||
type TargetView struct {
|
||||
Model string `json:"model"`
|
||||
// TokPerSec is what was asked for; PerReplica what one replica was
|
||||
// measured to serve at its SLO, on Profile.
|
||||
TokPerSec float64 `json:"tok_per_sec"`
|
||||
PerReplica float64 `json:"per_replica_tok_per_sec"`
|
||||
Profile string `json:"profile"`
|
||||
// Needed replicas in all; Current hold capacity now; More is how many of
|
||||
// Needed are not running, and Room how many more fit.
|
||||
Needed int `json:"needed"`
|
||||
Current int `json:"current"`
|
||||
More int `json:"more"`
|
||||
Room int `json:"room"`
|
||||
Fits bool `json:"fits"`
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
}
|
||||
|
||||
// Target sizes a model for a throughput.
|
||||
func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
|
||||
meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model)
|
||||
if !ok || meas.TokPerSec <= 0 {
|
||||
return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model)
|
||||
}
|
||||
v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile,
|
||||
Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)}
|
||||
h, ok := a.Headroom(model, v.Needed)
|
||||
if !ok {
|
||||
return TargetView{}, fmt.Errorf("no model %q", model)
|
||||
}
|
||||
v.Current = h.Current
|
||||
v.More = max(v.Needed-h.Current, 0)
|
||||
v.Room = h.More
|
||||
v.Fits = v.More <= h.More || h.Limited
|
||||
if !v.Fits {
|
||||
v.Blocked = h.Blocked
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (a *API) target(w http.ResponseWriter, r *http.Request) {
|
||||
tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64)
|
||||
if err != nil || tps <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number")
|
||||
return
|
||||
}
|
||||
v, err := a.Target(r.PathValue("name"), tps)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
func TestATargetIsSizedFromTheMeasurementAndTheRoom(t *testing.T) {
|
||||
// Three 8-GPU nodes; one tp8 replica was measured at 500 tokens/s.
|
||||
a, ctl := setup(t, testModels(), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
||||
if _, err := a.Target("llama", 1000); err == nil || !strings.Contains(err.Error(), "ctl bench llama --save") {
|
||||
t.Errorf("an unmeasured model should say how to measure it: %v", err)
|
||||
}
|
||||
ctl.Reconciler().SetMeasurements(func() map[string]reconciler.Measured {
|
||||
return map[string]reconciler.Measured{"llama/tp8": {MaxConcurrency: 16, TokPerSec: 500}}
|
||||
})
|
||||
v, err := a.Target("llama", 1200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Needed != 3 || v.PerReplica != 500 || v.Profile != "tp8" || v.More != 3 || !v.Fits {
|
||||
t.Errorf("1200 tok/s at 500 each is 3 replicas on 3 empty nodes: %+v", v)
|
||||
}
|
||||
v, _ = a.Target("llama", 2000)
|
||||
if v.Needed != 4 || v.Fits || v.Room != 3 || len(v.Blocked) == 0 {
|
||||
t.Errorf("a fourth does not fit on three nodes, and should say why: %+v", v)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user