Scale and place by what a replica was measured to serve
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9dc1b62fdc
commit
78e41460f9
16 files changed
+503
-15
No files matched your search
@@ -60,6 +60,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -399,6 +400,8 @@ type HeadroomView struct {
|
||||
Limited bool `json:"limited,omitempty"`
|
||||
// Blocked says why one more does not fit, a line per node considered.
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
|
||||
TokPerSec float64 `json:"tok_per_sec,omitempty"`
|
||||
}
|
||||
|
||||
// RejectionView is one rejected candidate.
|
||||
@@ -958,7 +961,7 @@ func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||
return HeadroomView{}, false
|
||||
}
|
||||
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||
Limited: h.Limited, Blocked: h.Blocked}, true
|
||||
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec}, true
|
||||
}
|
||||
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
|
||||
Reference in new issue
Block a user