Ilker ManapandClaude Opus 5.5
78e41460f9
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
64 lines
2.0 KiB
Go
64 lines
2.0 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"strconv"
|
|
)
|
|
|
|
// TargetView is how many replicas a model needs to serve a throughput, from
|
|
// what one replica was measured to serve, and whether the cluster has room.
|
|
type TargetView struct {
|
|
Model string `json:"model"`
|
|
// TokPerSec is what was asked for; PerReplica what one replica was
|
|
// measured to serve at its SLO, on Profile.
|
|
TokPerSec float64 `json:"tok_per_sec"`
|
|
PerReplica float64 `json:"per_replica_tok_per_sec"`
|
|
Profile string `json:"profile"`
|
|
// Needed replicas in all; Current hold capacity now; More is how many of
|
|
// Needed are not running, and Room how many more fit.
|
|
Needed int `json:"needed"`
|
|
Current int `json:"current"`
|
|
More int `json:"more"`
|
|
Room int `json:"room"`
|
|
Fits bool `json:"fits"`
|
|
Blocked []string `json:"blocked,omitempty"`
|
|
}
|
|
|
|
// Target sizes a model for a throughput.
|
|
func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
|
|
meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model)
|
|
if !ok || meas.TokPerSec <= 0 {
|
|
return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model)
|
|
}
|
|
v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile,
|
|
Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)}
|
|
h, ok := a.Headroom(model, v.Needed)
|
|
if !ok {
|
|
return TargetView{}, fmt.Errorf("no model %q", model)
|
|
}
|
|
v.Current = h.Current
|
|
v.More = max(v.Needed-h.Current, 0)
|
|
v.Room = h.More
|
|
v.Fits = v.More <= h.More || h.Limited
|
|
if !v.Fits {
|
|
v.Blocked = h.Blocked
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
func (a *API) target(w http.ResponseWriter, r *http.Request) {
|
|
tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64)
|
|
if err != nil || tps <= 0 {
|
|
writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number")
|
|
return
|
|
}
|
|
v, err := a.Target(r.PathValue("name"), tps)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, v)
|
|
}
|