package adminapi import ( "fmt" "math" "net/http" "strconv" ) // TargetView is how many replicas a model needs to serve a throughput, from // what one replica was measured to serve, and whether the cluster has room. type TargetView struct { Model string `json:"model"` // TokPerSec is what was asked for; PerReplica what one replica was // measured to serve at its SLO, on Profile. TokPerSec float64 `json:"tok_per_sec"` PerReplica float64 `json:"per_replica_tok_per_sec"` Profile string `json:"profile"` // Needed replicas in all; Current hold capacity now; More is how many of // Needed are not running, and Room how many more fit. Needed int `json:"needed"` Current int `json:"current"` More int `json:"more"` Room int `json:"room"` Fits bool `json:"fits"` Blocked []string `json:"blocked,omitempty"` // Preempts is whose replicas would be stopped to make the room, when // free room alone is not enough; Room counts that room too. Preempts []string `json:"preempts,omitempty"` } // Target sizes a model for a throughput. func (a *API) Target(model string, tokPerSec float64) (TargetView, error) { meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model) if !ok || meas.TokPerSec <= 0 { return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model) } v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile, Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)} h, ok := a.Headroom(model, v.Needed) if !ok { return TargetView{}, fmt.Errorf("no model %q", model) } v.Current = h.Current v.More = max(v.Needed-h.Current, 0) v.Room = h.More + h.ByPreempting v.Fits = v.More <= v.Room || h.Limited if v.More > h.More { v.Preempts = h.Preempts } if !v.Fits { v.Blocked = h.Blocked } return v, nil } func (a *API) target(w http.ResponseWriter, r *http.Request) { tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64) if err != nil || tps <= 0 { writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number") return } v, err := a.Target(r.PathValue("name"), tps) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } writeJSON(w, http.StatusOK, v) }