Ilker ManapandClaude Opus 5.5
f288f3c423
The quotas page gains a capacity section over the same API methods the CLI uses; a model's card sets its class, owner, memory and cores, applying only what changed. Headroom counts the replicas a model could get by preempting what its class outranks, and the card and ctl model target use it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
70 lines
2.3 KiB
Go
70 lines
2.3 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"strconv"
|
|
)
|
|
|
|
// TargetView is how many replicas a model needs to serve a throughput, from
|
|
// what one replica was measured to serve, and whether the cluster has room.
|
|
type TargetView struct {
|
|
Model string `json:"model"`
|
|
// TokPerSec is what was asked for; PerReplica what one replica was
|
|
// measured to serve at its SLO, on Profile.
|
|
TokPerSec float64 `json:"tok_per_sec"`
|
|
PerReplica float64 `json:"per_replica_tok_per_sec"`
|
|
Profile string `json:"profile"`
|
|
// Needed replicas in all; Current hold capacity now; More is how many of
|
|
// Needed are not running, and Room how many more fit.
|
|
Needed int `json:"needed"`
|
|
Current int `json:"current"`
|
|
More int `json:"more"`
|
|
Room int `json:"room"`
|
|
Fits bool `json:"fits"`
|
|
Blocked []string `json:"blocked,omitempty"`
|
|
// Preempts is whose replicas would be stopped to make the room, when
|
|
// free room alone is not enough; Room counts that room too.
|
|
Preempts []string `json:"preempts,omitempty"`
|
|
}
|
|
|
|
// Target sizes a model for a throughput.
|
|
func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
|
|
meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model)
|
|
if !ok || meas.TokPerSec <= 0 {
|
|
return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model)
|
|
}
|
|
v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile,
|
|
Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)}
|
|
h, ok := a.Headroom(model, v.Needed)
|
|
if !ok {
|
|
return TargetView{}, fmt.Errorf("no model %q", model)
|
|
}
|
|
v.Current = h.Current
|
|
v.More = max(v.Needed-h.Current, 0)
|
|
v.Room = h.More + h.ByPreempting
|
|
v.Fits = v.More <= v.Room || h.Limited
|
|
if v.More > h.More {
|
|
v.Preempts = h.Preempts
|
|
}
|
|
if !v.Fits {
|
|
v.Blocked = h.Blocked
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
func (a *API) target(w http.ResponseWriter, r *http.Request) {
|
|
tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64)
|
|
if err != nil || tps <= 0 {
|
|
writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number")
|
|
return
|
|
}
|
|
v, err := a.Target(r.PathValue("name"), tps)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, v)
|
|
}
|