Scale and place by what a replica was measured to serve
Release / build (push) Successful in 1m8s
CI / test (push) Successful in 4m24s
Release / notes (push) Successful in 3s

Stored ctl bench results reach the reconciler: requests per replica past
the measured SLO concurrency scale a model up, a replica is taken back only
if the rest stay under 70% of theirs, and among profiles of equal priority
the most measured tokens per GPU or core is tried first. ctl model target
sizes a model for a throughput against the room there is.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 18:21:08 +02:00
1 parent 9dc1b62fdc
commit 78e41460f9
16 files changed
+503 -15

No files matched your search

+1 -1
View File
@@ -137,7 +137,7 @@ func newCtlCmd() *cobra.Command {
// apply to it; it lives here because this is where operators look.
newBenchCmd(client, &asJSON),
newStatusCmd(client, &asJSON),
newModelCmd(client),
newModelCmd(client, &asJSON),
newScaleCmd(client),
newRestartCmd(client),
newRetryCmd(client),
+2 -1
View File
@@ -24,13 +24,14 @@ import (
// minutes of outage. For a fixed production set that is a reasonable trade; for
// a platform whose promise is that people can try things, it is the difference
// between trying one and not bothering.
func newModelCmd(client func() *ctlClient) *cobra.Command {
func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c := &cobra.Command{
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON),
newModelServeCmd(client))
return c
}
+66
View File
@@ -162,3 +162,69 @@ func newOwnerCmd(client func() *ctlClient) *cobra.Command {
},
}
}
func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var tps float64
var apply bool
c := &cobra.Command{
Use: "target <model> --tokens-per-sec N [--apply]",
Short: "How many replicas a model needs for a throughput, and whether they fit",
Long: "Sized from what one replica was measured to serve while meeting its SLO\n" +
"(`llmgw ctl bench MODEL --save`), on the profile its replicas run on.\n" +
"--apply makes that the number it always runs, if the cluster has room.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if tps <= 0 {
return fmt.Errorf("--tokens-per-sec is required")
}
var v adminapi.TargetView
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/target?tokens_per_sec=" +
strconv.FormatFloat(tps, 'f', -1, 64)
if err := client().get(cmd.Context(), path, &v); err != nil {
return err
}
if *asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s: %.0f tokens/s needs %d replicas at %.1f each (measured on %s); %d running\n",
v.Model, v.TokPerSec, v.Needed, v.PerReplica, v.Profile, v.Current)
switch {
case v.More == 0:
fmt.Fprintln(out, "It already has them.")
case v.Fits:
fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
default:
fmt.Fprintf(out, "%d more, and there is room for %d:\n", v.More, v.Room)
for _, b := range v.Blocked {
fmt.Fprintln(out, " "+b)
}
}
if !apply || v.More == 0 {
return nil
}
if !v.Fits {
return fmt.Errorf("not applied: the cluster has no room for them")
}
var models []adminapi.ModelView
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
return err
}
top := v.Needed
for _, m := range models {
if m.Name == v.Model {
top = max(top, m.Max)
}
}
body, _ := json.Marshal(adminapi.ScaleRequest{Min: v.Needed, Max: top})
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(v.Model)+"/scale", body, nil); err != nil {
return err
}
fmt.Fprintf(out, "Applied: %s runs %d (up to %d under load).\n", v.Model, v.Needed, top)
return nil
},
}
c.Flags().Float64Var(&tps, "tokens-per-sec", 0, "the throughput to size for")
c.Flags().BoolVar(&apply, "apply", false, "set it as the number of replicas the model always runs")
return c
}
+3
View File
@@ -488,6 +488,9 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
// The tenancy tree's capacity quotas: what each part is
// guaranteed and may hold.
ctl.Reconciler().SetQuotas(control.QuotaPolicy(st, log))
// And what each model was measured to serve, for scaling and for
// choosing between profiles.
ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
log.Info("competing to be the control plane that places work",
"owner", lead.Owner())
}