Scale and place by what a replica was measured to serve
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9dc1b62fdc
commit
78e41460f9
16 files changed
+503
-15
No files matched your search
+1
-1
@@ -137,7 +137,7 @@ func newCtlCmd() *cobra.Command {
|
||||
// apply to it; it lives here because this is where operators look.
|
||||
newBenchCmd(client, &asJSON),
|
||||
newStatusCmd(client, &asJSON),
|
||||
newModelCmd(client),
|
||||
newModelCmd(client, &asJSON),
|
||||
newScaleCmd(client),
|
||||
newRestartCmd(client),
|
||||
newRetryCmd(client),
|
||||
|
||||
@@ -24,13 +24,14 @@ import (
|
||||
// minutes of outage. For a fixed production set that is a reasonable trade; for
|
||||
// a platform whose promise is that people can try things, it is the difference
|
||||
// between trying one and not bothering.
|
||||
func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -162,3 +162,69 @@ func newOwnerCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var tps float64
|
||||
var apply bool
|
||||
c := &cobra.Command{
|
||||
Use: "target <model> --tokens-per-sec N [--apply]",
|
||||
Short: "How many replicas a model needs for a throughput, and whether they fit",
|
||||
Long: "Sized from what one replica was measured to serve while meeting its SLO\n" +
|
||||
"(`llmgw ctl bench MODEL --save`), on the profile its replicas run on.\n" +
|
||||
"--apply makes that the number it always runs, if the cluster has room.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if tps <= 0 {
|
||||
return fmt.Errorf("--tokens-per-sec is required")
|
||||
}
|
||||
var v adminapi.TargetView
|
||||
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/target?tokens_per_sec=" +
|
||||
strconv.FormatFloat(tps, 'f', -1, 64)
|
||||
if err := client().get(cmd.Context(), path, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %.0f tokens/s needs %d replicas at %.1f each (measured on %s); %d running\n",
|
||||
v.Model, v.TokPerSec, v.Needed, v.PerReplica, v.Profile, v.Current)
|
||||
switch {
|
||||
case v.More == 0:
|
||||
fmt.Fprintln(out, "It already has them.")
|
||||
case v.Fits:
|
||||
fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
|
||||
default:
|
||||
fmt.Fprintf(out, "%d more, and there is room for %d:\n", v.More, v.Room)
|
||||
for _, b := range v.Blocked {
|
||||
fmt.Fprintln(out, " "+b)
|
||||
}
|
||||
}
|
||||
if !apply || v.More == 0 {
|
||||
return nil
|
||||
}
|
||||
if !v.Fits {
|
||||
return fmt.Errorf("not applied: the cluster has no room for them")
|
||||
}
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||
return err
|
||||
}
|
||||
top := v.Needed
|
||||
for _, m := range models {
|
||||
if m.Name == v.Model {
|
||||
top = max(top, m.Max)
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.ScaleRequest{Min: v.Needed, Max: top})
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(v.Model)+"/scale", body, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "Applied: %s runs %d (up to %d under load).\n", v.Model, v.Needed, top)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().Float64Var(&tps, "tokens-per-sec", 0, "the throughput to size for")
|
||||
c.Flags().BoolVar(&apply, "apply", false, "set it as the number of replicas the model always runs")
|
||||
return c
|
||||
}
|
||||
@@ -488,6 +488,9 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
// The tenancy tree's capacity quotas: what each part is
|
||||
// guaranteed and may hold.
|
||||
ctl.Reconciler().SetQuotas(control.QuotaPolicy(st, log))
|
||||
// And what each model was measured to serve, for scaling and for
|
||||
// choosing between profiles.
|
||||
ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
|
||||
log.Info("competing to be the control plane that places work",
|
||||
"owner", lead.Owner())
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user