Add capacity classes and preempt lower ones to place a model
Release / build (push) Successful in 1m7s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m46s

A model's class (critical, standard, opportunistic; unset follows its
scaling) decides what may be stopped to make room for it: a lower class
for a higher, and within a class only what another grew into under load,
for a model below its minimum. Victims are tried on a copy of the
inventory and stopped only if they make the room. Preempted replicas are
drained with a short grace and their model is held back until the room is
used. ctl model resources and class move under ctl model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 18:02:59 +02:00
1 parent 990d82d801
commit ac24e204a6
16 files changed
+720 -12

No files matched your search

+5 -4
View File
@@ -139,7 +139,6 @@ func newCtlCmd() *cobra.Command {
newStatusCmd(client, &asJSON),
newModelCmd(client),
newScaleCmd(client),
newResourcesCmd(client),
newRestartCmd(client),
newRetryCmd(client),
newUpgradeCmd(client),
@@ -254,6 +253,8 @@ func modelStatus(m adminapi.ModelView) string {
switch {
case m.Parked:
return "parked"
case m.HeldFor != "":
return "gave room to " + m.HeldFor
case m.Unplaceable:
return "cannot place"
case m.Min == 0 && m.Total == 0:
@@ -324,15 +325,15 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tCLASS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
var warnings []string
for _, m := range models {
serving := "no"
if m.Ready > 0 {
serving = "yes"
}
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n",
m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m),
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
m.Name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m),
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
for _, x := range m.Warnings {
warnings = append(warnings, m.Name+": "+x)
+30
View File
@@ -30,6 +30,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client),
newModelServeCmd(client))
return c
}
@@ -233,3 +234,32 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command {
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
return c
}
func newClassCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "class <model> critical|standard|opportunistic|default",
Short: "Set how much a model's capacity is worth against another's",
Long: "When a model cannot be placed, replicas of a lower class are stopped to\n" +
"make room for it:\n\n" +
" critical never gives up the replicas of its minimum\n" +
" standard gives way to critical\n" +
" opportunistic gives way to both; the default for an on-demand model\n\n" +
"Within a class, a model below its minimum may take what another grew into\n" +
"under load, but not its minimum. \"default\" goes back to what the model's\n" +
"scaling implies: opportunistic when on demand, standard otherwise.",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
class := args[1]
if class == "default" {
class = ""
}
body, _ := json.Marshal(map[string]string{"class": class})
var res map[string]string
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/class", body, &res); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", res["model"], res["class"])
return nil
},
}
}