Add capacity classes and preempt lower ones to place a model
A model's class (critical, standard, opportunistic; unset follows its scaling) decides what may be stopped to make room for it: a lower class for a higher, and within a class only what another grew into under load, for a model below its minimum. Victims are tried on a copy of the inventory and stopped only if they make the room. Preempted replicas are drained with a short grace and their model is held back until the room is used. ctl model resources and class move under ctl model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
990d82d801
commit
ac24e204a6
16 files changed
+720
-12
No files matched your search
@@ -30,6 +30,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
@@ -233,3 +234,32 @@ func newResourcesCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
|
||||
return c
|
||||
}
|
||||
|
||||
func newClassCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "class <model> critical|standard|opportunistic|default",
|
||||
Short: "Set how much a model's capacity is worth against another's",
|
||||
Long: "When a model cannot be placed, replicas of a lower class are stopped to\n" +
|
||||
"make room for it:\n\n" +
|
||||
" critical never gives up the replicas of its minimum\n" +
|
||||
" standard gives way to critical\n" +
|
||||
" opportunistic gives way to both; the default for an on-demand model\n\n" +
|
||||
"Within a class, a model below its minimum may take what another grew into\n" +
|
||||
"under load, but not its minimum. \"default\" goes back to what the model's\n" +
|
||||
"scaling implies: opportunistic when on demand, standard otherwise.",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
class := args[1]
|
||||
if class == "default" {
|
||||
class = ""
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"class": class})
|
||||
var res map[string]string
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/class", body, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", res["model"], res["class"])
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user