llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
+55
-1
@@ -34,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -442,3 +442,57 @@ func newModelContextCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
|
||||
return c
|
||||
}
|
||||
|
||||
func newModelGPUsCmd(client func() *ctlClient) *cobra.Command {
|
||||
var profile string
|
||||
var inPlace bool
|
||||
c := &cobra.Command{
|
||||
Use: "gpus <model> <cards>",
|
||||
Short: "Spread a served model over more cards, or fewer",
|
||||
Long: "How many cards of one node each replica uses. vLLM, SGLang and TensorRT-LLM\n" +
|
||||
"spread the model by tensor parallel, which needs the count to divide the\n" +
|
||||
"model's KV heads or be divided by them; llama.cpp splits its layers over the\n" +
|
||||
"cards. It refuses a count a node does not have, and one whose cards would not\n" +
|
||||
"hold the model and its context, saying what each would need. Its replicas are\n" +
|
||||
"then started again on it, each new one beside the old before it stops.\n\n" +
|
||||
"With no node free enough beside the running replicas — a node of two cards,\n" +
|
||||
"the model on one — --in-place stops each replica first: the model does not\n" +
|
||||
"answer while the new one loads.\n\n" +
|
||||
"--profile names the profile for a model with more than one on cards.",
|
||||
Example: " llmgw ctl model gpus qwen3-coder-30b 2\n" +
|
||||
" llmgw ctl model gpus llama-70b 4 --profile h100 --in-place",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
n, err := strconv.Atoi(args[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("the cards are a whole number, not %q", args[1])
|
||||
}
|
||||
body, _ := json.Marshal(modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: inPlace})
|
||||
var v modelserve.GPUsChange
|
||||
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/gpus", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %d → %d %s card(s) a replica (%s, %s)\n", v.Model, v.Before, v.After, v.Class, v.Profile, v.Engine)
|
||||
if v.PerCard != "" {
|
||||
fmt.Fprintln(out, " "+v.PerCard)
|
||||
}
|
||||
if len(v.Args) > 0 {
|
||||
fmt.Fprintln(out, " args: "+strings.Join(v.Args, " "))
|
||||
}
|
||||
if v.Note != "" {
|
||||
fmt.Fprintln(out, " note: "+v.Note)
|
||||
}
|
||||
switch {
|
||||
case v.Restarted > 0 && v.InPlace:
|
||||
fmt.Fprintf(out, "%d replica(s) stopped and started again on it, one by one: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
case v.Restarted > 0:
|
||||
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&profile, "profile", "", "the profile to change, for a model with several on cards")
|
||||
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before its replacement starts, when there is no room beside it")
|
||||
return c
|
||||
}
|
||||
Reference in new issue
Block a user