A served model can be moved to more cards, or fewer

llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 16:40:49 +01:00
1 parent 51aeadae3e
commit e1cff0d4e3
16 files changed
+753 -13

No files matched your search

+55 -1
View File
@@ -34,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -442,3 +442,57 @@ func newModelContextCmd(client func() *ctlClient) *cobra.Command {
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
return c
}
func newModelGPUsCmd(client func() *ctlClient) *cobra.Command {
var profile string
var inPlace bool
c := &cobra.Command{
Use: "gpus <model> <cards>",
Short: "Spread a served model over more cards, or fewer",
Long: "How many cards of one node each replica uses. vLLM, SGLang and TensorRT-LLM\n" +
"spread the model by tensor parallel, which needs the count to divide the\n" +
"model's KV heads or be divided by them; llama.cpp splits its layers over the\n" +
"cards. It refuses a count a node does not have, and one whose cards would not\n" +
"hold the model and its context, saying what each would need. Its replicas are\n" +
"then started again on it, each new one beside the old before it stops.\n\n" +
"With no node free enough beside the running replicas — a node of two cards,\n" +
"the model on one — --in-place stops each replica first: the model does not\n" +
"answer while the new one loads.\n\n" +
"--profile names the profile for a model with more than one on cards.",
Example: " llmgw ctl model gpus qwen3-coder-30b 2\n" +
" llmgw ctl model gpus llama-70b 4 --profile h100 --in-place",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
n, err := strconv.Atoi(args[1])
if err != nil {
return fmt.Errorf("the cards are a whole number, not %q", args[1])
}
body, _ := json.Marshal(modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: inPlace})
var v modelserve.GPUsChange
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/gpus", body, &v); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s: %d → %d %s card(s) a replica (%s, %s)\n", v.Model, v.Before, v.After, v.Class, v.Profile, v.Engine)
if v.PerCard != "" {
fmt.Fprintln(out, " "+v.PerCard)
}
if len(v.Args) > 0 {
fmt.Fprintln(out, " args: "+strings.Join(v.Args, " "))
}
if v.Note != "" {
fmt.Fprintln(out, " note: "+v.Note)
}
switch {
case v.Restarted > 0 && v.InPlace:
fmt.Fprintf(out, "%d replica(s) stopped and started again on it, one by one: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
case v.Restarted > 0:
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
}
return nil
},
}
c.Flags().StringVar(&profile, "profile", "", "the profile to change, for a model with several on cards")
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before its replacement starts, when there is no room beside it")
return c
}
+1 -1
View File
@@ -607,7 +607,7 @@ func newServerCmd() *cobra.Command {
// Updates: the catalog in force, the service that
// changes it, and what replaces replicas.
Current: func() []catalog.Model { return admin.Catalog().Models() },
Models: catalogSvc, Restart: admin.Reconciler().Restart,
Models: catalogSvc, Restart: admin.Reconciler().Restart, RestartInPlace: admin.Reconciler().RestartInPlace,
Pool: adminAPI.PoolNodes, Notify: notifier,
// Canaries: how the router saw each side serve.
CanaryStats: rt.CanaryStats, ResetCanary: rt.ResetCanary,
+47
View File
@@ -356,6 +356,31 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
}})
return nil
}},
{"GPUs…", func(m *consoleModel) tea.Cmd {
profiles := append([]string{"(its only one on cards)"}, md.Profiles...)
m.openForm(&formDialog{title: "Cards of " + md.Name,
note: "How many cards of one node each replica uses: tensor parallel for vLLM, SGLang and TensorRT-LLM, " +
"the layers split for llama.cpp. Checked against the model's heads, its cards' memory and the nodes; " +
"its replicas are then started again on it, one by one.",
fields: []formField{
choiceF("profile", profiles, profiles[0]),
textF("cards", "", "a replica's, on one node"),
choiceF("in place", []string{"no", "yes"}, "no"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
n, err := strconv.Atoi(v[1])
if err != nil || n < 1 {
return nil, fmt.Errorf("the cards are a whole number from 1")
}
profile := v[0]
if profile == profiles[0] {
profile = ""
}
return m.call("Cards of "+md.Name, http.MethodPost, path+"/gpus",
modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: v[2] == "yes"}, gpusSays), nil
}})
return nil
}},
{"Resources…", func(m *consoleModel) tea.Cmd {
profiles := md.Profiles
if len(profiles) == 0 {
@@ -943,3 +968,25 @@ func contextSays(raw json.RawMessage) string {
}
return s
}
// gpusSays is a change of cards in a sentence or two.
func gpusSays(raw json.RawMessage) string {
var v modelserve.GPUsChange
if json.Unmarshal(raw, &v) != nil {
return "Done."
}
s := fmt.Sprintf("%s: %d → %d %s card(s) a replica.", v.Model, v.Before, v.After, v.Class)
if v.PerCard != "" {
s += "\n" + v.PerCard
}
if v.Note != "" {
s += "\n" + v.Note
}
switch {
case v.Restarted > 0 && v.InPlace:
s += fmt.Sprintf("\n%d replica(s) stopped and started again on it, one by one.", v.Restarted)
case v.Restarted > 0:
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
}
return s
}