A served model can be moved to more cards, or fewer
llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
+55
-1
@@ -34,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelContextCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -442,3 +442,57 @@ func newModelContextCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().IntVar(&slots, "slots", 0, "llama.cpp: requests a replica answers at once (default: as it is)")
|
||||
return c
|
||||
}
|
||||
|
||||
func newModelGPUsCmd(client func() *ctlClient) *cobra.Command {
|
||||
var profile string
|
||||
var inPlace bool
|
||||
c := &cobra.Command{
|
||||
Use: "gpus <model> <cards>",
|
||||
Short: "Spread a served model over more cards, or fewer",
|
||||
Long: "How many cards of one node each replica uses. vLLM, SGLang and TensorRT-LLM\n" +
|
||||
"spread the model by tensor parallel, which needs the count to divide the\n" +
|
||||
"model's KV heads or be divided by them; llama.cpp splits its layers over the\n" +
|
||||
"cards. It refuses a count a node does not have, and one whose cards would not\n" +
|
||||
"hold the model and its context, saying what each would need. Its replicas are\n" +
|
||||
"then started again on it, each new one beside the old before it stops.\n\n" +
|
||||
"With no node free enough beside the running replicas — a node of two cards,\n" +
|
||||
"the model on one — --in-place stops each replica first: the model does not\n" +
|
||||
"answer while the new one loads.\n\n" +
|
||||
"--profile names the profile for a model with more than one on cards.",
|
||||
Example: " llmgw ctl model gpus qwen3-coder-30b 2\n" +
|
||||
" llmgw ctl model gpus llama-70b 4 --profile h100 --in-place",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
n, err := strconv.Atoi(args[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("the cards are a whole number, not %q", args[1])
|
||||
}
|
||||
body, _ := json.Marshal(modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: inPlace})
|
||||
var v modelserve.GPUsChange
|
||||
if err := client().patient(time.Minute).post(cmd.Context(), "/api/v1/models/"+args[0]+"/gpus", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %d → %d %s card(s) a replica (%s, %s)\n", v.Model, v.Before, v.After, v.Class, v.Profile, v.Engine)
|
||||
if v.PerCard != "" {
|
||||
fmt.Fprintln(out, " "+v.PerCard)
|
||||
}
|
||||
if len(v.Args) > 0 {
|
||||
fmt.Fprintln(out, " args: "+strings.Join(v.Args, " "))
|
||||
}
|
||||
if v.Note != "" {
|
||||
fmt.Fprintln(out, " note: "+v.Note)
|
||||
}
|
||||
switch {
|
||||
case v.Restarted > 0 && v.InPlace:
|
||||
fmt.Fprintf(out, "%d replica(s) stopped and started again on it, one by one: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
case v.Restarted > 0:
|
||||
fmt.Fprintf(out, "%d replica(s) started again on it, each new one before the old stops: llmgw ctl replicas --model %s\n", v.Restarted, v.Model)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&profile, "profile", "", "the profile to change, for a model with several on cards")
|
||||
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before its replacement starts, when there is no room beside it")
|
||||
return c
|
||||
}
|
||||
@@ -607,7 +607,7 @@ func newServerCmd() *cobra.Command {
|
||||
// Updates: the catalog in force, the service that
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart,
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart, RestartInPlace: admin.Reconciler().RestartInPlace,
|
||||
Pool: adminAPI.PoolNodes, Notify: notifier,
|
||||
// Canaries: how the router saw each side serve.
|
||||
CanaryStats: rt.CanaryStats, ResetCanary: rt.ResetCanary,
|
||||
|
||||
@@ -356,6 +356,31 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"GPUs…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := append([]string{"(its only one on cards)"}, md.Profiles...)
|
||||
m.openForm(&formDialog{title: "Cards of " + md.Name,
|
||||
note: "How many cards of one node each replica uses: tensor parallel for vLLM, SGLang and TensorRT-LLM, " +
|
||||
"the layers split for llama.cpp. Checked against the model's heads, its cards' memory and the nodes; " +
|
||||
"its replicas are then started again on it, one by one.",
|
||||
fields: []formField{
|
||||
choiceF("profile", profiles, profiles[0]),
|
||||
textF("cards", "", "a replica's, on one node"),
|
||||
choiceF("in place", []string{"no", "yes"}, "no"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 1 {
|
||||
return nil, fmt.Errorf("the cards are a whole number from 1")
|
||||
}
|
||||
profile := v[0]
|
||||
if profile == profiles[0] {
|
||||
profile = ""
|
||||
}
|
||||
return m.call("Cards of "+md.Name, http.MethodPost, path+"/gpus",
|
||||
modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: v[2] == "yes"}, gpusSays), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Resources…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := md.Profiles
|
||||
if len(profiles) == 0 {
|
||||
@@ -943,3 +968,25 @@ func contextSays(raw json.RawMessage) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// gpusSays is a change of cards in a sentence or two.
|
||||
func gpusSays(raw json.RawMessage) string {
|
||||
var v modelserve.GPUsChange
|
||||
if json.Unmarshal(raw, &v) != nil {
|
||||
return "Done."
|
||||
}
|
||||
s := fmt.Sprintf("%s: %d → %d %s card(s) a replica.", v.Model, v.Before, v.After, v.Class)
|
||||
if v.PerCard != "" {
|
||||
s += "\n" + v.PerCard
|
||||
}
|
||||
if v.Note != "" {
|
||||
s += "\n" + v.Note
|
||||
}
|
||||
switch {
|
||||
case v.Restarted > 0 && v.InPlace:
|
||||
s += fmt.Sprintf("\n%d replica(s) stopped and started again on it, one by one.", v.Restarted)
|
||||
case v.Restarted > 0:
|
||||
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user