A served model can be moved to more cards, or fewer

llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 16:40:49 +01:00
1 parent 51aeadae3e
commit e1cff0d4e3
16 files changed
+753 -13

No files matched your search

+47
View File
@@ -356,6 +356,31 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
}})
return nil
}},
{"GPUs…", func(m *consoleModel) tea.Cmd {
profiles := append([]string{"(its only one on cards)"}, md.Profiles...)
m.openForm(&formDialog{title: "Cards of " + md.Name,
note: "How many cards of one node each replica uses: tensor parallel for vLLM, SGLang and TensorRT-LLM, " +
"the layers split for llama.cpp. Checked against the model's heads, its cards' memory and the nodes; " +
"its replicas are then started again on it, one by one.",
fields: []formField{
choiceF("profile", profiles, profiles[0]),
textF("cards", "", "a replica's, on one node"),
choiceF("in place", []string{"no", "yes"}, "no"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
n, err := strconv.Atoi(v[1])
if err != nil || n < 1 {
return nil, fmt.Errorf("the cards are a whole number from 1")
}
profile := v[0]
if profile == profiles[0] {
profile = ""
}
return m.call("Cards of "+md.Name, http.MethodPost, path+"/gpus",
modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: v[2] == "yes"}, gpusSays), nil
}})
return nil
}},
{"Resources…", func(m *consoleModel) tea.Cmd {
profiles := md.Profiles
if len(profiles) == 0 {
@@ -943,3 +968,25 @@ func contextSays(raw json.RawMessage) string {
}
return s
}
// gpusSays is a change of cards in a sentence or two.
func gpusSays(raw json.RawMessage) string {
var v modelserve.GPUsChange
if json.Unmarshal(raw, &v) != nil {
return "Done."
}
s := fmt.Sprintf("%s: %d → %d %s card(s) a replica.", v.Model, v.Before, v.After, v.Class)
if v.PerCard != "" {
s += "\n" + v.PerCard
}
if v.Note != "" {
s += "\n" + v.Note
}
switch {
case v.Restarted > 0 && v.InPlace:
s += fmt.Sprintf("\n%d replica(s) stopped and started again on it, one by one.", v.Restarted)
case v.Restarted > 0:
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
}
return s
}