llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
@@ -356,6 +356,31 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"GPUs…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := append([]string{"(its only one on cards)"}, md.Profiles...)
|
||||
m.openForm(&formDialog{title: "Cards of " + md.Name,
|
||||
note: "How many cards of one node each replica uses: tensor parallel for vLLM, SGLang and TensorRT-LLM, " +
|
||||
"the layers split for llama.cpp. Checked against the model's heads, its cards' memory and the nodes; " +
|
||||
"its replicas are then started again on it, one by one.",
|
||||
fields: []formField{
|
||||
choiceF("profile", profiles, profiles[0]),
|
||||
textF("cards", "", "a replica's, on one node"),
|
||||
choiceF("in place", []string{"no", "yes"}, "no"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 1 {
|
||||
return nil, fmt.Errorf("the cards are a whole number from 1")
|
||||
}
|
||||
profile := v[0]
|
||||
if profile == profiles[0] {
|
||||
profile = ""
|
||||
}
|
||||
return m.call("Cards of "+md.Name, http.MethodPost, path+"/gpus",
|
||||
modelserve.GPUsRequest{GPUs: n, Profile: profile, InPlace: v[2] == "yes"}, gpusSays), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Resources…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := md.Profiles
|
||||
if len(profiles) == 0 {
|
||||
@@ -943,3 +968,25 @@ func contextSays(raw json.RawMessage) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// gpusSays is a change of cards in a sentence or two.
|
||||
func gpusSays(raw json.RawMessage) string {
|
||||
var v modelserve.GPUsChange
|
||||
if json.Unmarshal(raw, &v) != nil {
|
||||
return "Done."
|
||||
}
|
||||
s := fmt.Sprintf("%s: %d → %d %s card(s) a replica.", v.Model, v.Before, v.After, v.Class)
|
||||
if v.PerCard != "" {
|
||||
s += "\n" + v.PerCard
|
||||
}
|
||||
if v.Note != "" {
|
||||
s += "\n" + v.Note
|
||||
}
|
||||
switch {
|
||||
case v.Restarted > 0 && v.InPlace:
|
||||
s += fmt.Sprintf("\n%d replica(s) stopped and started again on it, one by one.", v.Restarted)
|
||||
case v.Restarted > 0:
|
||||
s += fmt.Sprintf("\n%d replica(s) started again on it, one by one.", v.Restarted)
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user