A served model can be moved to more cards, or fewer
CI / test (push) Successful in 10m28s

llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 16:40:49 +01:00
1 parent 51aeadae3e
commit e1cff0d4e3
16 files changed
+753 -13

No files matched your search

+1 -4
View File
@@ -155,10 +155,7 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
if nodes > 1 && h.Fabric == "" {
continue
}
// The KV heads are split among the cards, or past their number each
// card holds a copy of one: the width divides them or they divide
// it. A model with 8 KV heads splits 2, 4, 8 or 16 ways, not 12.
if m.KVHeads > 0 && m.KVHeads%width != 0 && width%m.KVHeads != 0 {
if !SplitsHeads(m.KVHeads, width) {
continue
}
+14 -6
View File
@@ -71,6 +71,9 @@ func (n PoolNode) kinds() []PoolKind {
return []PoolKind{{Class: n.Class, GPUMemGB: n.GPUMemGB, GPUs: n.GPUs, FreeGPUs: n.FreeGPUs}}
}
// Kinds is a node's devices by class: its one class, or each of several.
func Kinds(n PoolNode) []PoolKind { return n.kinds() }
// isMIG reports whether a class names MIG instances: "a100-80g-mig-3g.40gb".
// A replica runs on exactly one, never a group of them, so a plan offers one
// instance and nothing wider. Read from the class label, as the card's memory
@@ -263,12 +266,7 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64)
perCard := func(width int) int64 {
return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width))
}
// Tensor parallel splits the KV heads among the cards, or past their
// number gives each card a copy of one: so the width divides the heads,
// or the heads divide the width. Qwen3-235B has four and runs on eight.
divides := func(w int) bool {
return sh.KVHeads == 0 || sh.KVHeads%w == 0 || w%sh.KVHeads == 0
}
divides := func(w int) bool { return SplitsHeads(sh.KVHeads, w) }
for _, c := range classes(nodes) {
budget := float64(c.mem) * gpuShare
opt := func(perNode, n int, fabric string) Option {
@@ -516,3 +514,13 @@ func CardNeed(engine string, weights, kv int64, cards int) float64 {
// CardBudget is what a card of a class may be planned to hold, in GB; 0 for
// a class whose memory its name does not say.
func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gpuShare }
// SplitsHeads reports whether tensor parallel can spread a model of kvHeads
// KV heads over width cards. It splits the heads among the cards, or past
// their number gives each card a copy of one: so the width divides the
// heads, or the heads divide the width. Qwen3-235B has four and runs on
// eight; a model with 8 splits 2, 4, 8 or 16 ways, not 12. 0 heads is
// unknown, and allowed.
func SplitsHeads(kvHeads, width int) bool {
return kvHeads == 0 || width <= 1 || kvHeads%width == 0 || width%kvHeads == 0
}