llmgw ctl model gpus MODEL N [--profile P] [--in-place],
POST /api/v1/models/{name}/gpus and the console's GPUs…: how many cards of
one node each replica uses. Tensor parallel for vLLM, SGLang and
TensorRT-LLM, held to the model's KV heads (modelplan.SplitsHeads, now one
rule for the planner and this); the layers split for llama.cpp, its old
--tensor-split dropped. Refused for a count no node has, for cards that
would not hold the weights and the context, and for engines on one card.
The replicas start again beside the old ones; with no room beside them the
command says so, and --in-place stops each first.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51aeadae3e
commit
e1cff0d4e3
16 files changed
+753
-13
No files matched your search
@@ -155,10 +155,7 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
|
||||
if nodes > 1 && h.Fabric == "" {
|
||||
continue
|
||||
}
|
||||
// The KV heads are split among the cards, or past their number each
|
||||
// card holds a copy of one: the width divides them or they divide
|
||||
// it. A model with 8 KV heads splits 2, 4, 8 or 16 ways, not 12.
|
||||
if m.KVHeads > 0 && m.KVHeads%width != 0 && width%m.KVHeads != 0 {
|
||||
if !SplitsHeads(m.KVHeads, width) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,9 @@ func (n PoolNode) kinds() []PoolKind {
|
||||
return []PoolKind{{Class: n.Class, GPUMemGB: n.GPUMemGB, GPUs: n.GPUs, FreeGPUs: n.FreeGPUs}}
|
||||
}
|
||||
|
||||
// Kinds is a node's devices by class: its one class, or each of several.
|
||||
func Kinds(n PoolNode) []PoolKind { return n.kinds() }
|
||||
|
||||
// isMIG reports whether a class names MIG instances: "a100-80g-mig-3g.40gb".
|
||||
// A replica runs on exactly one, never a group of them, so a plan offers one
|
||||
// instance and nothing wider. Read from the class label, as the card's memory
|
||||
@@ -263,12 +266,7 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64)
|
||||
perCard := func(width int) int64 {
|
||||
return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width))
|
||||
}
|
||||
// Tensor parallel splits the KV heads among the cards, or past their
|
||||
// number gives each card a copy of one: so the width divides the heads,
|
||||
// or the heads divide the width. Qwen3-235B has four and runs on eight.
|
||||
divides := func(w int) bool {
|
||||
return sh.KVHeads == 0 || sh.KVHeads%w == 0 || w%sh.KVHeads == 0
|
||||
}
|
||||
divides := func(w int) bool { return SplitsHeads(sh.KVHeads, w) }
|
||||
for _, c := range classes(nodes) {
|
||||
budget := float64(c.mem) * gpuShare
|
||||
opt := func(perNode, n int, fabric string) Option {
|
||||
@@ -516,3 +514,13 @@ func CardNeed(engine string, weights, kv int64, cards int) float64 {
|
||||
// CardBudget is what a card of a class may be planned to hold, in GB; 0 for
|
||||
// a class whose memory its name does not say.
|
||||
func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gpuShare }
|
||||
|
||||
// SplitsHeads reports whether tensor parallel can spread a model of kvHeads
|
||||
// KV heads over width cards. It splits the heads among the cards, or past
|
||||
// their number gives each card a copy of one: so the width divides the
|
||||
// heads, or the heads divide the width. Qwen3-235B has four and runs on
|
||||
// eight; a model with 8 splits 2, 4, 8 or 16 ways, not 12. 0 heads is
|
||||
// unknown, and allowed.
|
||||
func SplitsHeads(kvHeads, width int) bool {
|
||||
return kvHeads == 0 || width <= 1 || kvHeads%width == 0 || width%kvHeads == 0
|
||||
}
|
||||
Reference in new issue
Block a user