A plan for a big model: experts spread, any number of nodes, memory said

A mixture-of-experts model on more than one card is served with vLLM's
--enable-expert-parallel; config.json's experts are read by any of their
names, and a model that also reads pictures is sized by its text_config.
Tensor parallelism past the KV heads is allowed when it is a multiple of
them (Qwen3-235B, four KV heads, runs on eight cards), which the plan
had refused. Several nodes are offered in any number, not only powers of
two. Each way says what it needs of a card, part by part, and of all its
cards; the serve form checks a context longer than planned still fits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-20 08:45:33 +02:00
1 parent 314cc5e6a3
commit 4fe9b46451
12 files changed
+257 -34

No files matched your search

+81 -6
View File
@@ -33,6 +33,8 @@ type Shape struct {
KVHeads int
// Params is the parameter count, for bits per weight; 0 is unknown.
Params int64
// Experts is a mixture-of-experts model's routed experts; 0 for dense.
Experts int
}
// PoolNode is one ready node, as a plan sees it.
@@ -96,11 +98,71 @@ type Option struct {
FreeNow bool `json:"free_now"`
// BitsPerWeight is the artifact's precision, 0 when unknown.
BitsPerWeight float64 `json:"bits_per_weight,omitempty"`
// What NeedGB is made of, a card's share (or, on the CPU, the node's
// RAM): the weights, the context's KV cache, and the engine's own.
WeightsGB float64 `json:"weights_gb,omitempty"`
ContextGB float64 `json:"context_gb,omitempty"`
EngineGB float64 `json:"engine_gb,omitempty"`
// ExpertParallel spreads a mixture-of-experts model's experts over the
// cards, whole, rather than slicing each across them: vLLM's
// --enable-expert-parallel, for a model of experts on more than one card.
ExpertParallel bool `json:"expert_parallel,omitempty"`
}
// GPUs is every card one replica takes.
func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) }
// TotalGB is what one replica takes of every card together, or of the
// node's RAM on the CPU.
func (o Option) TotalGB() float64 {
if o.CPU {
return o.NeedGB
}
return o.NeedGB * float64(o.GPUs())
}
// Fits is whether a card, or the node, holds what it needs.
func (o Option) Fits() bool { return o.NeedGB <= o.HaveGB }
// WithContext is the option for another context: kvPerToken a token's KV
// cache, the whole model; the weights and the engine's share stay.
func (o Option) WithContext(kvPerToken int64, tokens int) Option {
if kvPerToken <= 0 || tokens <= 0 {
return o
}
ctx := GB(int64(tokens) * kvPerToken)
if !o.CPU {
ctx /= float64(max(o.GPUs(), 1))
}
o.NeedGB += ctx - o.ContextGB
o.ContextGB = ctx
return o
}
// MemoryText says what it needs, part by part.
func (o Option) MemoryText() string {
parts := fmt.Sprintf("weights %.1f + context %.1f + engine %.1f", o.WeightsGB, o.ContextGB, o.EngineGB)
if o.WeightsGB == 0 && o.ContextGB == 0 {
parts = ""
}
if o.CPU {
s := fmt.Sprintf("%.1f GB of RAM", o.NeedGB)
if parts != "" {
s += " (" + parts + ")"
}
return s + fmt.Sprintf(" of %.0f usable", o.HaveGB)
}
s := fmt.Sprintf("%.1f GB a card", o.NeedGB)
if parts != "" {
s += " (" + parts + ")"
}
s += fmt.Sprintf(" of %.0f usable", o.HaveGB)
if o.GPUs() > 1 {
s += fmt.Sprintf("; %.0f GB on %d cards", o.TotalGB(), o.GPUs())
}
return s
}
// Label says where it runs, in a few words.
func (o Option) Label() string {
if o.CPU {
@@ -151,7 +213,8 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option {
// The CPU: the whole replica in a node's RAM.
total := a.Bytes + kv + ggufOverhead(a.Bytes)
cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true,
NeedGB: GB(total), BitsPerWeight: bpw}
NeedGB: GB(total), BitsPerWeight: bpw,
WeightsGB: GB(a.Bytes), ContextGB: GB(kv), EngineGB: GB(ggufOverhead(a.Bytes))}
for _, n := range nodes {
have := float64(n.HostMemGB) * hostShare
if n.HostMemGB <= 0 || GB(total) > have {
@@ -175,7 +238,9 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option {
continue
}
o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class,
GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw}
GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw,
WeightsGB: GB(a.Bytes / int64(width)), ContextGB: GB(kv / int64(width)),
EngineGB: GB(ggufOverhead(a.Bytes / int64(width)))}
for _, fab := range c.fabrics {
for _, n := range fab.nodes {
if n.GPUs >= width {
@@ -198,13 +263,20 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64)
perCard := func(width int) int64 {
return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width))
}
// Tensor parallel within a node must divide the KV heads.
divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) }
// Tensor parallel splits the KV heads among the cards, or past their
// number gives each card a copy of one: so the width divides the heads,
// or the heads divide the width. Qwen3-235B has four and runs on eight.
divides := func(w int) bool {
return sh.KVHeads == 0 || sh.KVHeads%w == 0 || w%sh.KVHeads == 0
}
for _, c := range classes(nodes) {
budget := float64(c.mem) * gpuShare
opt := func(perNode, n int, fabric string) Option {
cards := int64(perNode * n)
return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric,
GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw}
GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw,
WeightsGB: GB(a.Bytes / cards), ContextGB: GB(kv / cards), EngineGB: GB(overheadBytes(a.Bytes / cards)),
ExpertParallel: sh.Experts > 1 && perNode*n > 1}
}
// One node: the narrowest power of two that holds it.
single := false
@@ -243,7 +315,10 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64)
if fab.name == "" {
continue
}
for n := 2; n <= len(fab.nodes); n *= 2 {
// Any number of nodes: pipeline stages need not be a power of
// two, and a third node is what a model just too big for two
// needs.
for n := 2; n <= len(fab.nodes); n++ {
if GB(perCard(w*n)) > budget {
continue
}