A plan for a big model: experts spread, any number of nodes, memory said

A mixture-of-experts model on more than one card is served with vLLM's
--enable-expert-parallel; config.json's experts are read by any of their
names, and a model that also reads pictures is sized by its text_config.
Tensor parallelism past the KV heads is allowed when it is a multiple of
them (Qwen3-235B, four KV heads, runs on eight cards), which the plan
had refused. Several nodes are offered in any number, not only powers of
two. Each way says what it needs of a card, part by part, and of all its
cards; the serve form checks a context longer than planned still fits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-20 08:45:33 +02:00
1 parent 314cc5e6a3
commit 4fe9b46451
12 files changed
+257 -34

No files matched your search

+70
View File
@@ -123,3 +123,73 @@ func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) {
t.Errorf("GPU: %s first", o[0].Artifact)
}
}
// A model of experts spread over more than one card has them spread whole
// (expert parallelism); on one card there is nothing to spread.
func TestAModelOfExpertsSpreadsItsExpertsOverItsCards(t *testing.T) {
nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}}
moe := Shape{KVBytesPerToken: 94 * 4 * 256 * 2, KVHeads: 4, Params: 235_000_000_000, Experts: 128}
opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 470 * gib, Quant: "bf16"}}, moe, nodes, 8192)
if len(opts) == 0 || opts[0].GPUsPerNode < 2 || !opts[0].ExpertParallel {
t.Fatalf("options %+v", opts)
}
small := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 30 * gib, Quant: "bf16"}},
Shape{KVHeads: 4, Experts: 128}, nodes, 8192)
if len(small) == 0 || small[0].GPUsPerNode != 1 || small[0].ExpertParallel {
t.Errorf("one card: %+v", small)
}
dense := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}},
Shape{KVHeads: 8}, nodes, 8192)
if len(dense) == 0 || dense[0].ExpertParallel {
t.Errorf("a dense model: %+v", dense)
}
}
// Too big for two nodes, it is offered on three, not only on four.
func TestAModelTooBigForTwoNodesIsOfferedOnThree(t *testing.T) {
var nodes []PoolNode
for _, id := range []string{"a", "b", "c", "d"} {
nodes = append(nodes, PoolNode{ID: id, Class: "l40s-48g", GPUMemGB: 48, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"})
}
// About 900 GB: more than two nodes' 16 cards hold, less than three's 24.
opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 900 * gib, Quant: "bf16"}},
Shape{KVHeads: 8}, nodes, 4096)
if len(opts) == 0 || opts[0].Nodes != 3 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" {
t.Fatalf("options %+v", labels(opts))
}
}
// Each way says what it needs part by part, and for another context.
func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) {
nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}}
kvPerToken := int64(80 * 8 * 128 * 2 * 2) // a 70B: 320 KB a token
opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}},
Shape{KVBytesPerToken: kvPerToken, KVHeads: 8}, nodes, 8192)
o := opts[0]
if o.GPUsPerNode != 4 {
t.Fatalf("expected four cards: %+v", o)
}
sum := o.WeightsGB + o.ContextGB + o.EngineGB
if d := sum - o.NeedGB; d > 0.01 || d < -0.01 || o.WeightsGB != 35 {
t.Errorf("parts %.2f+%.2f+%.2f do not make %.2f", o.WeightsGB, o.ContextGB, o.EngineGB, o.NeedGB)
}
text := o.MemoryText()
for _, want := range []string{"GB a card", "weights 35.0", "context 0.6", "of 72 usable", "on 4 cards"} {
if !strings.Contains(text, want) {
t.Errorf("%q lacks %q", text, want)
}
}
// Sixteen times the context: the weights stay, the context grows, and
// it no longer fits.
long := o.WithContext(kvPerToken, 8192*16)
if long.WeightsGB != o.WeightsGB || long.ContextGB < o.ContextGB*15.9 {
t.Errorf("for a longer context: %+v", long)
}
if !o.Fits() {
t.Error("the planned way does not fit")
}
huge := o.WithContext(kvPerToken, 8192*256)
if huge.Fits() {
t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText())
}
}