Ilker ManapandClaude Opus 5.5
f2c6f12cfb
For each layout modelplan.Options offers here — one card, tensor parallel across a node's cards, llama.cpp's layer split, a pipeline across nodes on a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes a token reads over each card's bandwidth, by its model name (a table, SXM and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on NVLink, 25 us and up on PCIe), turns between cards for a layer split, and a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a repository's files show the fastest, with the layout; mixed classes are judged class by class, FP8 on 8.9 cards only. Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB, 62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB, 144 measured, which says a model of experts reads 1.6 times its active share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics are not measured here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
71 lines
2.5 KiB
Go
71 lines
2.5 KiB
Go
package modelplan
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestACardsBandwidthIsKnownByItsName(t *testing.T) {
|
|
for name, want := range map[string]float64{
|
|
"Quadro RTX 6000": 672,
|
|
"NVIDIA RTX A6000": 768,
|
|
"NVIDIA RTX 6000 Ada Generation": 960,
|
|
"NVIDIA H100 80GB HBM3": 3350,
|
|
"NVIDIA H100 PCIe": 2000,
|
|
"NVIDIA A100-SXM4-80GB": 2039,
|
|
"NVIDIA A10": 600,
|
|
"Tesla T4": 320,
|
|
"Tesla V100-SXM2-32GB": 900,
|
|
"NVIDIA L4": 300,
|
|
"Some Card 9000": 0,
|
|
} {
|
|
if got := GPUBandwidthGBs(name); got != want {
|
|
t.Errorf("%q: %v GB/s, want %v", name, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGPUSpeedMatchesWhatCuda01Measured(t *testing.T) {
|
|
one := Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}
|
|
for _, c := range []struct {
|
|
name string
|
|
bytes int64
|
|
params int64
|
|
want float64
|
|
}{
|
|
{"qwen3-vl-8b", 8709519456, 8e9, 62.1},
|
|
{"Qwen3-Coder-30B-A3B-Instruct", 18556689568, 30.5e9, 144},
|
|
} {
|
|
got := GPUTokensPerSecond(one, c.bytes, ActiveShare(c.name, c.params), c.params, 672, Links{})
|
|
if math.Abs(got-c.want)/c.want > 0.15 {
|
|
t.Errorf("%s: %.1f tok/s, measured %.1f", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestALayoutsLinksDecideWhatMoreCardsBuy(t *testing.T) {
|
|
const bytes, params = 140e9, 70e9 // 70B in BF16
|
|
tp := func(cards, nodes int, l Links) float64 {
|
|
return GPUTokensPerSecond(Option{Engine: "vllm", GPUsPerNode: cards, Nodes: nodes}, bytes, 1, params, 3350, l)
|
|
}
|
|
nv, pcie := tp(4, 1, Links{NVLink: true}), tp(4, 1, Links{})
|
|
if !(nv > pcie) {
|
|
t.Errorf("four cards on NVLink (%.1f) should beat four on PCIe (%.1f)", nv, pcie)
|
|
}
|
|
if !(tp(8, 1, Links{NVLink: true}) > nv) {
|
|
t.Error("eight NVLinked cards should beat four")
|
|
}
|
|
// Pipeline across two nodes of eight is no faster than one node of
|
|
// eight, less the hops; InfiniBand's hop is the shorter.
|
|
one, ib, roce := tp(8, 1, Links{NVLink: true}), tp(8, 2, Links{NVLink: true, InfiniBand: true}), tp(8, 2, Links{NVLink: true})
|
|
if !(one > ib && ib > roce) {
|
|
t.Errorf("one node %.1f, two over InfiniBand %.1f, two over RoCE %.1f", one, ib, roce)
|
|
}
|
|
// llama.cpp across cards takes turns: no faster than one.
|
|
split := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 2, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{NVLink: true})
|
|
single := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{})
|
|
if split > single {
|
|
t.Errorf("a layer split (%.1f) should not beat one card (%.1f)", split, single)
|
|
}
|
|
}
|