package modelplan import ( "math" "testing" ) func TestACardsBandwidthIsKnownByItsName(t *testing.T) { for name, want := range map[string]float64{ "Quadro RTX 6000": 672, "NVIDIA RTX A6000": 768, "NVIDIA RTX 6000 Ada Generation": 960, "NVIDIA H100 80GB HBM3": 3350, "NVIDIA H100 PCIe": 2000, "NVIDIA A100-SXM4-80GB": 2039, "NVIDIA A10": 600, "Tesla T4": 320, "Tesla V100-SXM2-32GB": 900, "NVIDIA L4": 300, "NVIDIA GH200 480GB": 4000, "NVIDIA H200": 4800, "Some Card 9000": 0, } { if got := GPUBandwidthGBs(name); got != want { t.Errorf("%q: %v GB/s, want %v", name, got, want) } } } func TestGPUSpeedMatchesWhatCuda01Measured(t *testing.T) { one := Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1} for _, c := range []struct { name string bytes int64 params int64 want float64 }{ {"qwen3-vl-8b", 8709519456, 8e9, 62.1}, {"Qwen3-Coder-30B-A3B-Instruct", 18556689568, 30.5e9, 144}, } { got := GPUTokensPerSecond(one, c.bytes, ActiveShare(c.name, c.params), c.params, 672, Links{}) if math.Abs(got-c.want)/c.want > 0.15 { t.Errorf("%s: %.1f tok/s, measured %.1f", c.name, got, c.want) } } } func TestALayoutsLinksDecideWhatMoreCardsBuy(t *testing.T) { const bytes, params = 140e9, 70e9 // 70B in BF16 tp := func(cards, nodes int, l Links) float64 { return GPUTokensPerSecond(Option{Engine: "vllm", GPUsPerNode: cards, Nodes: nodes}, bytes, 1, params, 3350, l) } nv, pcie := tp(4, 1, Links{NVLink: true}), tp(4, 1, Links{}) if !(nv > pcie) { t.Errorf("four cards on NVLink (%.1f) should beat four on PCIe (%.1f)", nv, pcie) } if !(tp(8, 1, Links{NVLink: true}) > nv) { t.Error("eight NVLinked cards should beat four") } // Pipeline across two nodes of eight is no faster than one node of // eight, less the hops; InfiniBand's hop is the shorter. one, ib, roce := tp(8, 1, Links{NVLink: true}), tp(8, 2, Links{NVLink: true, InfiniBand: true}), tp(8, 2, Links{NVLink: true}) if !(one > ib && ib > roce) { t.Errorf("one node %.1f, two over InfiniBand %.1f, two over RoCE %.1f", one, ib, roce) } // llama.cpp across cards takes turns: no faster than one. split := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 2, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{NVLink: true}) single := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{}) if split > single { t.Errorf("a layer split (%.1f) should not beat one card (%.1f)", split, single) } }