Ilker ManapandClaude Opus 5.5
6a870a8c15
CI / test (push) Successful in 9m13s
From review. A MIG instance is given its share of the card's bandwidth, not the whole card's. NVLink between a layout's cards means all of them joined to each other, the largest such set on one ready node, not a count of cards with any peer: two bridged pairs are not four on NVLink. GH200 is matched before H200. An FP8 checkpoint is listed for cards of 8.0 and up, which vLLM runs with Marlin; 8.9 is the agent's bound for --quantization fp8. NVFP4 repositories tagged fp4 are sized at four bits. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
73 lines
2.6 KiB
Go
73 lines
2.6 KiB
Go
package modelplan
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestACardsBandwidthIsKnownByItsName(t *testing.T) {
|
|
for name, want := range map[string]float64{
|
|
"Quadro RTX 6000": 672,
|
|
"NVIDIA RTX A6000": 768,
|
|
"NVIDIA RTX 6000 Ada Generation": 960,
|
|
"NVIDIA H100 80GB HBM3": 3350,
|
|
"NVIDIA H100 PCIe": 2000,
|
|
"NVIDIA A100-SXM4-80GB": 2039,
|
|
"NVIDIA A10": 600,
|
|
"Tesla T4": 320,
|
|
"Tesla V100-SXM2-32GB": 900,
|
|
"NVIDIA L4": 300,
|
|
"NVIDIA GH200 480GB": 4000,
|
|
"NVIDIA H200": 4800,
|
|
"Some Card 9000": 0,
|
|
} {
|
|
if got := GPUBandwidthGBs(name); got != want {
|
|
t.Errorf("%q: %v GB/s, want %v", name, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGPUSpeedMatchesWhatCuda01Measured(t *testing.T) {
|
|
one := Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}
|
|
for _, c := range []struct {
|
|
name string
|
|
bytes int64
|
|
params int64
|
|
want float64
|
|
}{
|
|
{"qwen3-vl-8b", 8709519456, 8e9, 62.1},
|
|
{"Qwen3-Coder-30B-A3B-Instruct", 18556689568, 30.5e9, 144},
|
|
} {
|
|
got := GPUTokensPerSecond(one, c.bytes, ActiveShare(c.name, c.params), c.params, 672, Links{})
|
|
if math.Abs(got-c.want)/c.want > 0.15 {
|
|
t.Errorf("%s: %.1f tok/s, measured %.1f", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestALayoutsLinksDecideWhatMoreCardsBuy(t *testing.T) {
|
|
const bytes, params = 140e9, 70e9 // 70B in BF16
|
|
tp := func(cards, nodes int, l Links) float64 {
|
|
return GPUTokensPerSecond(Option{Engine: "vllm", GPUsPerNode: cards, Nodes: nodes}, bytes, 1, params, 3350, l)
|
|
}
|
|
nv, pcie := tp(4, 1, Links{NVLink: true}), tp(4, 1, Links{})
|
|
if !(nv > pcie) {
|
|
t.Errorf("four cards on NVLink (%.1f) should beat four on PCIe (%.1f)", nv, pcie)
|
|
}
|
|
if !(tp(8, 1, Links{NVLink: true}) > nv) {
|
|
t.Error("eight NVLinked cards should beat four")
|
|
}
|
|
// Pipeline across two nodes of eight is no faster than one node of
|
|
// eight, less the hops; InfiniBand's hop is the shorter.
|
|
one, ib, roce := tp(8, 1, Links{NVLink: true}), tp(8, 2, Links{NVLink: true, InfiniBand: true}), tp(8, 2, Links{NVLink: true})
|
|
if !(one > ib && ib > roce) {
|
|
t.Errorf("one node %.1f, two over InfiniBand %.1f, two over RoCE %.1f", one, ib, roce)
|
|
}
|
|
// llama.cpp across cards takes turns: no faster than one.
|
|
split := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 2, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{NVLink: true})
|
|
single := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{})
|
|
if split > single {
|
|
t.Errorf("a layer split (%.1f) should not beat one card (%.1f)", split, single)
|
|
}
|
|
}
|