Ilker ManapandClaude Opus 5.5
f2c6f12cfb
For each layout modelplan.Options offers here — one card, tensor parallel across a node's cards, llama.cpp's layer split, a pipeline across nodes on a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes a token reads over each card's bandwidth, by its model name (a table, SXM and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on NVLink, 25 us and up on PCIe), turns between cards for a layer split, and a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a repository's files show the fastest, with the layout; mixed classes are judged class by class, FP8 on 8.9 cards only. Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB, 62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB, 144 measured, which says a model of experts reads 1.6 times its active share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics are not measured here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
38 lines
1.1 KiB
Go
38 lines
1.1 KiB
Go
package modelplan
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) {
|
|
for _, c := range []struct {
|
|
bytes int64
|
|
bw, want float64
|
|
}{
|
|
{1117320736, 24.9, 14.5}, // node01, qwen2.5-1.5b
|
|
{9_000_000_000, 41.9, 3.3}, // node05, qwen2.5-coder-14b
|
|
} {
|
|
got := CPUTokensPerSecond(c.bytes, c.bw)
|
|
if math.Abs(got-c.want)/c.want > 0.15 {
|
|
t.Errorf("%d bytes at %.1f GB/s: %.1f tok/s, measured %.1f", c.bytes, c.bw, got, c.want)
|
|
}
|
|
}
|
|
if CPUTokensPerSecond(0, 40) != 0 || CPUTokensPerSecond(1e9, 0) != 0 {
|
|
t.Error("an unknown size or bandwidth should give no figure")
|
|
}
|
|
}
|
|
|
|
func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
|
|
for name, want := range map[string]float64{
|
|
"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16,
|
|
"inclusionAI/Ling-flash-A2.5B": 1.6 * 2.5 / 30,
|
|
"Qwen/Qwen3-8B": 1,
|
|
"meta-llama/Llama-3.1-8B-Instruct": 1,
|
|
} {
|
|
if got := ActiveShare(name, 30e9); math.Abs(got-want) > 0.001 && !(want == 1 && got == 1) {
|
|
t.Errorf("%s: %.3f, want %.3f", name, got, want)
|
|
}
|
|
}
|
|
}
|