How fast a model would answer on the cards, as this cluster would lay it out
CI / test (push) Successful in 9m1s
CI / test (push) Successful in 9m1s
For each layout modelplan.Options offers here — one card, tensor parallel across a node's cards, llama.cpp's layer split, a pipeline across nodes on a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes a token reads over each card's bandwidth, by its model name (a table, SXM and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on NVLink, 25 us and up on PCIe), turns between cards for a layer split, and a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a repository's files show the fastest, with the layout; mixed classes are judged class by class, FP8 on 8.9 cards only. Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB, 62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB, 144 measured, which says a model of experts reads 1.6 times its active share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics are not measured here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e70d6caa2f
commit
f2c6f12cfb
13 files changed
+449
-14
No files matched your search
@@ -38,14 +38,19 @@ func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 {
|
||||
return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed)
|
||||
}
|
||||
|
||||
// moeRead is what a token of a model of experts reads over its active
|
||||
// share.
|
||||
const moeRead = 1.6
|
||||
|
||||
// activeRE finds a model of experts' active parameters in its name, the
|
||||
// way they are written: Qwen3-30B-A3B, Ling-flash-A2.5B.
|
||||
var activeRE = regexp.MustCompile(`(?i)(?:^|[-_./])a(\d+(?:\.\d+)?)b(?:$|[-_.])`)
|
||||
|
||||
// ActiveShare is the part of a model's weights one token reads: for a model
|
||||
// of experts named with its active parameters (…-30B-A3B) that over its
|
||||
// total, otherwise 1. Rough — the shared layers are read for every token —
|
||||
// and what a name says is all a listing has.
|
||||
// total, otherwise 1 — and 1.6 times that, which is what Qwen3-Coder-30B-A3B
|
||||
// read on cuda01's card (144 tok/s from 18.6 GB): the shared layers, the
|
||||
// embedding and the routing are read for every token whatever the experts.
|
||||
func ActiveShare(name string, params int64) float64 {
|
||||
m := activeRE.FindStringSubmatch(name)
|
||||
if m == nil || params <= 0 {
|
||||
@@ -55,7 +60,7 @@ func ActiveShare(name string, params int64) float64 {
|
||||
if err != nil || b <= 0 {
|
||||
return 1
|
||||
}
|
||||
if s := b * 1e9 / float64(params); s < 1 {
|
||||
if s := moeRead * b * 1e9 / float64(params); s < 1 {
|
||||
return s
|
||||
}
|
||||
return 1
|
||||
|
||||
@@ -25,8 +25,8 @@ func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) {
|
||||
|
||||
func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
|
||||
for name, want := range map[string]float64{
|
||||
"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.1,
|
||||
"inclusionAI/Ling-flash-A2.5B": 2.5 / 30,
|
||||
"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16,
|
||||
"inclusionAI/Ling-flash-A2.5B": 1.6 * 2.5 / 30,
|
||||
"Qwen/Qwen3-8B": 1,
|
||||
"meta-llama/Llama-3.1-8B-Instruct": 1,
|
||||
} {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// How fast a model answers on cards, one reply at a time, for each way the
|
||||
// planner can place it. As on a CPU, a token reads every weight it uses
|
||||
// once, so a card's memory bandwidth decides it; what changes with the
|
||||
// layout is what is added:
|
||||
//
|
||||
// - tensor parallel across a node's cards: each card reads its share, and
|
||||
// every layer adds two all-reduces, a few microseconds over NVLink and
|
||||
// several times that over PCIe;
|
||||
// - llama.cpp across cards splits by layer, so the cards take turns and
|
||||
// the reply is no faster than on one;
|
||||
// - pipeline parallel across nodes: the stages take turns too, so a
|
||||
// reply is as fast as one node's share, less a hop over the fabric for
|
||||
// each node.
|
||||
//
|
||||
// Measured on cuda01's Quadro RTX 6000 (672 GB/s) on 2026-10-27, with
|
||||
// llama.cpp's predicted_per_second over 128 tokens:
|
||||
//
|
||||
// qwen3-vl-8b Q8_0 8.7 GB dense 62.1 tok/s 0.82 of the bandwidth
|
||||
// qwen3-coder-30b-A3B Q4_K_M 18.6 GB 144 tok/s 1.6× the active share read
|
||||
//
|
||||
// vLLM's figures are taken to be llama.cpp's; nothing here has measured
|
||||
// tensor parallel or a fabric, so those parts are from what NCCL is known
|
||||
// to take, and say so where they are shown.
|
||||
|
||||
// gpuReadShare is the part of a card's bandwidth a token is read at.
|
||||
const gpuReadShare = 0.82
|
||||
|
||||
// gpuTokenFixed is a token's cost that does not grow with the model.
|
||||
const gpuTokenFixed = 0.001
|
||||
|
||||
// gpuBandwidth is cards' memory bandwidth in GB/s, by what their name says,
|
||||
// most specific first: a class says "h100-80g" for an SXM card of 3.35 TB/s
|
||||
// and a PCIe one of 2, and only the name tells them apart.
|
||||
var gpuBandwidth = []struct {
|
||||
name string
|
||||
gbs float64
|
||||
}{
|
||||
{"gb300", 8000}, {"b300", 8000}, {"gb200", 8000}, {"b200", 8000},
|
||||
{"h200 nvl", 4800}, {"h200", 4800},
|
||||
{"h100 nvl", 3900}, {"h100 pcie", 2000}, {"h100", 3350}, {"h800 pcie", 2000}, {"h800", 3350},
|
||||
{"gh200", 4000}, {"h20", 4000},
|
||||
{"a100-sxm4-80gb", 2039}, {"a100 80gb pcie", 1935}, {"a100-pcie-80gb", 1935},
|
||||
{"a100-sxm4-40gb", 1555}, {"a100-pcie-40gb", 1555}, {"a100 80gb", 2039}, {"a100", 1555},
|
||||
{"a800", 2039}, {"a40", 696}, {"a30", 933}, {"a10g", 600}, {"a10", 600}, {"a16", 200}, {"a2", 200},
|
||||
{"l40s", 864}, {"l40", 864}, {"l20", 864}, {"l4", 300},
|
||||
{"rtx pro 6000", 1792}, {"rtx 6000 ada", 960}, {"6000 ada", 960}, {"rtx a6000", 768},
|
||||
{"rtx 5000 ada", 576}, {"rtx a5000", 768}, {"rtx 4000 ada", 360}, {"rtx a4000", 448},
|
||||
{"quadro rtx 8000", 672}, {"quadro rtx 6000", 672}, {"quadro rtx 5000", 448},
|
||||
{"rtx 5090", 1792}, {"rtx 5080", 960}, {"rtx 4090", 1008}, {"rtx 4080", 717},
|
||||
{"rtx 3090 ti", 1008}, {"rtx 3090", 936}, {"rtx 3080", 760}, {"titan rtx", 672},
|
||||
{"v100s", 1134}, {"v100", 900}, {"t4", 320}, {"p100", 732}, {"p40", 346},
|
||||
}
|
||||
|
||||
// GPUBandwidthGBs is a card's memory bandwidth in GB/s from its name as the
|
||||
// driver gives it ("NVIDIA H100 80GB HBM3"), 0 for a card not known.
|
||||
func GPUBandwidthGBs(name string) float64 {
|
||||
n := strings.ToLower(name)
|
||||
n = strings.TrimPrefix(strings.TrimPrefix(n, "nvidia "), "tesla ")
|
||||
for _, g := range gpuBandwidth {
|
||||
if strings.Contains(n, g.name) {
|
||||
// "a10" is in "a100": a bare short name must end a word.
|
||||
if i := strings.Index(n, g.name) + len(g.name); i < len(n) && n[i] >= '0' && n[i] <= '9' {
|
||||
continue
|
||||
}
|
||||
return g.gbs
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Links is what a layout's cards talk over.
|
||||
type Links struct {
|
||||
// NVLink is whether the cards of one node are joined by NVLink; without
|
||||
// it they reach each other over PCIe.
|
||||
NVLink bool
|
||||
// InfiniBand is whether the fabric between nodes is InfiniBand rather
|
||||
// than RoCE; RateGbps its ports' rate.
|
||||
InfiniBand bool
|
||||
RateGbps int
|
||||
}
|
||||
|
||||
// Microseconds an all-reduce of one token's activations takes, which is
|
||||
// latency rather than bandwidth at this size, and one hop between pipeline
|
||||
// stages on another node.
|
||||
const (
|
||||
nvlinkAllReduce = 8e-6
|
||||
pcieAllReduce = 25e-6
|
||||
ibHop = 25e-6
|
||||
roceHop = 40e-6
|
||||
// layerSplitHop is llama.cpp handing a token from one card to the next.
|
||||
layerSplitHop = 20e-6
|
||||
)
|
||||
|
||||
// GPUTokensPerSecond is about how many tokens a second one reply gets in a
|
||||
// layout, from cards of bwGBs: bytes is the model's weights, share the part
|
||||
// a token reads (ActiveShare), params its parameters (for its layers, which
|
||||
// set how many all-reduces a token takes). 0 when the card is not known.
|
||||
func GPUTokensPerSecond(o Option, bytes int64, share float64, params int64, bwGBs float64, l Links) float64 {
|
||||
if o.CPU || bytes <= 0 || bwGBs <= 0 {
|
||||
return 0
|
||||
}
|
||||
read := float64(bytes) * share
|
||||
cards := max(o.GPUsPerNode, 1)
|
||||
bw := bwGBs * 1e9 * gpuReadShare
|
||||
var t float64
|
||||
switch o.Engine {
|
||||
case "llamacpp":
|
||||
// By layer: the cards take turns, whichever holds the next layers.
|
||||
t = read/bw + gpuTokenFixed + float64(cards-1)*layerSplitHop
|
||||
default:
|
||||
t = read/(float64(cards)*bw) + gpuTokenFixed
|
||||
if cards > 1 {
|
||||
ar := pcieAllReduce + 5e-6*float64(cards-2)
|
||||
if l.NVLink {
|
||||
ar = nvlinkAllReduce
|
||||
}
|
||||
t += 2 * float64(Layers(params)) * ar
|
||||
}
|
||||
if o.Nodes > 1 {
|
||||
hop := roceHop
|
||||
if l.InfiniBand {
|
||||
hop = ibHop
|
||||
}
|
||||
t += float64(o.Nodes) * hop
|
||||
}
|
||||
}
|
||||
return 1 / t
|
||||
}
|
||||
|
||||
// Layers is about how many layers a model of params has, which a listing
|
||||
// does not say: 28 for 1.5B, 36 for 8B, 80 for 70B, as most are built.
|
||||
func Layers(params int64) int {
|
||||
if params <= 0 {
|
||||
return 32
|
||||
}
|
||||
return max(12, int(math.Round(16*math.Pow(float64(params)/1e9, 0.38))))
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestACardsBandwidthIsKnownByItsName(t *testing.T) {
|
||||
for name, want := range map[string]float64{
|
||||
"Quadro RTX 6000": 672,
|
||||
"NVIDIA RTX A6000": 768,
|
||||
"NVIDIA RTX 6000 Ada Generation": 960,
|
||||
"NVIDIA H100 80GB HBM3": 3350,
|
||||
"NVIDIA H100 PCIe": 2000,
|
||||
"NVIDIA A100-SXM4-80GB": 2039,
|
||||
"NVIDIA A10": 600,
|
||||
"Tesla T4": 320,
|
||||
"Tesla V100-SXM2-32GB": 900,
|
||||
"NVIDIA L4": 300,
|
||||
"Some Card 9000": 0,
|
||||
} {
|
||||
if got := GPUBandwidthGBs(name); got != want {
|
||||
t.Errorf("%q: %v GB/s, want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUSpeedMatchesWhatCuda01Measured(t *testing.T) {
|
||||
one := Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bytes int64
|
||||
params int64
|
||||
want float64
|
||||
}{
|
||||
{"qwen3-vl-8b", 8709519456, 8e9, 62.1},
|
||||
{"Qwen3-Coder-30B-A3B-Instruct", 18556689568, 30.5e9, 144},
|
||||
} {
|
||||
got := GPUTokensPerSecond(one, c.bytes, ActiveShare(c.name, c.params), c.params, 672, Links{})
|
||||
if math.Abs(got-c.want)/c.want > 0.15 {
|
||||
t.Errorf("%s: %.1f tok/s, measured %.1f", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestALayoutsLinksDecideWhatMoreCardsBuy(t *testing.T) {
|
||||
const bytes, params = 140e9, 70e9 // 70B in BF16
|
||||
tp := func(cards, nodes int, l Links) float64 {
|
||||
return GPUTokensPerSecond(Option{Engine: "vllm", GPUsPerNode: cards, Nodes: nodes}, bytes, 1, params, 3350, l)
|
||||
}
|
||||
nv, pcie := tp(4, 1, Links{NVLink: true}), tp(4, 1, Links{})
|
||||
if !(nv > pcie) {
|
||||
t.Errorf("four cards on NVLink (%.1f) should beat four on PCIe (%.1f)", nv, pcie)
|
||||
}
|
||||
if !(tp(8, 1, Links{NVLink: true}) > nv) {
|
||||
t.Error("eight NVLinked cards should beat four")
|
||||
}
|
||||
// Pipeline across two nodes of eight is no faster than one node of
|
||||
// eight, less the hops; InfiniBand's hop is the shorter.
|
||||
one, ib, roce := tp(8, 1, Links{NVLink: true}), tp(8, 2, Links{NVLink: true, InfiniBand: true}), tp(8, 2, Links{NVLink: true})
|
||||
if !(one > ib && ib > roce) {
|
||||
t.Errorf("one node %.1f, two over InfiniBand %.1f, two over RoCE %.1f", one, ib, roce)
|
||||
}
|
||||
// llama.cpp across cards takes turns: no faster than one.
|
||||
split := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 2, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{NVLink: true})
|
||||
single := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{})
|
||||
if split > single {
|
||||
t.Errorf("a layer split (%.1f) should not beat one card (%.1f)", split, single)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user