package modelplan import ( "math" "strings" ) // How fast a model answers on cards, one reply at a time, for each way the // planner can place it. As on a CPU, a token reads every weight it uses // once, so a card's memory bandwidth decides it; what changes with the // layout is what is added: // // - tensor parallel across a node's cards: each card reads its share, and // every layer adds two all-reduces, a few microseconds over NVLink and // several times that over PCIe; // - llama.cpp across cards splits by layer, so the cards take turns and // the reply is no faster than on one; // - pipeline parallel across nodes: the stages take turns too, so a // reply is as fast as one node's share, less a hop over the fabric for // each node. // // Measured on cuda01's Quadro RTX 6000 (672 GB/s) on 2026-10-27, with // llama.cpp's predicted_per_second over 128 tokens: // // qwen3-vl-8b Q8_0 8.7 GB dense 62.1 tok/s 0.82 of the bandwidth // qwen3-coder-30b-A3B Q4_K_M 18.6 GB 144 tok/s 1.6× the active share read // // vLLM's figures are taken to be llama.cpp's; nothing here has measured // tensor parallel or a fabric, so those parts are from what NCCL is known // to take, and say so where they are shown. // gpuReadShare is the part of a card's bandwidth a token is read at. const gpuReadShare = 0.82 // gpuTokenFixed is a token's cost that does not grow with the model. const gpuTokenFixed = 0.001 // gpuBandwidth is cards' memory bandwidth in GB/s, by what their name says, // most specific first: a class says "h100-80g" for an SXM card of 3.35 TB/s // and a PCIe one of 2, and only the name tells them apart. var gpuBandwidth = []struct { name string gbs float64 }{ {"gb300", 8000}, {"b300", 8000}, {"gb200", 8000}, {"b200", 8000}, {"gh200", 4000}, {"h200 nvl", 4800}, {"h200", 4800}, {"h100 nvl", 3900}, {"h100 pcie", 2000}, {"h100", 3350}, {"h800 pcie", 2000}, {"h800", 3350}, {"h20", 4000}, {"a100-sxm4-80gb", 2039}, {"a100 80gb pcie", 1935}, {"a100-pcie-80gb", 1935}, {"a100-sxm4-40gb", 1555}, {"a100-pcie-40gb", 1555}, {"a100 80gb", 2039}, {"a100", 1555}, {"a800", 2039}, {"a40", 696}, {"a30", 933}, {"a10g", 600}, {"a10", 600}, {"a16", 200}, {"a2", 200}, {"l40s", 864}, {"l40", 864}, {"l20", 864}, {"l4", 300}, {"rtx pro 6000", 1792}, {"rtx 6000 ada", 960}, {"6000 ada", 960}, {"rtx a6000", 768}, {"rtx 5000 ada", 576}, {"rtx a5000", 768}, {"rtx 4000 ada", 360}, {"rtx a4000", 448}, {"quadro rtx 8000", 672}, {"quadro rtx 6000", 672}, {"quadro rtx 5000", 448}, {"rtx 5090", 1792}, {"rtx 5080", 960}, {"rtx 4090", 1008}, {"rtx 4080", 717}, {"rtx 3090 ti", 1008}, {"rtx 3090", 936}, {"rtx 3080", 760}, {"titan rtx", 672}, {"v100s", 1134}, {"v100", 900}, {"t4", 320}, {"p100", 732}, {"p40", 346}, } // GPUBandwidthGBs is a card's memory bandwidth in GB/s from its name as the // driver gives it ("NVIDIA H100 80GB HBM3"), 0 for a card not known. func GPUBandwidthGBs(name string) float64 { n := strings.ToLower(name) n = strings.TrimPrefix(strings.TrimPrefix(n, "nvidia "), "tesla ") for _, g := range gpuBandwidth { if strings.Contains(n, g.name) { // "a10" is in "a100": a bare short name must end a word. if i := strings.Index(n, g.name) + len(g.name); i < len(n) && n[i] >= '0' && n[i] <= '9' { continue } return g.gbs } } return 0 } // Links is what a layout's cards talk over. type Links struct { // NVLink is whether the cards of one node are joined by NVLink; without // it they reach each other over PCIe. NVLink bool // InfiniBand is whether the fabric between nodes is InfiniBand rather // than RoCE; RateGbps its ports' rate. InfiniBand bool RateGbps int } // Microseconds an all-reduce of one token's activations takes, which is // latency rather than bandwidth at this size, and one hop between pipeline // stages on another node. const ( nvlinkAllReduce = 8e-6 pcieAllReduce = 25e-6 ibHop = 25e-6 roceHop = 40e-6 // layerSplitHop is llama.cpp handing a token from one card to the next. layerSplitHop = 20e-6 ) // GPUTokensPerSecond is about how many tokens a second one reply gets in a // layout, from cards of bwGBs: bytes is the model's weights, share the part // a token reads (ActiveShare), params its parameters (for its layers, which // set how many all-reduces a token takes). 0 when the card is not known. func GPUTokensPerSecond(o Option, bytes int64, share float64, params int64, bwGBs float64, l Links) float64 { if o.CPU || bytes <= 0 || bwGBs <= 0 { return 0 } read := float64(bytes) * share cards := max(o.GPUsPerNode, 1) bw := bwGBs * 1e9 * gpuReadShare var t float64 switch o.Engine { case "llamacpp": // By layer: the cards take turns, whichever holds the next layers. t = read/bw + gpuTokenFixed + float64(cards-1)*layerSplitHop default: t = read/(float64(cards)*bw) + gpuTokenFixed if cards > 1 { ar := pcieAllReduce + 5e-6*float64(cards-2) if l.NVLink { ar = nvlinkAllReduce } t += 2 * float64(Layers(params)) * ar } if o.Nodes > 1 { hop := roceHop if l.InfiniBand { hop = ibHop } t += float64(o.Nodes) * hop } } return 1 / t } // Layers is about how many layers a model of params has, which a listing // does not say: 28 for 1.5B, 36 for 8B, 80 for 70B, as most are built. func Layers(params int64) int { if params <= 0 { return 32 } return max(12, int(math.Round(16*math.Pow(float64(params)/1e9, 0.38)))) }