llmgw/internal/modelplan/gpuspeed.go

package modelplan

import (
	"math"
	"strings"
)

// How fast a model answers on cards, one reply at a time, for each way the
// planner can place it. As on a CPU, a token reads every weight it uses
// once, so a card's memory bandwidth decides it; what changes with the
// layout is what is added:
//
//   - tensor parallel across a node's cards: each card reads its share, and
//     every layer adds two all-reduces, a few microseconds over NVLink and
//     several times that over PCIe;
//   - llama.cpp across cards splits by layer, so the cards take turns and
//     the reply is no faster than on one;
//   - pipeline parallel across nodes: the stages take turns too, so a
//     reply is as fast as one node's share, less a hop over the fabric for
//     each node.
//
// Measured on cuda01's Quadro RTX 6000 (672 GB/s) on 2026-10-27, with
// llama.cpp's predicted_per_second over 128 tokens:
//
//	qwen3-vl-8b    Q8_0    8.7 GB dense  62.1 tok/s  0.82 of the bandwidth
//	qwen3-coder-30b-A3B Q4_K_M 18.6 GB   144 tok/s   1.6× the active share read
//
// vLLM's figures are taken to be llama.cpp's; nothing here has measured
// tensor parallel or a fabric, so those parts are from what NCCL is known
// to take, and say so where they are shown.

// gpuReadShare is the part of a card's bandwidth a token is read at.
const gpuReadShare = 0.82

// gpuTokenFixed is a token's cost that does not grow with the model.
const gpuTokenFixed = 0.001

// gpuBandwidth is cards' memory bandwidth in GB/s, by what their name says,
// most specific first: a class says "h100-80g" for an SXM card of 3.35 TB/s
// and a PCIe one of 2, and only the name tells them apart.
var gpuBandwidth = []struct {
	name string
	gbs  float64
}{
	{"gb300", 8000}, {"b300", 8000}, {"gb200", 8000}, {"b200", 8000},
	{"h200 nvl", 4800}, {"h200", 4800},
	{"h100 nvl", 3900}, {"h100 pcie", 2000}, {"h100", 3350}, {"h800 pcie", 2000}, {"h800", 3350},
	{"gh200", 4000}, {"h20", 4000},
	{"a100-sxm4-80gb", 2039}, {"a100 80gb pcie", 1935}, {"a100-pcie-80gb", 1935},
	{"a100-sxm4-40gb", 1555}, {"a100-pcie-40gb", 1555}, {"a100 80gb", 2039}, {"a100", 1555},
	{"a800", 2039}, {"a40", 696}, {"a30", 933}, {"a10g", 600}, {"a10", 600}, {"a16", 200}, {"a2", 200},
	{"l40s", 864}, {"l40", 864}, {"l20", 864}, {"l4", 300},
	{"rtx pro 6000", 1792}, {"rtx 6000 ada", 960}, {"6000 ada", 960}, {"rtx a6000", 768},
	{"rtx 5000 ada", 576}, {"rtx a5000", 768}, {"rtx 4000 ada", 360}, {"rtx a4000", 448},
	{"quadro rtx 8000", 672}, {"quadro rtx 6000", 672}, {"quadro rtx 5000", 448},
	{"rtx 5090", 1792}, {"rtx 5080", 960}, {"rtx 4090", 1008}, {"rtx 4080", 717},
	{"rtx 3090 ti", 1008}, {"rtx 3090", 936}, {"rtx 3080", 760}, {"titan rtx", 672},
	{"v100s", 1134}, {"v100", 900}, {"t4", 320}, {"p100", 732}, {"p40", 346},
}

// GPUBandwidthGBs is a card's memory bandwidth in GB/s from its name as the
// driver gives it ("NVIDIA H100 80GB HBM3"), 0 for a card not known.
func GPUBandwidthGBs(name string) float64 {
	n := strings.ToLower(name)
	n = strings.TrimPrefix(strings.TrimPrefix(n, "nvidia "), "tesla ")
	for _, g := range gpuBandwidth {
		if strings.Contains(n, g.name) {
			// "a10" is in "a100": a bare short name must end a word.
			if i := strings.Index(n, g.name) + len(g.name); i < len(n) && n[i] >= '0' && n[i] <= '9' {
				continue
			}
			return g.gbs
		}
	}
	return 0
}

// Links is what a layout's cards talk over.
type Links struct {
	// NVLink is whether the cards of one node are joined by NVLink; without
	// it they reach each other over PCIe.
	NVLink bool
	// InfiniBand is whether the fabric between nodes is InfiniBand rather
	// than RoCE; RateGbps its ports' rate.
	InfiniBand bool
	RateGbps   int
}

// Microseconds an all-reduce of one token's activations takes, which is
// latency rather than bandwidth at this size, and one hop between pipeline
// stages on another node.
const (
	nvlinkAllReduce = 8e-6
	pcieAllReduce   = 25e-6
	ibHop           = 25e-6
	roceHop         = 40e-6
	// layerSplitHop is llama.cpp handing a token from one card to the next.
	layerSplitHop = 20e-6
)

// GPUTokensPerSecond is about how many tokens a second one reply gets in a
// layout, from cards of bwGBs: bytes is the model's weights, share the part
// a token reads (ActiveShare), params its parameters (for its layers, which
// set how many all-reduces a token takes). 0 when the card is not known.
func GPUTokensPerSecond(o Option, bytes int64, share float64, params int64, bwGBs float64, l Links) float64 {
	if o.CPU || bytes <= 0 || bwGBs <= 0 {
		return 0
	}
	read := float64(bytes) * share
	cards := max(o.GPUsPerNode, 1)
	bw := bwGBs * 1e9 * gpuReadShare
	var t float64
	switch o.Engine {
	case "llamacpp":
		// By layer: the cards take turns, whichever holds the next layers.
		t = read/bw + gpuTokenFixed + float64(cards-1)*layerSplitHop
	default:
		t = read/(float64(cards)*bw) + gpuTokenFixed
		if cards > 1 {
			ar := pcieAllReduce + 5e-6*float64(cards-2)
			if l.NVLink {
				ar = nvlinkAllReduce
			}
			t += 2 * float64(Layers(params)) * ar
		}
		if o.Nodes > 1 {
			hop := roceHop
			if l.InfiniBand {
				hop = ibHop
			}
			t += float64(o.Nodes) * hop
		}
	}
	return 1 / t
}

// Layers is about how many layers a model of params has, which a listing
// does not say: 28 for 1.5B, 36 for 8B, 80 for 70B, as most are built.
func Layers(params int64) int {
	if params <= 0 {
		return 32
	}
	return max(12, int(math.Round(16*math.Pow(float64(params)/1e9, 0.38))))
}