diff --git a/CHANGELOG.md b/CHANGELOG.md index aec9e64..fdb6a58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,16 @@ APIs change between minors without a deprecation period. ### Added +- **And on the cards, as this cluster would lay it out.** The explorer's + **GPU tok/s** and a repository's files give one reply's speed for each + layout the planner offers here — one card, tensor parallel across a + node's cards, llama.cpp's layer split, a pipeline across nodes on a + fabric — the fastest of them, with the layout in the tooltip. From each + card's memory bandwidth by its model name, NVLink or PCIe between a + node's cards, and InfiniBand or RoCE between nodes; a cluster of mixed + classes is judged class by class. A model of experts reads 1.6 times its + active share, as Qwen3-Coder-30B-A3B did on cuda01's RTX 6000 (144 + tok/s; qwen3-vl-8b Q8: 62). - **How fast a model would answer on a CPU, before it is fetched.** Each agent measures how fast its node's memory reads when it starts (`Host` gets `mem_bandwidth_gbs`, the node's view `mem_bandwidth_gbs`); the diff --git a/docs/quick-install.md b/docs/quick-install.md index c474380..1bba62f 100644 --- a/docs/quick-install.md +++ b/docs/quick-install.md @@ -163,6 +163,19 @@ model's size — for a model of experts named like `30B-A3B`, its active part. On the test cluster it came within 15% of what llama.cpp measured. Several replies at once share the reads and each is slower. +**GPU tok/s** is the same on the cards, for the layout this cluster would +give the model — the fewest cards that hold it, of each class, the fastest of +those; its tooltip says which ("2× h100-80g, NVLink, vLLM", "2 nodes × 1× +a100-80g over ib0 (InfiniBand 200 Gb/s), vLLM"). A card's memory bandwidth is +known from its model name (an H100 SXM and an H100 PCIe differ by half); a card +not in the table gets no figure. Tensor parallel across a node's cards divides +what each reads, and adds two all-reduces a layer, short over NVLink and +longer over PCIe; llama.cpp across cards splits by layer, so the cards take +turns and a reply is no faster than on one; a pipeline across nodes takes turns +too, and adds a hop over the fabric for each node. One card's figures were +measured on the test cluster's RTX 6000; tensor parallel and the fabric are +from what NCCL is known to take, not measured here. + 1. Click a model to read its card. 2. Pick one of its files, usually a `q4_k_m` for a first try, and click **Fetch**. It is downloaded into the mirror once, for every node. diff --git a/internal/modelplan/cpuspeed.go b/internal/modelplan/cpuspeed.go index 4e2fe29..5c89fdd 100644 --- a/internal/modelplan/cpuspeed.go +++ b/internal/modelplan/cpuspeed.go @@ -38,14 +38,19 @@ func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 { return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed) } +// moeRead is what a token of a model of experts reads over its active +// share. +const moeRead = 1.6 + // activeRE finds a model of experts' active parameters in its name, the // way they are written: Qwen3-30B-A3B, Ling-flash-A2.5B. var activeRE = regexp.MustCompile(`(?i)(?:^|[-_./])a(\d+(?:\.\d+)?)b(?:$|[-_.])`) // ActiveShare is the part of a model's weights one token reads: for a model // of experts named with its active parameters (…-30B-A3B) that over its -// total, otherwise 1. Rough — the shared layers are read for every token — -// and what a name says is all a listing has. +// total, otherwise 1 — and 1.6 times that, which is what Qwen3-Coder-30B-A3B +// read on cuda01's card (144 tok/s from 18.6 GB): the shared layers, the +// embedding and the routing are read for every token whatever the experts. func ActiveShare(name string, params int64) float64 { m := activeRE.FindStringSubmatch(name) if m == nil || params <= 0 { @@ -55,7 +60,7 @@ func ActiveShare(name string, params int64) float64 { if err != nil || b <= 0 { return 1 } - if s := b * 1e9 / float64(params); s < 1 { + if s := moeRead * b * 1e9 / float64(params); s < 1 { return s } return 1 diff --git a/internal/modelplan/cpuspeed_test.go b/internal/modelplan/cpuspeed_test.go index f60a54a..d7aa8cf 100644 --- a/internal/modelplan/cpuspeed_test.go +++ b/internal/modelplan/cpuspeed_test.go @@ -25,8 +25,8 @@ func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) { func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) { for name, want := range map[string]float64{ - "Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.1, - "inclusionAI/Ling-flash-A2.5B": 2.5 / 30, + "Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16, + "inclusionAI/Ling-flash-A2.5B": 1.6 * 2.5 / 30, "Qwen/Qwen3-8B": 1, "meta-llama/Llama-3.1-8B-Instruct": 1, } { diff --git a/internal/modelplan/gpuspeed.go b/internal/modelplan/gpuspeed.go new file mode 100644 index 0000000..eb528b3 --- /dev/null +++ b/internal/modelplan/gpuspeed.go @@ -0,0 +1,144 @@ +package modelplan + +import ( + "math" + "strings" +) + +// How fast a model answers on cards, one reply at a time, for each way the +// planner can place it. As on a CPU, a token reads every weight it uses +// once, so a card's memory bandwidth decides it; what changes with the +// layout is what is added: +// +// - tensor parallel across a node's cards: each card reads its share, and +// every layer adds two all-reduces, a few microseconds over NVLink and +// several times that over PCIe; +// - llama.cpp across cards splits by layer, so the cards take turns and +// the reply is no faster than on one; +// - pipeline parallel across nodes: the stages take turns too, so a +// reply is as fast as one node's share, less a hop over the fabric for +// each node. +// +// Measured on cuda01's Quadro RTX 6000 (672 GB/s) on 2026-10-27, with +// llama.cpp's predicted_per_second over 128 tokens: +// +// qwen3-vl-8b Q8_0 8.7 GB dense 62.1 tok/s 0.82 of the bandwidth +// qwen3-coder-30b-A3B Q4_K_M 18.6 GB 144 tok/s 1.6× the active share read +// +// vLLM's figures are taken to be llama.cpp's; nothing here has measured +// tensor parallel or a fabric, so those parts are from what NCCL is known +// to take, and say so where they are shown. + +// gpuReadShare is the part of a card's bandwidth a token is read at. +const gpuReadShare = 0.82 + +// gpuTokenFixed is a token's cost that does not grow with the model. +const gpuTokenFixed = 0.001 + +// gpuBandwidth is cards' memory bandwidth in GB/s, by what their name says, +// most specific first: a class says "h100-80g" for an SXM card of 3.35 TB/s +// and a PCIe one of 2, and only the name tells them apart. +var gpuBandwidth = []struct { + name string + gbs float64 +}{ + {"gb300", 8000}, {"b300", 8000}, {"gb200", 8000}, {"b200", 8000}, + {"h200 nvl", 4800}, {"h200", 4800}, + {"h100 nvl", 3900}, {"h100 pcie", 2000}, {"h100", 3350}, {"h800 pcie", 2000}, {"h800", 3350}, + {"gh200", 4000}, {"h20", 4000}, + {"a100-sxm4-80gb", 2039}, {"a100 80gb pcie", 1935}, {"a100-pcie-80gb", 1935}, + {"a100-sxm4-40gb", 1555}, {"a100-pcie-40gb", 1555}, {"a100 80gb", 2039}, {"a100", 1555}, + {"a800", 2039}, {"a40", 696}, {"a30", 933}, {"a10g", 600}, {"a10", 600}, {"a16", 200}, {"a2", 200}, + {"l40s", 864}, {"l40", 864}, {"l20", 864}, {"l4", 300}, + {"rtx pro 6000", 1792}, {"rtx 6000 ada", 960}, {"6000 ada", 960}, {"rtx a6000", 768}, + {"rtx 5000 ada", 576}, {"rtx a5000", 768}, {"rtx 4000 ada", 360}, {"rtx a4000", 448}, + {"quadro rtx 8000", 672}, {"quadro rtx 6000", 672}, {"quadro rtx 5000", 448}, + {"rtx 5090", 1792}, {"rtx 5080", 960}, {"rtx 4090", 1008}, {"rtx 4080", 717}, + {"rtx 3090 ti", 1008}, {"rtx 3090", 936}, {"rtx 3080", 760}, {"titan rtx", 672}, + {"v100s", 1134}, {"v100", 900}, {"t4", 320}, {"p100", 732}, {"p40", 346}, +} + +// GPUBandwidthGBs is a card's memory bandwidth in GB/s from its name as the +// driver gives it ("NVIDIA H100 80GB HBM3"), 0 for a card not known. +func GPUBandwidthGBs(name string) float64 { + n := strings.ToLower(name) + n = strings.TrimPrefix(strings.TrimPrefix(n, "nvidia "), "tesla ") + for _, g := range gpuBandwidth { + if strings.Contains(n, g.name) { + // "a10" is in "a100": a bare short name must end a word. + if i := strings.Index(n, g.name) + len(g.name); i < len(n) && n[i] >= '0' && n[i] <= '9' { + continue + } + return g.gbs + } + } + return 0 +} + +// Links is what a layout's cards talk over. +type Links struct { + // NVLink is whether the cards of one node are joined by NVLink; without + // it they reach each other over PCIe. + NVLink bool + // InfiniBand is whether the fabric between nodes is InfiniBand rather + // than RoCE; RateGbps its ports' rate. + InfiniBand bool + RateGbps int +} + +// Microseconds an all-reduce of one token's activations takes, which is +// latency rather than bandwidth at this size, and one hop between pipeline +// stages on another node. +const ( + nvlinkAllReduce = 8e-6 + pcieAllReduce = 25e-6 + ibHop = 25e-6 + roceHop = 40e-6 + // layerSplitHop is llama.cpp handing a token from one card to the next. + layerSplitHop = 20e-6 +) + +// GPUTokensPerSecond is about how many tokens a second one reply gets in a +// layout, from cards of bwGBs: bytes is the model's weights, share the part +// a token reads (ActiveShare), params its parameters (for its layers, which +// set how many all-reduces a token takes). 0 when the card is not known. +func GPUTokensPerSecond(o Option, bytes int64, share float64, params int64, bwGBs float64, l Links) float64 { + if o.CPU || bytes <= 0 || bwGBs <= 0 { + return 0 + } + read := float64(bytes) * share + cards := max(o.GPUsPerNode, 1) + bw := bwGBs * 1e9 * gpuReadShare + var t float64 + switch o.Engine { + case "llamacpp": + // By layer: the cards take turns, whichever holds the next layers. + t = read/bw + gpuTokenFixed + float64(cards-1)*layerSplitHop + default: + t = read/(float64(cards)*bw) + gpuTokenFixed + if cards > 1 { + ar := pcieAllReduce + 5e-6*float64(cards-2) + if l.NVLink { + ar = nvlinkAllReduce + } + t += 2 * float64(Layers(params)) * ar + } + if o.Nodes > 1 { + hop := roceHop + if l.InfiniBand { + hop = ibHop + } + t += float64(o.Nodes) * hop + } + } + return 1 / t +} + +// Layers is about how many layers a model of params has, which a listing +// does not say: 28 for 1.5B, 36 for 8B, 80 for 70B, as most are built. +func Layers(params int64) int { + if params <= 0 { + return 32 + } + return max(12, int(math.Round(16*math.Pow(float64(params)/1e9, 0.38)))) +} diff --git a/internal/modelplan/gpuspeed_test.go b/internal/modelplan/gpuspeed_test.go new file mode 100644 index 0000000..e116c5d --- /dev/null +++ b/internal/modelplan/gpuspeed_test.go @@ -0,0 +1,70 @@ +package modelplan + +import ( + "math" + "testing" +) + +func TestACardsBandwidthIsKnownByItsName(t *testing.T) { + for name, want := range map[string]float64{ + "Quadro RTX 6000": 672, + "NVIDIA RTX A6000": 768, + "NVIDIA RTX 6000 Ada Generation": 960, + "NVIDIA H100 80GB HBM3": 3350, + "NVIDIA H100 PCIe": 2000, + "NVIDIA A100-SXM4-80GB": 2039, + "NVIDIA A10": 600, + "Tesla T4": 320, + "Tesla V100-SXM2-32GB": 900, + "NVIDIA L4": 300, + "Some Card 9000": 0, + } { + if got := GPUBandwidthGBs(name); got != want { + t.Errorf("%q: %v GB/s, want %v", name, got, want) + } + } +} + +func TestGPUSpeedMatchesWhatCuda01Measured(t *testing.T) { + one := Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1} + for _, c := range []struct { + name string + bytes int64 + params int64 + want float64 + }{ + {"qwen3-vl-8b", 8709519456, 8e9, 62.1}, + {"Qwen3-Coder-30B-A3B-Instruct", 18556689568, 30.5e9, 144}, + } { + got := GPUTokensPerSecond(one, c.bytes, ActiveShare(c.name, c.params), c.params, 672, Links{}) + if math.Abs(got-c.want)/c.want > 0.15 { + t.Errorf("%s: %.1f tok/s, measured %.1f", c.name, got, c.want) + } + } +} + +func TestALayoutsLinksDecideWhatMoreCardsBuy(t *testing.T) { + const bytes, params = 140e9, 70e9 // 70B in BF16 + tp := func(cards, nodes int, l Links) float64 { + return GPUTokensPerSecond(Option{Engine: "vllm", GPUsPerNode: cards, Nodes: nodes}, bytes, 1, params, 3350, l) + } + nv, pcie := tp(4, 1, Links{NVLink: true}), tp(4, 1, Links{}) + if !(nv > pcie) { + t.Errorf("four cards on NVLink (%.1f) should beat four on PCIe (%.1f)", nv, pcie) + } + if !(tp(8, 1, Links{NVLink: true}) > nv) { + t.Error("eight NVLinked cards should beat four") + } + // Pipeline across two nodes of eight is no faster than one node of + // eight, less the hops; InfiniBand's hop is the shorter. + one, ib, roce := tp(8, 1, Links{NVLink: true}), tp(8, 2, Links{NVLink: true, InfiniBand: true}), tp(8, 2, Links{NVLink: true}) + if !(one > ib && ib > roce) { + t.Errorf("one node %.1f, two over InfiniBand %.1f, two over RoCE %.1f", one, ib, roce) + } + // llama.cpp across cards takes turns: no faster than one. + split := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 2, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{NVLink: true}) + single := GPUTokensPerSecond(Option{Engine: "llamacpp", GPUsPerNode: 1, Nodes: 1}, 40e9, 1, 70e9, 3350, Links{}) + if split > single { + t.Errorf("a layer split (%.1f) should not beat one card (%.1f)", split, single) + } +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index dc6d2bb..058e0a5 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -61,6 +61,8 @@ type repoSet struct { // CPUSpeed is about how many tokens a second one reply gets from a GGUF // on the CPU of CPUNode, the fastest node that holds it; empty otherwise. CPUSpeed, CPUNode string + // GPUSpeed is the same on the cards, laid out as GPULayout says. + GPUSpeed, GPULayout string hf.FileSet // Sources are every file in the set, in the order the registry listed them. // A shard set goes as one action — a fifteen-shard model is not fifteen @@ -828,6 +830,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params)) rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node } + if format := artifactFormat(set.Name); rs.Verdict.Servable && format != "" { + tps, layout := fit.gpuSpeed(langOf(r), modelplan.Artifact{Name: set.Name, Format: format, Bytes: set.Bytes, + Quant: modelplan.QuantOf(set.Name)}, got.Params, modelplan.ActiveShare(got.ID, got.Params), false) + rs.GPUSpeed, rs.GPULayout = speedLabel(tps), layout + } } for _, f := range set.Files { rs.Sources = append(rs.Sources, diff --git a/internal/web/explore_fit.go b/internal/web/explore_fit.go index dced133..8c440d4 100644 --- a/internal/web/explore_fit.go +++ b/internal/web/explore_fit.go @@ -32,6 +32,12 @@ type clusterFit struct { // cpus are the nodes whose memory's speed their agent measured, for // how fast a model would answer on one. cpus []cpuHost + // classBW is each card class's memory bandwidth in GB/s, from its + // cards' names; nvlink how many cards of a class one node has joined + // by NVLink, at most; fabrics what each fabric is, by name. + classBW map[string]float64 + nvlink map[string]int + fabrics map[string]modelplan.Links } // cpuHost is a node as a model on its CPU sees it. @@ -55,8 +61,31 @@ func (p *AccountPages) fitOf() *clusterFit { if len(c.pool) == 0 { return nil } + c.classBW, c.nvlink, c.fabrics = map[string]float64{}, map[string]int{}, map[string]modelplan.Links{} newest := map[string]float64{} for _, n := range nodes { + joined := map[string]int{} + for _, g := range n.GPUs { + if bw := modelplan.GPUBandwidthGBs(g.Name); bw > c.classBW[g.Class] { + c.classBW[g.Class] = bw + } + if len(g.NVLinkPeers) > 0 { + joined[g.Class]++ + } + } + for class, k := range joined { + c.nvlink[class] = max(c.nvlink[class], k) + } + for _, p := range n.IBPorts { + if p.Fabric != "" && p.Active { + l := c.fabrics[p.Fabric] + l.InfiniBand = p.LinkLayer == "InfiniBand" + if l.RateGbps == 0 || p.RateGbps < l.RateGbps { + l.RateGbps = p.RateGbps + } + c.fabrics[p.Fabric] = l + } + } if n.State == "ready" && n.MemBandwidthGBs > 0 { c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs}) } @@ -156,13 +185,7 @@ func (c *clusterFit) note(l *Lang) string { // holds judges one file set of an open repository, whose size is known: a // verdict that it cannot be served here, and false, when no node holds it. func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) { - format := "" - switch lower := strings.ToLower(name); { - case strings.HasSuffix(lower, ".gguf"): - format = "gguf" - case strings.HasSuffix(lower, ".safetensors"): - format = "safetensors" - } + format := artifactFormat(name) if format == "" || bytes <= 0 { return gguf.Verdict{}, true } @@ -234,3 +257,96 @@ func (c *clusterFit) rowSpeed(m hf.Model) (string, string) { tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params())) return speedLabel(tps), node } + +// gpuSpeed is about how many tokens a second one reply gets from art on +// the cards, laid out as the planner would place it — the fewest cards +// that hold it, for each class — the fastest of those, and how it is laid +// out. 0 when nothing here holds it or no card's speed is known. +func (c *clusterFit) gpuSpeed(l *Lang, art modelplan.Artifact, params int64, share float64, fp8 bool) (float64, string) { + pool := c.pool + if fp8 { + pool = c.fp8 + } + best, layout := 0.0, "" + for _, o := range modelplan.Options([]modelplan.Artifact{art}, modelplan.Shape{Params: params}, pool, 0) { + if o.CPU || !o.Fits() { + continue + } + links := modelplan.Links{NVLink: c.nvlink[o.Class] >= o.GPUsPerNode && o.GPUsPerNode > 1} + if o.Nodes > 1 { + f := c.fabrics[o.Fabric] + links.InfiniBand, links.RateGbps = f.InfiniBand, f.RateGbps + } + tps := modelplan.GPUTokensPerSecond(o, art.Bytes, share, params, c.classBW[o.Class], links) + if tps > best { + best, layout = tps, layoutLabel(l, o, links) + } + } + return best, layout +} + +// layoutLabel says how a replica would be laid out: "1× h100-80g", +// "4× h100-80g, NVLink, vLLM", "2 nodes × 8× h100-80g over ib0 (InfiniBand +// 400 Gb/s), vLLM". +func layoutLabel(l *Lang, o modelplan.Option, links modelplan.Links) string { + s := strconv.Itoa(max(o.GPUsPerNode, 1)) + "× " + o.Class + if o.GPUsPerNode > 1 { + if links.NVLink { + s += ", NVLink" + } else { + s += ", PCIe" + } + } + if o.Nodes > 1 { + kind := "RoCE" + if links.InfiniBand { + kind = "InfiniBand" + } + if links.RateGbps > 0 { + kind += " " + strconv.Itoa(links.RateGbps) + " Gb/s" + } + s = l.T("explore.layout.nodes", o.Nodes, s, o.Fabric, kind) + } + engine := "llama.cpp" + if o.Engine == "vllm" { + engine = "vLLM" + } + return s + ", " + engine +} + +// rowGPU is a listed repository's speed on the cards: its GGUF at Q4_K_M +// on llama.cpp, its safetensors on vLLM, whichever is faster here. +func (c *clusterFit) rowGPU(l *Lang, m hf.Model) (string, string) { + if !c.gpus() || m.Params() <= 0 { + return "", "" + } + params, share := m.Params(), modelplan.ActiveShare(m.ID, m.Params()) + best, layout := 0.0, "" + if m.GGUF != nil || m.HasTag("gguf") { + b := int64(float64(params) * ggufBitsPerWeight / 8) + best, layout = c.gpuSpeed(l, modelplan.Artifact{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b, Quant: "Q4_K_M"}, + params, share, false) + } + if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") { + b := m.WeightBytes() + if b == 0 { + b = params * 2 + } + if tps, lay := c.gpuSpeed(l, modelplan.Artifact{Name: "m.safetensors", Format: "safetensors", Bytes: b}, + params, share, m.FP8()); tps > best { + best, layout = tps, lay + } + } + return speedLabel(best), layout +} + +// artifactFormat is "gguf" or "safetensors" by a file set's name, or "". +func artifactFormat(name string) string { + switch lower := strings.ToLower(name); { + case strings.HasSuffix(lower, ".gguf"): + return "gguf" + case strings.HasSuffix(lower, ".safetensors"): + return "safetensors" + } + return "" +} diff --git a/internal/web/explore_fit_test.go b/internal/web/explore_fit_test.go index adc58bd..ffcfbce 100644 --- a/internal/web/explore_fit_test.go +++ b/internal/web/explore_fit_test.go @@ -153,8 +153,8 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) { ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()), ) // 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second. - // The 30B model of experts reads its 3B active part: about 15. - for _, want := range []string{">~5.8<", ">~15<", "on node05's CPU"} { + // The 30B model of experts reads 1.6 times its 3B active part: about 9.4. + for _, want := range []string{">~5.8<", ">~9.4<", "on node05's CPU"} { if !strings.Contains(body, want) { t.Errorf("the listing lacks %q: %s", want, body) } @@ -169,3 +169,59 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) { t.Errorf("each file should say its speed on the CPU, by its own size: %s", page) } } + +// cards is a node of n cards of a class, named as the driver names them, +// joined by NVLink or not, on a fabric or none. +func cards(id, class, name string, n int, cc string, nvlink bool, fabric string) adminapi.NodeView { + v := gpuNode(id, class, n, cc, 1024) + for i := 0; i < n; i++ { + g := adminapi.GPUView{Index: i, Name: name, Class: class} + if nvlink { + for j := 0; j < n; j++ { + if j != i { + g.NVLinkPeers = append(g.NVLinkPeers, j) + } + } + } + v.GPUs = append(v.GPUs, g) + } + if fabric != "" { + v.Fabrics = []string{fabric} + v.IBPorts = []adminapi.IBPortView{{HCA: "mlx5_0", Port: 1, Fabric: fabric, LinkLayer: "InfiniBand", + RateGbps: 200, Active: true}} + } + return v +} + +func TestTheExplorerSaysHowFastAModelAnswersOnTheCards(t *testing.T) { + // The PoC's one Quadro RTX 6000: an 8B GGUF at Q4_K_M, 4.8 GB, on + // 672 GB/s. + poc := []adminapi.NodeView{cards("cuda01", "6000-24g", "Quadro RTX 6000", 1, "7.5", false, "")} + body, _ := listing(t, poc, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now())) + if !strings.Contains(body, `title="1× 6000-24g, llama.cpp">~103<`) { + t.Errorf("an 8B on cuda01's card should be about 103 a second: %s", body) + } + + // A mixed site: eight H100s on NVLink, four L40S on PCIe. 70B in BF16 + // fits two H100s or four L40S, and the planner takes the fewest cards: + // the H100s are the faster. + mixed := []adminapi.NodeView{ + cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 8, "9.0", true, ""), + cards("l", "l40s-48g", "NVIDIA L40S", 4, "8.9", false, ""), + } + body, _ = listing(t, mixed, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9})) + if !strings.Contains(body, "2× h100-80g, NVLink, vLLM") { + t.Errorf("70B should run fastest on two NVLinked H100s: %s", body) + } + + // Two nodes of one A100 each on InfiniBand: 70B fits neither, and the + // pipeline across them is what the planner offers. + ib := []adminapi.NodeView{ + cards("a", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"), + cards("b", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"), + } + body, _ = listing(t, ib, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9})) + if !strings.Contains(body, "2 nodes × 1× a100-80g over ib0 (InfiniBand 200 Gb/s), vLLM") { + t.Errorf("70B should be laid out across both nodes over ib0: %s", body) + } +} diff --git a/internal/web/locales/en/mirror.json b/internal/web/locales/en/mirror.json index 4eb8aad..8f1f946 100644 --- a/internal/web/locales/en/mirror.json +++ b/internal/web/locales/en/mirror.json @@ -82,6 +82,9 @@ "explore.col.downloads": "Downloads", "explore.col.likes": "Likes", "explore.col.updated": "Updated", + "explore.col.gpu": "GPU tok/s", + "explore.col.gpu_title": "About how many tokens a second one reply gets on the cards, laid out as this cluster would place it — the fewest cards that hold it: each card's memory bandwidth (from its model), tensor parallel's all-reduces over NVLink or PCIe, a hop over the fabric for each node of a pipeline. The layout is in each figure's tooltip.", + "explore.layout.nodes": "%d nodes × %s over %s (%s)", "explore.col.cpu": "CPU tok/s", "explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).", "explore.cpu_on": "on %s's CPU", @@ -142,6 +145,7 @@ "card.model": "model", "card.cannot_serve": "cannot be served", "card.imatrix": "it is an importance matrix for quantising, not a model", + "card.gpu_speed": "%s tok/s on %s", "card.cpu_speed": "%s tok/s on %s's CPU", "card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB", "card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF", diff --git a/internal/web/locales/tr/mirror.json b/internal/web/locales/tr/mirror.json index 803c271..6283943 100644 --- a/internal/web/locales/tr/mirror.json +++ b/internal/web/locales/tr/mirror.json @@ -80,6 +80,9 @@ "explore.col.downloads": "İndirme", "explore.col.likes": "Beğeni", "explore.col.updated": "Güncellendi", + "explore.col.gpu": "GPU token/sn", + "explore.col.gpu_title": "Bu kümenin onu yerleştireceği biçimde — modeli alan en az kartla — tek bir yanıtın kartlarda saniyede aşağı yukarı kaç token alacağı: her kartın bellek bant genişliği (modelinden), tensör paralelinin NVLink ya da PCIe üzerindeki all-reduce'ları, boru hattının her düğümü için ağ üzerinden bir atlama. Yerleşim her rakamın ipucunda.", + "explore.layout.nodes": "%d düğüm × %s, %s üzerinden (%s)", "explore.col.cpu": "CPU token/sn", "explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).", "explore.cpu_on": "%s düğümünün CPU'sunda", @@ -137,6 +140,7 @@ "card.model": "model", "card.cannot_serve": "sunulamaz", "card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi", + "card.gpu_speed": "%[2]s üzerinde %[1]s token/sn", "card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn", "card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB", "card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin", diff --git a/internal/web/mirror_explore.go b/internal/web/mirror_explore.go index 13c3d60..0132572 100644 --- a/internal/web/mirror_explore.go +++ b/internal/web/mirror_explore.go @@ -96,6 +96,8 @@ type exploreRow struct { // CPUSpeed is about how many tokens a second one reply would get on the // CPU of CPUNode, the fastest node that holds it; empty when unknown. CPUSpeed, CPUNode string + // GPUSpeed is the same on the cards, laid out as GPULayout says. + GPUSpeed, GPULayout string } // ParamsLabel is the parameter count as people say it: 7.6B, 494M. @@ -316,6 +318,7 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) { } if fit != nil { row.CPUSpeed, row.CPUNode = fit.rowSpeed(m) + row.GPUSpeed, row.GPULayout = fit.rowGPU(l, m) } if e.Here && row.Verdict.Checked && !row.Verdict.Servable { e.Hidden++ diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index c2aba7d..aaf26a7 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -302,6 +302,7 @@ {{if .Rows}} + {{range .Rows}} @@ -311,6 +312,7 @@ + @@ -470,6 +472,7 @@ document.querySelector("table.explore").addEventListener("submit", async functio {{with .Companions}}{{t "card.companions" (len .)}}{{end}} {{if eq .Kind "adapter"}}{{t "card.adapter" .Arch}} {{else if eq .Kind "model"}}{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}{{end}} + {{if .GPUSpeed}}{{t "card.gpu_speed" .GPUSpeed .GPULayout}}{{end}} {{if .CPUSpeed}}{{t "card.cpu_speed" .CPUSpeed .CPUNode}}{{end}}
{{t "explore.col.repo"}}{{t "explore.col.size"}}{{t "explore.col.arch"}}{{t "explore.col.context"}}{{t "explore.col.gpu"}} {{t "explore.col.cpu"}} {{t "explore.col.downloads"}}{{t "explore.col.likes"}}{{t "explore.col.updated"}}
{{.ParamsLabel}} {{if .Architecture}}{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}{{else}}—{{end}} {{.ContextLabel}}{{if .GPUSpeed}}{{.GPUSpeed}}{{else}}—{{end}} {{if .CPUSpeed}}{{.CPUSpeed}}{{else}}—{{end}} {{compact .Downloads}} {{compact .Likes}} {{bytes .Bytes}} {{with .SHA256}}{{slice . 0 12}}{{else}}—{{end}}