How fast a model would answer on the cards, as this cluster would lay it out
CI / test (push) Successful in 9m1s

For each layout modelplan.Options offers here — one card, tensor parallel
across a node's cards, llama.cpp's layer split, a pipeline across nodes on
a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes
a token reads over each card's bandwidth, by its model name (a table, SXM
and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on
NVLink, 25 us and up on PCIe), turns between cards for a layer split, and
a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a
repository's files show the fastest, with the layout; mixed classes are
judged class by class, FP8 on 8.9 cards only.

Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB,
62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB,
144 measured, which says a model of experts reads 1.6 times its active
share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics
are not measured here.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-28 00:19:36 +01:00
1 parent e70d6caa2f
commit f2c6f12cfb
13 files changed
+449 -14

No files matched your search

+7
View File
@@ -61,6 +61,8 @@ type repoSet struct {
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
CPUSpeed, CPUNode string
// GPUSpeed is the same on the cards, laid out as GPULayout says.
GPUSpeed, GPULayout string
hf.FileSet
// Sources are every file in the set, in the order the registry listed them.
// A shard set goes as one action — a fifteen-shard model is not fifteen
@@ -828,6 +830,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
}
if format := artifactFormat(set.Name); rs.Verdict.Servable && format != "" {
tps, layout := fit.gpuSpeed(langOf(r), modelplan.Artifact{Name: set.Name, Format: format, Bytes: set.Bytes,
Quant: modelplan.QuantOf(set.Name)}, got.Params, modelplan.ActiveShare(got.ID, got.Params), false)
rs.GPUSpeed, rs.GPULayout = speedLabel(tps), layout
}
}
for _, f := range set.Files {
rs.Sources = append(rs.Sources,
+123 -7
View File
@@ -32,6 +32,12 @@ type clusterFit struct {
// cpus are the nodes whose memory's speed their agent measured, for
// how fast a model would answer on one.
cpus []cpuHost
// classBW is each card class's memory bandwidth in GB/s, from its
// cards' names; nvlink how many cards of a class one node has joined
// by NVLink, at most; fabrics what each fabric is, by name.
classBW map[string]float64
nvlink map[string]int
fabrics map[string]modelplan.Links
}
// cpuHost is a node as a model on its CPU sees it.
@@ -55,8 +61,31 @@ func (p *AccountPages) fitOf() *clusterFit {
if len(c.pool) == 0 {
return nil
}
c.classBW, c.nvlink, c.fabrics = map[string]float64{}, map[string]int{}, map[string]modelplan.Links{}
newest := map[string]float64{}
for _, n := range nodes {
joined := map[string]int{}
for _, g := range n.GPUs {
if bw := modelplan.GPUBandwidthGBs(g.Name); bw > c.classBW[g.Class] {
c.classBW[g.Class] = bw
}
if len(g.NVLinkPeers) > 0 {
joined[g.Class]++
}
}
for class, k := range joined {
c.nvlink[class] = max(c.nvlink[class], k)
}
for _, p := range n.IBPorts {
if p.Fabric != "" && p.Active {
l := c.fabrics[p.Fabric]
l.InfiniBand = p.LinkLayer == "InfiniBand"
if l.RateGbps == 0 || p.RateGbps < l.RateGbps {
l.RateGbps = p.RateGbps
}
c.fabrics[p.Fabric] = l
}
}
if n.State == "ready" && n.MemBandwidthGBs > 0 {
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
}
@@ -156,13 +185,7 @@ func (c *clusterFit) note(l *Lang) string {
// holds judges one file set of an open repository, whose size is known: a
// verdict that it cannot be served here, and false, when no node holds it.
func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) {
format := ""
switch lower := strings.ToLower(name); {
case strings.HasSuffix(lower, ".gguf"):
format = "gguf"
case strings.HasSuffix(lower, ".safetensors"):
format = "safetensors"
}
format := artifactFormat(name)
if format == "" || bytes <= 0 {
return gguf.Verdict{}, true
}
@@ -234,3 +257,96 @@ func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
return speedLabel(tps), node
}
// gpuSpeed is about how many tokens a second one reply gets from art on
// the cards, laid out as the planner would place it — the fewest cards
// that hold it, for each class — the fastest of those, and how it is laid
// out. 0 when nothing here holds it or no card's speed is known.
func (c *clusterFit) gpuSpeed(l *Lang, art modelplan.Artifact, params int64, share float64, fp8 bool) (float64, string) {
pool := c.pool
if fp8 {
pool = c.fp8
}
best, layout := 0.0, ""
for _, o := range modelplan.Options([]modelplan.Artifact{art}, modelplan.Shape{Params: params}, pool, 0) {
if o.CPU || !o.Fits() {
continue
}
links := modelplan.Links{NVLink: c.nvlink[o.Class] >= o.GPUsPerNode && o.GPUsPerNode > 1}
if o.Nodes > 1 {
f := c.fabrics[o.Fabric]
links.InfiniBand, links.RateGbps = f.InfiniBand, f.RateGbps
}
tps := modelplan.GPUTokensPerSecond(o, art.Bytes, share, params, c.classBW[o.Class], links)
if tps > best {
best, layout = tps, layoutLabel(l, o, links)
}
}
return best, layout
}
// layoutLabel says how a replica would be laid out: "1× h100-80g",
// "4× h100-80g, NVLink, vLLM", "2 nodes × 8× h100-80g over ib0 (InfiniBand
// 400 Gb/s), vLLM".
func layoutLabel(l *Lang, o modelplan.Option, links modelplan.Links) string {
s := strconv.Itoa(max(o.GPUsPerNode, 1)) + "× " + o.Class
if o.GPUsPerNode > 1 {
if links.NVLink {
s += ", NVLink"
} else {
s += ", PCIe"
}
}
if o.Nodes > 1 {
kind := "RoCE"
if links.InfiniBand {
kind = "InfiniBand"
}
if links.RateGbps > 0 {
kind += " " + strconv.Itoa(links.RateGbps) + " Gb/s"
}
s = l.T("explore.layout.nodes", o.Nodes, s, o.Fabric, kind)
}
engine := "llama.cpp"
if o.Engine == "vllm" {
engine = "vLLM"
}
return s + ", " + engine
}
// rowGPU is a listed repository's speed on the cards: its GGUF at Q4_K_M
// on llama.cpp, its safetensors on vLLM, whichever is faster here.
func (c *clusterFit) rowGPU(l *Lang, m hf.Model) (string, string) {
if !c.gpus() || m.Params() <= 0 {
return "", ""
}
params, share := m.Params(), modelplan.ActiveShare(m.ID, m.Params())
best, layout := 0.0, ""
if m.GGUF != nil || m.HasTag("gguf") {
b := int64(float64(params) * ggufBitsPerWeight / 8)
best, layout = c.gpuSpeed(l, modelplan.Artifact{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b, Quant: "Q4_K_M"},
params, share, false)
}
if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") {
b := m.WeightBytes()
if b == 0 {
b = params * 2
}
if tps, lay := c.gpuSpeed(l, modelplan.Artifact{Name: "m.safetensors", Format: "safetensors", Bytes: b},
params, share, m.FP8()); tps > best {
best, layout = tps, lay
}
}
return speedLabel(best), layout
}
// artifactFormat is "gguf" or "safetensors" by a file set's name, or "".
func artifactFormat(name string) string {
switch lower := strings.ToLower(name); {
case strings.HasSuffix(lower, ".gguf"):
return "gguf"
case strings.HasSuffix(lower, ".safetensors"):
return "safetensors"
}
return ""
}
+58 -2
View File
@@ -153,8 +153,8 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()),
)
// 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second.
// The 30B model of experts reads its 3B active part: about 15.
for _, want := range []string{">~5.8<", ">~15<", "on node05&#39;s CPU"} {
// The 30B model of experts reads 1.6 times its 3B active part: about 9.4.
for _, want := range []string{">~5.8<", ">~9.4<", "on node05&#39;s CPU"} {
if !strings.Contains(body, want) {
t.Errorf("the listing lacks %q: %s", want, body)
}
@@ -169,3 +169,59 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
t.Errorf("each file should say its speed on the CPU, by its own size: %s", page)
}
}
// cards is a node of n cards of a class, named as the driver names them,
// joined by NVLink or not, on a fabric or none.
func cards(id, class, name string, n int, cc string, nvlink bool, fabric string) adminapi.NodeView {
v := gpuNode(id, class, n, cc, 1024)
for i := 0; i < n; i++ {
g := adminapi.GPUView{Index: i, Name: name, Class: class}
if nvlink {
for j := 0; j < n; j++ {
if j != i {
g.NVLinkPeers = append(g.NVLinkPeers, j)
}
}
}
v.GPUs = append(v.GPUs, g)
}
if fabric != "" {
v.Fabrics = []string{fabric}
v.IBPorts = []adminapi.IBPortView{{HCA: "mlx5_0", Port: 1, Fabric: fabric, LinkLayer: "InfiniBand",
RateGbps: 200, Active: true}}
}
return v
}
func TestTheExplorerSaysHowFastAModelAnswersOnTheCards(t *testing.T) {
// The PoC's one Quadro RTX 6000: an 8B GGUF at Q4_K_M, 4.8 GB, on
// 672 GB/s.
poc := []adminapi.NodeView{cards("cuda01", "6000-24g", "Quadro RTX 6000", 1, "7.5", false, "")}
body, _ := listing(t, poc, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()))
if !strings.Contains(body, `title="1× 6000-24g, llama.cpp">~103<`) {
t.Errorf("an 8B on cuda01's card should be about 103 a second: %s", body)
}
// A mixed site: eight H100s on NVLink, four L40S on PCIe. 70B in BF16
// fits two H100s or four L40S, and the planner takes the fewest cards:
// the H100s are the faster.
mixed := []adminapi.NodeView{
cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 8, "9.0", true, ""),
cards("l", "l40s-48g", "NVIDIA L40S", 4, "8.9", false, ""),
}
body, _ = listing(t, mixed, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9}))
if !strings.Contains(body, "2× h100-80g, NVLink, vLLM") {
t.Errorf("70B should run fastest on two NVLinked H100s: %s", body)
}
// Two nodes of one A100 each on InfiniBand: 70B fits neither, and the
// pipeline across them is what the planner offers.
ib := []adminapi.NodeView{
cards("a", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"),
cards("b", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"),
}
body, _ = listing(t, ib, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9}))
if !strings.Contains(body, "2 nodes × 1× a100-80g over ib0 (InfiniBand 200 Gb/s), vLLM") {
t.Errorf("70B should be laid out across both nodes over ib0: %s", body)
}
}
+4
View File
@@ -82,6 +82,9 @@
"explore.col.downloads": "Downloads",
"explore.col.likes": "Likes",
"explore.col.updated": "Updated",
"explore.col.gpu": "GPU tok/s",
"explore.col.gpu_title": "About how many tokens a second one reply gets on the cards, laid out as this cluster would place it — the fewest cards that hold it: each card's memory bandwidth (from its model), tensor parallel's all-reduces over NVLink or PCIe, a hop over the fabric for each node of a pipeline. The layout is in each figure's tooltip.",
"explore.layout.nodes": "%d nodes × %s over %s (%s)",
"explore.col.cpu": "CPU tok/s",
"explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).",
"explore.cpu_on": "on %s's CPU",
@@ -142,6 +145,7 @@
"card.model": "model",
"card.cannot_serve": "cannot be served",
"card.imatrix": "it is an importance matrix for quantising, not a model",
"card.gpu_speed": "%s tok/s on %s",
"card.cpu_speed": "%s tok/s on %s's CPU",
"card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB",
"card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF",
+4
View File
@@ -80,6 +80,9 @@
"explore.col.downloads": "İndirme",
"explore.col.likes": "Beğeni",
"explore.col.updated": "Güncellendi",
"explore.col.gpu": "GPU token/sn",
"explore.col.gpu_title": "Bu kümenin onu yerleştireceği biçimde — modeli alan en az kartla — tek bir yanıtın kartlarda saniyede aşağı yukarı kaç token alacağı: her kartın bellek bant genişliği (modelinden), tensör paralelinin NVLink ya da PCIe üzerindeki all-reduce'ları, boru hattının her düğümü için ağ üzerinden bir atlama. Yerleşim her rakamın ipucunda.",
"explore.layout.nodes": "%d düğüm × %s, %s üzerinden (%s)",
"explore.col.cpu": "CPU token/sn",
"explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).",
"explore.cpu_on": "%s düğümünün CPU'sunda",
@@ -137,6 +140,7 @@
"card.model": "model",
"card.cannot_serve": "sunulamaz",
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
"card.gpu_speed": "%[2]s üzerinde %[1]s token/sn",
"card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn",
"card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB",
"card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin",
+3
View File
@@ -96,6 +96,8 @@ type exploreRow struct {
// CPUSpeed is about how many tokens a second one reply would get on the
// CPU of CPUNode, the fastest node that holds it; empty when unknown.
CPUSpeed, CPUNode string
// GPUSpeed is the same on the cards, laid out as GPULayout says.
GPUSpeed, GPULayout string
}
// ParamsLabel is the parameter count as people say it: 7.6B, 494M.
@@ -316,6 +318,7 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
}
if fit != nil {
row.CPUSpeed, row.CPUNode = fit.rowSpeed(m)
row.GPUSpeed, row.GPULayout = fit.rowGPU(l, m)
}
if e.Here && row.Verdict.Checked && !row.Verdict.Servable {
e.Hidden++
+3
View File
@@ -302,6 +302,7 @@
{{if .Rows}}
<table class="rows explore">
<tr><th>{{t "explore.col.repo"}}</th><th class="num">{{t "explore.col.size"}}</th><th>{{t "explore.col.arch"}}</th><th class="num">{{t "explore.col.context"}}</th>
<th class="num" title="{{t "explore.col.gpu_title"}}">{{t "explore.col.gpu"}}</th>
<th class="num" title="{{t "explore.col.cpu_title"}}">{{t "explore.col.cpu"}}</th>
<th class="num">{{t "explore.col.downloads"}}</th><th class="num">{{t "explore.col.likes"}}</th><th class="num">{{t "explore.col.updated"}}</th></tr>
{{range .Rows}}<tr class="explore-row">
@@ -311,6 +312,7 @@
<td class="num">{{.ParamsLabel}}</td>
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
<td class="num">{{.ContextLabel}}</td>
<td class="num">{{if .GPUSpeed}}<span title="{{.GPULayout}}">{{.GPUSpeed}}</span>{{else}}—{{end}}</td>
<td class="num">{{if .CPUSpeed}}<span title="{{t "explore.cpu_on" .CPUNode}}">{{.CPUSpeed}}</span>{{else}}—{{end}}</td>
<td class="num">{{compact .Downloads}}</td>
<td class="num">{{compact .Likes}}</td>
@@ -470,6 +472,7 @@ document.querySelector("table.explore").addEventListener("submit", async functio
{{with .Companions}}<span class="opt" title="{{join .}}">{{t "card.companions" (len .)}}</span>{{end}}
{{if eq .Kind "adapter"}}<span class="tag new" title="{{t "card.adapter_title" .Arch}}">{{t "card.adapter" .Arch}}</span>
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}
{{if .GPUSpeed}}<span class="opt" title="{{t "explore.col.gpu_title"}}">{{t "card.gpu_speed" .GPUSpeed .GPULayout}}</span>{{end}}
{{if .CPUSpeed}}<span class="opt" title="{{t "explore.col.cpu_title"}}">{{t "card.cpu_speed" .CPUSpeed .CPUNode}}</span>{{end}}</td>
<td class="num">{{bytes .Bytes}}</td>
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>