How fast a model would answer on the cards, as this cluster would lay it out
CI / test (push) Successful in 9m1s
CI / test (push) Successful in 9m1s
For each layout modelplan.Options offers here — one card, tensor parallel across a node's cards, llama.cpp's layer split, a pipeline across nodes on a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes a token reads over each card's bandwidth, by its model name (a table, SXM and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on NVLink, 25 us and up on PCIe), turns between cards for a layer split, and a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a repository's files show the fastest, with the layout; mixed classes are judged class by class, FP8 on 8.9 cards only. Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB, 62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB, 144 measured, which says a model of experts reads 1.6 times its active share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics are not measured here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e70d6caa2f
commit
f2c6f12cfb
13 files changed
+449
-14
No files matched your search
@@ -61,6 +61,8 @@ type repoSet struct {
|
||||
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
|
||||
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
|
||||
CPUSpeed, CPUNode string
|
||||
// GPUSpeed is the same on the cards, laid out as GPULayout says.
|
||||
GPUSpeed, GPULayout string
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -828,6 +830,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
|
||||
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
|
||||
}
|
||||
if format := artifactFormat(set.Name); rs.Verdict.Servable && format != "" {
|
||||
tps, layout := fit.gpuSpeed(langOf(r), modelplan.Artifact{Name: set.Name, Format: format, Bytes: set.Bytes,
|
||||
Quant: modelplan.QuantOf(set.Name)}, got.Params, modelplan.ActiveShare(got.ID, got.Params), false)
|
||||
rs.GPUSpeed, rs.GPULayout = speedLabel(tps), layout
|
||||
}
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
|
||||
+123
-7
@@ -32,6 +32,12 @@ type clusterFit struct {
|
||||
// cpus are the nodes whose memory's speed their agent measured, for
|
||||
// how fast a model would answer on one.
|
||||
cpus []cpuHost
|
||||
// classBW is each card class's memory bandwidth in GB/s, from its
|
||||
// cards' names; nvlink how many cards of a class one node has joined
|
||||
// by NVLink, at most; fabrics what each fabric is, by name.
|
||||
classBW map[string]float64
|
||||
nvlink map[string]int
|
||||
fabrics map[string]modelplan.Links
|
||||
}
|
||||
|
||||
// cpuHost is a node as a model on its CPU sees it.
|
||||
@@ -55,8 +61,31 @@ func (p *AccountPages) fitOf() *clusterFit {
|
||||
if len(c.pool) == 0 {
|
||||
return nil
|
||||
}
|
||||
c.classBW, c.nvlink, c.fabrics = map[string]float64{}, map[string]int{}, map[string]modelplan.Links{}
|
||||
newest := map[string]float64{}
|
||||
for _, n := range nodes {
|
||||
joined := map[string]int{}
|
||||
for _, g := range n.GPUs {
|
||||
if bw := modelplan.GPUBandwidthGBs(g.Name); bw > c.classBW[g.Class] {
|
||||
c.classBW[g.Class] = bw
|
||||
}
|
||||
if len(g.NVLinkPeers) > 0 {
|
||||
joined[g.Class]++
|
||||
}
|
||||
}
|
||||
for class, k := range joined {
|
||||
c.nvlink[class] = max(c.nvlink[class], k)
|
||||
}
|
||||
for _, p := range n.IBPorts {
|
||||
if p.Fabric != "" && p.Active {
|
||||
l := c.fabrics[p.Fabric]
|
||||
l.InfiniBand = p.LinkLayer == "InfiniBand"
|
||||
if l.RateGbps == 0 || p.RateGbps < l.RateGbps {
|
||||
l.RateGbps = p.RateGbps
|
||||
}
|
||||
c.fabrics[p.Fabric] = l
|
||||
}
|
||||
}
|
||||
if n.State == "ready" && n.MemBandwidthGBs > 0 {
|
||||
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
|
||||
}
|
||||
@@ -156,13 +185,7 @@ func (c *clusterFit) note(l *Lang) string {
|
||||
// holds judges one file set of an open repository, whose size is known: a
|
||||
// verdict that it cannot be served here, and false, when no node holds it.
|
||||
func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) {
|
||||
format := ""
|
||||
switch lower := strings.ToLower(name); {
|
||||
case strings.HasSuffix(lower, ".gguf"):
|
||||
format = "gguf"
|
||||
case strings.HasSuffix(lower, ".safetensors"):
|
||||
format = "safetensors"
|
||||
}
|
||||
format := artifactFormat(name)
|
||||
if format == "" || bytes <= 0 {
|
||||
return gguf.Verdict{}, true
|
||||
}
|
||||
@@ -234,3 +257,96 @@ func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
|
||||
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
|
||||
return speedLabel(tps), node
|
||||
}
|
||||
|
||||
// gpuSpeed is about how many tokens a second one reply gets from art on
|
||||
// the cards, laid out as the planner would place it — the fewest cards
|
||||
// that hold it, for each class — the fastest of those, and how it is laid
|
||||
// out. 0 when nothing here holds it or no card's speed is known.
|
||||
func (c *clusterFit) gpuSpeed(l *Lang, art modelplan.Artifact, params int64, share float64, fp8 bool) (float64, string) {
|
||||
pool := c.pool
|
||||
if fp8 {
|
||||
pool = c.fp8
|
||||
}
|
||||
best, layout := 0.0, ""
|
||||
for _, o := range modelplan.Options([]modelplan.Artifact{art}, modelplan.Shape{Params: params}, pool, 0) {
|
||||
if o.CPU || !o.Fits() {
|
||||
continue
|
||||
}
|
||||
links := modelplan.Links{NVLink: c.nvlink[o.Class] >= o.GPUsPerNode && o.GPUsPerNode > 1}
|
||||
if o.Nodes > 1 {
|
||||
f := c.fabrics[o.Fabric]
|
||||
links.InfiniBand, links.RateGbps = f.InfiniBand, f.RateGbps
|
||||
}
|
||||
tps := modelplan.GPUTokensPerSecond(o, art.Bytes, share, params, c.classBW[o.Class], links)
|
||||
if tps > best {
|
||||
best, layout = tps, layoutLabel(l, o, links)
|
||||
}
|
||||
}
|
||||
return best, layout
|
||||
}
|
||||
|
||||
// layoutLabel says how a replica would be laid out: "1× h100-80g",
|
||||
// "4× h100-80g, NVLink, vLLM", "2 nodes × 8× h100-80g over ib0 (InfiniBand
|
||||
// 400 Gb/s), vLLM".
|
||||
func layoutLabel(l *Lang, o modelplan.Option, links modelplan.Links) string {
|
||||
s := strconv.Itoa(max(o.GPUsPerNode, 1)) + "× " + o.Class
|
||||
if o.GPUsPerNode > 1 {
|
||||
if links.NVLink {
|
||||
s += ", NVLink"
|
||||
} else {
|
||||
s += ", PCIe"
|
||||
}
|
||||
}
|
||||
if o.Nodes > 1 {
|
||||
kind := "RoCE"
|
||||
if links.InfiniBand {
|
||||
kind = "InfiniBand"
|
||||
}
|
||||
if links.RateGbps > 0 {
|
||||
kind += " " + strconv.Itoa(links.RateGbps) + " Gb/s"
|
||||
}
|
||||
s = l.T("explore.layout.nodes", o.Nodes, s, o.Fabric, kind)
|
||||
}
|
||||
engine := "llama.cpp"
|
||||
if o.Engine == "vllm" {
|
||||
engine = "vLLM"
|
||||
}
|
||||
return s + ", " + engine
|
||||
}
|
||||
|
||||
// rowGPU is a listed repository's speed on the cards: its GGUF at Q4_K_M
|
||||
// on llama.cpp, its safetensors on vLLM, whichever is faster here.
|
||||
func (c *clusterFit) rowGPU(l *Lang, m hf.Model) (string, string) {
|
||||
if !c.gpus() || m.Params() <= 0 {
|
||||
return "", ""
|
||||
}
|
||||
params, share := m.Params(), modelplan.ActiveShare(m.ID, m.Params())
|
||||
best, layout := 0.0, ""
|
||||
if m.GGUF != nil || m.HasTag("gguf") {
|
||||
b := int64(float64(params) * ggufBitsPerWeight / 8)
|
||||
best, layout = c.gpuSpeed(l, modelplan.Artifact{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b, Quant: "Q4_K_M"},
|
||||
params, share, false)
|
||||
}
|
||||
if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") {
|
||||
b := m.WeightBytes()
|
||||
if b == 0 {
|
||||
b = params * 2
|
||||
}
|
||||
if tps, lay := c.gpuSpeed(l, modelplan.Artifact{Name: "m.safetensors", Format: "safetensors", Bytes: b},
|
||||
params, share, m.FP8()); tps > best {
|
||||
best, layout = tps, lay
|
||||
}
|
||||
}
|
||||
return speedLabel(best), layout
|
||||
}
|
||||
|
||||
// artifactFormat is "gguf" or "safetensors" by a file set's name, or "".
|
||||
func artifactFormat(name string) string {
|
||||
switch lower := strings.ToLower(name); {
|
||||
case strings.HasSuffix(lower, ".gguf"):
|
||||
return "gguf"
|
||||
case strings.HasSuffix(lower, ".safetensors"):
|
||||
return "safetensors"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -153,8 +153,8 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
|
||||
ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()),
|
||||
)
|
||||
// 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second.
|
||||
// The 30B model of experts reads its 3B active part: about 15.
|
||||
for _, want := range []string{">~5.8<", ">~15<", "on node05's CPU"} {
|
||||
// The 30B model of experts reads 1.6 times its 3B active part: about 9.4.
|
||||
for _, want := range []string{">~5.8<", ">~9.4<", "on node05's CPU"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the listing lacks %q: %s", want, body)
|
||||
}
|
||||
@@ -169,3 +169,59 @@ func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
|
||||
t.Errorf("each file should say its speed on the CPU, by its own size: %s", page)
|
||||
}
|
||||
}
|
||||
|
||||
// cards is a node of n cards of a class, named as the driver names them,
|
||||
// joined by NVLink or not, on a fabric or none.
|
||||
func cards(id, class, name string, n int, cc string, nvlink bool, fabric string) adminapi.NodeView {
|
||||
v := gpuNode(id, class, n, cc, 1024)
|
||||
for i := 0; i < n; i++ {
|
||||
g := adminapi.GPUView{Index: i, Name: name, Class: class}
|
||||
if nvlink {
|
||||
for j := 0; j < n; j++ {
|
||||
if j != i {
|
||||
g.NVLinkPeers = append(g.NVLinkPeers, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
v.GPUs = append(v.GPUs, g)
|
||||
}
|
||||
if fabric != "" {
|
||||
v.Fabrics = []string{fabric}
|
||||
v.IBPorts = []adminapi.IBPortView{{HCA: "mlx5_0", Port: 1, Fabric: fabric, LinkLayer: "InfiniBand",
|
||||
RateGbps: 200, Active: true}}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func TestTheExplorerSaysHowFastAModelAnswersOnTheCards(t *testing.T) {
|
||||
// The PoC's one Quadro RTX 6000: an 8B GGUF at Q4_K_M, 4.8 GB, on
|
||||
// 672 GB/s.
|
||||
poc := []adminapi.NodeView{cards("cuda01", "6000-24g", "Quadro RTX 6000", 1, "7.5", false, "")}
|
||||
body, _ := listing(t, poc, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()))
|
||||
if !strings.Contains(body, `title="1× 6000-24g, llama.cpp">~103<`) {
|
||||
t.Errorf("an 8B on cuda01's card should be about 103 a second: %s", body)
|
||||
}
|
||||
|
||||
// A mixed site: eight H100s on NVLink, four L40S on PCIe. 70B in BF16
|
||||
// fits two H100s or four L40S, and the planner takes the fewest cards:
|
||||
// the H100s are the faster.
|
||||
mixed := []adminapi.NodeView{
|
||||
cards("h", "h100-80g", "NVIDIA H100 80GB HBM3", 8, "9.0", true, ""),
|
||||
cards("l", "l40s-48g", "NVIDIA L40S", 4, "8.9", false, ""),
|
||||
}
|
||||
body, _ = listing(t, mixed, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9}))
|
||||
if !strings.Contains(body, "2× h100-80g, NVLink, vLLM") {
|
||||
t.Errorf("70B should run fastest on two NVLinked H100s: %s", body)
|
||||
}
|
||||
|
||||
// Two nodes of one A100 each on InfiniBand: 70B fits neither, and the
|
||||
// pipeline across them is what the planner offers.
|
||||
ib := []adminapi.NodeView{
|
||||
cards("a", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"),
|
||||
cards("b", "a100-80g", "NVIDIA A100-SXM4-80GB", 1, "8.0", false, "ib0"),
|
||||
}
|
||||
body, _ = listing(t, ib, stModel("meta-llama/Llama-3.3-70B-Instruct", map[string]int64{"BF16": 70e9}))
|
||||
if !strings.Contains(body, "2 nodes × 1× a100-80g over ib0 (InfiniBand 200 Gb/s), vLLM") {
|
||||
t.Errorf("70B should be laid out across both nodes over ib0: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -82,6 +82,9 @@
|
||||
"explore.col.downloads": "Downloads",
|
||||
"explore.col.likes": "Likes",
|
||||
"explore.col.updated": "Updated",
|
||||
"explore.col.gpu": "GPU tok/s",
|
||||
"explore.col.gpu_title": "About how many tokens a second one reply gets on the cards, laid out as this cluster would place it — the fewest cards that hold it: each card's memory bandwidth (from its model), tensor parallel's all-reduces over NVLink or PCIe, a hop over the fabric for each node of a pipeline. The layout is in each figure's tooltip.",
|
||||
"explore.layout.nodes": "%d nodes × %s over %s (%s)",
|
||||
"explore.col.cpu": "CPU tok/s",
|
||||
"explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).",
|
||||
"explore.cpu_on": "on %s's CPU",
|
||||
@@ -142,6 +145,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "cannot be served",
|
||||
"card.imatrix": "it is an importance matrix for quantising, not a model",
|
||||
"card.gpu_speed": "%s tok/s on %s",
|
||||
"card.cpu_speed": "%s tok/s on %s's CPU",
|
||||
"card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB",
|
||||
"card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF",
|
||||
|
||||
@@ -80,6 +80,9 @@
|
||||
"explore.col.downloads": "İndirme",
|
||||
"explore.col.likes": "Beğeni",
|
||||
"explore.col.updated": "Güncellendi",
|
||||
"explore.col.gpu": "GPU token/sn",
|
||||
"explore.col.gpu_title": "Bu kümenin onu yerleştireceği biçimde — modeli alan en az kartla — tek bir yanıtın kartlarda saniyede aşağı yukarı kaç token alacağı: her kartın bellek bant genişliği (modelinden), tensör paralelinin NVLink ya da PCIe üzerindeki all-reduce'ları, boru hattının her düğümü için ağ üzerinden bir atlama. Yerleşim her rakamın ipucunda.",
|
||||
"explore.layout.nodes": "%d düğüm × %s, %s üzerinden (%s)",
|
||||
"explore.col.cpu": "CPU token/sn",
|
||||
"explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).",
|
||||
"explore.cpu_on": "%s düğümünün CPU'sunda",
|
||||
@@ -137,6 +140,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "sunulamaz",
|
||||
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
|
||||
"card.gpu_speed": "%[2]s üzerinde %[1]s token/sn",
|
||||
"card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn",
|
||||
"card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB",
|
||||
"card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin",
|
||||
|
||||
@@ -96,6 +96,8 @@ type exploreRow struct {
|
||||
// CPUSpeed is about how many tokens a second one reply would get on the
|
||||
// CPU of CPUNode, the fastest node that holds it; empty when unknown.
|
||||
CPUSpeed, CPUNode string
|
||||
// GPUSpeed is the same on the cards, laid out as GPULayout says.
|
||||
GPUSpeed, GPULayout string
|
||||
}
|
||||
|
||||
// ParamsLabel is the parameter count as people say it: 7.6B, 494M.
|
||||
@@ -316,6 +318,7 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
|
||||
}
|
||||
if fit != nil {
|
||||
row.CPUSpeed, row.CPUNode = fit.rowSpeed(m)
|
||||
row.GPUSpeed, row.GPULayout = fit.rowGPU(l, m)
|
||||
}
|
||||
if e.Here && row.Verdict.Checked && !row.Verdict.Servable {
|
||||
e.Hidden++
|
||||
|
||||
@@ -302,6 +302,7 @@
|
||||
{{if .Rows}}
|
||||
<table class="rows explore">
|
||||
<tr><th>{{t "explore.col.repo"}}</th><th class="num">{{t "explore.col.size"}}</th><th>{{t "explore.col.arch"}}</th><th class="num">{{t "explore.col.context"}}</th>
|
||||
<th class="num" title="{{t "explore.col.gpu_title"}}">{{t "explore.col.gpu"}}</th>
|
||||
<th class="num" title="{{t "explore.col.cpu_title"}}">{{t "explore.col.cpu"}}</th>
|
||||
<th class="num">{{t "explore.col.downloads"}}</th><th class="num">{{t "explore.col.likes"}}</th><th class="num">{{t "explore.col.updated"}}</th></tr>
|
||||
{{range .Rows}}<tr class="explore-row">
|
||||
@@ -311,6 +312,7 @@
|
||||
<td class="num">{{.ParamsLabel}}</td>
|
||||
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{.ContextLabel}}</td>
|
||||
<td class="num">{{if .GPUSpeed}}<span title="{{.GPULayout}}">{{.GPUSpeed}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .CPUSpeed}}<span title="{{t "explore.cpu_on" .CPUNode}}">{{.CPUSpeed}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{compact .Downloads}}</td>
|
||||
<td class="num">{{compact .Likes}}</td>
|
||||
@@ -470,6 +472,7 @@ document.querySelector("table.explore").addEventListener("submit", async functio
|
||||
{{with .Companions}}<span class="opt" title="{{join .}}">{{t "card.companions" (len .)}}</span>{{end}}
|
||||
{{if eq .Kind "adapter"}}<span class="tag new" title="{{t "card.adapter_title" .Arch}}">{{t "card.adapter" .Arch}}</span>
|
||||
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}
|
||||
{{if .GPUSpeed}}<span class="opt" title="{{t "explore.col.gpu_title"}}">{{t "card.gpu_speed" .GPUSpeed .GPULayout}}</span>{{end}}
|
||||
{{if .CPUSpeed}}<span class="opt" title="{{t "explore.col.cpu_title"}}">{{t "card.cpu_speed" .CPUSpeed .CPUNode}}</span>{{end}}</td>
|
||||
<td class="num">{{bytes .Bytes}}</td>
|
||||
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
|
||||
|
||||
Reference in new issue
Block a user