How fast a model would answer on the cards, as this cluster would lay it out
For each layout modelplan.Options offers here — one card, tensor parallel across a node's cards, llama.cpp's layer split, a pipeline across nodes on a fabric — modelplan.GPUTokensPerSecond gives one reply's speed: the bytes a token reads over each card's bandwidth, by its model name (a table, SXM and PCIe apart), plus two all-reduces a layer for tensor parallel (8 us on NVLink, 25 us and up on PCIe), turns between cards for a layer split, and a hop per node over InfiniBand or RoCE for a pipeline. The explorer and a repository's files show the fastest, with the layout; mixed classes are judged class by class, FP8 on 8.9 cards only. Calibrated on cuda01's Quadro RTX 6000 (672 GB/s): qwen3-vl-8b Q8, 8.7 GB, 62.1 tok/s measured, 59 estimated; Qwen3-Coder-30B-A3B Q4_K_M, 18.6 GB, 144 measured, which says a model of experts reads 1.6 times its active share (ActiveShare now does, on the CPU too). Tensor parallel and fabrics are not measured here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e70d6caa2f
commit
f2c6f12cfb
13 files changed
+449
-14
No files matched your search
@@ -61,6 +61,8 @@ type repoSet struct {
|
||||
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
|
||||
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
|
||||
CPUSpeed, CPUNode string
|
||||
// GPUSpeed is the same on the cards, laid out as GPULayout says.
|
||||
GPUSpeed, GPULayout string
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -828,6 +830,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
|
||||
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
|
||||
}
|
||||
if format := artifactFormat(set.Name); rs.Verdict.Servable && format != "" {
|
||||
tps, layout := fit.gpuSpeed(langOf(r), modelplan.Artifact{Name: set.Name, Format: format, Bytes: set.Bytes,
|
||||
Quant: modelplan.QuantOf(set.Name)}, got.Params, modelplan.ActiveShare(got.ID, got.Params), false)
|
||||
rs.GPUSpeed, rs.GPULayout = speedLabel(tps), layout
|
||||
}
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
|
||||
Reference in new issue
Block a user