How fast a model would answer on a CPU, before it is fetched
CI / test (push) Failing after 6m37s

Each agent measures its node's memory read rate when it starts: every CPU
sums a share of a buffer past any cache, a tenth of free memory at most,
the best of four passes, in a fraction of a second; reported in Host
(mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it
uses once, so one reply's speed is about that rate over those bytes:
modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token,
calibrated on the PoC against llama.cpp's predicted_per_second (node01
24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s,
qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active
part (ActiveShare).

The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that
holds the model; a repository's page gives it for each GGUF by its size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-27 00:09:47 +01:00
1 parent 2a45068461
commit 84444e71c8
20 files changed
+388 -7

No files matched your search

+7
View File
@@ -20,6 +20,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/safedial"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -57,6 +58,9 @@ type repoSet struct {
// architecture the registry reports; a set that cannot be served is
// shown with the reason and no fetch button.
Verdict gguf.Verdict
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
CPUSpeed, CPUNode string
hf.FileSet
// Sources are every file in the set, in the order the registry listed them.
// A shard set goes as one action — a fifteen-shard model is not fifteen
@@ -820,6 +824,9 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
if fit != nil && rs.Verdict.Servable {
if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok {
rs.Verdict = v
} else if gguf.IsGGUF(set.Name) {
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
}
}
for _, f := range set.Files {
+55
View File
@@ -29,6 +29,16 @@ type clusterFit struct {
card int
cc string
host int
// cpus are the nodes whose memory's speed their agent measured, for
// how fast a model would answer on one.
cpus []cpuHost
}
// cpuHost is a node as a model on its CPU sees it.
type cpuHost struct {
id string
ramGB int
bwGBs float64
}
// fp8Capability is the oldest card vLLM runs FP8 weights on.
@@ -47,6 +57,9 @@ func (p *AccountPages) fitOf() *clusterFit {
}
newest := map[string]float64{}
for _, n := range nodes {
if n.State == "ready" && n.MemBandwidthGBs > 0 {
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
}
for _, s := range n.ComputeCapabilities {
if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
newest[n.ID] = v
@@ -179,3 +192,45 @@ func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verd
}
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false
}
// cpuSpeed is about how many tokens a second one reply gets from a GGUF of
// bytes on the fastest node whose RAM holds it, and that node; share is the
// part a token reads (ActiveShare). 0 when no measured node holds it.
func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) {
var best cpuHost
for _, n := range c.cpus {
need := float64(bytes) + float64(1<<30) + float64(bytes)/20
if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs {
best = n
}
}
if best.id == "" {
return 0, ""
}
return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id
}
// speedLabel is a rate as the page shows it: whole tokens from ten up, one
// decimal below, and under one said so.
func speedLabel(tps float64) string {
switch {
case tps <= 0:
return ""
case tps < 1:
return "<1"
case tps < 10:
return "~" + strconv.FormatFloat(tps, 'f', 1, 64)
}
return "~" + strconv.FormatFloat(tps, 'f', 0, 64)
}
// rowSpeed is a listed repository's speed on the CPU, judged at Q4_K_M:
// GGUF only, which is what runs there.
func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
if m.GGUF == nil && !m.HasTag("gguf") || m.Params() <= 0 {
return "", ""
}
b := int64(float64(m.Params()) * ggufBitsPerWeight / 8)
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
return speedLabel(tps), node
}
+26
View File
@@ -1,6 +1,7 @@
package web
import (
"html"
"strings"
"testing"
"time"
@@ -143,3 +144,28 @@ func TestPackedWeightsAreSizedAsTheyAreStored(t *testing.T) {
t.Errorf("both run on cuda01: %s", body)
}
}
func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
slow, fast := cpuNode("node01", 7), cpuNode("node05", 31)
slow.MemBandwidthGBs, fast.MemBandwidthGBs = 24.9, 41.9
body, _ := listing(t, []adminapi.NodeView{slow, fast},
ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()),
ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()),
)
// 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second.
// The 30B model of experts reads its 3B active part: about 15.
for _, want := range []string{">~5.8<", ">~15<", "on node05&#39;s CPU"} {
if !strings.Contains(body, want) {
t.Errorf("the listing lacks %q: %s", want, body)
}
}
reg := &fakeRegistry{repo: hf.Repo{ID: "Qwen/Qwen3-8B-GGUF", Revision: "abc123def456", Params: 8e9,
Files: []hf.File{{Path: "Qwen3-8B-Q4_K_M.gguf", Bytes: 5 << 30}, {Path: "Qwen3-8B-Q8_0.gguf", Bytes: 8 << 30}}}}
mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg,
Fleet: nodesFleet{nodes: []adminapi.NodeView{slow, fast}}})
page := html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=Qwen/Qwen3-8B-GGUF").Body.String())
if !strings.Contains(page, "~5.2 tok/s on node05's CPU") || !strings.Contains(page, "~3.3 tok/s on node05's CPU") {
t.Errorf("each file should say its speed on the CPU, by its own size: %s", page)
}
}
+4
View File
@@ -82,6 +82,9 @@
"explore.col.downloads": "Downloads",
"explore.col.likes": "Likes",
"explore.col.updated": "Updated",
"explore.col.cpu": "CPU tok/s",
"explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).",
"explore.cpu_on": "on %s's CPU",
"explore.new": "new",
"explore.gated": "gated",
"explore.from": "from %s",
@@ -139,6 +142,7 @@
"card.model": "model",
"card.cannot_serve": "cannot be served",
"card.imatrix": "it is an importance matrix for quantising, not a model",
"card.cpu_speed": "%s tok/s on %s's CPU",
"card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB",
"card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF",
"card.pickle_hidden.one": "%d file is not shown: it is in a pickle format, which runs code when it is loaded, and nodes refuse it.",
+4
View File
@@ -80,6 +80,9 @@
"explore.col.downloads": "İndirme",
"explore.col.likes": "Beğeni",
"explore.col.updated": "Güncellendi",
"explore.col.cpu": "CPU token/sn",
"explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).",
"explore.cpu_on": "%s düğümünün CPU'sunda",
"explore.new": "yeni",
"explore.gated": "erişimi kısıtlı",
"explore.from": "temeli: %s",
@@ -134,6 +137,7 @@
"card.model": "model",
"card.cannot_serve": "sunulamaz",
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
"card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn",
"card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB",
"card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin",
"card.pickle_hidden.other": "%d dosya gösterilmiyor: pickle biçiminde, yüklenirken kod çalıştırır ve düğümler onu almaz.",
+6
View File
@@ -93,6 +93,9 @@ type exploreRow struct {
New bool
// Updated is how long ago the repository changed, in the page's language.
Updated string
// CPUSpeed is about how many tokens a second one reply would get on the
// CPU of CPUNode, the fastest node that holds it; empty when unknown.
CPUSpeed, CPUNode string
}
// ParamsLabel is the parameter count as people say it: 7.6B, 494M.
@@ -311,6 +314,9 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
e.Hidden++
continue
}
if fit != nil {
row.CPUSpeed, row.CPUNode = fit.rowSpeed(m)
}
if e.Here && row.Verdict.Checked && !row.Verdict.Servable {
e.Hidden++
continue
+4 -1
View File
@@ -302,6 +302,7 @@
{{if .Rows}}
<table class="rows explore">
<tr><th>{{t "explore.col.repo"}}</th><th class="num">{{t "explore.col.size"}}</th><th>{{t "explore.col.arch"}}</th><th class="num">{{t "explore.col.context"}}</th>
<th class="num" title="{{t "explore.col.cpu_title"}}">{{t "explore.col.cpu"}}</th>
<th class="num">{{t "explore.col.downloads"}}</th><th class="num">{{t "explore.col.likes"}}</th><th class="num">{{t "explore.col.updated"}}</th></tr>
{{range .Rows}}<tr class="explore-row">
<td><a href="{{.Href}}" class="open-repo" data-repo="{{.ID}}" aria-expanded="false"><code>{{.ID}}</code></a>
@@ -310,6 +311,7 @@
<td class="num">{{.ParamsLabel}}</td>
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
<td class="num">{{.ContextLabel}}</td>
<td class="num">{{if .CPUSpeed}}<span title="{{t "explore.cpu_on" .CPUNode}}">{{.CPUSpeed}}</span>{{else}}—{{end}}</td>
<td class="num">{{compact .Downloads}}</td>
<td class="num">{{compact .Likes}}</td>
<td class="num">{{.Updated}}</td>
@@ -467,7 +469,8 @@ document.querySelector("table.explore").addEventListener("submit", async functio
{{if .Shards}}<span class="opt">{{t "card.shards" .Shards}}</span>{{end}}
{{with .Companions}}<span class="opt" title="{{join .}}">{{t "card.companions" (len .)}}</span>{{end}}
{{if eq .Kind "adapter"}}<span class="tag new" title="{{t "card.adapter_title" .Arch}}">{{t "card.adapter" .Arch}}</span>
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}</td>
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}
{{if .CPUSpeed}}<span class="opt" title="{{t "explore.col.cpu_title"}}">{{t "card.cpu_speed" .CPUSpeed .CPUNode}}</span>{{end}}</td>
<td class="num">{{bytes .Bytes}}</td>
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
<td>