Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a45068461
commit
84444e71c8
20 files changed
+388
-7
No files matched your search
@@ -20,6 +20,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/safedial"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -57,6 +58,9 @@ type repoSet struct {
|
||||
// architecture the registry reports; a set that cannot be served is
|
||||
// shown with the reason and no fetch button.
|
||||
Verdict gguf.Verdict
|
||||
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
|
||||
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
|
||||
CPUSpeed, CPUNode string
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -820,6 +824,9 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
if fit != nil && rs.Verdict.Servable {
|
||||
if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok {
|
||||
rs.Verdict = v
|
||||
} else if gguf.IsGGUF(set.Name) {
|
||||
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
|
||||
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
|
||||
}
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
|
||||
@@ -29,6 +29,16 @@ type clusterFit struct {
|
||||
card int
|
||||
cc string
|
||||
host int
|
||||
// cpus are the nodes whose memory's speed their agent measured, for
|
||||
// how fast a model would answer on one.
|
||||
cpus []cpuHost
|
||||
}
|
||||
|
||||
// cpuHost is a node as a model on its CPU sees it.
|
||||
type cpuHost struct {
|
||||
id string
|
||||
ramGB int
|
||||
bwGBs float64
|
||||
}
|
||||
|
||||
// fp8Capability is the oldest card vLLM runs FP8 weights on.
|
||||
@@ -47,6 +57,9 @@ func (p *AccountPages) fitOf() *clusterFit {
|
||||
}
|
||||
newest := map[string]float64{}
|
||||
for _, n := range nodes {
|
||||
if n.State == "ready" && n.MemBandwidthGBs > 0 {
|
||||
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
|
||||
}
|
||||
for _, s := range n.ComputeCapabilities {
|
||||
if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
|
||||
newest[n.ID] = v
|
||||
@@ -179,3 +192,45 @@ func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verd
|
||||
}
|
||||
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false
|
||||
}
|
||||
|
||||
// cpuSpeed is about how many tokens a second one reply gets from a GGUF of
|
||||
// bytes on the fastest node whose RAM holds it, and that node; share is the
|
||||
// part a token reads (ActiveShare). 0 when no measured node holds it.
|
||||
func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) {
|
||||
var best cpuHost
|
||||
for _, n := range c.cpus {
|
||||
need := float64(bytes) + float64(1<<30) + float64(bytes)/20
|
||||
if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs {
|
||||
best = n
|
||||
}
|
||||
}
|
||||
if best.id == "" {
|
||||
return 0, ""
|
||||
}
|
||||
return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id
|
||||
}
|
||||
|
||||
// speedLabel is a rate as the page shows it: whole tokens from ten up, one
|
||||
// decimal below, and under one said so.
|
||||
func speedLabel(tps float64) string {
|
||||
switch {
|
||||
case tps <= 0:
|
||||
return ""
|
||||
case tps < 1:
|
||||
return "<1"
|
||||
case tps < 10:
|
||||
return "~" + strconv.FormatFloat(tps, 'f', 1, 64)
|
||||
}
|
||||
return "~" + strconv.FormatFloat(tps, 'f', 0, 64)
|
||||
}
|
||||
|
||||
// rowSpeed is a listed repository's speed on the CPU, judged at Q4_K_M:
|
||||
// GGUF only, which is what runs there.
|
||||
func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
|
||||
if m.GGUF == nil && !m.HasTag("gguf") || m.Params() <= 0 {
|
||||
return "", ""
|
||||
}
|
||||
b := int64(float64(m.Params()) * ggufBitsPerWeight / 8)
|
||||
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
|
||||
return speedLabel(tps), node
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"html"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -143,3 +144,28 @@ func TestPackedWeightsAreSizedAsTheyAreStored(t *testing.T) {
|
||||
t.Errorf("both run on cuda01: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) {
|
||||
slow, fast := cpuNode("node01", 7), cpuNode("node05", 31)
|
||||
slow.MemBandwidthGBs, fast.MemBandwidthGBs = 24.9, 41.9
|
||||
body, _ := listing(t, []adminapi.NodeView{slow, fast},
|
||||
ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()),
|
||||
ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()),
|
||||
)
|
||||
// 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second.
|
||||
// The 30B model of experts reads its 3B active part: about 15.
|
||||
for _, want := range []string{">~5.8<", ">~15<", "on node05's CPU"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the listing lacks %q: %s", want, body)
|
||||
}
|
||||
}
|
||||
|
||||
reg := &fakeRegistry{repo: hf.Repo{ID: "Qwen/Qwen3-8B-GGUF", Revision: "abc123def456", Params: 8e9,
|
||||
Files: []hf.File{{Path: "Qwen3-8B-Q4_K_M.gguf", Bytes: 5 << 30}, {Path: "Qwen3-8B-Q8_0.gguf", Bytes: 8 << 30}}}}
|
||||
mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg,
|
||||
Fleet: nodesFleet{nodes: []adminapi.NodeView{slow, fast}}})
|
||||
page := html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=Qwen/Qwen3-8B-GGUF").Body.String())
|
||||
if !strings.Contains(page, "~5.2 tok/s on node05's CPU") || !strings.Contains(page, "~3.3 tok/s on node05's CPU") {
|
||||
t.Errorf("each file should say its speed on the CPU, by its own size: %s", page)
|
||||
}
|
||||
}
|
||||
@@ -82,6 +82,9 @@
|
||||
"explore.col.downloads": "Downloads",
|
||||
"explore.col.likes": "Likes",
|
||||
"explore.col.updated": "Updated",
|
||||
"explore.col.cpu": "CPU tok/s",
|
||||
"explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).",
|
||||
"explore.cpu_on": "on %s's CPU",
|
||||
"explore.new": "new",
|
||||
"explore.gated": "gated",
|
||||
"explore.from": "from %s",
|
||||
@@ -139,6 +142,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "cannot be served",
|
||||
"card.imatrix": "it is an importance matrix for quantising, not a model",
|
||||
"card.cpu_speed": "%s tok/s on %s's CPU",
|
||||
"card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB",
|
||||
"card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF",
|
||||
"card.pickle_hidden.one": "%d file is not shown: it is in a pickle format, which runs code when it is loaded, and nodes refuse it.",
|
||||
|
||||
@@ -80,6 +80,9 @@
|
||||
"explore.col.downloads": "İndirme",
|
||||
"explore.col.likes": "Beğeni",
|
||||
"explore.col.updated": "Güncellendi",
|
||||
"explore.col.cpu": "CPU token/sn",
|
||||
"explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).",
|
||||
"explore.cpu_on": "%s düğümünün CPU'sunda",
|
||||
"explore.new": "yeni",
|
||||
"explore.gated": "erişimi kısıtlı",
|
||||
"explore.from": "temeli: %s",
|
||||
@@ -134,6 +137,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "sunulamaz",
|
||||
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
|
||||
"card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn",
|
||||
"card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB",
|
||||
"card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin",
|
||||
"card.pickle_hidden.other": "%d dosya gösterilmiyor: pickle biçiminde, yüklenirken kod çalıştırır ve düğümler onu almaz.",
|
||||
|
||||
@@ -93,6 +93,9 @@ type exploreRow struct {
|
||||
New bool
|
||||
// Updated is how long ago the repository changed, in the page's language.
|
||||
Updated string
|
||||
// CPUSpeed is about how many tokens a second one reply would get on the
|
||||
// CPU of CPUNode, the fastest node that holds it; empty when unknown.
|
||||
CPUSpeed, CPUNode string
|
||||
}
|
||||
|
||||
// ParamsLabel is the parameter count as people say it: 7.6B, 494M.
|
||||
@@ -311,6 +314,9 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
|
||||
e.Hidden++
|
||||
continue
|
||||
}
|
||||
if fit != nil {
|
||||
row.CPUSpeed, row.CPUNode = fit.rowSpeed(m)
|
||||
}
|
||||
if e.Here && row.Verdict.Checked && !row.Verdict.Servable {
|
||||
e.Hidden++
|
||||
continue
|
||||
|
||||
@@ -302,6 +302,7 @@
|
||||
{{if .Rows}}
|
||||
<table class="rows explore">
|
||||
<tr><th>{{t "explore.col.repo"}}</th><th class="num">{{t "explore.col.size"}}</th><th>{{t "explore.col.arch"}}</th><th class="num">{{t "explore.col.context"}}</th>
|
||||
<th class="num" title="{{t "explore.col.cpu_title"}}">{{t "explore.col.cpu"}}</th>
|
||||
<th class="num">{{t "explore.col.downloads"}}</th><th class="num">{{t "explore.col.likes"}}</th><th class="num">{{t "explore.col.updated"}}</th></tr>
|
||||
{{range .Rows}}<tr class="explore-row">
|
||||
<td><a href="{{.Href}}" class="open-repo" data-repo="{{.ID}}" aria-expanded="false"><code>{{.ID}}</code></a>
|
||||
@@ -310,6 +311,7 @@
|
||||
<td class="num">{{.ParamsLabel}}</td>
|
||||
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{.ContextLabel}}</td>
|
||||
<td class="num">{{if .CPUSpeed}}<span title="{{t "explore.cpu_on" .CPUNode}}">{{.CPUSpeed}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{compact .Downloads}}</td>
|
||||
<td class="num">{{compact .Likes}}</td>
|
||||
<td class="num">{{.Updated}}</td>
|
||||
@@ -467,7 +469,8 @@ document.querySelector("table.explore").addEventListener("submit", async functio
|
||||
{{if .Shards}}<span class="opt">{{t "card.shards" .Shards}}</span>{{end}}
|
||||
{{with .Companions}}<span class="opt" title="{{join .}}">{{t "card.companions" (len .)}}</span>{{end}}
|
||||
{{if eq .Kind "adapter"}}<span class="tag new" title="{{t "card.adapter_title" .Arch}}">{{t "card.adapter" .Arch}}</span>
|
||||
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}</td>
|
||||
{{else if eq .Kind "model"}}<span class="opt">{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}</span>{{end}}
|
||||
{{if .CPUSpeed}}<span class="opt" title="{{t "explore.col.cpu_title"}}">{{t "card.cpu_speed" .CPUSpeed .CPUNode}}</span>{{end}}</td>
|
||||
<td class="num">{{bytes .Bytes}}</td>
|
||||
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
|
||||
<td>
|
||||
|
||||
Reference in new issue
Block a user