Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a45068461
commit
84444e71c8
20 files changed
+388
-7
No files matched your search
@@ -20,6 +20,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/safedial"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -57,6 +58,9 @@ type repoSet struct {
|
||||
// architecture the registry reports; a set that cannot be served is
|
||||
// shown with the reason and no fetch button.
|
||||
Verdict gguf.Verdict
|
||||
// CPUSpeed is about how many tokens a second one reply gets from a GGUF
|
||||
// on the CPU of CPUNode, the fastest node that holds it; empty otherwise.
|
||||
CPUSpeed, CPUNode string
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -820,6 +824,9 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
if fit != nil && rs.Verdict.Servable {
|
||||
if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok {
|
||||
rs.Verdict = v
|
||||
} else if gguf.IsGGUF(set.Name) {
|
||||
tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params))
|
||||
rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node
|
||||
}
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
|
||||
Reference in new issue
Block a user