Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a45068461
commit
84444e71c8
20 files changed
+388
-7
No files matched your search
@@ -0,0 +1,62 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// How fast a model answers on a CPU, from its size and the node's memory.
|
||||
//
|
||||
// Writing a token reads every weight it uses once, so on a CPU the memory
|
||||
// decides the speed: a token takes the bytes read over the rate llama.cpp
|
||||
// reads them at, plus a little that does not shrink with the model. Measured
|
||||
// on the PoC on 2026-10-26, each node's bandwidth by MeasureMemoryBandwidth
|
||||
// against llama.cpp's own predicted_per_second for 128 tokens:
|
||||
//
|
||||
// node01 4 vCPU 24.9 GB/s qwen2.5-1.5b Q4_K_M 1.1 GB 14.5 tok/s 0.65 of the bandwidth
|
||||
// node05 8 vCPU 41.9 GB/s qwen2.5-14b Q4_K_M 9.0 GB 3.3 tok/s 0.71
|
||||
// cuda01 12 vCPU 42.4 GB/s qwen2.5-0.5b Q4_K_M 0.5 GB 36.0 tok/s (the fixed cost shows)
|
||||
//
|
||||
// So 0.68 of the measured rate, and 5 ms a token besides. One reply at a
|
||||
// time; several at once share the same reads and each is slower. The PoC's
|
||||
// host has far fewer cores than its VMs, so these are what a busy host
|
||||
// gives, and a machine of its own does better.
|
||||
|
||||
// cpuReadShare is the part of the measured bandwidth llama.cpp reads at.
|
||||
const cpuReadShare = 0.68
|
||||
|
||||
// cpuTokenFixed is a token's cost that does not grow with the model.
|
||||
const cpuTokenFixed = 0.005
|
||||
|
||||
// CPUTokensPerSecond is about how many tokens a second one reply gets on a
|
||||
// CPU whose memory reads bwGBs, from a model whose tokens each read bytes.
|
||||
// 0 when either is unknown.
|
||||
func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 {
|
||||
if bytes <= 0 || bwGBs <= 0 {
|
||||
return 0
|
||||
}
|
||||
return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed)
|
||||
}
|
||||
|
||||
// activeRE finds a model of experts' active parameters in its name, the
|
||||
// way they are written: Qwen3-30B-A3B, Ling-flash-A2.5B.
|
||||
var activeRE = regexp.MustCompile(`(?i)(?:^|[-_./])a(\d+(?:\.\d+)?)b(?:$|[-_.])`)
|
||||
|
||||
// ActiveShare is the part of a model's weights one token reads: for a model
|
||||
// of experts named with its active parameters (…-30B-A3B) that over its
|
||||
// total, otherwise 1. Rough — the shared layers are read for every token —
|
||||
// and what a name says is all a listing has.
|
||||
func ActiveShare(name string, params int64) float64 {
|
||||
m := activeRE.FindStringSubmatch(name)
|
||||
if m == nil || params <= 0 {
|
||||
return 1
|
||||
}
|
||||
b, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil || b <= 0 {
|
||||
return 1
|
||||
}
|
||||
if s := b * 1e9 / float64(params); s < 1 {
|
||||
return s
|
||||
}
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
bytes int64
|
||||
bw, want float64
|
||||
}{
|
||||
{1117320736, 24.9, 14.5}, // node01, qwen2.5-1.5b
|
||||
{9_000_000_000, 41.9, 3.3}, // node05, qwen2.5-coder-14b
|
||||
} {
|
||||
got := CPUTokensPerSecond(c.bytes, c.bw)
|
||||
if math.Abs(got-c.want)/c.want > 0.15 {
|
||||
t.Errorf("%d bytes at %.1f GB/s: %.1f tok/s, measured %.1f", c.bytes, c.bw, got, c.want)
|
||||
}
|
||||
}
|
||||
if CPUTokensPerSecond(0, 40) != 0 || CPUTokensPerSecond(1e9, 0) != 0 {
|
||||
t.Error("an unknown size or bandwidth should give no figure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
|
||||
for name, want := range map[string]float64{
|
||||
"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.1,
|
||||
"inclusionAI/Ling-flash-A2.5B": 2.5 / 30,
|
||||
"Qwen/Qwen3-8B": 1,
|
||||
"meta-llama/Llama-3.1-8B-Instruct": 1,
|
||||
} {
|
||||
if got := ActiveShare(name, 30e9); math.Abs(got-want) > 0.001 && !(want == 1 && got == 1) {
|
||||
t.Errorf("%s: %.3f, want %.3f", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user