How fast a model would answer on a CPU, before it is fetched
CI / test (push) Failing after 6m37s

Each agent measures its node's memory read rate when it starts: every CPU
sums a share of a buffer past any cache, a tenth of free memory at most,
the best of four passes, in a fraction of a second; reported in Host
(mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it
uses once, so one reply's speed is about that rate over those bytes:
modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token,
calibrated on the PoC against llama.cpp's predicted_per_second (node01
24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s,
qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active
part (ActiveShare).

The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that
holds the model; a repository's page gives it for each GGUF by its size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-27 00:09:47 +01:00
1 parent 2a45068461
commit 84444e71c8
20 files changed
+388 -7

No files matched your search

+5 -1
View File
@@ -12,6 +12,7 @@ import (
"io"
"log/slog"
"maps"
"math"
"math/rand/v2"
"net"
"net/http"
@@ -513,12 +514,14 @@ func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, opts, a.log)
host := discovery.DiscoverHost()
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth()
a.mu.Lock()
a.inv = inv
a.prov.Host = host
a.mu.Unlock()
a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver,
"memory_gbs", math.Round(host.MemBandwidthGBs*10)/10)
return err
}
@@ -569,6 +572,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
reg.Host = &llmgwv1.Host{
OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename,
Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver,
MemBandwidthGbs: h.MemBandwidthGBs,
}
if err := stream.Send(&llmgwv1.AgentMessage{
+4
View File
@@ -31,6 +31,10 @@ type Host struct {
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
// is not known. Not read here but from the GPU inventory, by the agent.
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
// measured: what decides a model's speed on the CPU. Not read here but
// measured by the agent when it starts (MeasureMemoryBandwidth).
MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
}
// Family is the packaging family: "deb", "rpm", or "" when neither is known.
+113
View File
@@ -0,0 +1,113 @@
package discovery
import (
"bufio"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
// How fast this machine reads its memory, which on a CPU is what decides how
// fast a model answers: every token of a reply reads all the weights it uses
// once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten
// times a second, whatever its core count. Measured once when the agent
// starts, in a fraction of a second, by every CPU summing a share of a buffer
// too large for any cache — a VM sharing its host's memory with others gets
// its share, which is the number that matters for it.
// bandwidthBuffer is what is read: well past any processor's cache.
const bandwidthBuffer = 512 << 20
// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a
// few passes; 0 when the machine has too little free memory to measure.
func MeasureMemoryBandwidth() float64 {
size := int64(bandwidthBuffer)
// Never more than a tenth of what is free: a node already holding models
// must not be pushed into swap by a measurement.
if avail := memAvailable("/proc/meminfo"); avail > 0 {
size = min(size, avail/10)
}
if size < 64<<20 {
return 0
}
buf := make([]uint64, size/8)
for i := range buf {
buf[i] = uint64(i)
}
workers := runtime.NumCPU()
chunk := (len(buf) + workers - 1) / workers
best := 0.0
for pass := 0; pass < 4; pass++ {
var wg sync.WaitGroup
start := time.Now()
for w := 0; w < workers; w++ {
lo, hi := w*chunk, min((w+1)*chunk, len(buf))
if lo >= hi {
continue
}
wg.Add(1)
go func(part []uint64) {
defer wg.Done()
sink(sum(part))
}(buf[lo:hi])
}
wg.Wait()
if s := time.Since(start).Seconds(); s > 0 {
best = max(best, float64(size)/s/1e9)
}
}
return best
}
// sum reads every word, eight at a time so the loop is not what limits it.
func sum(p []uint64) uint64 {
var a, b, c, d, e, f, g, h uint64
i := 0
for ; i+8 <= len(p); i += 8 {
a += p[i]
b += p[i+1]
c += p[i+2]
d += p[i+3]
e += p[i+4]
f += p[i+5]
g += p[i+6]
h += p[i+7]
}
for ; i < len(p); i++ {
a += p[i]
}
return a + b + c + d + e + f + g + h
}
// sinkValue keeps the sums alive, so the reads are not optimised away.
var sinkValue struct {
sync.Mutex
v uint64
}
func sink(v uint64) {
sinkValue.Lock()
sinkValue.v += v
sinkValue.Unlock()
}
// memAvailable is MemAvailable in bytes, 0 when unknown.
func memAvailable(path string) int64 {
f, err := os.Open(path) //nolint:gosec // a fixed system path
if err != nil {
return 0
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
kb, _ := strconv.ParseInt(fields[1], 10, 64)
return kb << 10
}
}
return 0
}
+16
View File
@@ -0,0 +1,16 @@
package discovery
import "testing"
func TestMemoryBandwidthIsMeasured(t *testing.T) {
if testing.Short() {
t.Skip("reads half a gigabyte")
}
bw := MeasureMemoryBandwidth()
// Any machine running this reads more than a gigabyte a second and less
// than a few terabytes.
if bw < 1 || bw > 5000 {
t.Errorf("measured %.1f GB/s", bw)
}
t.Logf("%.1f GB/s", bw)
}