Files
llmgw/internal/agent/discovery/membw.go
T
Ilker ManapandClaude Opus 5.5 84444e71c8
CI / test (push) Failing after 6m37s
How fast a model would answer on a CPU, before it is fetched
Each agent measures its node's memory read rate when it starts: every CPU
sums a share of a buffer past any cache, a tenth of free memory at most,
the best of four passes, in a fraction of a second; reported in Host
(mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it
uses once, so one reply's speed is about that rate over those bytes:
modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token,
calibrated on the PoC against llama.cpp's predicted_per_second (node01
24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s,
qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active
part (ActiveShare).

The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that
holds the model; a repository's page gives it for each GGUF by its size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 00:09:47 +01:00

114 lines
2.7 KiB
Go

package discovery
import (
"bufio"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
// How fast this machine reads its memory, which on a CPU is what decides how
// fast a model answers: every token of a reply reads all the weights it uses
// once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten
// times a second, whatever its core count. Measured once when the agent
// starts, in a fraction of a second, by every CPU summing a share of a buffer
// too large for any cache — a VM sharing its host's memory with others gets
// its share, which is the number that matters for it.
// bandwidthBuffer is what is read: well past any processor's cache.
const bandwidthBuffer = 512 << 20
// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a
// few passes; 0 when the machine has too little free memory to measure.
func MeasureMemoryBandwidth() float64 {
size := int64(bandwidthBuffer)
// Never more than a tenth of what is free: a node already holding models
// must not be pushed into swap by a measurement.
if avail := memAvailable("/proc/meminfo"); avail > 0 {
size = min(size, avail/10)
}
if size < 64<<20 {
return 0
}
buf := make([]uint64, size/8)
for i := range buf {
buf[i] = uint64(i)
}
workers := runtime.NumCPU()
chunk := (len(buf) + workers - 1) / workers
best := 0.0
for pass := 0; pass < 4; pass++ {
var wg sync.WaitGroup
start := time.Now()
for w := 0; w < workers; w++ {
lo, hi := w*chunk, min((w+1)*chunk, len(buf))
if lo >= hi {
continue
}
wg.Add(1)
go func(part []uint64) {
defer wg.Done()
sink(sum(part))
}(buf[lo:hi])
}
wg.Wait()
if s := time.Since(start).Seconds(); s > 0 {
best = max(best, float64(size)/s/1e9)
}
}
return best
}
// sum reads every word, eight at a time so the loop is not what limits it.
func sum(p []uint64) uint64 {
var a, b, c, d, e, f, g, h uint64
i := 0
for ; i+8 <= len(p); i += 8 {
a += p[i]
b += p[i+1]
c += p[i+2]
d += p[i+3]
e += p[i+4]
f += p[i+5]
g += p[i+6]
h += p[i+7]
}
for ; i < len(p); i++ {
a += p[i]
}
return a + b + c + d + e + f + g + h
}
// sinkValue keeps the sums alive, so the reads are not optimised away.
var sinkValue struct {
sync.Mutex
v uint64
}
func sink(v uint64) {
sinkValue.Lock()
sinkValue.v += v
sinkValue.Unlock()
}
// memAvailable is MemAvailable in bytes, 0 when unknown.
func memAvailable(path string) int64 {
f, err := os.Open(path) //nolint:gosec // a fixed system path
if err != nil {
return 0
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Fields(sc.Text())
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
kb, _ := strconv.ParseInt(fields[1], 10, 64)
return kb << 10
}
}
return 0
}