Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
114 lines
2.7 KiB
Go
114 lines
2.7 KiB
Go
package discovery
|
|
|
|
import (
|
|
"bufio"
|
|
"os"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// How fast this machine reads its memory, which on a CPU is what decides how
|
|
// fast a model answers: every token of a reply reads all the weights it uses
|
|
// once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten
|
|
// times a second, whatever its core count. Measured once when the agent
|
|
// starts, in a fraction of a second, by every CPU summing a share of a buffer
|
|
// too large for any cache — a VM sharing its host's memory with others gets
|
|
// its share, which is the number that matters for it.
|
|
|
|
// bandwidthBuffer is what is read: well past any processor's cache.
|
|
const bandwidthBuffer = 512 << 20
|
|
|
|
// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a
|
|
// few passes; 0 when the machine has too little free memory to measure.
|
|
func MeasureMemoryBandwidth() float64 {
|
|
size := int64(bandwidthBuffer)
|
|
// Never more than a tenth of what is free: a node already holding models
|
|
// must not be pushed into swap by a measurement.
|
|
if avail := memAvailable("/proc/meminfo"); avail > 0 {
|
|
size = min(size, avail/10)
|
|
}
|
|
if size < 64<<20 {
|
|
return 0
|
|
}
|
|
buf := make([]uint64, size/8)
|
|
for i := range buf {
|
|
buf[i] = uint64(i)
|
|
}
|
|
workers := runtime.NumCPU()
|
|
chunk := (len(buf) + workers - 1) / workers
|
|
best := 0.0
|
|
for pass := 0; pass < 4; pass++ {
|
|
var wg sync.WaitGroup
|
|
start := time.Now()
|
|
for w := 0; w < workers; w++ {
|
|
lo, hi := w*chunk, min((w+1)*chunk, len(buf))
|
|
if lo >= hi {
|
|
continue
|
|
}
|
|
wg.Add(1)
|
|
go func(part []uint64) {
|
|
defer wg.Done()
|
|
sink(sum(part))
|
|
}(buf[lo:hi])
|
|
}
|
|
wg.Wait()
|
|
if s := time.Since(start).Seconds(); s > 0 {
|
|
best = max(best, float64(size)/s/1e9)
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
// sum reads every word, eight at a time so the loop is not what limits it.
|
|
func sum(p []uint64) uint64 {
|
|
var a, b, c, d, e, f, g, h uint64
|
|
i := 0
|
|
for ; i+8 <= len(p); i += 8 {
|
|
a += p[i]
|
|
b += p[i+1]
|
|
c += p[i+2]
|
|
d += p[i+3]
|
|
e += p[i+4]
|
|
f += p[i+5]
|
|
g += p[i+6]
|
|
h += p[i+7]
|
|
}
|
|
for ; i < len(p); i++ {
|
|
a += p[i]
|
|
}
|
|
return a + b + c + d + e + f + g + h
|
|
}
|
|
|
|
// sinkValue keeps the sums alive, so the reads are not optimised away.
|
|
var sinkValue struct {
|
|
sync.Mutex
|
|
v uint64
|
|
}
|
|
|
|
func sink(v uint64) {
|
|
sinkValue.Lock()
|
|
sinkValue.v += v
|
|
sinkValue.Unlock()
|
|
}
|
|
|
|
// memAvailable is MemAvailable in bytes, 0 when unknown.
|
|
func memAvailable(path string) int64 {
|
|
f, err := os.Open(path) //nolint:gosec // a fixed system path
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
fields := strings.Fields(sc.Text())
|
|
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
|
|
kb, _ := strconv.ParseInt(fields[1], 10, 64)
|
|
return kb << 10
|
|
}
|
|
}
|
|
return 0
|
|
}
|