Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2a45068461
commit
84444e71c8
20 files changed
+388
-7
No files matched your search
@@ -12,6 +12,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -513,12 +514,14 @@ func (a *Agent) scan(ctx context.Context) error {
|
||||
inv, err := discovery.Discover(ctx, opts, a.log)
|
||||
host := discovery.DiscoverHost()
|
||||
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
|
||||
host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth()
|
||||
a.mu.Lock()
|
||||
a.inv = inv
|
||||
a.prov.Host = host
|
||||
a.mu.Unlock()
|
||||
a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
|
||||
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
|
||||
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver,
|
||||
"memory_gbs", math.Round(host.MemBandwidthGBs*10)/10)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -569,6 +572,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
reg.Host = &llmgwv1.Host{
|
||||
OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename,
|
||||
Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver,
|
||||
MemBandwidthGbs: h.MemBandwidthGBs,
|
||||
}
|
||||
|
||||
if err := stream.Send(&llmgwv1.AgentMessage{
|
||||
|
||||
@@ -31,6 +31,10 @@ type Host struct {
|
||||
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
|
||||
// is not known. Not read here but from the GPU inventory, by the agent.
|
||||
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
|
||||
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
|
||||
// measured: what decides a model's speed on the CPU. Not read here but
|
||||
// measured by the agent when it starts (MeasureMemoryBandwidth).
|
||||
MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
|
||||
}
|
||||
|
||||
// Family is the packaging family: "deb", "rpm", or "" when neither is known.
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// How fast this machine reads its memory, which on a CPU is what decides how
|
||||
// fast a model answers: every token of a reply reads all the weights it uses
|
||||
// once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten
|
||||
// times a second, whatever its core count. Measured once when the agent
|
||||
// starts, in a fraction of a second, by every CPU summing a share of a buffer
|
||||
// too large for any cache — a VM sharing its host's memory with others gets
|
||||
// its share, which is the number that matters for it.
|
||||
|
||||
// bandwidthBuffer is what is read: well past any processor's cache.
|
||||
const bandwidthBuffer = 512 << 20
|
||||
|
||||
// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a
|
||||
// few passes; 0 when the machine has too little free memory to measure.
|
||||
func MeasureMemoryBandwidth() float64 {
|
||||
size := int64(bandwidthBuffer)
|
||||
// Never more than a tenth of what is free: a node already holding models
|
||||
// must not be pushed into swap by a measurement.
|
||||
if avail := memAvailable("/proc/meminfo"); avail > 0 {
|
||||
size = min(size, avail/10)
|
||||
}
|
||||
if size < 64<<20 {
|
||||
return 0
|
||||
}
|
||||
buf := make([]uint64, size/8)
|
||||
for i := range buf {
|
||||
buf[i] = uint64(i)
|
||||
}
|
||||
workers := runtime.NumCPU()
|
||||
chunk := (len(buf) + workers - 1) / workers
|
||||
best := 0.0
|
||||
for pass := 0; pass < 4; pass++ {
|
||||
var wg sync.WaitGroup
|
||||
start := time.Now()
|
||||
for w := 0; w < workers; w++ {
|
||||
lo, hi := w*chunk, min((w+1)*chunk, len(buf))
|
||||
if lo >= hi {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(part []uint64) {
|
||||
defer wg.Done()
|
||||
sink(sum(part))
|
||||
}(buf[lo:hi])
|
||||
}
|
||||
wg.Wait()
|
||||
if s := time.Since(start).Seconds(); s > 0 {
|
||||
best = max(best, float64(size)/s/1e9)
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// sum reads every word, eight at a time so the loop is not what limits it.
|
||||
func sum(p []uint64) uint64 {
|
||||
var a, b, c, d, e, f, g, h uint64
|
||||
i := 0
|
||||
for ; i+8 <= len(p); i += 8 {
|
||||
a += p[i]
|
||||
b += p[i+1]
|
||||
c += p[i+2]
|
||||
d += p[i+3]
|
||||
e += p[i+4]
|
||||
f += p[i+5]
|
||||
g += p[i+6]
|
||||
h += p[i+7]
|
||||
}
|
||||
for ; i < len(p); i++ {
|
||||
a += p[i]
|
||||
}
|
||||
return a + b + c + d + e + f + g + h
|
||||
}
|
||||
|
||||
// sinkValue keeps the sums alive, so the reads are not optimised away.
|
||||
var sinkValue struct {
|
||||
sync.Mutex
|
||||
v uint64
|
||||
}
|
||||
|
||||
func sink(v uint64) {
|
||||
sinkValue.Lock()
|
||||
sinkValue.v += v
|
||||
sinkValue.Unlock()
|
||||
}
|
||||
|
||||
// memAvailable is MemAvailable in bytes, 0 when unknown.
|
||||
func memAvailable(path string) int64 {
|
||||
f, err := os.Open(path) //nolint:gosec // a fixed system path
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
|
||||
kb, _ := strconv.ParseInt(fields[1], 10, 64)
|
||||
return kb << 10
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package discovery
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMemoryBandwidthIsMeasured(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("reads half a gigabyte")
|
||||
}
|
||||
bw := MeasureMemoryBandwidth()
|
||||
// Any machine running this reads more than a gigabyte a second and less
|
||||
// than a few terabytes.
|
||||
if bw < 1 || bw > 5000 {
|
||||
t.Errorf("measured %.1f GB/s", bw)
|
||||
}
|
||||
t.Logf("%.1f GB/s", bw)
|
||||
}
|
||||
Reference in new issue
Block a user