package discovery import ( "bufio" "os" "runtime" "strconv" "strings" "sync" "time" ) // How fast this machine reads its memory, which on a CPU is what decides how // fast a model answers: every token of a reply reads all the weights it uses // once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten // times a second, whatever its core count. Measured once when the agent // starts, in a fraction of a second, by every CPU summing a share of a buffer // too large for any cache — a VM sharing its host's memory with others gets // its share, which is the number that matters for it. // bandwidthBuffer is what is read: well past any processor's cache. const bandwidthBuffer = 512 << 20 // MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a // few passes; 0 when the machine has too little free memory to measure. func MeasureMemoryBandwidth() float64 { size := int64(bandwidthBuffer) // Never more than a tenth of what is free: a node already holding models // must not be pushed into swap by a measurement. if avail := memAvailable("/proc/meminfo"); avail > 0 { size = min(size, avail/10) } if size < 64<<20 { return 0 } buf := make([]uint64, size/8) for i := range buf { buf[i] = uint64(i) } workers := runtime.NumCPU() chunk := (len(buf) + workers - 1) / workers best := 0.0 for pass := 0; pass < 4; pass++ { var wg sync.WaitGroup start := time.Now() for w := 0; w < workers; w++ { lo, hi := w*chunk, min((w+1)*chunk, len(buf)) if lo >= hi { continue } wg.Add(1) go func(part []uint64) { defer wg.Done() sink(sum(part)) }(buf[lo:hi]) } wg.Wait() if s := time.Since(start).Seconds(); s > 0 { best = max(best, float64(size)/s/1e9) } } return best } // sum reads every word, eight at a time so the loop is not what limits it. func sum(p []uint64) uint64 { var a, b, c, d, e, f, g, h uint64 i := 0 for ; i+8 <= len(p); i += 8 { a += p[i] b += p[i+1] c += p[i+2] d += p[i+3] e += p[i+4] f += p[i+5] g += p[i+6] h += p[i+7] } for ; i < len(p); i++ { a += p[i] } return a + b + c + d + e + f + g + h } // sinkValue keeps the sums alive, so the reads are not optimised away. var sinkValue struct { sync.Mutex v uint64 } func sink(v uint64) { sinkValue.Lock() sinkValue.v += v sinkValue.Unlock() } // memAvailable is MemAvailable in bytes, 0 when unknown. func memAvailable(path string) int64 { f, err := os.Open(path) //nolint:gosec // a fixed system path if err != nil { return 0 } defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { fields := strings.Fields(sc.Text()) if len(fields) >= 2 && fields[0] == "MemAvailable:" { kb, _ := strconv.ParseInt(fields[1], 10, 64) return kb << 10 } } return 0 }