The memory is measured after the agent reaches its control plane
CI / test (push) Successful in 9m5s

Measuring it in the startup scan held every agent half a second or more
before registering, which on a shared CI runner was past the failover
tests' wait. Now it runs once a process, in the background, the first time
it is asked for, and Heartbeat.mem_bandwidth_gbs carries it when known;
the control plane keeps the last figure and a heartbeat without one leaves
it. The agent's tests do not measure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-27 15:31:21 +01:00
1 parent 84444e71c8
commit e70d6caa2f
6 files changed
+83 -8

No files matched your search

+29 -3
View File
@@ -514,14 +514,13 @@ func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, opts, a.log)
host := discovery.DiscoverHost()
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth()
host.MemBandwidthGBs = a.memBandwidth()
a.mu.Lock()
a.inv = inv
a.prov.Host = host
a.mu.Unlock()
a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver,
"memory_gbs", math.Round(host.MemBandwidthGBs*10)/10)
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
return err
}
@@ -670,6 +669,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
// without anybody restarting anything.
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
hb.CpuCount = int32(runtime.NumCPU())
hb.MemBandwidthGbs = a.memBandwidth()
a.reprobeEngines()
a.mu.Lock()
@@ -1543,3 +1543,29 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess
}
}
}
// memBandwidth is the node's memory read rate in GB/s, 0 until measured.
// Measured once a process, in the background, the first time it is asked
// for: it reads half a gigabyte, which must not hold up an agent reaching
// its control plane, and the heartbeat carries it when it is known. Once a
// process rather than an agent, so tests running many agents in one read it
// once.
func (a *Agent) memBandwidth() float64 {
bwOnce.Do(func() {
go func() {
bw := measureBandwidth()
bwBits.Store(math.Float64bits(bw))
if bw > 0 {
a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10)
}
}()
})
return math.Float64frombits(bwBits.Load())
}
var (
bwOnce sync.Once
bwBits atomic.Uint64
// measureBandwidth is replaced in tests, which have no use for it.
measureBandwidth = discovery.MeasureMemoryBandwidth
)
+13
View File
@@ -0,0 +1,13 @@
package agent
import (
"os"
"testing"
)
func TestMain(m *testing.M) {
// Half a gigabyte read per test binary, and contending with the tests
// for the CPUs: these never look at the figure.
measureBandwidth = func() float64 { return 0 }
os.Exit(m.Run())
}