The memory is measured after the agent reaches its control plane
CI / test (push) Successful in 9m5s

Measuring it in the startup scan held every agent half a second or more
before registering, which on a shared CI runner was past the failover
tests' wait. Now it runs once a process, in the background, the first time
it is asked for, and Heartbeat.mem_bandwidth_gbs carries it when known;
the control plane keeps the last figure and a heartbeat without one leaves
it. The agent's tests do not measure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-27 15:31:21 +01:00
1 parent 84444e71c8
commit e70d6caa2f
6 files changed
+83 -8

No files matched your search

+29 -3
View File
@@ -514,14 +514,13 @@ func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, opts, a.log)
host := discovery.DiscoverHost()
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth()
host.MemBandwidthGBs = a.memBandwidth()
a.mu.Lock()
a.inv = inv
a.prov.Host = host
a.mu.Unlock()
a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver,
"memory_gbs", math.Round(host.MemBandwidthGBs*10)/10)
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
return err
}
@@ -670,6 +669,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
// without anybody restarting anything.
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
hb.CpuCount = int32(runtime.NumCPU())
hb.MemBandwidthGbs = a.memBandwidth()
a.reprobeEngines()
a.mu.Lock()
@@ -1543,3 +1543,29 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess
}
}
}
// memBandwidth is the node's memory read rate in GB/s, 0 until measured.
// Measured once a process, in the background, the first time it is asked
// for: it reads half a gigabyte, which must not hold up an agent reaching
// its control plane, and the heartbeat carries it when it is known. Once a
// process rather than an agent, so tests running many agents in one read it
// once.
func (a *Agent) memBandwidth() float64 {
bwOnce.Do(func() {
go func() {
bw := measureBandwidth()
bwBits.Store(math.Float64bits(bw))
if bw > 0 {
a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10)
}
}()
})
return math.Float64frombits(bwBits.Load())
}
var (
bwOnce sync.Once
bwBits atomic.Uint64
// measureBandwidth is replaced in tests, which have no use for it.
measureBandwidth = discovery.MeasureMemoryBandwidth
)
+13
View File
@@ -0,0 +1,13 @@
package agent
import (
"os"
"testing"
)
func TestMain(m *testing.M) {
// Half a gigabyte read per test binary, and contending with the tests
// for the CPUs: these never look at the figure.
measureBandwidth = func() float64 { return 0 }
os.Exit(m.Run())
}
@@ -325,3 +325,18 @@ func TestAProblemOnANodeWithoutGPUsDoesNotPanic(t *testing.T) {
t.Fatal("the node is gone")
}
}
func TestAHeartbeatCarriesTheMemorysMeasuredSpeed(t *testing.T) {
// Measured in the background after the agent starts, so it arrives in a
// heartbeat, not in the registration.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemBandwidthGbs: 41.9})
if n, _ := s.inv.Get("n1"); n.Host.MemBandwidthGBs != 41.9 {
t.Errorf("bandwidth %v, want 41.9", n.Host.MemBandwidthGBs)
}
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
if n, _ := s.inv.Get("n1"); n.Host.MemBandwidthGBs != 41.9 {
t.Errorf("a heartbeat without it erased it: %v", n.Host.MemBandwidthGBs)
}
}
+5 -1
View File
@@ -547,10 +547,11 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
storage := hb.GetStorage()
memMB := int(hb.GetMemoryTotalMb())
cpus := int(hb.GetCpuCount())
bw := hb.GetMemBandwidthGbs()
if len(used) == 0 && len(foreign) == 0 && len(problems) == 0 && len(storage) == 0 &&
!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil && bw == 0 {
return
}
@@ -622,6 +623,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
n.MemoryTotalMB = memMB
}
if bw > 0 {
n.Host.MemBandwidthGBs = bw
}
if cpus > 0 && cpus != n.CPUCount {
if n.CPUCount > 0 {
s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus)