diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 31d311a..e72cee5 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1545,24 +1545,33 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess } // memBandwidth is the node's memory read rate in GB/s, 0 until measured. -// Measured once a process, in the background, the first time it is asked -// for: it reads half a gigabyte, which must not hold up an agent reaching -// its control plane, and the heartbeat carries it when it is known. Once a -// process rather than an agent, so tests running many agents in one read it -// once. +// Measured in the background, the first time it is asked for: it reads half +// a gigabyte, which must not hold up an agent reaching its control plane, +// and the heartbeat carries it when it is known. Not at once but after a +// minute or so, chosen at random, and again every hour, keeping the best: +// VMs on one host that all restart together measure each other, and read a +// third less than any of them gets alone. Once a process rather than an +// agent, so tests running many agents in one read it once. func (a *Agent) memBandwidth() float64 { bwOnce.Do(func() { go func() { - bw := measureBandwidth() - bwBits.Store(math.Float64bits(bw)) - if bw > 0 { - a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10) + time.Sleep(bandwidthDelay()) + for { + if bw := measureBandwidth(); bw > math.Float64frombits(bwBits.Load()) { + bwBits.Store(math.Float64bits(bw)) + a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10) + } + time.Sleep(time.Hour) } }() }) return math.Float64frombits(bwBits.Load()) } +// bandwidthDelay is how long an agent waits before measuring, so agents +// started together measure apart. +var bandwidthDelay = func() time.Duration { return 10*time.Second + rand.N(110*time.Second) } + var ( bwOnce sync.Once bwBits atomic.Uint64 diff --git a/internal/agent/main_test.go b/internal/agent/main_test.go index 76842a1..1e62bbe 100644 --- a/internal/agent/main_test.go +++ b/internal/agent/main_test.go @@ -3,11 +3,13 @@ package agent import ( "os" "testing" + "time" ) func TestMain(m *testing.M) { // Half a gigabyte read per test binary, and contending with the tests // for the CPUs: these never look at the figure. measureBandwidth = func() float64 { return 0 } + bandwidthDelay = func() time.Duration { return 0 } os.Exit(m.Run()) }