The memory is measured after the agent reaches its control plane
CI / test (push) Successful in 9m5s
CI / test (push) Successful in 9m5s
Measuring it in the startup scan held every agent half a second or more before registering, which on a shared CI runner was past the failover tests' wait. Now it runs once a process, in the background, the first time it is asked for, and Heartbeat.mem_bandwidth_gbs carries it when known; the control plane keeps the last figure and a heartbeat without one leaves it. The agent's tests do not measure. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
84444e71c8
commit
e70d6caa2f
6 files changed
+83
-8
No files matched your search
+29
-3
@@ -514,14 +514,13 @@ func (a *Agent) scan(ctx context.Context) error {
|
||||
inv, err := discovery.Discover(ctx, opts, a.log)
|
||||
host := discovery.DiscoverHost()
|
||||
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
|
||||
host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth()
|
||||
host.MemBandwidthGBs = a.memBandwidth()
|
||||
a.mu.Lock()
|
||||
a.inv = inv
|
||||
a.prov.Host = host
|
||||
a.mu.Unlock()
|
||||
a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
|
||||
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver,
|
||||
"memory_gbs", math.Round(host.MemBandwidthGBs*10)/10)
|
||||
"cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -670,6 +669,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
// without anybody restarting anything.
|
||||
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
|
||||
hb.CpuCount = int32(runtime.NumCPU())
|
||||
hb.MemBandwidthGbs = a.memBandwidth()
|
||||
|
||||
a.reprobeEngines()
|
||||
a.mu.Lock()
|
||||
@@ -1543,3 +1543,29 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// memBandwidth is the node's memory read rate in GB/s, 0 until measured.
|
||||
// Measured once a process, in the background, the first time it is asked
|
||||
// for: it reads half a gigabyte, which must not hold up an agent reaching
|
||||
// its control plane, and the heartbeat carries it when it is known. Once a
|
||||
// process rather than an agent, so tests running many agents in one read it
|
||||
// once.
|
||||
func (a *Agent) memBandwidth() float64 {
|
||||
bwOnce.Do(func() {
|
||||
go func() {
|
||||
bw := measureBandwidth()
|
||||
bwBits.Store(math.Float64bits(bw))
|
||||
if bw > 0 {
|
||||
a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10)
|
||||
}
|
||||
}()
|
||||
})
|
||||
return math.Float64frombits(bwBits.Load())
|
||||
}
|
||||
|
||||
var (
|
||||
bwOnce sync.Once
|
||||
bwBits atomic.Uint64
|
||||
// measureBandwidth is replaced in tests, which have no use for it.
|
||||
measureBandwidth = discovery.MeasureMemoryBandwidth
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Half a gigabyte read per test binary, and contending with the tests
|
||||
// for the CPUs: these never look at the figure.
|
||||
measureBandwidth = func() float64 { return 0 }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
Reference in new issue
Block a user