Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -285,6 +285,25 @@ message NodeLoad {
|
||||
int64 memory_used_bytes = 6;
|
||||
int64 memory_available_bytes = 7;
|
||||
repeated ShardLoad shards = 8;
|
||||
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
|
||||
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
|
||||
// inlet_temp_c the board's inlet or ambient sensor where it has one.
|
||||
double cpu_temp_c = 9;
|
||||
double cpu_power_w = 10;
|
||||
double inlet_temp_c = 11;
|
||||
repeated GPUSensor gpus = 12;
|
||||
}
|
||||
|
||||
// GPUSensor is one card's heat, power, and what is holding it back.
|
||||
message GPUSensor {
|
||||
int32 index = 1;
|
||||
double temp_c = 2;
|
||||
double memory_temp_c = 3;
|
||||
double power_w = 4;
|
||||
double power_limit_w = 5;
|
||||
// throttle names what is holding the clocks down now: sw_power_cap,
|
||||
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
|
||||
repeated string throttle = 6;
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
|
||||
Reference in new issue
Block a user