Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL

Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 19:49:08 +02:00
1 parent 8724dc027a
commit cd10e086c8
15 files changed
+996 -172

No files matched your search

+31
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
t.Error("the cluster page does not lead to the racks")
}
}
// hotSource is a GPU node whose sensors report, one card held back.
type hotSource struct{ fakeSource }
func (hotSource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
GPUs: []inventory.GPUSensor{
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
}}
return []adminapi.NodeView{n}
}
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
if strings.Contains(vm, absent) {
t.Errorf("a node with no sensors shows %q", absent)
}
}
}