Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Release / build (push) Successful in 1m10s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s

Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 19:49:08 +02:00
1 parent 8724dc027a
commit cd10e086c8
15 files changed
+996 -172

No files matched your search

+31
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
t.Error("the cluster page does not lead to the racks")
}
}
// hotSource is a GPU node whose sensors report, one card held back.
type hotSource struct{ fakeSource }
func (hotSource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
GPUs: []inventory.GPUSensor{
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
}}
return []adminapi.NodeView{n}
}
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
if strings.Contains(vm, absent) {
t.Errorf("a node with no sensors shows %q", absent)
}
}
}
+15
View File
@@ -21,6 +21,10 @@
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
{{/* Only what this machine has sensors for: a VM has none, and a tile
saying 0 °C would be a reading nobody took. */}}
{{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{printf "%.0f" .CPUTempC}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{printf "%.0f" .CPUPowerW}} W{{end}}</div></div>{{end}}
{{if .InletTempC}}<div class="tile"><div class="v">{{printf "%.1f" .InletTempC}} °C</div><div class="k">inlet air</div></div>{{end}}
{{else}}
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
{{end}}
@@ -29,6 +33,17 @@
</div>
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}}
<h2>GPUs</h2>
<table class="rows"><tr><th class="num">GPU</th><th class="num">Temperature</th><th class="num">Memory</th><th>Power</th><th>Held back by</th></tr>
{{range .GPUs}}<tr>
<td class="num">{{.Index}}</td>
<td class="num">{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}}</td>
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}</td>
</tr>{{end}}</table>
{{end}}{{end}}
{{end}}
{{if or .Node.IBPorts .Node.Fabrics}}
+7
View File
@@ -109,6 +109,13 @@ var funcs = template.FuncMap{
return p
},
"ago": ago,
// percent is part of whole, 0 to 100, for a bar.
"percent": func(part, whole float64) float64 {
if whole <= 0 {
return 0
}
return part * 100 / whole
},
// shortNode is a node's name without what every node's name starts with.
"shortNode": strings.TrimPrefix,
"age": age,