Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
|
||||
t.Error("the cluster page does not lead to the racks")
|
||||
}
|
||||
}
|
||||
|
||||
// hotSource is a GPU node whose sensors report, one card held back.
|
||||
type hotSource struct{ fakeSource }
|
||||
|
||||
func (hotSource) Nodes() []adminapi.NodeView {
|
||||
n := fakeSource{}.Nodes()[0]
|
||||
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
|
||||
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
|
||||
GPUs: []inventory.GPUSensor{
|
||||
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
|
||||
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
|
||||
}}
|
||||
return []adminapi.NodeView{n}
|
||||
}
|
||||
|
||||
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
|
||||
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
|
||||
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page lacks %q", want)
|
||||
}
|
||||
}
|
||||
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
|
||||
if strings.Contains(vm, absent) {
|
||||
t.Errorf("a node with no sensors shows %q", absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,10 @@
|
||||
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
|
||||
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
|
||||
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
|
||||
{{/* Only what this machine has sensors for: a VM has none, and a tile
|
||||
saying 0 °C would be a reading nobody took. */}}
|
||||
{{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{printf "%.0f" .CPUTempC}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{printf "%.0f" .CPUPowerW}} W{{end}}</div></div>{{end}}
|
||||
{{if .InletTempC}}<div class="tile"><div class="v">{{printf "%.1f" .InletTempC}} °C</div><div class="k">inlet air</div></div>{{end}}
|
||||
{{else}}
|
||||
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
|
||||
{{end}}
|
||||
@@ -29,6 +33,17 @@
|
||||
</div>
|
||||
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
|
||||
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
|
||||
{{with .Load}}{{if .GPUs}}
|
||||
<h2>GPUs</h2>
|
||||
<table class="rows"><tr><th class="num">GPU</th><th class="num">Temperature</th><th class="num">Memory</th><th>Power</th><th>Held back by</th></tr>
|
||||
{{range .GPUs}}<tr>
|
||||
<td class="num">{{.Index}}</td>
|
||||
<td class="num">{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
|
||||
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
|
||||
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if or .Node.IBPorts .Node.Fabrics}}
|
||||
|
||||
@@ -109,6 +109,13 @@ var funcs = template.FuncMap{
|
||||
return p
|
||||
},
|
||||
"ago": ago,
|
||||
// percent is part of whole, 0 to 100, for a bar.
|
||||
"percent": func(part, whole float64) float64 {
|
||||
if whole <= 0 {
|
||||
return 0
|
||||
}
|
||||
return part * 100 / whole
|
||||
},
|
||||
// shortNode is a node's name without what every node's name starts with.
|
||||
"shortNode": strings.TrimPrefix,
|
||||
"age": age,
|
||||
|
||||
Reference in new issue
Block a user