Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
|
||||
t.Error("the cluster page does not lead to the racks")
|
||||
}
|
||||
}
|
||||
|
||||
// hotSource is a GPU node whose sensors report, one card held back.
|
||||
type hotSource struct{ fakeSource }
|
||||
|
||||
func (hotSource) Nodes() []adminapi.NodeView {
|
||||
n := fakeSource{}.Nodes()[0]
|
||||
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
|
||||
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
|
||||
GPUs: []inventory.GPUSensor{
|
||||
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
|
||||
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
|
||||
}}
|
||||
return []adminapi.NodeView{n}
|
||||
}
|
||||
|
||||
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
|
||||
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
|
||||
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page lacks %q", want)
|
||||
}
|
||||
}
|
||||
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
|
||||
if strings.Contains(vm, absent) {
|
||||
t.Errorf("a node with no sensors shows %q", absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user