Files
llmgw/internal/control/fleetmetrics/sensors_test.go
T
Ilker ManapandClaude Opus 5.5 cd10e086c8 Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 19:49:08 +02:00

42 lines
1.7 KiB
Go

package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
for _, want := range []string{
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
"llmgw_node_cpu_power_watts node=g1 -> 65",
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
// The fresh reading, not the 40 °C from registration.
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q", want)
}
}
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
t.Errorf("published what was not measured: %s", line)
}
}
}
}