Files
llmgw/internal/agent/discovery/sensors_test.go
T
Ilker ManapandClaude Opus 5.5 cd10e086c8 Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 19:49:08 +02:00

170 lines
6.2 KiB
Go

package discovery
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func put(t *testing.T, root, rel, body string) {
t.Helper()
p := filepath.Join(root, rel)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// dellHost is the layout of the Proxmox host these tests were written
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
// and cores, and RAPL with a package and its dram subdomain.
func dellHost(t *testing.T) string {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
return root
}
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
}
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
s := sampler(root, noSMI, &now)
first := s.Sample(t.Context())
// The package reading, not the hotter core: the package is what the
// chip itself reports as its temperature.
if first.CPUTempC != 58 {
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
}
if first.CPUPowerW != 0 {
t.Error("power from one reading of a counter is invented")
}
if first.InletTempC != 0 || first.GPUs != nil {
t.Errorf("sensors this host does not have were reported: %+v", first)
}
// 10 s later the package has used 650 J, and its dram subdomain some of
// that — counted once.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
t.Errorf("CPU power %.2f W, expected 65", got)
}
}
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
s := sampler(root, noSMI, &now)
s.Sample(t.Context())
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
t.Errorf("across a wrap: %.2f W, expected 50", got)
}
}
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
root := t.TempDir()
put(t, root, "class/thermal/cooling_device0/type", "Processor")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
t.Errorf("a machine with no sensors reported %+v", got)
}
}
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
}
}
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
var asked []string
smi := func(_ context.Context, fields string) ([]byte, error) {
asked = append(asked, fields)
return []byte(
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
}
now := time.Now()
ss := sampler(t.TempDir(), smi, &now)
got := ss.Sample(t.Context())
if len(got.GPUs) != 3 {
t.Fatalf("%+v", got.GPUs)
}
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
t.Errorf("card 0 %+v", g0)
}
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
t.Errorf("card 1 throttled by %v", g1.Throttle)
}
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
}
// Within ten seconds the last reading is reused rather than running
// nvidia-smi on every heartbeat; after, it is asked again.
now = now.Add(5 * time.Second)
ss.Sample(t.Context())
if len(asked) != 1 {
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
}
now = now.Add(6 * time.Second)
ss.Sample(t.Context())
if len(asked) != 2 {
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
}
}
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
smi := func(_ context.Context, fields string) ([]byte, error) {
if strings.Contains(fields, "temperature.memory") {
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
}
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
t.Errorf("%+v", got.GPUs)
}
}