Files
llmgw/internal/agent/discovery/sensors_test.go
T
Ilker ManapandClaude Opus 5.5 bd36798b15 Sample GPU utilisation and memory every heartbeat, and chart them
They were read once, at agent start, and published unchanged since.
Each sample now carries utilisation, memory and bandwidth in use, the SM
clock and uncorrected ECC errors. The node page charts each card, the
cluster page the fleet by class, Grafana has a GPU row, and three GPU
alerts are added.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 15:42:17 +02:00

211 lines
7.9 KiB
Go

package discovery
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func put(t *testing.T, root, rel, body string) {
t.Helper()
p := filepath.Join(root, rel)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// dellHost is the layout of the Proxmox host these tests were written
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
// and cores, and RAPL with a package and its dram subdomain.
func dellHost(t *testing.T) string {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
return root
}
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
}
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
s := sampler(root, noSMI, &now)
first := s.Sample(t.Context())
// The package reading, not the hotter core: the package is what the
// chip itself reports as its temperature.
if first.CPUTempC != 58 {
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
}
if first.CPUPowerW != 0 {
t.Error("power from one reading of a counter is invented")
}
if first.InletTempC != 0 || first.GPUs != nil {
t.Errorf("sensors this host does not have were reported: %+v", first)
}
// 10 s later the package has used 650 J, and its dram subdomain some of
// that — counted once.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
t.Errorf("CPU power %.2f W, expected 65", got)
}
}
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
s := sampler(root, noSMI, &now)
s.Sample(t.Context())
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
t.Errorf("across a wrap: %.2f W, expected 50", got)
}
}
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
root := t.TempDir()
put(t, root, "class/thermal/cooling_device0/type", "Processor")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
t.Errorf("a machine with no sensors reported %+v", got)
}
}
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
}
}
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
var asked []string
smi := func(_ context.Context, fields string) ([]byte, error) {
asked = append(asked, fields)
return []byte(
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
}
now := time.Now()
ss := sampler(t.TempDir(), smi, &now)
got := ss.Sample(t.Context())
if len(got.GPUs) != 3 {
t.Fatalf("%+v", got.GPUs)
}
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
t.Errorf("card 0 %+v", g0)
}
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
t.Errorf("card 1 throttled by %v", g1.Throttle)
}
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
}
// Within ten seconds the last reading is reused rather than running
// nvidia-smi on every heartbeat; after, it is asked again.
now = now.Add(5 * time.Second)
ss.Sample(t.Context())
if len(asked) != 1 {
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
}
now = now.Add(6 * time.Second)
ss.Sample(t.Context())
if len(asked) != 2 {
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
}
}
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
smi := func(_ context.Context, fields string) ([]byte, error) {
if strings.Contains(fields, "temperature.memory") {
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
}
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
t.Errorf("%+v", got.GPUs)
}
}
func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) {
// What the card is doing now, every sample: discovery read it once, at
// start, and a card busy since looked as idle as it was then.
smi := func(_ context.Context, fields string) ([]byte, error) {
if !strings.Contains(fields, "utilization.gpu") {
return nil, errors.New("asked without how busy the cards are")
}
return []byte(
"0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" +
"1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
if len(got) != 2 {
t.Fatalf("%+v", got)
}
g0, g1 := got[0], got[1]
if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 ||
g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 {
t.Errorf("card 0 %+v", g0)
}
// Idle is zero; a count the card does not keep is unknown, not zero.
if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 {
t.Errorf("card 1 %+v", g1)
}
}
func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) {
smi := func(_ context.Context, fields string) ([]byte, error) {
if strings.Contains(fields, "utilization.gpu") {
return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`)
}
return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 {
t.Errorf("%+v", got)
}
}