They were read once, at agent start, and published unchanged since. Each sample now carries utilisation, memory and bandwidth in use, the SM clock and uncorrected ECC errors. The node page charts each card, the cluster page the fleet by class, Grafana has a GPU row, and three GPU alerts are added. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
211 lines
7.9 KiB
Go
211 lines
7.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func put(t *testing.T, root, rel, body string) {
|
|
t.Helper()
|
|
p := filepath.Join(root, rel)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// dellHost is the layout of the Proxmox host these tests were written
|
|
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
|
|
// and cores, and RAPL with a package and its dram subdomain.
|
|
func dellHost(t *testing.T) string {
|
|
root := t.TempDir()
|
|
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
|
|
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
|
|
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
|
|
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
|
|
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
|
|
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
|
|
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
|
|
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
|
|
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
|
|
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
|
|
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
|
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
|
|
return root
|
|
}
|
|
|
|
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
|
|
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
|
|
}
|
|
|
|
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
|
|
|
|
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
|
|
root := dellHost(t)
|
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
s := sampler(root, noSMI, &now)
|
|
|
|
first := s.Sample(t.Context())
|
|
// The package reading, not the hotter core: the package is what the
|
|
// chip itself reports as its temperature.
|
|
if first.CPUTempC != 58 {
|
|
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
|
|
}
|
|
if first.CPUPowerW != 0 {
|
|
t.Error("power from one reading of a counter is invented")
|
|
}
|
|
if first.InletTempC != 0 || first.GPUs != nil {
|
|
t.Errorf("sensors this host does not have were reported: %+v", first)
|
|
}
|
|
|
|
// 10 s later the package has used 650 J, and its dram subdomain some of
|
|
// that — counted once.
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
|
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
|
|
now = now.Add(10 * time.Second)
|
|
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
|
|
t.Errorf("CPU power %.2f W, expected 65", got)
|
|
}
|
|
}
|
|
|
|
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
|
|
root := dellHost(t)
|
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
|
|
s := sampler(root, noSMI, &now)
|
|
s.Sample(t.Context())
|
|
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
|
|
now = now.Add(10 * time.Second)
|
|
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
|
|
t.Errorf("across a wrap: %.2f W, expected 50", got)
|
|
}
|
|
}
|
|
|
|
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
|
|
root := t.TempDir()
|
|
put(t, root, "class/thermal/cooling_device0/type", "Processor")
|
|
now := time.Now()
|
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
|
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
|
|
t.Errorf("a machine with no sensors reported %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
|
|
root := t.TempDir()
|
|
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
|
|
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
|
|
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
|
|
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
|
|
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
|
|
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
|
|
now := time.Now()
|
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
|
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
|
|
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
|
|
}
|
|
}
|
|
|
|
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
|
|
var asked []string
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
asked = append(asked, fields)
|
|
return []byte(
|
|
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
|
|
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
|
|
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
|
|
}
|
|
now := time.Now()
|
|
ss := sampler(t.TempDir(), smi, &now)
|
|
got := ss.Sample(t.Context())
|
|
if len(got.GPUs) != 3 {
|
|
t.Fatalf("%+v", got.GPUs)
|
|
}
|
|
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
|
|
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
|
|
t.Errorf("card 0 %+v", g0)
|
|
}
|
|
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
|
|
t.Errorf("card 1 throttled by %v", g1.Throttle)
|
|
}
|
|
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
|
|
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
|
|
}
|
|
// Within ten seconds the last reading is reused rather than running
|
|
// nvidia-smi on every heartbeat; after, it is asked again.
|
|
now = now.Add(5 * time.Second)
|
|
ss.Sample(t.Context())
|
|
if len(asked) != 1 {
|
|
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
|
|
}
|
|
now = now.Add(6 * time.Second)
|
|
ss.Sample(t.Context())
|
|
if len(asked) != 2 {
|
|
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
|
|
}
|
|
}
|
|
|
|
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
if strings.Contains(fields, "temperature.memory") {
|
|
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
|
|
}
|
|
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
|
|
}
|
|
now := time.Now()
|
|
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
|
|
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
|
|
t.Errorf("%+v", got.GPUs)
|
|
}
|
|
}
|
|
|
|
func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) {
|
|
// What the card is doing now, every sample: discovery read it once, at
|
|
// start, and a card busy since looked as idle as it was then.
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
if !strings.Contains(fields, "utilization.gpu") {
|
|
return nil, errors.New("asked without how busy the cards are")
|
|
}
|
|
return []byte(
|
|
"0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" +
|
|
"1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil
|
|
}
|
|
now := time.Now()
|
|
got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
|
|
if len(got) != 2 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
g0, g1 := got[0], got[1]
|
|
if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 ||
|
|
g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 {
|
|
t.Errorf("card 0 %+v", g0)
|
|
}
|
|
// Idle is zero; a count the card does not keep is unknown, not zero.
|
|
if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 {
|
|
t.Errorf("card 1 %+v", g1)
|
|
}
|
|
}
|
|
|
|
func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) {
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
if strings.Contains(fields, "utilization.gpu") {
|
|
return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`)
|
|
}
|
|
return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil
|
|
}
|
|
now := time.Now()
|
|
got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
|
|
if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 {
|
|
t.Errorf("%+v", got)
|
|
}
|
|
}
|