package discovery import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" ) func put(t *testing.T, root, rel, body string) { t.Helper() p := filepath.Join(root, rel) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil { t.Fatal(err) } } // dellHost is the layout of the Proxmox host these tests were written // against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package // and cores, and RAPL with a package and its dram subdomain. func dellHost(t *testing.T) string { root := t.TempDir() put(t, root, "class/hwmon/hwmon0/name", "dell_smm") put(t, root, "class/hwmon/hwmon0/temp1_input", "55000") put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU") put(t, root, "class/hwmon/hwmon0/temp2_input", "47000") put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM") put(t, root, "class/hwmon/hwmon1/name", "coretemp") put(t, root, "class/hwmon/hwmon1/temp1_input", "58000") put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0") put(t, root, "class/hwmon/hwmon1/temp10_input", "61000") put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8") put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000") put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850") put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000") return root } func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler { return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }} } func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") } func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) { root := dellHost(t) now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) s := sampler(root, noSMI, &now) first := s.Sample(t.Context()) // The package reading, not the hotter core: the package is what the // chip itself reports as its temperature. if first.CPUTempC != 58 { t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC) } if first.CPUPowerW != 0 { t.Error("power from one reading of a counter is invented") } if first.InletTempC != 0 || first.GPUs != nil { t.Errorf("sensors this host does not have were reported: %+v", first) } // 10 s later the package has used 650 J, and its dram subdomain some of // that — counted once. put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000") put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000") now = now.Add(10 * time.Second) if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 { t.Errorf("CPU power %.2f W, expected 65", got) } } func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) { root := dellHost(t) now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000") s := sampler(root, noSMI, &now) s.Sample(t.Context()) // 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s. put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150") now = now.Add(10 * time.Second) if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 { t.Errorf("across a wrap: %.2f W, expected 50", got) } } func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) { root := t.TempDir() put(t, root, "class/thermal/cooling_device0/type", "Processor") now := time.Now() got := sampler(root, noSMI, &now).Sample(t.Context()) if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil { t.Errorf("a machine with no sensors reported %+v", got) } } func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) { root := t.TempDir() put(t, root, "class/hwmon/hwmon2/name", "nct6798") put(t, root, "class/hwmon/hwmon2/temp1_input", "24500") put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN") put(t, root, "class/hwmon/hwmon3/name", "k10temp") put(t, root, "class/hwmon/hwmon3/temp1_input", "71250") put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl") now := time.Now() got := sampler(root, noSMI, &now).Sample(t.Context()) if got.InletTempC != 24.5 || got.CPUTempC != 71.25 { t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC) } } func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) { var asked []string smi := func(_ context.Context, fields string) ([]byte, error) { asked = append(asked, fields) return []byte( "0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" + "1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal "2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only } now := time.Now() ss := sampler(t.TempDir(), smi, &now) got := ss.Sample(t.Context()) if len(got.GPUs) != 3 { t.Fatalf("%+v", got.GPUs) } g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2] if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil { t.Errorf("card 0 %+v", g0) } if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" { t.Errorf("card 1 throttled by %v", g1.Throttle) } if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil { t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2) } // Within ten seconds the last reading is reused rather than running // nvidia-smi on every heartbeat; after, it is asked again. now = now.Add(5 * time.Second) ss.Sample(t.Context()) if len(asked) != 1 { t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked)) } now = now.Add(6 * time.Second) ss.Sample(t.Context()) if len(asked) != 2 { t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked)) } } func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) { smi := func(_ context.Context, fields string) ([]byte, error) { if strings.Contains(fields, "temperature.memory") { return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`) } return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil } now := time.Now() got := sampler(t.TempDir(), smi, &now).Sample(t.Context()) if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" { t.Errorf("%+v", got.GPUs) } } func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) { // What the card is doing now, every sample: discovery read it once, at // start, and a card busy since looked as idle as it was then. smi := func(_ context.Context, fields string) ([]byte, error) { if !strings.Contains(fields, "utilization.gpu") { return nil, errors.New("asked without how busy the cards are") } return []byte( "0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" + "1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil } now := time.Now() got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs if len(got) != 2 { t.Fatalf("%+v", got) } g0, g1 := got[0], got[1] if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 || g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 { t.Errorf("card 0 %+v", g0) } // Idle is zero; a count the card does not keep is unknown, not zero. if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 { t.Errorf("card 1 %+v", g1) } } func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) { smi := func(_ context.Context, fields string) ([]byte, error) { if strings.Contains(fields, "utilization.gpu") { return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`) } return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil } now := time.Now() got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 { t.Errorf("%+v", got) } }