Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
170 lines
6.2 KiB
Go
170 lines
6.2 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func put(t *testing.T, root, rel, body string) {
|
|
t.Helper()
|
|
p := filepath.Join(root, rel)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// dellHost is the layout of the Proxmox host these tests were written
|
|
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
|
|
// and cores, and RAPL with a package and its dram subdomain.
|
|
func dellHost(t *testing.T) string {
|
|
root := t.TempDir()
|
|
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
|
|
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
|
|
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
|
|
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
|
|
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
|
|
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
|
|
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
|
|
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
|
|
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
|
|
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
|
|
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
|
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
|
|
return root
|
|
}
|
|
|
|
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
|
|
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
|
|
}
|
|
|
|
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
|
|
|
|
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
|
|
root := dellHost(t)
|
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
s := sampler(root, noSMI, &now)
|
|
|
|
first := s.Sample(t.Context())
|
|
// The package reading, not the hotter core: the package is what the
|
|
// chip itself reports as its temperature.
|
|
if first.CPUTempC != 58 {
|
|
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
|
|
}
|
|
if first.CPUPowerW != 0 {
|
|
t.Error("power from one reading of a counter is invented")
|
|
}
|
|
if first.InletTempC != 0 || first.GPUs != nil {
|
|
t.Errorf("sensors this host does not have were reported: %+v", first)
|
|
}
|
|
|
|
// 10 s later the package has used 650 J, and its dram subdomain some of
|
|
// that — counted once.
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
|
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
|
|
now = now.Add(10 * time.Second)
|
|
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
|
|
t.Errorf("CPU power %.2f W, expected 65", got)
|
|
}
|
|
}
|
|
|
|
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
|
|
root := dellHost(t)
|
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
|
|
s := sampler(root, noSMI, &now)
|
|
s.Sample(t.Context())
|
|
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
|
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
|
|
now = now.Add(10 * time.Second)
|
|
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
|
|
t.Errorf("across a wrap: %.2f W, expected 50", got)
|
|
}
|
|
}
|
|
|
|
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
|
|
root := t.TempDir()
|
|
put(t, root, "class/thermal/cooling_device0/type", "Processor")
|
|
now := time.Now()
|
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
|
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
|
|
t.Errorf("a machine with no sensors reported %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
|
|
root := t.TempDir()
|
|
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
|
|
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
|
|
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
|
|
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
|
|
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
|
|
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
|
|
now := time.Now()
|
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
|
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
|
|
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
|
|
}
|
|
}
|
|
|
|
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
|
|
var asked []string
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
asked = append(asked, fields)
|
|
return []byte(
|
|
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
|
|
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
|
|
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
|
|
}
|
|
now := time.Now()
|
|
ss := sampler(t.TempDir(), smi, &now)
|
|
got := ss.Sample(t.Context())
|
|
if len(got.GPUs) != 3 {
|
|
t.Fatalf("%+v", got.GPUs)
|
|
}
|
|
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
|
|
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
|
|
t.Errorf("card 0 %+v", g0)
|
|
}
|
|
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
|
|
t.Errorf("card 1 throttled by %v", g1.Throttle)
|
|
}
|
|
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
|
|
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
|
|
}
|
|
// Within ten seconds the last reading is reused rather than running
|
|
// nvidia-smi on every heartbeat; after, it is asked again.
|
|
now = now.Add(5 * time.Second)
|
|
ss.Sample(t.Context())
|
|
if len(asked) != 1 {
|
|
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
|
|
}
|
|
now = now.Add(6 * time.Second)
|
|
ss.Sample(t.Context())
|
|
if len(asked) != 2 {
|
|
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
|
|
}
|
|
}
|
|
|
|
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
|
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
|
if strings.Contains(fields, "temperature.memory") {
|
|
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
|
|
}
|
|
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
|
|
}
|
|
now := time.Now()
|
|
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
|
|
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
|
|
t.Errorf("%+v", got.GPUs)
|
|
}
|
|
}
|