Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -101,6 +101,9 @@ type Agent struct {
|
||||
smsErr map[string]string
|
||||
// load measures the machine between heartbeats.
|
||||
load *discovery.LoadSampler
|
||||
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
|
||||
// from nvidia-smi.
|
||||
sensors *discovery.SensorSampler
|
||||
|
||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||
// download reaches every shard that asked for it. A fetch is single-flight
|
||||
@@ -146,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
logRing: ring,
|
||||
provKick: make(chan struct{}, 1),
|
||||
load: discovery.NewLoadSampler(),
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -638,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
||||
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
|
||||
})
|
||||
}
|
||||
if a.sensors != nil {
|
||||
se := a.sensors.Sample(context.Background())
|
||||
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
|
||||
for _, g := range se.GPUs {
|
||||
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
|
||||
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
|
||||
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sensors is what a node's own hardware says about heat and power: the CPUs,
|
||||
// the board where it has an inlet sensor, and each GPU. A zero is not
|
||||
// measured, never a reading of zero — a VM has none of these, and must not
|
||||
// look like a machine running at 0 °C on no power.
|
||||
type Sensors struct {
|
||||
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
|
||||
// where no package reading exists.
|
||||
CPUTempC float64
|
||||
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
|
||||
// energy counters over the time between two samples.
|
||||
CPUPowerW float64
|
||||
// InletTempC is the air coming in, where the board has a sensor it calls
|
||||
// that. Most do not, and a CPU's temperature is not a substitute: it
|
||||
// follows the load, not the room.
|
||||
InletTempC float64
|
||||
GPUs []GPUSensor
|
||||
}
|
||||
|
||||
// GPUSensor is one card's heat, power and whether it is slowing itself down.
|
||||
type GPUSensor struct {
|
||||
Index int
|
||||
TempC float64
|
||||
MemoryTempC float64
|
||||
PowerW float64
|
||||
PowerLimitW float64
|
||||
// Throttle names what is holding the card's clocks down now:
|
||||
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
|
||||
// is running free. Idle and application clock settings are left out;
|
||||
// they are not the card being held back.
|
||||
Throttle []string
|
||||
}
|
||||
|
||||
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
|
||||
// than temperatures change, and each run costs a few hundred milliseconds.
|
||||
const sensorEvery = 10 * time.Second
|
||||
|
||||
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
|
||||
// two readings of the energy counters.
|
||||
type SensorSampler struct {
|
||||
sys string // /sys, or a test's copy of it
|
||||
smi func(ctx context.Context, fields string) ([]byte, error)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
last Sensors
|
||||
lastAt time.Time
|
||||
prevEnergy map[string]uint64
|
||||
prevAt time.Time
|
||||
}
|
||||
|
||||
// NewSensorSampler reads this machine.
|
||||
func NewSensorSampler() *SensorSampler {
|
||||
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
|
||||
}
|
||||
|
||||
func runSMI(ctx context.Context, fields string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
//nolint:gosec // fixed binary, fields from this file
|
||||
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
|
||||
"--format=csv,noheader,nounits").Output()
|
||||
}
|
||||
|
||||
// Sample reads the sensors, or returns the last reading when that is recent.
|
||||
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := s.now()
|
||||
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
|
||||
return s.last
|
||||
}
|
||||
out := Sensors{}
|
||||
out.CPUTempC, out.InletTempC = s.hwmon()
|
||||
out.CPUPowerW = s.rapl(now)
|
||||
out.GPUs = s.gpus(ctx)
|
||||
s.last, s.lastAt = out, now
|
||||
return out
|
||||
}
|
||||
|
||||
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
|
||||
// device the kernel exposes — what lm_sensors reads, without it installed.
|
||||
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
|
||||
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
|
||||
var pkg, core float64
|
||||
for _, d := range devs {
|
||||
name := readTrim(filepath.Join(d, "name"))
|
||||
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
|
||||
for _, in := range inputs {
|
||||
milli, err := strconv.ParseFloat(readTrim(in), 64)
|
||||
if err != nil || milli <= 0 {
|
||||
continue
|
||||
}
|
||||
c := milli / 1000
|
||||
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
|
||||
switch {
|
||||
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
|
||||
pkg = max(pkg, c)
|
||||
case isCPUSensor(name):
|
||||
core = max(core, c)
|
||||
case label == "cpu":
|
||||
// A board's own reading of the CPU, as Dell's SMM gives.
|
||||
core = max(core, c)
|
||||
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
|
||||
strings.Contains(label, "intake") || label == "systin":
|
||||
inlet = max(inlet, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if pkg > 0 {
|
||||
return pkg, inlet
|
||||
}
|
||||
return core, inlet
|
||||
}
|
||||
|
||||
func isCPUSensor(name string) bool {
|
||||
switch name {
|
||||
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rapl is the CPU packages' power from their energy counters. Package
|
||||
// domains only: the dram and core subdomains are parts of a package, and
|
||||
// adding them would count the same energy twice.
|
||||
func (s *SensorSampler) rapl(now time.Time) float64 {
|
||||
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
|
||||
energy := map[string]uint64{}
|
||||
wrap := map[string]uint64{}
|
||||
for _, d := range doms {
|
||||
if strings.Count(filepath.Base(d), ":") != 1 {
|
||||
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
|
||||
}
|
||||
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
|
||||
if err != nil {
|
||||
continue // unreadable without root on newer kernels
|
||||
}
|
||||
energy[d] = e
|
||||
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
|
||||
}
|
||||
prev, prevAt := s.prevEnergy, s.prevAt
|
||||
s.prevEnergy, s.prevAt = energy, now
|
||||
dt := now.Sub(prevAt).Seconds()
|
||||
if prev == nil || dt <= 0 || len(energy) == 0 {
|
||||
return 0
|
||||
}
|
||||
var joules float64
|
||||
for d, e := range energy {
|
||||
p, ok := prev[d]
|
||||
if !ok {
|
||||
return 0 // a domain appeared: no fair difference this time
|
||||
}
|
||||
if e < p { // the counter wrapped
|
||||
e += wrap[d]
|
||||
}
|
||||
joules += float64(e-p) / 1e6
|
||||
}
|
||||
return joules / dt
|
||||
}
|
||||
|
||||
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
|
||||
// card is being held back are named.
|
||||
var throttleBits = []struct {
|
||||
bit uint64
|
||||
name string
|
||||
}{
|
||||
{0x4, "sw_power_cap"},
|
||||
{0x8, "hw_slowdown"},
|
||||
{0x20, "sw_thermal"},
|
||||
{0x40, "hw_thermal"},
|
||||
{0x80, "power_brake"},
|
||||
}
|
||||
|
||||
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
|
||||
// is not known to every driver, and a field nvidia-smi does not know fails the
|
||||
// whole query, so it is asked for first and dropped on failure.
|
||||
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
|
||||
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
|
||||
out, err := s.smi(ctx, withMem)
|
||||
if err != nil {
|
||||
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
var gs []GPUSensor
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Split(line, ",")
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
for i := range f {
|
||||
f[i] = strings.TrimSpace(f[i])
|
||||
}
|
||||
idx, err := strconv.Atoi(f[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
|
||||
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
|
||||
for _, b := range throttleBits {
|
||||
if bits&b.bit != 0 {
|
||||
g.Throttle = append(g.Throttle, b.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(f) > 5 {
|
||||
g.MemoryTempC = num(f[5])
|
||||
}
|
||||
gs = append(gs, g)
|
||||
}
|
||||
return gs
|
||||
}
|
||||
|
||||
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
|
||||
func num(s string) float64 {
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil || v < 0 {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func readTrim(path string) string {
|
||||
b, err := os.ReadFile(path) //nolint:gosec // sysfs
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func put(t *testing.T, root, rel, body string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// dellHost is the layout of the Proxmox host these tests were written
|
||||
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
|
||||
// and cores, and RAPL with a package and its dram subdomain.
|
||||
func dellHost(t *testing.T) string {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
|
||||
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
|
||||
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
|
||||
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
|
||||
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
|
||||
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
|
||||
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
|
||||
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
|
||||
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
|
||||
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
|
||||
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
|
||||
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
|
||||
return root
|
||||
}
|
||||
|
||||
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
|
||||
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
|
||||
}
|
||||
|
||||
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
|
||||
|
||||
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
|
||||
root := dellHost(t)
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
s := sampler(root, noSMI, &now)
|
||||
|
||||
first := s.Sample(t.Context())
|
||||
// The package reading, not the hotter core: the package is what the
|
||||
// chip itself reports as its temperature.
|
||||
if first.CPUTempC != 58 {
|
||||
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
|
||||
}
|
||||
if first.CPUPowerW != 0 {
|
||||
t.Error("power from one reading of a counter is invented")
|
||||
}
|
||||
if first.InletTempC != 0 || first.GPUs != nil {
|
||||
t.Errorf("sensors this host does not have were reported: %+v", first)
|
||||
}
|
||||
|
||||
// 10 s later the package has used 650 J, and its dram subdomain some of
|
||||
// that — counted once.
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
|
||||
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
|
||||
now = now.Add(10 * time.Second)
|
||||
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
|
||||
t.Errorf("CPU power %.2f W, expected 65", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
|
||||
root := dellHost(t)
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
|
||||
s := sampler(root, noSMI, &now)
|
||||
s.Sample(t.Context())
|
||||
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
|
||||
now = now.Add(10 * time.Second)
|
||||
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
|
||||
t.Errorf("across a wrap: %.2f W, expected 50", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/thermal/cooling_device0/type", "Processor")
|
||||
now := time.Now()
|
||||
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
|
||||
t.Errorf("a machine with no sensors reported %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
|
||||
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
|
||||
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
|
||||
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
|
||||
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
|
||||
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
|
||||
now := time.Now()
|
||||
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
|
||||
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
|
||||
var asked []string
|
||||
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||
asked = append(asked, fields)
|
||||
return []byte(
|
||||
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
|
||||
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
|
||||
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
|
||||
}
|
||||
now := time.Now()
|
||||
ss := sampler(t.TempDir(), smi, &now)
|
||||
got := ss.Sample(t.Context())
|
||||
if len(got.GPUs) != 3 {
|
||||
t.Fatalf("%+v", got.GPUs)
|
||||
}
|
||||
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
|
||||
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
|
||||
t.Errorf("card 0 %+v", g0)
|
||||
}
|
||||
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
|
||||
t.Errorf("card 1 throttled by %v", g1.Throttle)
|
||||
}
|
||||
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
|
||||
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
|
||||
}
|
||||
// Within ten seconds the last reading is reused rather than running
|
||||
// nvidia-smi on every heartbeat; after, it is asked again.
|
||||
now = now.Add(5 * time.Second)
|
||||
ss.Sample(t.Context())
|
||||
if len(asked) != 1 {
|
||||
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
|
||||
}
|
||||
now = now.Add(6 * time.Second)
|
||||
ss.Sample(t.Context())
|
||||
if len(asked) != 2 {
|
||||
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
|
||||
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||
if strings.Contains(fields, "temperature.memory") {
|
||||
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
|
||||
}
|
||||
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
|
||||
}
|
||||
now := time.Now()
|
||||
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
|
||||
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
|
||||
t.Errorf("%+v", got.GPUs)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user