Files
llmgw/internal/agent/discovery/sensors.go
T
Ilker ManapandClaude Opus 5.5 cd10e086c8
Release / build (push) Successful in 1m10s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s
Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 19:49:08 +02:00

243 lines
7.2 KiB
Go

package discovery
import (
"context"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// Sensors is what a node's own hardware says about heat and power: the CPUs,
// the board where it has an inlet sensor, and each GPU. A zero is not
// measured, never a reading of zero — a VM has none of these, and must not
// look like a machine running at 0 °C on no power.
type Sensors struct {
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
// where no package reading exists.
CPUTempC float64
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
// energy counters over the time between two samples.
CPUPowerW float64
// InletTempC is the air coming in, where the board has a sensor it calls
// that. Most do not, and a CPU's temperature is not a substitute: it
// follows the load, not the room.
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat, power and whether it is slowing itself down.
type GPUSensor struct {
Index int
TempC float64
MemoryTempC float64
PowerW float64
PowerLimitW float64
// Throttle names what is holding the card's clocks down now:
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
// is running free. Idle and application clock settings are left out;
// they are not the card being held back.
Throttle []string
}
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
// than temperatures change, and each run costs a few hundred milliseconds.
const sensorEvery = 10 * time.Second
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
// two readings of the energy counters.
type SensorSampler struct {
sys string // /sys, or a test's copy of it
smi func(ctx context.Context, fields string) ([]byte, error)
now func() time.Time
mu sync.Mutex
last Sensors
lastAt time.Time
prevEnergy map[string]uint64
prevAt time.Time
}
// NewSensorSampler reads this machine.
func NewSensorSampler() *SensorSampler {
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
}
func runSMI(ctx context.Context, fields string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
//nolint:gosec // fixed binary, fields from this file
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
"--format=csv,noheader,nounits").Output()
}
// Sample reads the sensors, or returns the last reading when that is recent.
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
return s.last
}
out := Sensors{}
out.CPUTempC, out.InletTempC = s.hwmon()
out.CPUPowerW = s.rapl(now)
out.GPUs = s.gpus(ctx)
s.last, s.lastAt = out, now
return out
}
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
// device the kernel exposes — what lm_sensors reads, without it installed.
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
var pkg, core float64
for _, d := range devs {
name := readTrim(filepath.Join(d, "name"))
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
for _, in := range inputs {
milli, err := strconv.ParseFloat(readTrim(in), 64)
if err != nil || milli <= 0 {
continue
}
c := milli / 1000
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
switch {
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
pkg = max(pkg, c)
case isCPUSensor(name):
core = max(core, c)
case label == "cpu":
// A board's own reading of the CPU, as Dell's SMM gives.
core = max(core, c)
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
strings.Contains(label, "intake") || label == "systin":
inlet = max(inlet, c)
}
}
}
if pkg > 0 {
return pkg, inlet
}
return core, inlet
}
func isCPUSensor(name string) bool {
switch name {
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
return true
}
return false
}
// rapl is the CPU packages' power from their energy counters. Package
// domains only: the dram and core subdomains are parts of a package, and
// adding them would count the same energy twice.
func (s *SensorSampler) rapl(now time.Time) float64 {
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
energy := map[string]uint64{}
wrap := map[string]uint64{}
for _, d := range doms {
if strings.Count(filepath.Base(d), ":") != 1 {
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
}
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
if err != nil {
continue // unreadable without root on newer kernels
}
energy[d] = e
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
}
prev, prevAt := s.prevEnergy, s.prevAt
s.prevEnergy, s.prevAt = energy, now
dt := now.Sub(prevAt).Seconds()
if prev == nil || dt <= 0 || len(energy) == 0 {
return 0
}
var joules float64
for d, e := range energy {
p, ok := prev[d]
if !ok {
return 0 // a domain appeared: no fair difference this time
}
if e < p { // the counter wrapped
e += wrap[d]
}
joules += float64(e-p) / 1e6
}
return joules / dt
}
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
// card is being held back are named.
var throttleBits = []struct {
bit uint64
name string
}{
{0x4, "sw_power_cap"},
{0x8, "hw_slowdown"},
{0x20, "sw_thermal"},
{0x40, "hw_thermal"},
{0x80, "power_brake"},
}
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
// is not known to every driver, and a field nvidia-smi does not know fails the
// whole query, so it is asked for first and dropped on failure.
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
out, err := s.smi(ctx, withMem)
if err != nil {
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
return nil
}
}
var gs []GPUSensor
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
f := strings.Split(line, ",")
if len(f) < 5 {
continue
}
for i := range f {
f[i] = strings.TrimSpace(f[i])
}
idx, err := strconv.Atoi(f[0])
if err != nil {
continue
}
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
for _, b := range throttleBits {
if bits&b.bit != 0 {
g.Throttle = append(g.Throttle, b.name)
}
}
}
if len(f) > 5 {
g.MemoryTempC = num(f[5])
}
gs = append(gs, g)
}
return gs
}
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
func num(s string) float64 {
v, err := strconv.ParseFloat(s, 64)
if err != nil || v < 0 {
return 0
}
return v
}
func readTrim(path string) string {
b, err := os.ReadFile(path) //nolint:gosec // sysfs
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}