Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
243 lines
7.2 KiB
Go
243 lines
7.2 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Sensors is what a node's own hardware says about heat and power: the CPUs,
|
|
// the board where it has an inlet sensor, and each GPU. A zero is not
|
|
// measured, never a reading of zero — a VM has none of these, and must not
|
|
// look like a machine running at 0 °C on no power.
|
|
type Sensors struct {
|
|
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
|
|
// where no package reading exists.
|
|
CPUTempC float64
|
|
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
|
|
// energy counters over the time between two samples.
|
|
CPUPowerW float64
|
|
// InletTempC is the air coming in, where the board has a sensor it calls
|
|
// that. Most do not, and a CPU's temperature is not a substitute: it
|
|
// follows the load, not the room.
|
|
InletTempC float64
|
|
GPUs []GPUSensor
|
|
}
|
|
|
|
// GPUSensor is one card's heat, power and whether it is slowing itself down.
|
|
type GPUSensor struct {
|
|
Index int
|
|
TempC float64
|
|
MemoryTempC float64
|
|
PowerW float64
|
|
PowerLimitW float64
|
|
// Throttle names what is holding the card's clocks down now:
|
|
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
|
|
// is running free. Idle and application clock settings are left out;
|
|
// they are not the card being held back.
|
|
Throttle []string
|
|
}
|
|
|
|
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
|
|
// than temperatures change, and each run costs a few hundred milliseconds.
|
|
const sensorEvery = 10 * time.Second
|
|
|
|
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
|
|
// two readings of the energy counters.
|
|
type SensorSampler struct {
|
|
sys string // /sys, or a test's copy of it
|
|
smi func(ctx context.Context, fields string) ([]byte, error)
|
|
now func() time.Time
|
|
|
|
mu sync.Mutex
|
|
last Sensors
|
|
lastAt time.Time
|
|
prevEnergy map[string]uint64
|
|
prevAt time.Time
|
|
}
|
|
|
|
// NewSensorSampler reads this machine.
|
|
func NewSensorSampler() *SensorSampler {
|
|
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
|
|
}
|
|
|
|
func runSMI(ctx context.Context, fields string) ([]byte, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
//nolint:gosec // fixed binary, fields from this file
|
|
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
|
|
"--format=csv,noheader,nounits").Output()
|
|
}
|
|
|
|
// Sample reads the sensors, or returns the last reading when that is recent.
|
|
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
now := s.now()
|
|
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
|
|
return s.last
|
|
}
|
|
out := Sensors{}
|
|
out.CPUTempC, out.InletTempC = s.hwmon()
|
|
out.CPUPowerW = s.rapl(now)
|
|
out.GPUs = s.gpus(ctx)
|
|
s.last, s.lastAt = out, now
|
|
return out
|
|
}
|
|
|
|
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
|
|
// device the kernel exposes — what lm_sensors reads, without it installed.
|
|
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
|
|
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
|
|
var pkg, core float64
|
|
for _, d := range devs {
|
|
name := readTrim(filepath.Join(d, "name"))
|
|
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
|
|
for _, in := range inputs {
|
|
milli, err := strconv.ParseFloat(readTrim(in), 64)
|
|
if err != nil || milli <= 0 {
|
|
continue
|
|
}
|
|
c := milli / 1000
|
|
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
|
|
switch {
|
|
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
|
|
pkg = max(pkg, c)
|
|
case isCPUSensor(name):
|
|
core = max(core, c)
|
|
case label == "cpu":
|
|
// A board's own reading of the CPU, as Dell's SMM gives.
|
|
core = max(core, c)
|
|
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
|
|
strings.Contains(label, "intake") || label == "systin":
|
|
inlet = max(inlet, c)
|
|
}
|
|
}
|
|
}
|
|
if pkg > 0 {
|
|
return pkg, inlet
|
|
}
|
|
return core, inlet
|
|
}
|
|
|
|
func isCPUSensor(name string) bool {
|
|
switch name {
|
|
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// rapl is the CPU packages' power from their energy counters. Package
|
|
// domains only: the dram and core subdomains are parts of a package, and
|
|
// adding them would count the same energy twice.
|
|
func (s *SensorSampler) rapl(now time.Time) float64 {
|
|
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
|
|
energy := map[string]uint64{}
|
|
wrap := map[string]uint64{}
|
|
for _, d := range doms {
|
|
if strings.Count(filepath.Base(d), ":") != 1 {
|
|
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
|
|
}
|
|
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
|
|
if err != nil {
|
|
continue // unreadable without root on newer kernels
|
|
}
|
|
energy[d] = e
|
|
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
|
|
}
|
|
prev, prevAt := s.prevEnergy, s.prevAt
|
|
s.prevEnergy, s.prevAt = energy, now
|
|
dt := now.Sub(prevAt).Seconds()
|
|
if prev == nil || dt <= 0 || len(energy) == 0 {
|
|
return 0
|
|
}
|
|
var joules float64
|
|
for d, e := range energy {
|
|
p, ok := prev[d]
|
|
if !ok {
|
|
return 0 // a domain appeared: no fair difference this time
|
|
}
|
|
if e < p { // the counter wrapped
|
|
e += wrap[d]
|
|
}
|
|
joules += float64(e-p) / 1e6
|
|
}
|
|
return joules / dt
|
|
}
|
|
|
|
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
|
|
// card is being held back are named.
|
|
var throttleBits = []struct {
|
|
bit uint64
|
|
name string
|
|
}{
|
|
{0x4, "sw_power_cap"},
|
|
{0x8, "hw_slowdown"},
|
|
{0x20, "sw_thermal"},
|
|
{0x40, "hw_thermal"},
|
|
{0x80, "power_brake"},
|
|
}
|
|
|
|
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
|
|
// is not known to every driver, and a field nvidia-smi does not know fails the
|
|
// whole query, so it is asked for first and dropped on failure.
|
|
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
|
|
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
|
|
out, err := s.smi(ctx, withMem)
|
|
if err != nil {
|
|
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
|
|
return nil
|
|
}
|
|
}
|
|
var gs []GPUSensor
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
f := strings.Split(line, ",")
|
|
if len(f) < 5 {
|
|
continue
|
|
}
|
|
for i := range f {
|
|
f[i] = strings.TrimSpace(f[i])
|
|
}
|
|
idx, err := strconv.Atoi(f[0])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
|
|
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
|
|
for _, b := range throttleBits {
|
|
if bits&b.bit != 0 {
|
|
g.Throttle = append(g.Throttle, b.name)
|
|
}
|
|
}
|
|
}
|
|
if len(f) > 5 {
|
|
g.MemoryTempC = num(f[5])
|
|
}
|
|
gs = append(gs, g)
|
|
}
|
|
return gs
|
|
}
|
|
|
|
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
|
|
func num(s string) float64 {
|
|
v, err := strconv.ParseFloat(s, 64)
|
|
if err != nil || v < 0 {
|
|
return 0
|
|
}
|
|
return v
|
|
}
|
|
|
|
func readTrim(path string) string {
|
|
b, err := os.ReadFile(path) //nolint:gosec // sysfs
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(string(b))
|
|
}
|