Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Release / build (push) Successful in 1m10s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s

Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 19:49:08 +02:00
1 parent 8724dc027a
commit cd10e086c8
15 files changed
+996 -172

No files matched your search

+14
View File
@@ -101,6 +101,9 @@ type Agent struct {
smsErr map[string]string
// load measures the machine between heartbeats.
load *discovery.LoadSampler
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
// from nvidia-smi.
sensors *discovery.SensorSampler
// fetchWatchers are the shards waiting on each source, so progress on one
// download reaches every shard that asked for it. A fetch is single-flight
@@ -146,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
logRing: ring,
provKick: make(chan struct{}, 1),
load: discovery.NewLoadSampler(),
sensors: discovery.NewSensorSampler(),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -638,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
})
}
if a.sensors != nil {
se := a.sensors.Sample(context.Background())
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
for _, g := range se.GPUs {
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
})
}
}
return out
}
+242
View File
@@ -0,0 +1,242 @@
package discovery
import (
"context"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// Sensors is what a node's own hardware says about heat and power: the CPUs,
// the board where it has an inlet sensor, and each GPU. A zero is not
// measured, never a reading of zero — a VM has none of these, and must not
// look like a machine running at 0 °C on no power.
type Sensors struct {
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
// where no package reading exists.
CPUTempC float64
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
// energy counters over the time between two samples.
CPUPowerW float64
// InletTempC is the air coming in, where the board has a sensor it calls
// that. Most do not, and a CPU's temperature is not a substitute: it
// follows the load, not the room.
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat, power and whether it is slowing itself down.
type GPUSensor struct {
Index int
TempC float64
MemoryTempC float64
PowerW float64
PowerLimitW float64
// Throttle names what is holding the card's clocks down now:
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
// is running free. Idle and application clock settings are left out;
// they are not the card being held back.
Throttle []string
}
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
// than temperatures change, and each run costs a few hundred milliseconds.
const sensorEvery = 10 * time.Second
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
// two readings of the energy counters.
type SensorSampler struct {
sys string // /sys, or a test's copy of it
smi func(ctx context.Context, fields string) ([]byte, error)
now func() time.Time
mu sync.Mutex
last Sensors
lastAt time.Time
prevEnergy map[string]uint64
prevAt time.Time
}
// NewSensorSampler reads this machine.
func NewSensorSampler() *SensorSampler {
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
}
func runSMI(ctx context.Context, fields string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
//nolint:gosec // fixed binary, fields from this file
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
"--format=csv,noheader,nounits").Output()
}
// Sample reads the sensors, or returns the last reading when that is recent.
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
return s.last
}
out := Sensors{}
out.CPUTempC, out.InletTempC = s.hwmon()
out.CPUPowerW = s.rapl(now)
out.GPUs = s.gpus(ctx)
s.last, s.lastAt = out, now
return out
}
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
// device the kernel exposes — what lm_sensors reads, without it installed.
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
var pkg, core float64
for _, d := range devs {
name := readTrim(filepath.Join(d, "name"))
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
for _, in := range inputs {
milli, err := strconv.ParseFloat(readTrim(in), 64)
if err != nil || milli <= 0 {
continue
}
c := milli / 1000
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
switch {
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
pkg = max(pkg, c)
case isCPUSensor(name):
core = max(core, c)
case label == "cpu":
// A board's own reading of the CPU, as Dell's SMM gives.
core = max(core, c)
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
strings.Contains(label, "intake") || label == "systin":
inlet = max(inlet, c)
}
}
}
if pkg > 0 {
return pkg, inlet
}
return core, inlet
}
func isCPUSensor(name string) bool {
switch name {
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
return true
}
return false
}
// rapl is the CPU packages' power from their energy counters. Package
// domains only: the dram and core subdomains are parts of a package, and
// adding them would count the same energy twice.
func (s *SensorSampler) rapl(now time.Time) float64 {
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
energy := map[string]uint64{}
wrap := map[string]uint64{}
for _, d := range doms {
if strings.Count(filepath.Base(d), ":") != 1 {
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
}
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
if err != nil {
continue // unreadable without root on newer kernels
}
energy[d] = e
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
}
prev, prevAt := s.prevEnergy, s.prevAt
s.prevEnergy, s.prevAt = energy, now
dt := now.Sub(prevAt).Seconds()
if prev == nil || dt <= 0 || len(energy) == 0 {
return 0
}
var joules float64
for d, e := range energy {
p, ok := prev[d]
if !ok {
return 0 // a domain appeared: no fair difference this time
}
if e < p { // the counter wrapped
e += wrap[d]
}
joules += float64(e-p) / 1e6
}
return joules / dt
}
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
// card is being held back are named.
var throttleBits = []struct {
bit uint64
name string
}{
{0x4, "sw_power_cap"},
{0x8, "hw_slowdown"},
{0x20, "sw_thermal"},
{0x40, "hw_thermal"},
{0x80, "power_brake"},
}
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
// is not known to every driver, and a field nvidia-smi does not know fails the
// whole query, so it is asked for first and dropped on failure.
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
out, err := s.smi(ctx, withMem)
if err != nil {
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
return nil
}
}
var gs []GPUSensor
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
f := strings.Split(line, ",")
if len(f) < 5 {
continue
}
for i := range f {
f[i] = strings.TrimSpace(f[i])
}
idx, err := strconv.Atoi(f[0])
if err != nil {
continue
}
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
for _, b := range throttleBits {
if bits&b.bit != 0 {
g.Throttle = append(g.Throttle, b.name)
}
}
}
if len(f) > 5 {
g.MemoryTempC = num(f[5])
}
gs = append(gs, g)
}
return gs
}
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
func num(s string) float64 {
v, err := strconv.ParseFloat(s, 64)
if err != nil || v < 0 {
return 0
}
return v
}
func readTrim(path string) string {
b, err := os.ReadFile(path) //nolint:gosec // sysfs
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}
+169
View File
@@ -0,0 +1,169 @@
package discovery
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func put(t *testing.T, root, rel, body string) {
t.Helper()
p := filepath.Join(root, rel)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// dellHost is the layout of the Proxmox host these tests were written
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
// and cores, and RAPL with a package and its dram subdomain.
func dellHost(t *testing.T) string {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
return root
}
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
}
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
s := sampler(root, noSMI, &now)
first := s.Sample(t.Context())
// The package reading, not the hotter core: the package is what the
// chip itself reports as its temperature.
if first.CPUTempC != 58 {
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
}
if first.CPUPowerW != 0 {
t.Error("power from one reading of a counter is invented")
}
if first.InletTempC != 0 || first.GPUs != nil {
t.Errorf("sensors this host does not have were reported: %+v", first)
}
// 10 s later the package has used 650 J, and its dram subdomain some of
// that — counted once.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
t.Errorf("CPU power %.2f W, expected 65", got)
}
}
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
s := sampler(root, noSMI, &now)
s.Sample(t.Context())
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
t.Errorf("across a wrap: %.2f W, expected 50", got)
}
}
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
root := t.TempDir()
put(t, root, "class/thermal/cooling_device0/type", "Processor")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
t.Errorf("a machine with no sensors reported %+v", got)
}
}
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
}
}
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
var asked []string
smi := func(_ context.Context, fields string) ([]byte, error) {
asked = append(asked, fields)
return []byte(
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
}
now := time.Now()
ss := sampler(t.TempDir(), smi, &now)
got := ss.Sample(t.Context())
if len(got.GPUs) != 3 {
t.Fatalf("%+v", got.GPUs)
}
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
t.Errorf("card 0 %+v", g0)
}
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
t.Errorf("card 1 throttled by %v", g1.Throttle)
}
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
}
// Within ten seconds the last reading is reused rather than running
// nvidia-smi on every heartbeat; after, it is asked again.
now = now.Add(5 * time.Second)
ss.Sample(t.Context())
if len(asked) != 1 {
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
}
now = now.Add(6 * time.Second)
ss.Sample(t.Context())
if len(asked) != 2 {
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
}
}
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
smi := func(_ context.Context, fields string) ([]byte, error) {
if strings.Contains(fields, "temperature.memory") {
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
}
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
t.Errorf("%+v", got.GPUs)
}
}
+7 -1
View File
@@ -249,6 +249,11 @@ type LoadView struct {
// MemPercent is used over used plus available.
MemPercent float64 `json:"memory_percent"`
AgeSec int64 `json:"age_seconds"`
// What the node's sensors say; zero is not measured.
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
InletTempC float64 `json:"inlet_temp_c,omitempty"`
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
}
// loadFresh is how old a measurement may be and still be shown as current.
@@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
AgeSec: int64(now.Sub(l.At).Seconds()),
AgeSec: int64(now.Sub(l.At).Seconds()),
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
}
if v.MemTotal > 0 {
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
+7
View File
@@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
}
}
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
for _, g := range l.GetGpus() {
out.GPUs = append(out.GPUs, inventory.GPUSensor{
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
})
}
return out
}
+65 -1
View File
@@ -74,6 +74,13 @@ type Collector struct {
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuTemp *prometheus.Desc
gpuMemTemp *prometheus.Desc
gpuPower *prometheus.Desc
gpuPowerLimit *prometheus.Desc
gpuThrottled *prometheus.Desc
nodeCPUTemp *prometheus.Desc
nodeCPUPower *prometheus.Desc
nodeInletTemp *prometheus.Desc
gpuHealthy *prometheus.Desc
gpuNVLink *prometheus.Desc
gpuECC *prometheus.Desc
@@ -142,6 +149,21 @@ func New(src Source) *Collector {
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
"A GPU's power draw now.", gpu, nil),
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
"The power limit a GPU is held to.", gpu, nil),
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
append(gpu, "reason"), nil),
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
"A node's hottest CPU sensor.", node, nil),
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
"A node's CPU packages' power draw, from RAPL.", node, nil),
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
"A GPU's reported temperature.", gpu, nil),
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
@@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
@@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
}
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
// Each only when measured: a VM has none of these, and a zero
// would read as a machine at 0 °C drawing nothing.
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
}
}
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
if l.CPUCount > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
@@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
float64(g.UtilizationPercent), labels...)
}
if g.Health.TemperatureC > 0 {
// The sensors' fresh reading when there is one; the temperature
// read at registration otherwise, which is all an older agent sends.
if s, ok := sensorFor(n.Load, g.Index, now); ok {
c.gpuSensor(ch, s, labels)
} else if g.Health.TemperatureC > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
float64(g.Health.TemperatureC), labels...)
}
@@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
}
}
}
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
if l == nil || now.Sub(l.At) >= loadFresh {
return inventory.GPUSensor{}, false
}
for _, g := range l.GPUs {
if g.Index == index {
return g, true
}
}
return inventory.GPUSensor{}, false
}
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
}
}
for _, reason := range s.Throttle {
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
}
}
@@ -0,0 +1,41 @@
package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
for _, want := range []string{
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
"llmgw_node_cpu_power_watts node=g1 -> 65",
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
// The fresh reading, not the 40 °C from registration.
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q", want)
}
}
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
t.Errorf("published what was not measured: %s", line)
}
}
}
}
+15
View File
@@ -304,6 +304,21 @@ type Load struct {
MemAvailBytes int64
// Shards is each engine's use, by shard ID.
Shards map[string]ShardLoad
// What the node's sensors say; zero is not measured.
CPUTempC float64
CPUPowerW float64
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat and power, and what is throttling it.
type GPUSensor struct {
Index int `json:"index"`
TempC float64 `json:"temp_c,omitempty"`
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
PowerW float64 `json:"power_w,omitempty"`
PowerLimitW float64 `json:"power_limit_w,omitempty"`
Throttle []string `json:"throttle,omitempty"`
}
// ShardLoad is one engine's use: cores busy, and resident memory.
+31
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
t.Error("the cluster page does not lead to the racks")
}
}
// hotSource is a GPU node whose sensors report, one card held back.
type hotSource struct{ fakeSource }
func (hotSource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
GPUs: []inventory.GPUSensor{
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
}}
return []adminapi.NodeView{n}
}
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
if strings.Contains(vm, absent) {
t.Errorf("a node with no sensors shows %q", absent)
}
}
}
+15
View File
@@ -21,6 +21,10 @@
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
{{/* Only what this machine has sensors for: a VM has none, and a tile
saying 0 °C would be a reading nobody took. */}}
{{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{printf "%.0f" .CPUTempC}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{printf "%.0f" .CPUPowerW}} W{{end}}</div></div>{{end}}
{{if .InletTempC}}<div class="tile"><div class="v">{{printf "%.1f" .InletTempC}} °C</div><div class="k">inlet air</div></div>{{end}}
{{else}}
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
{{end}}
@@ -29,6 +33,17 @@
</div>
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}}
<h2>GPUs</h2>
<table class="rows"><tr><th class="num">GPU</th><th class="num">Temperature</th><th class="num">Memory</th><th>Power</th><th>Held back by</th></tr>
{{range .GPUs}}<tr>
<td class="num">{{.Index}}</td>
<td class="num">{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}}</td>
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}</td>
</tr>{{end}}</table>
{{end}}{{end}}
{{end}}
{{if or .Node.IBPorts .Node.Fabrics}}
+7
View File
@@ -109,6 +109,13 @@ var funcs = template.FuncMap{
return p
},
"ago": ago,
// percent is part of whole, 0 to 100, for a bar.
"percent": func(part, whole float64) float64 {
if whole <= 0 {
return 0
}
return part * 100 / whole
},
// shortNode is a node's name without what every node's name starts with.
"shortNode": strings.TrimPrefix,
"age": age,