Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL

Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 19:49:08 +02:00
1 parent 8724dc027a
commit cd10e086c8
15 files changed
+996 -172

No files matched your search

+7 -1
View File
@@ -249,6 +249,11 @@ type LoadView struct {
// MemPercent is used over used plus available.
MemPercent float64 `json:"memory_percent"`
AgeSec int64 `json:"age_seconds"`
// What the node's sensors say; zero is not measured.
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
InletTempC float64 `json:"inlet_temp_c,omitempty"`
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
}
// loadFresh is how old a measurement may be and still be shown as current.
@@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
AgeSec: int64(now.Sub(l.At).Seconds()),
AgeSec: int64(now.Sub(l.At).Seconds()),
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
}
if v.MemTotal > 0 {
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
+7
View File
@@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
}
}
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
for _, g := range l.GetGpus() {
out.GPUs = append(out.GPUs, inventory.GPUSensor{
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
})
}
return out
}
+65 -1
View File
@@ -74,6 +74,13 @@ type Collector struct {
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuTemp *prometheus.Desc
gpuMemTemp *prometheus.Desc
gpuPower *prometheus.Desc
gpuPowerLimit *prometheus.Desc
gpuThrottled *prometheus.Desc
nodeCPUTemp *prometheus.Desc
nodeCPUPower *prometheus.Desc
nodeInletTemp *prometheus.Desc
gpuHealthy *prometheus.Desc
gpuNVLink *prometheus.Desc
gpuECC *prometheus.Desc
@@ -142,6 +149,21 @@ func New(src Source) *Collector {
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
"A GPU's power draw now.", gpu, nil),
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
"The power limit a GPU is held to.", gpu, nil),
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
append(gpu, "reason"), nil),
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
"A node's hottest CPU sensor.", node, nil),
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
"A node's CPU packages' power draw, from RAPL.", node, nil),
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
"A GPU's reported temperature.", gpu, nil),
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
@@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
@@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
}
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
// Each only when measured: a VM has none of these, and a zero
// would read as a machine at 0 °C drawing nothing.
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
}
}
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
if l.CPUCount > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
@@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
float64(g.UtilizationPercent), labels...)
}
if g.Health.TemperatureC > 0 {
// The sensors' fresh reading when there is one; the temperature
// read at registration otherwise, which is all an older agent sends.
if s, ok := sensorFor(n.Load, g.Index, now); ok {
c.gpuSensor(ch, s, labels)
} else if g.Health.TemperatureC > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
float64(g.Health.TemperatureC), labels...)
}
@@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
}
}
}
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
if l == nil || now.Sub(l.At) >= loadFresh {
return inventory.GPUSensor{}, false
}
for _, g := range l.GPUs {
if g.Index == index {
return g, true
}
}
return inventory.GPUSensor{}, false
}
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
}
}
for _, reason := range s.Throttle {
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
}
}
@@ -0,0 +1,41 @@
package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
for _, want := range []string{
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
"llmgw_node_cpu_power_watts node=g1 -> 65",
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
// The fresh reading, not the 40 °C from registration.
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q", want)
}
}
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
t.Errorf("published what was not measured: %s", line)
}
}
}
}
+15
View File
@@ -304,6 +304,21 @@ type Load struct {
MemAvailBytes int64
// Shards is each engine's use, by shard ID.
Shards map[string]ShardLoad
// What the node's sensors say; zero is not measured.
CPUTempC float64
CPUPowerW float64
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat and power, and what is throttling it.
type GPUSensor struct {
Index int `json:"index"`
TempC float64 `json:"temp_c,omitempty"`
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
PowerW float64 `json:"power_w,omitempty"`
PowerLimitW float64 `json:"power_limit_w,omitempty"`
Throttle []string `json:"throttle,omitempty"`
}
// ShardLoad is one engine's use: cores busy, and resident memory.