Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -249,6 +249,11 @@ type LoadView struct {
|
||||
// MemPercent is used over used plus available.
|
||||
MemPercent float64 `json:"memory_percent"`
|
||||
AgeSec int64 `json:"age_seconds"`
|
||||
// What the node's sensors say; zero is not measured.
|
||||
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
|
||||
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
|
||||
InletTempC float64 `json:"inlet_temp_c,omitempty"`
|
||||
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
|
||||
}
|
||||
|
||||
// loadFresh is how old a measurement may be and still be shown as current.
|
||||
@@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
|
||||
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
|
||||
}
|
||||
if v.MemTotal > 0 {
|
||||
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
||||
|
||||
@@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
|
||||
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
|
||||
}
|
||||
}
|
||||
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
|
||||
for _, g := range l.GetGpus() {
|
||||
out.GPUs = append(out.GPUs, inventory.GPUSensor{
|
||||
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
|
||||
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -74,6 +74,13 @@ type Collector struct {
|
||||
gpuMemoryUsed *prometheus.Desc
|
||||
gpuUtil *prometheus.Desc
|
||||
gpuTemp *prometheus.Desc
|
||||
gpuMemTemp *prometheus.Desc
|
||||
gpuPower *prometheus.Desc
|
||||
gpuPowerLimit *prometheus.Desc
|
||||
gpuThrottled *prometheus.Desc
|
||||
nodeCPUTemp *prometheus.Desc
|
||||
nodeCPUPower *prometheus.Desc
|
||||
nodeInletTemp *prometheus.Desc
|
||||
gpuHealthy *prometheus.Desc
|
||||
gpuNVLink *prometheus.Desc
|
||||
gpuECC *prometheus.Desc
|
||||
@@ -142,6 +149,21 @@ func New(src Source) *Collector {
|
||||
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
||||
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
||||
"this says whether it is doing anything.", gpu, nil),
|
||||
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
|
||||
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
|
||||
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
|
||||
"A GPU's power draw now.", gpu, nil),
|
||||
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
|
||||
"The power limit a GPU is held to.", gpu, nil),
|
||||
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
|
||||
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
|
||||
append(gpu, "reason"), nil),
|
||||
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
|
||||
"A node's hottest CPU sensor.", node, nil),
|
||||
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
|
||||
"A node's CPU packages' power draw, from RAPL.", node, nil),
|
||||
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
|
||||
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
|
||||
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
||||
"A GPU's reported temperature.", gpu, nil),
|
||||
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
||||
@@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuNVLJoined, c.imexUp,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
|
||||
@@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
|
||||
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
|
||||
// Each only when measured: a VM has none of these, and a zero
|
||||
// would read as a machine at 0 °C drawing nothing.
|
||||
for _, v := range []struct {
|
||||
d *prometheus.Desc
|
||||
val float64
|
||||
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
|
||||
if v.val > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
|
||||
}
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
|
||||
if l.CPUCount > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
|
||||
@@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
||||
float64(g.UtilizationPercent), labels...)
|
||||
}
|
||||
if g.Health.TemperatureC > 0 {
|
||||
// The sensors' fresh reading when there is one; the temperature
|
||||
// read at registration otherwise, which is all an older agent sends.
|
||||
if s, ok := sensorFor(n.Load, g.Index, now); ok {
|
||||
c.gpuSensor(ch, s, labels)
|
||||
} else if g.Health.TemperatureC > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
||||
float64(g.Health.TemperatureC), labels...)
|
||||
}
|
||||
@@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
|
||||
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
|
||||
if l == nil || now.Sub(l.At) >= loadFresh {
|
||||
return inventory.GPUSensor{}, false
|
||||
}
|
||||
for _, g := range l.GPUs {
|
||||
if g.Index == index {
|
||||
return g, true
|
||||
}
|
||||
}
|
||||
return inventory.GPUSensor{}, false
|
||||
}
|
||||
|
||||
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
|
||||
for _, v := range []struct {
|
||||
d *prometheus.Desc
|
||||
val float64
|
||||
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
|
||||
if v.val > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
|
||||
}
|
||||
}
|
||||
for _, reason := range s.Throttle {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package fleetmetrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
||||
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
||||
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
||||
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
||||
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
||||
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
||||
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
||||
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
||||
|
||||
for _, want := range []string{
|
||||
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
||||
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
||||
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
||||
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
||||
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
||||
// The fresh reading, not the 40 °C from registration.
|
||||
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
||||
t.Errorf("published what was not measured: %s", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -304,6 +304,21 @@ type Load struct {
|
||||
MemAvailBytes int64
|
||||
// Shards is each engine's use, by shard ID.
|
||||
Shards map[string]ShardLoad
|
||||
// What the node's sensors say; zero is not measured.
|
||||
CPUTempC float64
|
||||
CPUPowerW float64
|
||||
InletTempC float64
|
||||
GPUs []GPUSensor
|
||||
}
|
||||
|
||||
// GPUSensor is one card's heat and power, and what is throttling it.
|
||||
type GPUSensor struct {
|
||||
Index int `json:"index"`
|
||||
TempC float64 `json:"temp_c,omitempty"`
|
||||
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
|
||||
PowerW float64 `json:"power_w,omitempty"`
|
||||
PowerLimitW float64 `json:"power_limit_w,omitempty"`
|
||||
Throttle []string `json:"throttle,omitempty"`
|
||||
}
|
||||
|
||||
// ShardLoad is one engine's use: cores busy, and resident memory.
|
||||
|
||||
Reference in new issue
Block a user