Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8724dc027a
commit
cd10e086c8
15 files changed
+996
-172
No files matched your search
@@ -101,6 +101,9 @@ type Agent struct {
|
||||
smsErr map[string]string
|
||||
// load measures the machine between heartbeats.
|
||||
load *discovery.LoadSampler
|
||||
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
|
||||
// from nvidia-smi.
|
||||
sensors *discovery.SensorSampler
|
||||
|
||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||
// download reaches every shard that asked for it. A fetch is single-flight
|
||||
@@ -146,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
logRing: ring,
|
||||
provKick: make(chan struct{}, 1),
|
||||
load: discovery.NewLoadSampler(),
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -638,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
||||
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
|
||||
})
|
||||
}
|
||||
if a.sensors != nil {
|
||||
se := a.sensors.Sample(context.Background())
|
||||
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
|
||||
for _, g := range se.GPUs {
|
||||
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
|
||||
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
|
||||
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sensors is what a node's own hardware says about heat and power: the CPUs,
|
||||
// the board where it has an inlet sensor, and each GPU. A zero is not
|
||||
// measured, never a reading of zero — a VM has none of these, and must not
|
||||
// look like a machine running at 0 °C on no power.
|
||||
type Sensors struct {
|
||||
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
|
||||
// where no package reading exists.
|
||||
CPUTempC float64
|
||||
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
|
||||
// energy counters over the time between two samples.
|
||||
CPUPowerW float64
|
||||
// InletTempC is the air coming in, where the board has a sensor it calls
|
||||
// that. Most do not, and a CPU's temperature is not a substitute: it
|
||||
// follows the load, not the room.
|
||||
InletTempC float64
|
||||
GPUs []GPUSensor
|
||||
}
|
||||
|
||||
// GPUSensor is one card's heat, power and whether it is slowing itself down.
|
||||
type GPUSensor struct {
|
||||
Index int
|
||||
TempC float64
|
||||
MemoryTempC float64
|
||||
PowerW float64
|
||||
PowerLimitW float64
|
||||
// Throttle names what is holding the card's clocks down now:
|
||||
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
|
||||
// is running free. Idle and application clock settings are left out;
|
||||
// they are not the card being held back.
|
||||
Throttle []string
|
||||
}
|
||||
|
||||
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
|
||||
// than temperatures change, and each run costs a few hundred milliseconds.
|
||||
const sensorEvery = 10 * time.Second
|
||||
|
||||
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
|
||||
// two readings of the energy counters.
|
||||
type SensorSampler struct {
|
||||
sys string // /sys, or a test's copy of it
|
||||
smi func(ctx context.Context, fields string) ([]byte, error)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
last Sensors
|
||||
lastAt time.Time
|
||||
prevEnergy map[string]uint64
|
||||
prevAt time.Time
|
||||
}
|
||||
|
||||
// NewSensorSampler reads this machine.
|
||||
func NewSensorSampler() *SensorSampler {
|
||||
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
|
||||
}
|
||||
|
||||
func runSMI(ctx context.Context, fields string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
//nolint:gosec // fixed binary, fields from this file
|
||||
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
|
||||
"--format=csv,noheader,nounits").Output()
|
||||
}
|
||||
|
||||
// Sample reads the sensors, or returns the last reading when that is recent.
|
||||
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := s.now()
|
||||
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
|
||||
return s.last
|
||||
}
|
||||
out := Sensors{}
|
||||
out.CPUTempC, out.InletTempC = s.hwmon()
|
||||
out.CPUPowerW = s.rapl(now)
|
||||
out.GPUs = s.gpus(ctx)
|
||||
s.last, s.lastAt = out, now
|
||||
return out
|
||||
}
|
||||
|
||||
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
|
||||
// device the kernel exposes — what lm_sensors reads, without it installed.
|
||||
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
|
||||
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
|
||||
var pkg, core float64
|
||||
for _, d := range devs {
|
||||
name := readTrim(filepath.Join(d, "name"))
|
||||
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
|
||||
for _, in := range inputs {
|
||||
milli, err := strconv.ParseFloat(readTrim(in), 64)
|
||||
if err != nil || milli <= 0 {
|
||||
continue
|
||||
}
|
||||
c := milli / 1000
|
||||
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
|
||||
switch {
|
||||
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
|
||||
pkg = max(pkg, c)
|
||||
case isCPUSensor(name):
|
||||
core = max(core, c)
|
||||
case label == "cpu":
|
||||
// A board's own reading of the CPU, as Dell's SMM gives.
|
||||
core = max(core, c)
|
||||
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
|
||||
strings.Contains(label, "intake") || label == "systin":
|
||||
inlet = max(inlet, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if pkg > 0 {
|
||||
return pkg, inlet
|
||||
}
|
||||
return core, inlet
|
||||
}
|
||||
|
||||
func isCPUSensor(name string) bool {
|
||||
switch name {
|
||||
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rapl is the CPU packages' power from their energy counters. Package
|
||||
// domains only: the dram and core subdomains are parts of a package, and
|
||||
// adding them would count the same energy twice.
|
||||
func (s *SensorSampler) rapl(now time.Time) float64 {
|
||||
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
|
||||
energy := map[string]uint64{}
|
||||
wrap := map[string]uint64{}
|
||||
for _, d := range doms {
|
||||
if strings.Count(filepath.Base(d), ":") != 1 {
|
||||
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
|
||||
}
|
||||
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
|
||||
if err != nil {
|
||||
continue // unreadable without root on newer kernels
|
||||
}
|
||||
energy[d] = e
|
||||
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
|
||||
}
|
||||
prev, prevAt := s.prevEnergy, s.prevAt
|
||||
s.prevEnergy, s.prevAt = energy, now
|
||||
dt := now.Sub(prevAt).Seconds()
|
||||
if prev == nil || dt <= 0 || len(energy) == 0 {
|
||||
return 0
|
||||
}
|
||||
var joules float64
|
||||
for d, e := range energy {
|
||||
p, ok := prev[d]
|
||||
if !ok {
|
||||
return 0 // a domain appeared: no fair difference this time
|
||||
}
|
||||
if e < p { // the counter wrapped
|
||||
e += wrap[d]
|
||||
}
|
||||
joules += float64(e-p) / 1e6
|
||||
}
|
||||
return joules / dt
|
||||
}
|
||||
|
||||
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
|
||||
// card is being held back are named.
|
||||
var throttleBits = []struct {
|
||||
bit uint64
|
||||
name string
|
||||
}{
|
||||
{0x4, "sw_power_cap"},
|
||||
{0x8, "hw_slowdown"},
|
||||
{0x20, "sw_thermal"},
|
||||
{0x40, "hw_thermal"},
|
||||
{0x80, "power_brake"},
|
||||
}
|
||||
|
||||
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
|
||||
// is not known to every driver, and a field nvidia-smi does not know fails the
|
||||
// whole query, so it is asked for first and dropped on failure.
|
||||
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
|
||||
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
|
||||
out, err := s.smi(ctx, withMem)
|
||||
if err != nil {
|
||||
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
var gs []GPUSensor
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Split(line, ",")
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
for i := range f {
|
||||
f[i] = strings.TrimSpace(f[i])
|
||||
}
|
||||
idx, err := strconv.Atoi(f[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
|
||||
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
|
||||
for _, b := range throttleBits {
|
||||
if bits&b.bit != 0 {
|
||||
g.Throttle = append(g.Throttle, b.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(f) > 5 {
|
||||
g.MemoryTempC = num(f[5])
|
||||
}
|
||||
gs = append(gs, g)
|
||||
}
|
||||
return gs
|
||||
}
|
||||
|
||||
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
|
||||
func num(s string) float64 {
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil || v < 0 {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func readTrim(path string) string {
|
||||
b, err := os.ReadFile(path) //nolint:gosec // sysfs
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func put(t *testing.T, root, rel, body string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// dellHost is the layout of the Proxmox host these tests were written
|
||||
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
|
||||
// and cores, and RAPL with a package and its dram subdomain.
|
||||
func dellHost(t *testing.T) string {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
|
||||
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
|
||||
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
|
||||
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
|
||||
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
|
||||
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
|
||||
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
|
||||
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
|
||||
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
|
||||
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
|
||||
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
|
||||
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
|
||||
return root
|
||||
}
|
||||
|
||||
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
|
||||
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
|
||||
}
|
||||
|
||||
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
|
||||
|
||||
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
|
||||
root := dellHost(t)
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
s := sampler(root, noSMI, &now)
|
||||
|
||||
first := s.Sample(t.Context())
|
||||
// The package reading, not the hotter core: the package is what the
|
||||
// chip itself reports as its temperature.
|
||||
if first.CPUTempC != 58 {
|
||||
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
|
||||
}
|
||||
if first.CPUPowerW != 0 {
|
||||
t.Error("power from one reading of a counter is invented")
|
||||
}
|
||||
if first.InletTempC != 0 || first.GPUs != nil {
|
||||
t.Errorf("sensors this host does not have were reported: %+v", first)
|
||||
}
|
||||
|
||||
// 10 s later the package has used 650 J, and its dram subdomain some of
|
||||
// that — counted once.
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
|
||||
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
|
||||
now = now.Add(10 * time.Second)
|
||||
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
|
||||
t.Errorf("CPU power %.2f W, expected 65", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
|
||||
root := dellHost(t)
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
|
||||
s := sampler(root, noSMI, &now)
|
||||
s.Sample(t.Context())
|
||||
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
|
||||
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
|
||||
now = now.Add(10 * time.Second)
|
||||
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
|
||||
t.Errorf("across a wrap: %.2f W, expected 50", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/thermal/cooling_device0/type", "Processor")
|
||||
now := time.Now()
|
||||
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
|
||||
t.Errorf("a machine with no sensors reported %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
|
||||
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
|
||||
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
|
||||
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
|
||||
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
|
||||
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
|
||||
now := time.Now()
|
||||
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
|
||||
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
|
||||
var asked []string
|
||||
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||
asked = append(asked, fields)
|
||||
return []byte(
|
||||
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
|
||||
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
|
||||
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
|
||||
}
|
||||
now := time.Now()
|
||||
ss := sampler(t.TempDir(), smi, &now)
|
||||
got := ss.Sample(t.Context())
|
||||
if len(got.GPUs) != 3 {
|
||||
t.Fatalf("%+v", got.GPUs)
|
||||
}
|
||||
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
|
||||
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
|
||||
t.Errorf("card 0 %+v", g0)
|
||||
}
|
||||
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
|
||||
t.Errorf("card 1 throttled by %v", g1.Throttle)
|
||||
}
|
||||
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
|
||||
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
|
||||
}
|
||||
// Within ten seconds the last reading is reused rather than running
|
||||
// nvidia-smi on every heartbeat; after, it is asked again.
|
||||
now = now.Add(5 * time.Second)
|
||||
ss.Sample(t.Context())
|
||||
if len(asked) != 1 {
|
||||
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
|
||||
}
|
||||
now = now.Add(6 * time.Second)
|
||||
ss.Sample(t.Context())
|
||||
if len(asked) != 2 {
|
||||
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
|
||||
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||
if strings.Contains(fields, "temperature.memory") {
|
||||
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
|
||||
}
|
||||
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
|
||||
}
|
||||
now := time.Now()
|
||||
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
|
||||
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
|
||||
t.Errorf("%+v", got.GPUs)
|
||||
}
|
||||
}
|
||||
@@ -249,6 +249,11 @@ type LoadView struct {
|
||||
// MemPercent is used over used plus available.
|
||||
MemPercent float64 `json:"memory_percent"`
|
||||
AgeSec int64 `json:"age_seconds"`
|
||||
// What the node's sensors say; zero is not measured.
|
||||
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
|
||||
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
|
||||
InletTempC float64 `json:"inlet_temp_c,omitempty"`
|
||||
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
|
||||
}
|
||||
|
||||
// loadFresh is how old a measurement may be and still be shown as current.
|
||||
@@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
|
||||
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
|
||||
}
|
||||
if v.MemTotal > 0 {
|
||||
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
||||
|
||||
@@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
|
||||
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
|
||||
}
|
||||
}
|
||||
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
|
||||
for _, g := range l.GetGpus() {
|
||||
out.GPUs = append(out.GPUs, inventory.GPUSensor{
|
||||
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
|
||||
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -74,6 +74,13 @@ type Collector struct {
|
||||
gpuMemoryUsed *prometheus.Desc
|
||||
gpuUtil *prometheus.Desc
|
||||
gpuTemp *prometheus.Desc
|
||||
gpuMemTemp *prometheus.Desc
|
||||
gpuPower *prometheus.Desc
|
||||
gpuPowerLimit *prometheus.Desc
|
||||
gpuThrottled *prometheus.Desc
|
||||
nodeCPUTemp *prometheus.Desc
|
||||
nodeCPUPower *prometheus.Desc
|
||||
nodeInletTemp *prometheus.Desc
|
||||
gpuHealthy *prometheus.Desc
|
||||
gpuNVLink *prometheus.Desc
|
||||
gpuECC *prometheus.Desc
|
||||
@@ -142,6 +149,21 @@ func New(src Source) *Collector {
|
||||
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
||||
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
||||
"this says whether it is doing anything.", gpu, nil),
|
||||
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
|
||||
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
|
||||
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
|
||||
"A GPU's power draw now.", gpu, nil),
|
||||
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
|
||||
"The power limit a GPU is held to.", gpu, nil),
|
||||
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
|
||||
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
|
||||
append(gpu, "reason"), nil),
|
||||
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
|
||||
"A node's hottest CPU sensor.", node, nil),
|
||||
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
|
||||
"A node's CPU packages' power draw, from RAPL.", node, nil),
|
||||
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
|
||||
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
|
||||
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
||||
"A GPU's reported temperature.", gpu, nil),
|
||||
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
||||
@@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuNVLJoined, c.imexUp,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
|
||||
@@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
|
||||
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
|
||||
// Each only when measured: a VM has none of these, and a zero
|
||||
// would read as a machine at 0 °C drawing nothing.
|
||||
for _, v := range []struct {
|
||||
d *prometheus.Desc
|
||||
val float64
|
||||
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
|
||||
if v.val > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
|
||||
}
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
|
||||
if l.CPUCount > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
|
||||
@@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
||||
float64(g.UtilizationPercent), labels...)
|
||||
}
|
||||
if g.Health.TemperatureC > 0 {
|
||||
// The sensors' fresh reading when there is one; the temperature
|
||||
// read at registration otherwise, which is all an older agent sends.
|
||||
if s, ok := sensorFor(n.Load, g.Index, now); ok {
|
||||
c.gpuSensor(ch, s, labels)
|
||||
} else if g.Health.TemperatureC > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
||||
float64(g.Health.TemperatureC), labels...)
|
||||
}
|
||||
@@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
|
||||
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
|
||||
if l == nil || now.Sub(l.At) >= loadFresh {
|
||||
return inventory.GPUSensor{}, false
|
||||
}
|
||||
for _, g := range l.GPUs {
|
||||
if g.Index == index {
|
||||
return g, true
|
||||
}
|
||||
}
|
||||
return inventory.GPUSensor{}, false
|
||||
}
|
||||
|
||||
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
|
||||
for _, v := range []struct {
|
||||
d *prometheus.Desc
|
||||
val float64
|
||||
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
|
||||
if v.val > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
|
||||
}
|
||||
}
|
||||
for _, reason := range s.Throttle {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package fleetmetrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
||||
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
||||
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
||||
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
||||
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
||||
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
||||
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
||||
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
||||
|
||||
for _, want := range []string{
|
||||
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
||||
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
||||
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
||||
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
||||
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
||||
// The fresh reading, not the 40 °C from registration.
|
||||
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
||||
t.Errorf("published what was not measured: %s", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -304,6 +304,21 @@ type Load struct {
|
||||
MemAvailBytes int64
|
||||
// Shards is each engine's use, by shard ID.
|
||||
Shards map[string]ShardLoad
|
||||
// What the node's sensors say; zero is not measured.
|
||||
CPUTempC float64
|
||||
CPUPowerW float64
|
||||
InletTempC float64
|
||||
GPUs []GPUSensor
|
||||
}
|
||||
|
||||
// GPUSensor is one card's heat and power, and what is throttling it.
|
||||
type GPUSensor struct {
|
||||
Index int `json:"index"`
|
||||
TempC float64 `json:"temp_c,omitempty"`
|
||||
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
|
||||
PowerW float64 `json:"power_w,omitempty"`
|
||||
PowerLimitW float64 `json:"power_limit_w,omitempty"`
|
||||
Throttle []string `json:"throttle,omitempty"`
|
||||
}
|
||||
|
||||
// ShardLoad is one engine's use: cores busy, and resident memory.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
|
||||
t.Error("the cluster page does not lead to the racks")
|
||||
}
|
||||
}
|
||||
|
||||
// hotSource is a GPU node whose sensors report, one card held back.
|
||||
type hotSource struct{ fakeSource }
|
||||
|
||||
func (hotSource) Nodes() []adminapi.NodeView {
|
||||
n := fakeSource{}.Nodes()[0]
|
||||
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
|
||||
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
|
||||
GPUs: []inventory.GPUSensor{
|
||||
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
|
||||
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
|
||||
}}
|
||||
return []adminapi.NodeView{n}
|
||||
}
|
||||
|
||||
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
|
||||
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
|
||||
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page lacks %q", want)
|
||||
}
|
||||
}
|
||||
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
|
||||
if strings.Contains(vm, absent) {
|
||||
t.Errorf("a node with no sensors shows %q", absent)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,10 @@
|
||||
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
|
||||
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
|
||||
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
|
||||
{{/* Only what this machine has sensors for: a VM has none, and a tile
|
||||
saying 0 °C would be a reading nobody took. */}}
|
||||
{{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{printf "%.0f" .CPUTempC}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{printf "%.0f" .CPUPowerW}} W{{end}}</div></div>{{end}}
|
||||
{{if .InletTempC}}<div class="tile"><div class="v">{{printf "%.1f" .InletTempC}} °C</div><div class="k">inlet air</div></div>{{end}}
|
||||
{{else}}
|
||||
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
|
||||
{{end}}
|
||||
@@ -29,6 +33,17 @@
|
||||
</div>
|
||||
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
|
||||
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
|
||||
{{with .Load}}{{if .GPUs}}
|
||||
<h2>GPUs</h2>
|
||||
<table class="rows"><tr><th class="num">GPU</th><th class="num">Temperature</th><th class="num">Memory</th><th>Power</th><th>Held back by</th></tr>
|
||||
{{range .GPUs}}<tr>
|
||||
<td class="num">{{.Index}}</td>
|
||||
<td class="num">{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
|
||||
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
|
||||
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if or .Node.IBPorts .Node.Fabrics}}
|
||||
|
||||
@@ -109,6 +109,13 @@ var funcs = template.FuncMap{
|
||||
return p
|
||||
},
|
||||
"ago": ago,
|
||||
// percent is part of whole, 0 to 100, for a bar.
|
||||
"percent": func(part, whole float64) float64 {
|
||||
if whole <= 0 {
|
||||
return 0
|
||||
}
|
||||
return part * 100 / whole
|
||||
},
|
||||
// shortNode is a node's name without what every node's name starts with.
|
||||
"shortNode": strings.TrimPrefix,
|
||||
"age": age,
|
||||
|
||||
Reference in new issue
Block a user