Every agent samples, at most every ten seconds: nvidia-smi's temperature, HBM temperature, power draw, enforced limit and throttle reasons per GPU; hwmon's CPU package and board inlet temperatures; and CPU package power from RAPL's energy counters. Sent with heartbeats, published as metrics and shown on the node page; unmeasured sensors are absent, never zero. Observe-only, step 0 of docs/facility-aware-placement.md, which now describes the agent source and what these readings can and cannot say. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
42 lines
1.7 KiB
Go
42 lines
1.7 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
|
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
|
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
|
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
|
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
|
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
|
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
|
|
|
for _, want := range []string{
|
|
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
|
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
|
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
|
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
|
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
|
// The fresh reading, not the 40 °C from registration.
|
|
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
|
t.Errorf("published what was not measured: %s", line)
|
|
}
|
|
}
|
|
}
|
|
}
|