Every minute the agent reads nvidia-smi's full report, its NVLink error counters and the kernel's XID lines. They are published per card, shown on the node page and the GPU map, charted in Grafana, and five alerts watch them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
130 lines
6.0 KiB
Go
130 lines
6.0 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
|
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
|
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
|
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
|
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
|
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
|
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
|
|
|
for _, want := range []string{
|
|
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
|
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
|
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
|
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
|
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
|
// The fresh reading, not the 40 °C from registration.
|
|
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
|
t.Errorf("published what was not measured: %s", line)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
|
|
// Discovery read 87% and 70 GB used when the agent came up; the card has
|
|
// been idle since. The sample says so, zero included.
|
|
reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
|
|
MemoryUsedMB: 70000, UtilizationPercent: 87}
|
|
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg},
|
|
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35,
|
|
UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559,
|
|
SMClockMHz: 345, ECCUncorrected: 2}}}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
for _, want := range []string{
|
|
"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
|
|
"llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
|
|
"llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n",
|
|
"llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n",
|
|
"llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s in:\n%s", want, out)
|
|
}
|
|
}
|
|
if strings.Contains(out, "-> 87\n") {
|
|
t.Error("the utilization read at registration was published over the sample's")
|
|
}
|
|
|
|
// An agent too old to sample it: registration's figures, and no zero
|
|
// standing in for a utilization the driver did not give.
|
|
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}}
|
|
out = gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") {
|
|
t.Errorf("an older agent's registration figure was dropped:\n%s", out)
|
|
}
|
|
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}}
|
|
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") {
|
|
t.Errorf("a utilization the driver did not report was published:\n%s", out)
|
|
}
|
|
}
|
|
|
|
// ownedSource adds owners to fakeSource.
|
|
type ownedSource struct{ fakeSource }
|
|
|
|
func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
|
|
|
|
func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
|
|
src := ownedSource{fakeSource{replicas: []reconciler.Replica{
|
|
{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
|
|
{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
|
|
}}}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
|
|
"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s in:\n%s", want, out)
|
|
}
|
|
}
|
|
if strings.Contains(out, "replica=old-1") {
|
|
t.Error("a stopped replica's GPU was counted as held")
|
|
}
|
|
}
|
|
|
|
func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) {
|
|
n := inventory.Node{ID: "g1", State: inventory.Ready,
|
|
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
|
|
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 40, MemoryTotalMB: 81559,
|
|
Wear: &inventory.GPUWear{PCIeReplays: 12, RetiredSBE: -1, RetiredDBE: -1, RemappedCorrectable: 2,
|
|
RemappedUncorrectable: 1, RemapPending: true, NVLinkCRC: 7, XIDs: 3, LastXID: 79}}}}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
for _, want := range []string{
|
|
"llmgw_gpu_pcie_replays class=h100-80g gpu=0 node=g1 -> 12\n",
|
|
"llmgw_gpu_remapped_rows class=h100-80g gpu=0 kind=uncorrectable node=g1 -> 1\n",
|
|
"llmgw_gpu_remap_pending class=h100-80g gpu=0 node=g1 -> 1\n",
|
|
"llmgw_gpu_nvlink_errors class=h100-80g gpu=0 kind=crc node=g1 -> 7\n",
|
|
"llmgw_gpu_xid_errors class=h100-80g gpu=0 node=g1 -> 3\n",
|
|
"llmgw_gpu_last_xid class=h100-80g gpu=0 node=g1 -> 79\n",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s", want)
|
|
}
|
|
}
|
|
// An H100 keeps no retired pages; none are published.
|
|
if strings.Contains(out, "llmgw_gpu_retired_pages") {
|
|
t.Errorf("a count the card does not keep was published:\n%s", out)
|
|
}
|
|
}
|