Files
llmgw/internal/control/fleetmetrics/sensors_test.go
T
Ilker ManapandClaude Opus 5.5 b6a802c03b Map the fleet's GPUs, and name the cards held and idle
The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 15:54:58 +02:00

105 lines
4.7 KiB
Go

package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
for _, want := range []string{
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
"llmgw_node_cpu_power_watts node=g1 -> 65",
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
// The fresh reading, not the 40 °C from registration.
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q", want)
}
}
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
t.Errorf("published what was not measured: %s", line)
}
}
}
}
func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
// Discovery read 87% and 70 GB used when the agent came up; the card has
// been idle since. The sample says so, zero included.
reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
MemoryUsedMB: 70000, UtilizationPercent: 87}
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg},
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35,
UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559,
SMClockMHz: 345, ECCUncorrected: 2}}}}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
for _, want := range []string{
"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
"llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
"llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n",
"llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n",
"llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
if strings.Contains(out, "-> 87\n") {
t.Error("the utilization read at registration was published over the sample's")
}
// An agent too old to sample it: registration's figures, and no zero
// standing in for a utilization the driver did not give.
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}}
out = gather(t, fakeSource{nodes: []inventory.Node{n}})
if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") {
t.Errorf("an older agent's registration figure was dropped:\n%s", out)
}
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}}
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") {
t.Errorf("a utilization the driver did not report was published:\n%s", out)
}
}
// ownedSource adds owners to fakeSource.
type ownedSource struct{ fakeSource }
func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
src := ownedSource{fakeSource{replicas: []reconciler.Replica{
{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
}}}
out := gather(t, src)
for _, want := range []string{
"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
if strings.Contains(out, "replica=old-1") {
t.Error("a stopped replica's GPU was counted as held")
}
}