The cluster page draws every card coloured by utilisation, memory or temperature, and lists replicas whose every card is under 5% busy. llmgw_gpu_assigned says which replica, model and owner hold each card. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
105 lines
4.7 KiB
Go
105 lines
4.7 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
|
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
|
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
|
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
|
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
|
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
|
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
|
|
|
for _, want := range []string{
|
|
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
|
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
|
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
|
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
|
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
|
// The fresh reading, not the 40 °C from registration.
|
|
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
|
t.Errorf("published what was not measured: %s", line)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
|
|
// Discovery read 87% and 70 GB used when the agent came up; the card has
|
|
// been idle since. The sample says so, zero included.
|
|
reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
|
|
MemoryUsedMB: 70000, UtilizationPercent: 87}
|
|
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg},
|
|
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35,
|
|
UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559,
|
|
SMClockMHz: 345, ECCUncorrected: 2}}}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
for _, want := range []string{
|
|
"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
|
|
"llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
|
|
"llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n",
|
|
"llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n",
|
|
"llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s in:\n%s", want, out)
|
|
}
|
|
}
|
|
if strings.Contains(out, "-> 87\n") {
|
|
t.Error("the utilization read at registration was published over the sample's")
|
|
}
|
|
|
|
// An agent too old to sample it: registration's figures, and no zero
|
|
// standing in for a utilization the driver did not give.
|
|
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}}
|
|
out = gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") {
|
|
t.Errorf("an older agent's registration figure was dropped:\n%s", out)
|
|
}
|
|
n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}}
|
|
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") {
|
|
t.Errorf("a utilization the driver did not report was published:\n%s", out)
|
|
}
|
|
}
|
|
|
|
// ownedSource adds owners to fakeSource.
|
|
type ownedSource struct{ fakeSource }
|
|
|
|
func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
|
|
|
|
func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
|
|
src := ownedSource{fakeSource{replicas: []reconciler.Replica{
|
|
{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
|
|
{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
|
|
}}}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
|
|
"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s in:\n%s", want, out)
|
|
}
|
|
}
|
|
if strings.Contains(out, "replica=old-1") {
|
|
t.Error("a stopped replica's GPU was counted as held")
|
|
}
|
|
}
|