package fleetmetrics import ( "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { gpuNode := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}}, Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65, GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}}}}} vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}} out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}}) for _, want := range []string{ "llmgw_node_cpu_temperature_celsius node=g1 -> 58", "llmgw_node_cpu_power_watts node=g1 -> 65", "llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698", "llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700", "llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1", // The fresh reading, not the 40 °C from registration. "llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88", } { if !strings.Contains(out, want) { t.Errorf("missing %q", want) } } for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} { for _, line := range strings.Split(out, "\n") { if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) { t.Errorf("published what was not measured: %s", line) } } } } func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) { // Discovery read 87% and 70 GB used when the agent came up; the card has // been idle since. The sample says so, zero included. reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, MemoryUsedMB: 70000, UtilizationPercent: 87} n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg}, Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559, SMClockMHz: 345, ECCUncorrected: 2}}}} out := gather(t, fakeSource{nodes: []inventory.Node{n}}) for _, want := range []string{ "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", "llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", "llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n", "llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n", "llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n", } { if !strings.Contains(out, want) { t.Errorf("missing %s in:\n%s", want, out) } } if strings.Contains(out, "-> 87\n") { t.Error("the utilization read at registration was published over the sample's") } // An agent too old to sample it: registration's figures, and no zero // standing in for a utilization the driver did not give. n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}} out = gather(t, fakeSource{nodes: []inventory.Node{n}}) if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") { t.Errorf("an older agent's registration figure was dropped:\n%s", out) } n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}} if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") { t.Errorf("a utilization the driver did not report was published:\n%s", out) } } // ownedSource adds owners to fakeSource. type ownedSource struct{ fakeSource } func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} } func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) { src := ownedSource{fakeSource{replicas: []reconciler.Replica{ {ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}}, {ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}}, }}} out := gather(t, src) for _, want := range []string{ "llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", "llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", } { if !strings.Contains(out, want) { t.Errorf("missing %s in:\n%s", want, out) } } if strings.Contains(out, "replica=old-1") { t.Error("a stopped replica's GPU was counted as held") } } func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) { n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}}, Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 40, MemoryTotalMB: 81559, Wear: &inventory.GPUWear{PCIeReplays: 12, RetiredSBE: -1, RetiredDBE: -1, RemappedCorrectable: 2, RemappedUncorrectable: 1, RemapPending: true, NVLinkCRC: 7, XIDs: 3, LastXID: 79}}}}} out := gather(t, fakeSource{nodes: []inventory.Node{n}}) for _, want := range []string{ "llmgw_gpu_pcie_replays class=h100-80g gpu=0 node=g1 -> 12\n", "llmgw_gpu_remapped_rows class=h100-80g gpu=0 kind=uncorrectable node=g1 -> 1\n", "llmgw_gpu_remap_pending class=h100-80g gpu=0 node=g1 -> 1\n", "llmgw_gpu_nvlink_errors class=h100-80g gpu=0 kind=crc node=g1 -> 7\n", "llmgw_gpu_xid_errors class=h100-80g gpu=0 node=g1 -> 3\n", "llmgw_gpu_last_xid class=h100-80g gpu=0 node=g1 -> 79\n", } { if !strings.Contains(out, want) { t.Errorf("missing %s", want) } } // An H100 keeps no retired pages; none are published. if strings.Contains(out, "llmgw_gpu_retired_pages") { t.Errorf("a count the card does not keep was published:\n%s", out) } }