llmgw/internal/control/fleetmetrics/sensors_test.go

package fleetmetrics

import (
	"strings"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)

func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
	gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
		GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
		Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
			GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
				Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
	vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
	out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})

	for _, want := range []string{
		"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
		"llmgw_node_cpu_power_watts node=g1 -> 65",
		"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
		"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
		"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
		// The fresh reading, not the 40 °C from registration.
		"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %q", want)
		}
	}
	for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
		for _, line := range strings.Split(out, "\n") {
			if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
				t.Errorf("published what was not measured: %s", line)
			}
		}
	}
}

func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
	// Discovery read 87% and 70 GB used when the agent came up; the card has
	// been idle since. The sample says so, zero included.
	reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
		MemoryUsedMB: 70000, UtilizationPercent: 87}
	n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg},
		Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35,
			UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559,
			SMClockMHz: 345, ECCUncorrected: 2}}}}
	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
	for _, want := range []string{
		"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
		"llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n",
		"llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n",
		"llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n",
		"llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %s in:\n%s", want, out)
		}
	}
	if strings.Contains(out, "-> 87\n") {
		t.Error("the utilization read at registration was published over the sample's")
	}

	// An agent too old to sample it: registration's figures, and no zero
	// standing in for a utilization the driver did not give.
	n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}}
	out = gather(t, fakeSource{nodes: []inventory.Node{n}})
	if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") {
		t.Errorf("an older agent's registration figure was dropped:\n%s", out)
	}
	n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}}
	if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") {
		t.Errorf("a utilization the driver did not report was published:\n%s", out)
	}
}

// ownedSource adds owners to fakeSource.
type ownedSource struct{ fakeSource }

func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }

func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
	src := ownedSource{fakeSource{replicas: []reconciler.Replica{
		{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
		{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
	}}}
	out := gather(t, src)
	for _, want := range []string{
		"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
		"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %s in:\n%s", want, out)
		}
	}
	if strings.Contains(out, "replica=old-1") {
		t.Error("a stopped replica's GPU was counted as held")
	}
}

func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) {
	n := inventory.Node{ID: "g1", State: inventory.Ready,
		GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
		Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 40, MemoryTotalMB: 81559,
			Wear: &inventory.GPUWear{PCIeReplays: 12, RetiredSBE: -1, RetiredDBE: -1, RemappedCorrectable: 2,
				RemappedUncorrectable: 1, RemapPending: true, NVLinkCRC: 7, XIDs: 3, LastXID: 79}}}}}
	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
	for _, want := range []string{
		"llmgw_gpu_pcie_replays class=h100-80g gpu=0 node=g1 -> 12\n",
		"llmgw_gpu_remapped_rows class=h100-80g gpu=0 kind=uncorrectable node=g1 -> 1\n",
		"llmgw_gpu_remap_pending class=h100-80g gpu=0 node=g1 -> 1\n",
		"llmgw_gpu_nvlink_errors class=h100-80g gpu=0 kind=crc node=g1 -> 7\n",
		"llmgw_gpu_xid_errors class=h100-80g gpu=0 node=g1 -> 3\n",
		"llmgw_gpu_last_xid class=h100-80g gpu=0 node=g1 -> 79\n",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %s", want)
		}
	}
	// An H100 keeps no retired pages; none are published.
	if strings.Contains(out, "llmgw_gpu_retired_pages") {
		t.Errorf("a count the card does not keep was published:\n%s", out)
	}
}

func TestHowMuchOfTheChipIsPublishedBesideHowBusy(t *testing.T) {
	// The Quadro RTX 6000 in decode: 92% busy, its SMs at work 69% of the
	// time. The DRAM figure was not read, and is left out rather than zero.
	n := inventory.Node{ID: "g1", State: inventory.Ready,
		GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
		Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, UtilPercent: 92, MemoryTotalMB: 81559,
			Profile: &inventory.GPUProfile{SMActive: 69, SMOccupancy: 45, TensorActive: 0, DRAMActive: -1, Source: "gpm"}}}}}
	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
	for _, want := range []string{
		"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 92\n",
		"llmgw_gpu_sm_active_percent class=h100-80g gpu=0 node=g1 -> 69\n",
		"llmgw_gpu_sm_occupancy_percent class=h100-80g gpu=0 node=g1 -> 45\n",
		// Idle tensor cores are a reading.
		"llmgw_gpu_tensor_active_percent class=h100-80g gpu=0 node=g1 -> 0\n",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %s in:\n%s", want, out)
		}
	}
	if strings.Contains(out, "llmgw_gpu_dram_active_percent") {
		t.Errorf("a figure not read was published:\n%s", out)
	}
}