llmgw/internal/control/fleetmetrics/fleetmetrics_test.go

package fleetmetrics

import (
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)

type fakeSource struct {
	nodes    []inventory.Node
	free     map[string][]int
	replicas []reconciler.Replica
}

func (f fakeSource) Nodes() []inventory.Node        { return f.nodes }
func (f fakeSource) FreeGPUs(id string) []int       { return f.free[id] }
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }

func gather(t *testing.T, src Source) string {
	t.Helper()
	reg := prometheus.NewRegistry()
	if err := reg.Register(New(src)); err != nil {
		t.Fatal(err)
	}
	var b strings.Builder
	mfs, err := reg.Gather()
	if err != nil {
		t.Fatal(err)
	}
	for _, mf := range mfs {
		for _, m := range mf.GetMetric() {
			b.WriteString(mf.GetName())
			for _, l := range m.GetLabel() {
				b.WriteString(" " + l.GetName() + "=" + l.GetValue())
			}
			if g := m.GetGauge(); g != nil {
				b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
				continue
			}
			b.WriteString("\n")
		}
	}
	return b.String()
}

func gpuNode(id string, gpus int, fabric string) inventory.Node {
	n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
	for i := range gpus {
		n.GPUs = append(n.GPUs, discovery.GPU{
			Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
			MemoryUsedMB: 1024, UtilizationPercent: 40,
			Health: discovery.GPUHealth{TemperatureC: 45},
		})
	}
	if fabric != "" {
		n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
	}
	return n
}

func TestTheFleetIsReported(t *testing.T) {
	// The gap this closes: twenty-two metric families about traffic, and none
	// about the machines serving it — so "was the cluster full at the time",
	// the first question after an incident, had no answer.
	src := fakeSource{
		nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
		free:  map[string][]int{"n1": {1}},
		replicas: []reconciler.Replica{
			{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
		},
	}
	out := gather(t, src)
	for _, want := range []string{
		"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
		"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
		"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("%s is missing from:\n%s", want, out)
		}
	}
	// The fabric travels as a label, because with isolated fabrics "how full is
	// the cluster" is the wrong question and "how full is this fabric" is the
	// right one.
	if !strings.Contains(out, "fabric=ib-a") {
		t.Errorf("the fabric label is missing:\n%s", out)
	}
}

func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
	// A replica the router has said nothing about would otherwise look like the
	// most idle thing in the cluster, and after a router restart that would be
	// all of them at once.
	src := fakeSource{
		replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
	}
	if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
		t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
	}
}

func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
	// A driver too old to give it would otherwise show every card at a flat
	// zero, which reads as an idle fleet rather than a missing measurement.
	n := gpuNode("n1", 1, "")
	n.GPUs[0].UtilizationPercent = 0
	src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
	if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
		t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
	}
}

func TestNodeStateIsALabelNotANumber(t *testing.T) {
	// A cordoned node and an unreachable one need different responses, and a
	// single gauge counting "not ready" would hide which it is.
	n := gpuNode("n1", 1, "")
	n.State = inventory.Cordoned
	src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
	out := gather(t, src)
	if !strings.Contains(out, "state=cordoned") {
		t.Errorf("the node state is not a label:\n%s", out)
	}
	if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
		t.Errorf("unexpected shape:\n%s", out)
	}
}

func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
	// A control plane with no agents yet must still serve /metrics; failing
	// there would take monitoring down exactly when a cluster is being built.
	if out := gather(t, fakeSource{}); out == "" {
		t.Log("no metrics for an empty fleet, which is correct")
	}
}

func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
	// The nodes are CPU machines: without this, how busy one was had no
	// answer anywhere.
	n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
		At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
		MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
		Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
	}}
	src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
		ID: "r1", Model: "qwen", State: reconciler.Ready,
		Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
	}}}
	out := gather(t, src)
	for _, want := range []string{
		"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
		"llmgw_node_cpus node=n1 -> 8",
		"llmgw_node_load node=n1 window=1m -> 3.5",
		"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
		"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
		"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %q in:\n%s", want, out)
		}
	}
}

func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
	// A node that stopped reporting has an unknown load. Repeating its last
	// figure would draw a busy machine long after it went away.
	n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
		Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
	if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
		t.Errorf("a stale load was reported:\n%s", out)
	}
}

func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
	n := gpuNode("n1", 0, "ib-a")
	n.IBPorts = []inventory.IBPort{
		{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand",
			Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}},
		{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"},
	}
	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
	for _, want := range []string{
		"llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1",
		"llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0",
		"llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400",
		"llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %q in:\n%s", want, out)
		}
	}
}

type healthSource struct {
	fakeSource
	health []reconciler.ModelHealth
}

func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health }
func (h healthSource) IsLeader() bool                        { return true }

func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) {
	// ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and
	// the series was only ever published while a replica was ready — so the
	// alert could not fire for the case it was written for.
	src := healthSource{health: []reconciler.ModelHealth{
		{Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"},
		{Model: "stopped", Desired: 0},
	}}
	out := gather(t, src)
	for _, want := range []string{
		"llmgw_model_replicas_ready model=wanted -> 0",
		"llmgw_model_parked model=wanted -> 1",
		"llmgw_model_start_failures model=wanted -> 5",
		"llmgw_control_plane_leader -> 1",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("missing %s", want)
		}
	}
	if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") {
		t.Error("a stopped model publishes a ready count, so the alert would fire for it")
	}
}