package fleetmetrics import ( "strconv" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) type fakeSource struct { nodes []inventory.Node free map[string][]int replicas []reconciler.Replica } func (f fakeSource) Nodes() []inventory.Node { return f.nodes } func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] } func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas } func gather(t *testing.T, src Source) string { t.Helper() reg := prometheus.NewRegistry() if err := reg.Register(New(src)); err != nil { t.Fatal(err) } var b strings.Builder mfs, err := reg.Gather() if err != nil { t.Fatal(err) } for _, mf := range mfs { for _, m := range mf.GetMetric() { b.WriteString(mf.GetName()) for _, l := range m.GetLabel() { b.WriteString(" " + l.GetName() + "=" + l.GetValue()) } if g := m.GetGauge(); g != nil { b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n") continue } b.WriteString("\n") } } return b.String() } func gpuNode(id string, gpus int, fabric string) inventory.Node { n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()} for i := range gpus { n.GPUs = append(n.GPUs, discovery.GPU{ Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, MemoryUsedMB: 1024, UtilizationPercent: 40, Health: discovery.GPUHealth{TemperatureC: 45}, }) } if fabric != "" { n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}} } return n } func TestTheFleetIsReported(t *testing.T) { // The gap this closes: twenty-two metric families about traffic, and none // about the machines serving it — so "was the cluster full at the time", // the first question after an incident, had no answer. src := fakeSource{ nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")}, free: map[string][]int{"n1": {1}}, replicas: []reconciler.Replica{ {ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)}, }, } out := gather(t, src) for _, want := range []string{ "llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes", "llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas", "llmgw_model_replicas_ready", "llmgw_replica_idle_seconds", } { if !strings.Contains(out, want) { t.Errorf("%s is missing from:\n%s", want, out) } } // The fabric travels as a label, because with isolated fabrics "how full is // the cluster" is the wrong question and "how full is this fabric" is the // right one. if !strings.Contains(out, "fabric=ib-a") { t.Errorf("the fabric label is missing:\n%s", out) } } func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) { // A replica the router has said nothing about would otherwise look like the // most idle thing in the cluster, and after a router restart that would be // all of them at once. src := fakeSource{ replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}}, } if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") { t.Errorf("idleness was reported for a replica nothing has used:\n%s", out) } } func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) { // A driver too old to give it would otherwise show every card at a flat // zero, which reads as an idle fleet rather than a missing measurement. n := gpuNode("n1", 1, "") n.GPUs[0].UtilizationPercent = 0 src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}} if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") { t.Errorf("utilisation was reported when the driver gave none:\n%s", out) } } func TestNodeStateIsALabelNotANumber(t *testing.T) { // A cordoned node and an unreachable one need different responses, and a // single gauge counting "not ready" would hide which it is. n := gpuNode("n1", 1, "") n.State = inventory.Cordoned src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}} out := gather(t, src) if !strings.Contains(out, "state=cordoned") { t.Errorf("the node state is not a label:\n%s", out) } if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") { t.Errorf("unexpected shape:\n%s", out) } } func TestAnEmptyFleetCollectsWithoutError(t *testing.T) { // A control plane with no agents yet must still serve /metrics; failing // there would take monitoring down exactly when a cluster is being built. if out := gather(t, fakeSource{}); out == "" { t.Log("no metrics for an empty fleet, which is correct") } } func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) { // The nodes are CPU machines: without this, how busy one was had no // answer anywhere. n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{ At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1, MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30, Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}}, }} src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{ ID: "r1", Model: "qwen", State: reconciler.Ready, Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}}, }}} out := gather(t, src) for _, want := range []string{ "llmgw_node_cpu_utilization_percent node=n1 -> 62.5", "llmgw_node_cpus node=n1 -> 8", "llmgw_node_load node=n1 window=1m -> 3.5", "llmgw_node_memory_used_bytes node=n1 -> 6442450944", "llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5", "llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120", } { if !strings.Contains(out, want) { t.Errorf("missing %q in:\n%s", want, out) } } } func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) { // A node that stopped reporting has an unknown load. Repeating its last // figure would draw a busy machine long after it went away. n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}} if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") { t.Errorf("a stale load was reported:\n%s", out) } } func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) { n := gpuNode("n1", 0, "ib-a") n.IBPorts = []inventory.IBPort{ {HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand", Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}}, {HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"}, } out := gather(t, fakeSource{nodes: []inventory.Node{n}}) for _, want := range []string{ "llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1", "llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0", "llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400", "llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1", } { if !strings.Contains(out, want) { t.Errorf("missing %q in:\n%s", want, out) } } } type healthSource struct { fakeSource health []reconciler.ModelHealth } func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health } func (h healthSource) IsLeader() bool { return true } func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) { // ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and // the series was only ever published while a replica was ready — so the // alert could not fire for the case it was written for. src := healthSource{health: []reconciler.ModelHealth{ {Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"}, {Model: "stopped", Desired: 0}, }} out := gather(t, src) for _, want := range []string{ "llmgw_model_replicas_ready model=wanted -> 0", "llmgw_model_parked model=wanted -> 1", "llmgw_model_start_failures model=wanted -> 5", "llmgw_control_plane_leader -> 1", } { if !strings.Contains(out, want) { t.Errorf("missing %s", want) } } if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") { t.Error("a stopped model publishes a ready count, so the alert would fire for it") } }