package fleetmetrics import ( "strconv" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) type fakeSource struct { nodes []inventory.Node free map[string][]int replicas []reconciler.Replica } func (f fakeSource) Nodes() []inventory.Node { return f.nodes } func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] } func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas } func gather(t *testing.T, src Source) string { t.Helper() reg := prometheus.NewRegistry() if err := reg.Register(New(src)); err != nil { t.Fatal(err) } var b strings.Builder mfs, err := reg.Gather() if err != nil { t.Fatal(err) } for _, mf := range mfs { for _, m := range mf.GetMetric() { b.WriteString(mf.GetName()) for _, l := range m.GetLabel() { b.WriteString(" " + l.GetName() + "=" + l.GetValue()) } if g := m.GetGauge(); g != nil { b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n") continue } b.WriteString("\n") } } return b.String() } func gpuNode(id string, gpus int, fabric string) inventory.Node { n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()} for i := range gpus { n.GPUs = append(n.GPUs, discovery.GPU{ Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, MemoryUsedMB: 1024, UtilizationPercent: 40, Health: discovery.GPUHealth{TemperatureC: 45}, }) } if fabric != "" { n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}} } return n } func TestTheFleetIsReported(t *testing.T) { // The gap this closes: twenty-two metric families about traffic, and none // about the machines serving it — so "was the cluster full at the time", // the first question after an incident, had no answer. src := fakeSource{ nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")}, free: map[string][]int{"n1": {1}}, replicas: []reconciler.Replica{ {ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)}, }, } out := gather(t, src) for _, want := range []string{ "llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes", "llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas", "llmgw_model_replicas_ready", "llmgw_replica_idle_seconds", } { if !strings.Contains(out, want) { t.Errorf("%s is missing from:\n%s", want, out) } } // The fabric travels as a label, because with isolated fabrics "how full is // the cluster" is the wrong question and "how full is this fabric" is the // right one. if !strings.Contains(out, "fabric=ib-a") { t.Errorf("the fabric label is missing:\n%s", out) } } func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) { // A replica the router has said nothing about would otherwise look like the // most idle thing in the cluster, and after a router restart that would be // all of them at once. src := fakeSource{ replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}}, } if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") { t.Errorf("idleness was reported for a replica nothing has used:\n%s", out) } } func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) { // A driver too old to give it would otherwise show every card at a flat // zero, which reads as an idle fleet rather than a missing measurement. n := gpuNode("n1", 1, "") n.GPUs[0].UtilizationPercent = 0 src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}} if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") { t.Errorf("utilisation was reported when the driver gave none:\n%s", out) } } func TestNodeStateIsALabelNotANumber(t *testing.T) { // A cordoned node and an unreachable one need different responses, and a // single gauge counting "not ready" would hide which it is. n := gpuNode("n1", 1, "") n.State = inventory.Cordoned src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}} out := gather(t, src) if !strings.Contains(out, "state=cordoned") { t.Errorf("the node state is not a label:\n%s", out) } if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") { t.Errorf("unexpected shape:\n%s", out) } } func TestAnEmptyFleetCollectsWithoutError(t *testing.T) { // A control plane with no agents yet must still serve /metrics; failing // there would take monitoring down exactly when a cluster is being built. if out := gather(t, fakeSource{}); out == "" { t.Log("no metrics for an empty fleet, which is correct") } }