Files
llmgw/internal/control/fleetmetrics/fleetmetrics_test.go
T
Ilker ManapandClaude Opus 5.5 daed835d88 Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 23:21:23 +02:00

179 lines
6.4 KiB
Go

package fleetmetrics
import (
"strconv"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
type fakeSource struct {
nodes []inventory.Node
free map[string][]int
replicas []reconciler.Replica
}
func (f fakeSource) Nodes() []inventory.Node { return f.nodes }
func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] }
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }
func gather(t *testing.T, src Source) string {
t.Helper()
reg := prometheus.NewRegistry()
if err := reg.Register(New(src)); err != nil {
t.Fatal(err)
}
var b strings.Builder
mfs, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
for _, mf := range mfs {
for _, m := range mf.GetMetric() {
b.WriteString(mf.GetName())
for _, l := range m.GetLabel() {
b.WriteString(" " + l.GetName() + "=" + l.GetValue())
}
if g := m.GetGauge(); g != nil {
b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
continue
}
b.WriteString("\n")
}
}
return b.String()
}
func gpuNode(id string, gpus int, fabric string) inventory.Node {
n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
for i := range gpus {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
MemoryUsedMB: 1024, UtilizationPercent: 40,
Health: discovery.GPUHealth{TemperatureC: 45},
})
}
if fabric != "" {
n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
}
return n
}
func TestTheFleetIsReported(t *testing.T) {
// The gap this closes: twenty-two metric families about traffic, and none
// about the machines serving it — so "was the cluster full at the time",
// the first question after an incident, had no answer.
src := fakeSource{
nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
free: map[string][]int{"n1": {1}},
replicas: []reconciler.Replica{
{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
},
}
out := gather(t, src)
for _, want := range []string{
"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
} {
if !strings.Contains(out, want) {
t.Errorf("%s is missing from:\n%s", want, out)
}
}
// The fabric travels as a label, because with isolated fabrics "how full is
// the cluster" is the wrong question and "how full is this fabric" is the
// right one.
if !strings.Contains(out, "fabric=ib-a") {
t.Errorf("the fabric label is missing:\n%s", out)
}
}
func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
// A replica the router has said nothing about would otherwise look like the
// most idle thing in the cluster, and after a router restart that would be
// all of them at once.
src := fakeSource{
replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
}
if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
}
}
func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
// A driver too old to give it would otherwise show every card at a flat
// zero, which reads as an idle fleet rather than a missing measurement.
n := gpuNode("n1", 1, "")
n.GPUs[0].UtilizationPercent = 0
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
}
}
func TestNodeStateIsALabelNotANumber(t *testing.T) {
// A cordoned node and an unreachable one need different responses, and a
// single gauge counting "not ready" would hide which it is.
n := gpuNode("n1", 1, "")
n.State = inventory.Cordoned
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
out := gather(t, src)
if !strings.Contains(out, "state=cordoned") {
t.Errorf("the node state is not a label:\n%s", out)
}
if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
t.Errorf("unexpected shape:\n%s", out)
}
}
func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
// A control plane with no agents yet must still serve /metrics; failing
// there would take monitoring down exactly when a cluster is being built.
if out := gather(t, fakeSource{}); out == "" {
t.Log("no metrics for an empty fleet, which is correct")
}
}
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
// The nodes are CPU machines: without this, how busy one was had no
// answer anywhere.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
}}
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
ID: "r1", Model: "qwen", State: reconciler.Ready,
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
}}}
out := gather(t, src)
for _, want := range []string{
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
"llmgw_node_cpus node=n1 -> 8",
"llmgw_node_load node=n1 window=1m -> 3.5",
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
// A node that stopped reporting has an unknown load. Repeating its last
// figure would draw a busy machine long after it went away.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
t.Errorf("a stale load was reported:\n%s", out)
}
}