Files
llmgw/internal/control/fleetmetrics/fleetmetrics_test.go
T
Ilker ManapandClaude Opus 5.5 93bd16f00e Watch InfiniBand ports while the agent runs, not only when it starts
A port's state was read once, at registration: one that went down later
left its node on the fabric, offered multi-node work that would hang on
its first collective, until somebody restarted the agent. The agent now
re-reads every IB and RoCE port on each heartbeat - state, physical
state, rate and the error and data counters - and when the fabrics it
can carry traffic on change, says so once; the control plane replaces
the node's fabrics, which is what placement reads.

The ports are exported as llmgw_ib_port_active (with state and
phys_state, so an unplugged cable reads differently from a missing
subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total,
and a node's page lists them with the fabric each carries and any error
counter that is not zero.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 00:13:54 +02:00

199 lines
7.3 KiB
Go

package fleetmetrics
import (
"strconv"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
type fakeSource struct {
nodes []inventory.Node
free map[string][]int
replicas []reconciler.Replica
}
func (f fakeSource) Nodes() []inventory.Node { return f.nodes }
func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] }
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }
func gather(t *testing.T, src Source) string {
t.Helper()
reg := prometheus.NewRegistry()
if err := reg.Register(New(src)); err != nil {
t.Fatal(err)
}
var b strings.Builder
mfs, err := reg.Gather()
if err != nil {
t.Fatal(err)
}
for _, mf := range mfs {
for _, m := range mf.GetMetric() {
b.WriteString(mf.GetName())
for _, l := range m.GetLabel() {
b.WriteString(" " + l.GetName() + "=" + l.GetValue())
}
if g := m.GetGauge(); g != nil {
b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
continue
}
b.WriteString("\n")
}
}
return b.String()
}
func gpuNode(id string, gpus int, fabric string) inventory.Node {
n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
for i := range gpus {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
MemoryUsedMB: 1024, UtilizationPercent: 40,
Health: discovery.GPUHealth{TemperatureC: 45},
})
}
if fabric != "" {
n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
}
return n
}
func TestTheFleetIsReported(t *testing.T) {
// The gap this closes: twenty-two metric families about traffic, and none
// about the machines serving it — so "was the cluster full at the time",
// the first question after an incident, had no answer.
src := fakeSource{
nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
free: map[string][]int{"n1": {1}},
replicas: []reconciler.Replica{
{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
},
}
out := gather(t, src)
for _, want := range []string{
"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
} {
if !strings.Contains(out, want) {
t.Errorf("%s is missing from:\n%s", want, out)
}
}
// The fabric travels as a label, because with isolated fabrics "how full is
// the cluster" is the wrong question and "how full is this fabric" is the
// right one.
if !strings.Contains(out, "fabric=ib-a") {
t.Errorf("the fabric label is missing:\n%s", out)
}
}
func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
// A replica the router has said nothing about would otherwise look like the
// most idle thing in the cluster, and after a router restart that would be
// all of them at once.
src := fakeSource{
replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
}
if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
}
}
func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
// A driver too old to give it would otherwise show every card at a flat
// zero, which reads as an idle fleet rather than a missing measurement.
n := gpuNode("n1", 1, "")
n.GPUs[0].UtilizationPercent = 0
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
}
}
func TestNodeStateIsALabelNotANumber(t *testing.T) {
// A cordoned node and an unreachable one need different responses, and a
// single gauge counting "not ready" would hide which it is.
n := gpuNode("n1", 1, "")
n.State = inventory.Cordoned
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
out := gather(t, src)
if !strings.Contains(out, "state=cordoned") {
t.Errorf("the node state is not a label:\n%s", out)
}
if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
t.Errorf("unexpected shape:\n%s", out)
}
}
func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
// A control plane with no agents yet must still serve /metrics; failing
// there would take monitoring down exactly when a cluster is being built.
if out := gather(t, fakeSource{}); out == "" {
t.Log("no metrics for an empty fleet, which is correct")
}
}
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
// The nodes are CPU machines: without this, how busy one was had no
// answer anywhere.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
}}
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
ID: "r1", Model: "qwen", State: reconciler.Ready,
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
}}}
out := gather(t, src)
for _, want := range []string{
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
"llmgw_node_cpus node=n1 -> 8",
"llmgw_node_load node=n1 window=1m -> 3.5",
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
// A node that stopped reporting has an unknown load. Repeating its last
// figure would draw a busy machine long after it went away.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
t.Errorf("a stale load was reported:\n%s", out)
}
}
func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
n := gpuNode("n1", 0, "ib-a")
n.IBPorts = []inventory.IBPort{
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand",
Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}},
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"},
}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
for _, want := range []string{
"llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1",
"llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0",
"llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400",
"llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}