A port's state was read once, at registration: one that went down later left its node on the fabric, offered multi-node work that would hang on its first collective, until somebody restarted the agent. The agent now re-reads every IB and RoCE port on each heartbeat - state, physical state, rate and the error and data counters - and when the fabrics it can carry traffic on change, says so once; the control plane replaces the node's fabrics, which is what placement reads. The ports are exported as llmgw_ib_port_active (with state and phys_state, so an unplugged cable reads differently from a missing subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total, and a node's page lists them with the fabric each carries and any error counter that is not zero. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
199 lines
7.3 KiB
Go
199 lines
7.3 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
type fakeSource struct {
|
|
nodes []inventory.Node
|
|
free map[string][]int
|
|
replicas []reconciler.Replica
|
|
}
|
|
|
|
func (f fakeSource) Nodes() []inventory.Node { return f.nodes }
|
|
func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] }
|
|
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }
|
|
|
|
func gather(t *testing.T, src Source) string {
|
|
t.Helper()
|
|
reg := prometheus.NewRegistry()
|
|
if err := reg.Register(New(src)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var b strings.Builder
|
|
mfs, err := reg.Gather()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, mf := range mfs {
|
|
for _, m := range mf.GetMetric() {
|
|
b.WriteString(mf.GetName())
|
|
for _, l := range m.GetLabel() {
|
|
b.WriteString(" " + l.GetName() + "=" + l.GetValue())
|
|
}
|
|
if g := m.GetGauge(); g != nil {
|
|
b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
|
|
continue
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func gpuNode(id string, gpus int, fabric string) inventory.Node {
|
|
n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
|
|
for i := range gpus {
|
|
n.GPUs = append(n.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
|
|
MemoryUsedMB: 1024, UtilizationPercent: 40,
|
|
Health: discovery.GPUHealth{TemperatureC: 45},
|
|
})
|
|
}
|
|
if fabric != "" {
|
|
n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestTheFleetIsReported(t *testing.T) {
|
|
// The gap this closes: twenty-two metric families about traffic, and none
|
|
// about the machines serving it — so "was the cluster full at the time",
|
|
// the first question after an incident, had no answer.
|
|
src := fakeSource{
|
|
nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
|
|
free: map[string][]int{"n1": {1}},
|
|
replicas: []reconciler.Replica{
|
|
{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
|
|
},
|
|
}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
|
|
"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
|
|
"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("%s is missing from:\n%s", want, out)
|
|
}
|
|
}
|
|
// The fabric travels as a label, because with isolated fabrics "how full is
|
|
// the cluster" is the wrong question and "how full is this fabric" is the
|
|
// right one.
|
|
if !strings.Contains(out, "fabric=ib-a") {
|
|
t.Errorf("the fabric label is missing:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
|
|
// A replica the router has said nothing about would otherwise look like the
|
|
// most idle thing in the cluster, and after a router restart that would be
|
|
// all of them at once.
|
|
src := fakeSource{
|
|
replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
|
|
}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
|
|
t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
|
|
// A driver too old to give it would otherwise show every card at a flat
|
|
// zero, which reads as an idle fleet rather than a missing measurement.
|
|
n := gpuNode("n1", 1, "")
|
|
n.GPUs[0].UtilizationPercent = 0
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
|
|
t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestNodeStateIsALabelNotANumber(t *testing.T) {
|
|
// A cordoned node and an unreachable one need different responses, and a
|
|
// single gauge counting "not ready" would hide which it is.
|
|
n := gpuNode("n1", 1, "")
|
|
n.State = inventory.Cordoned
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
|
|
out := gather(t, src)
|
|
if !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("the node state is not a label:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("unexpected shape:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
|
|
// A control plane with no agents yet must still serve /metrics; failing
|
|
// there would take monitoring down exactly when a cluster is being built.
|
|
if out := gather(t, fakeSource{}); out == "" {
|
|
t.Log("no metrics for an empty fleet, which is correct")
|
|
}
|
|
}
|
|
|
|
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
|
|
// The nodes are CPU machines: without this, how busy one was had no
|
|
// answer anywhere.
|
|
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
|
|
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
|
|
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
|
|
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
|
|
}}
|
|
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
|
|
ID: "r1", Model: "qwen", State: reconciler.Ready,
|
|
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
|
|
}}}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
|
|
"llmgw_node_cpus node=n1 -> 8",
|
|
"llmgw_node_load node=n1 window=1m -> 3.5",
|
|
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
|
|
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
|
|
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q in:\n%s", want, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
|
|
// A node that stopped reporting has an unknown load. Repeating its last
|
|
// figure would draw a busy machine long after it went away.
|
|
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
|
|
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
|
|
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
|
|
t.Errorf("a stale load was reported:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
|
|
n := gpuNode("n1", 0, "ib-a")
|
|
n.IBPorts = []inventory.IBPort{
|
|
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: "InfiniBand",
|
|
Counters: map[string]uint64{"symbol_error": 12, "port_xmit_data": 4096}},
|
|
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", LinkLayer: "InfiniBand"},
|
|
}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
for _, want := range []string{
|
|
"llmgw_ib_port_active hca=mlx5_0 link_layer=InfiniBand node=n1 phys_state=LinkUp port=1 state=ACTIVE -> 1",
|
|
"llmgw_ib_port_active hca=mlx5_1 link_layer=InfiniBand node=n1 phys_state=Polling port=1 state=DOWN -> 0",
|
|
"llmgw_ib_port_rate_gbps hca=mlx5_0 node=n1 port=1 -> 400",
|
|
"llmgw_ib_port_counter_total counter=symbol_error hca=mlx5_0 node=n1 port=1",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q in:\n%s", want, out)
|
|
}
|
|
}
|
|
}
|