The router published twenty-two metric families about requests, endpoints, admission and scoring — everything about traffic and nothing about the machines carrying it. There was no way to graph GPU usage, watch a node go unhealthy, see a fabric fill up, or answer "was the cluster full at the time", which is the first question anyone asks after an incident. The agents exported nothing at all. Exported by the control plane rather than by each agent. It already holds every number: hardware arrives at registration and state on each heartbeat, so this is a map read away. One scrape target for a fleet of hundreds, no Prometheus configuration per node, no firewall rule per machine, and nothing new running on the nodes. On the same /metrics as the traffic figures, because correlating "the cluster was full" with "requests were queueing" should not need two systems. A collector rather than gauges kept current by a ticker: the numbers are derived from state that already exists, so computing them when asked is simpler than maintaining a copy that can drift, and nothing runs between scrapes. GPU utilisation is new, and is the number people actually watch. Memory used says a model is loaded; this says whether it is doing anything, and a fleet at ninety percent occupancy and two percent utilisation is a fleet holding models nobody is asking for. Adding it caught two things. utilization.gpu goes last in the nvidia-smi query and is optional, because a driver too old to know it returns a shorter row — requiring it made an llmgw upgrade silently stop discovering GPUs on an older node, which is far worse than not knowing how busy a card is. And I gave the new protobuf field number 14, which ecc_errors already had; buf lint is why that was a moment rather than a corrupted wire format. Absent is kept distinct from zero throughout, for the same reason as everywhere else here: an unreported utilisation shown as a flat zero reads as an idle fleet rather than a missing measurement, and a replica nothing has reported would be the most idle thing in the cluster. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
142 lines
4.7 KiB
Go
142 lines
4.7 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
type fakeSource struct {
|
|
nodes []inventory.Node
|
|
free map[string][]int
|
|
replicas []reconciler.Replica
|
|
}
|
|
|
|
func (f fakeSource) Nodes() []inventory.Node { return f.nodes }
|
|
func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] }
|
|
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }
|
|
|
|
func gather(t *testing.T, src Source) string {
|
|
t.Helper()
|
|
reg := prometheus.NewRegistry()
|
|
if err := reg.Register(New(src)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var b strings.Builder
|
|
mfs, err := reg.Gather()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, mf := range mfs {
|
|
for _, m := range mf.GetMetric() {
|
|
b.WriteString(mf.GetName())
|
|
for _, l := range m.GetLabel() {
|
|
b.WriteString(" " + l.GetName() + "=" + l.GetValue())
|
|
}
|
|
if g := m.GetGauge(); g != nil {
|
|
b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
|
|
continue
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func gpuNode(id string, gpus int, fabric string) inventory.Node {
|
|
n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
|
|
for i := range gpus {
|
|
n.GPUs = append(n.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
|
|
MemoryUsedMB: 1024, UtilizationPercent: 40,
|
|
Health: discovery.GPUHealth{TemperatureC: 45},
|
|
})
|
|
}
|
|
if fabric != "" {
|
|
n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestTheFleetIsReported(t *testing.T) {
|
|
// The gap this closes: twenty-two metric families about traffic, and none
|
|
// about the machines serving it — so "was the cluster full at the time",
|
|
// the first question after an incident, had no answer.
|
|
src := fakeSource{
|
|
nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
|
|
free: map[string][]int{"n1": {1}},
|
|
replicas: []reconciler.Replica{
|
|
{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
|
|
},
|
|
}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
|
|
"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
|
|
"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("%s is missing from:\n%s", want, out)
|
|
}
|
|
}
|
|
// The fabric travels as a label, because with isolated fabrics "how full is
|
|
// the cluster" is the wrong question and "how full is this fabric" is the
|
|
// right one.
|
|
if !strings.Contains(out, "fabric=ib-a") {
|
|
t.Errorf("the fabric label is missing:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
|
|
// A replica the router has said nothing about would otherwise look like the
|
|
// most idle thing in the cluster, and after a router restart that would be
|
|
// all of them at once.
|
|
src := fakeSource{
|
|
replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
|
|
}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
|
|
t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
|
|
// A driver too old to give it would otherwise show every card at a flat
|
|
// zero, which reads as an idle fleet rather than a missing measurement.
|
|
n := gpuNode("n1", 1, "")
|
|
n.GPUs[0].UtilizationPercent = 0
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
|
|
t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestNodeStateIsALabelNotANumber(t *testing.T) {
|
|
// A cordoned node and an unreachable one need different responses, and a
|
|
// single gauge counting "not ready" would hide which it is.
|
|
n := gpuNode("n1", 1, "")
|
|
n.State = inventory.Cordoned
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
|
|
out := gather(t, src)
|
|
if !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("the node state is not a label:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("unexpected shape:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
|
|
// A control plane with no agents yet must still serve /metrics; failing
|
|
// there would take monitoring down exactly when a cluster is being built.
|
|
if out := gather(t, fakeSource{}); out == "" {
|
|
t.Log("no metrics for an empty fleet, which is correct")
|
|
}
|
|
}
|