The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
179 lines
6.4 KiB
Go
179 lines
6.4 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
type fakeSource struct {
|
|
nodes []inventory.Node
|
|
free map[string][]int
|
|
replicas []reconciler.Replica
|
|
}
|
|
|
|
func (f fakeSource) Nodes() []inventory.Node { return f.nodes }
|
|
func (f fakeSource) FreeGPUs(id string) []int { return f.free[id] }
|
|
func (f fakeSource) Replicas() []reconciler.Replica { return f.replicas }
|
|
|
|
func gather(t *testing.T, src Source) string {
|
|
t.Helper()
|
|
reg := prometheus.NewRegistry()
|
|
if err := reg.Register(New(src)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var b strings.Builder
|
|
mfs, err := reg.Gather()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, mf := range mfs {
|
|
for _, m := range mf.GetMetric() {
|
|
b.WriteString(mf.GetName())
|
|
for _, l := range m.GetLabel() {
|
|
b.WriteString(" " + l.GetName() + "=" + l.GetValue())
|
|
}
|
|
if g := m.GetGauge(); g != nil {
|
|
b.WriteString(" -> " + strconv.FormatFloat(g.GetValue(), 'f', -1, 64) + "\n")
|
|
continue
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func gpuNode(id string, gpus int, fabric string) inventory.Node {
|
|
n := inventory.Node{ID: id, State: inventory.Ready, LastSeen: time.Now()}
|
|
for i := range gpus {
|
|
n.GPUs = append(n.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
|
|
MemoryUsedMB: 1024, UtilizationPercent: 40,
|
|
Health: discovery.GPUHealth{TemperatureC: 45},
|
|
})
|
|
}
|
|
if fabric != "" {
|
|
n.Fabrics = []inventory.FabricAttachment{{Fabric: fabric, HCAs: []string{"mlx5_0"}}}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestTheFleetIsReported(t *testing.T) {
|
|
// The gap this closes: twenty-two metric families about traffic, and none
|
|
// about the machines serving it — so "was the cluster full at the time",
|
|
// the first question after an incident, had no answer.
|
|
src := fakeSource{
|
|
nodes: []inventory.Node{gpuNode("n1", 2, "ib-a")},
|
|
free: map[string][]int{"n1": {1}},
|
|
replicas: []reconciler.Replica{
|
|
{ID: "r1", Model: "m", State: reconciler.Ready, LastUsedAt: time.Now().Add(-90 * time.Second)},
|
|
},
|
|
}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_node_up", "llmgw_node_gpus_free", "llmgw_gpu_memory_used_bytes",
|
|
"llmgw_gpu_utilization_percent", "llmgw_gpu_healthy", "llmgw_replicas",
|
|
"llmgw_model_replicas_ready", "llmgw_replica_idle_seconds",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("%s is missing from:\n%s", want, out)
|
|
}
|
|
}
|
|
// The fabric travels as a label, because with isolated fabrics "how full is
|
|
// the cluster" is the wrong question and "how full is this fabric" is the
|
|
// right one.
|
|
if !strings.Contains(out, "fabric=ib-a") {
|
|
t.Errorf("the fabric label is missing:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnIdleReplicaNothingHasReportedIsAbsentNotZero(t *testing.T) {
|
|
// A replica the router has said nothing about would otherwise look like the
|
|
// most idle thing in the cluster, and after a router restart that would be
|
|
// all of them at once.
|
|
src := fakeSource{
|
|
replicas: []reconciler.Replica{{ID: "r1", Model: "m", State: reconciler.Ready}},
|
|
}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_replica_idle_seconds") {
|
|
t.Errorf("idleness was reported for a replica nothing has used:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnUnreportedUtilisationIsAbsentNotZero(t *testing.T) {
|
|
// A driver too old to give it would otherwise show every card at a flat
|
|
// zero, which reads as an idle fleet rather than a missing measurement.
|
|
n := gpuNode("n1", 1, "")
|
|
n.GPUs[0].UtilizationPercent = 0
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{"n1": {0}}}
|
|
if out := gather(t, src); strings.Contains(out, "llmgw_gpu_utilization_percent") {
|
|
t.Errorf("utilisation was reported when the driver gave none:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestNodeStateIsALabelNotANumber(t *testing.T) {
|
|
// A cordoned node and an unreachable one need different responses, and a
|
|
// single gauge counting "not ready" would hide which it is.
|
|
n := gpuNode("n1", 1, "")
|
|
n.State = inventory.Cordoned
|
|
src := fakeSource{nodes: []inventory.Node{n}, free: map[string][]int{}}
|
|
out := gather(t, src)
|
|
if !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("the node state is not a label:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "llmgw_node_up state=cordoned") && !strings.Contains(out, "state=cordoned") {
|
|
t.Errorf("unexpected shape:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
|
|
// A control plane with no agents yet must still serve /metrics; failing
|
|
// there would take monitoring down exactly when a cluster is being built.
|
|
if out := gather(t, fakeSource{}); out == "" {
|
|
t.Log("no metrics for an empty fleet, which is correct")
|
|
}
|
|
}
|
|
|
|
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
|
|
// The nodes are CPU machines: without this, how busy one was had no
|
|
// answer anywhere.
|
|
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
|
|
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
|
|
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
|
|
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
|
|
}}
|
|
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
|
|
ID: "r1", Model: "qwen", State: reconciler.Ready,
|
|
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
|
|
}}}
|
|
out := gather(t, src)
|
|
for _, want := range []string{
|
|
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
|
|
"llmgw_node_cpus node=n1 -> 8",
|
|
"llmgw_node_load node=n1 window=1m -> 3.5",
|
|
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
|
|
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
|
|
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %q in:\n%s", want, out)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
|
|
// A node that stopped reporting has an unknown load. Repeating its last
|
|
// figure would draw a busy machine long after it went away.
|
|
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
|
|
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
|
|
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
|
|
t.Errorf("a stale load was reported:\n%s", out)
|
|
}
|
|
}
|