Files
llmgw/internal/control/fleetmetrics/fleetmetrics.go
T
Ilker ManapandClaude Opus 5.5 daed835d88 Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 23:21:23 +02:00

291 lines
13 KiB
Go

// Package fleetmetrics exports what the cluster is made of and how much of it
// is in use.
//
// The router already exported twenty-two metric families about requests,
// endpoints, admission and scoring — everything about traffic, and nothing
// about the machines serving it. There was no way to graph GPU usage, spot a
// node going unhealthy, watch a fabric fill up, or answer "was the cluster full
// at the time" after an incident, which is the first question anyone asks.
//
// Exported by the control plane rather than by each agent, deliberately. The
// control plane already holds all of it: agents report their hardware at
// registration and their state on every heartbeat, so the data is a map read
// away. One scrape target for a fleet of hundreds, no per-node Prometheus
// configuration, no firewall rule per machine, and nothing new on the nodes.
package fleetmetrics
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// Source is the control plane state this reads.
type Source interface {
Nodes() []inventory.Node
FreeGPUs(nodeID string) []int
Replicas() []reconciler.Replica
}
// Collector reports the fleet on each scrape.
//
// A collector rather than gauges kept up to date by a ticker: the numbers are
// derived from state that already exists, so computing them when asked is both
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
// runs between scrapes.
type Collector struct {
src Source
nodeUp *prometheus.Desc
nodeGPUs *prometheus.Desc
nodeGPUsFree *prometheus.Desc
nodeMemory *prometheus.Desc
nodeSeen *prometheus.Desc
nodeDiskFree *prometheus.Desc
nodeDiskSize *prometheus.Desc
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuTemp *prometheus.Desc
gpuHealthy *prometheus.Desc
gpuNVLink *prometheus.Desc
gpuECC *prometheus.Desc
replicaState *prometheus.Desc
replicaIdle *prometheus.Desc
replicaReady *prometheus.Desc
nodeCPU *prometheus.Desc
nodeCPUs *prometheus.Desc
nodeLoad *prometheus.Desc
nodeMemUsed *prometheus.Desc
nodeMemAvail *prometheus.Desc
shardCPU *prometheus.Desc
shardMemory *prometheus.Desc
}
// loadFresh is how old a node's load may be and still be reported. A node
// that stopped sending it has an unknown load, and repeating its last figure
// would draw a busy machine long after it went away.
const loadFresh = time.Minute
// New builds the collector.
func New(src Source) *Collector {
node := []string{"node"}
gpu := []string{"node", "gpu", "class"}
return &Collector{
src: src,
nodeUp: prometheus.NewDesc("llmgw_node_up",
"1 when a node is ready to take work, 0 otherwise.",
append(node, "state", "fabric"), nil),
nodeGPUs: prometheus.NewDesc("llmgw_node_gpus_total",
"GPUs a node reports.", append(node, "class"), nil),
nodeGPUsFree: prometheus.NewDesc("llmgw_node_gpus_free",
"GPUs on a node that are healthy and unallocated.", node, nil),
nodeMemory: prometheus.NewDesc("llmgw_node_memory_bytes",
"A node's total RAM, 0 when it does not report one.", node, nil),
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
// Only the disk a node fills itself: space on a shared filer is not
// this node's to spend, and summing it per node would have a fleet of
// twenty each report the same free terabyte. Absent rather than zero
// when nothing was reported, so an alert on "free is low" does not
// fire for every node that was never asked about its disk.
nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes",
"Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil),
nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes",
"Size of those filesystems, so free space can be read as a fraction.", node, nil),
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
"A GPU's framebuffer size.", gpu, nil),
gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
"A GPU's reported temperature.", gpu, nil),
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
"1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil),
gpuNVLink: prometheus.NewDesc("llmgw_gpu_nvlink_peers",
"GPUs reachable over NVLink from this one. Always 0 on an agent built without "+
"the nvml tag, which is every portable binary — so a flat zero across a fleet "+
"means the topology is unknown rather than absent, and tensor groups are being "+
"chosen by index.", gpu, nil),
gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors",
"Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil),
replicaState: prometheus.NewDesc("llmgw_replicas",
"Replicas by model and state.", []string{"model", "state"}, nil),
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
"Replicas of a model that are serving.", []string{"model"}, nil),
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
"which is not the same as idle.", []string{"model", "replica"}, nil),
// A node's own use, from its heartbeat. Absent when the agent has
// not measured it lately, which is not the same as idle.
nodeCPU: prometheus.NewDesc("llmgw_node_cpu_utilization_percent",
"How busy a node's CPUs were between its last two heartbeats, 0 to 100 across all cores.", node, nil),
nodeCPUs: prometheus.NewDesc("llmgw_node_cpus",
"CPUs a node has, so a load average can be read against it.", node, nil),
nodeLoad: prometheus.NewDesc("llmgw_node_load",
"The kernel's load average over 1, 5 and 15 minutes.", append(node, "window"), nil),
nodeMemUsed: prometheus.NewDesc("llmgw_node_memory_used_bytes",
"RAM in use: total less what the kernel could hand to a new process.", node, nil),
nodeMemAvail: prometheus.NewDesc("llmgw_node_memory_available_bytes",
"RAM the kernel could hand to a new process without swapping.", node, nil),
shardCPU: prometheus.NewDesc("llmgw_shard_cpu_cores",
"Cores an engine process group kept busy, the way top counts: 2.5 is two and a half.",
[]string{"node", "model", "replica", "shard"}, nil),
shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes",
"Resident memory of an engine process group.",
[]string{"node", "model", "replica", "shard"}, nil),
}
}
// Describe implements prometheus.Collector.
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
} {
ch <- d
}
}
// Collect implements prometheus.Collector.
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
now := time.Now()
loads := map[string]*inventory.Load{}
for _, n := range c.src.Nodes() {
fabric := ""
if len(n.Fabrics) > 0 {
fabric = n.Fabrics[0].Fabric
}
up := 0.0
if n.State == inventory.Ready {
up = 1
}
// The state is a label rather than a number, because a cordoned node
// and an unreachable one need different responses and a single gauge
// counting "not ready" would hide which it is.
ch <- prometheus.MustNewConstMetric(c.nodeUp, prometheus.GaugeValue, up,
n.ID, n.State.String(), fabric)
ch <- prometheus.MustNewConstMetric(c.nodeGPUsFree, prometheus.GaugeValue,
float64(len(c.src.FreeGPUs(n.ID))), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeMemory, prometheus.GaugeValue,
float64(n.MemoryTotalMB)*1024*1024, n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
now.Sub(n.LastSeen).Seconds(), n.ID)
// Nothing reported means nothing is known, so no sample at all. A zero
// here would read as a full disk and page someone about a node that
// was simply never asked.
if free, total := n.LocalDisk(); total > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue,
float64(free), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue,
float64(total), n.ID)
}
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
if l.CPUCount > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
}
for w, v := range map[string]float64{"1m": l.Load1, "5m": l.Load5, "15m": l.Load15} {
ch <- prometheus.MustNewConstMetric(c.nodeLoad, prometheus.GaugeValue, v, n.ID, w)
}
if l.MemUsedBytes > 0 || l.MemAvailBytes > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeMemUsed, prometheus.GaugeValue, float64(l.MemUsedBytes), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeMemAvail, prometheus.GaugeValue, float64(l.MemAvailBytes), n.ID)
}
loads[n.ID] = l
}
byClass := map[string]int{}
for _, g := range n.GPUs {
class := g.Class()
byClass[class]++
labels := []string{n.ID, strconv.Itoa(g.Index), class}
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
float64(g.MemoryTotalMB)*1024*1024, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
float64(g.MemoryUsedMB)*1024*1024, labels...)
if g.UtilizationPercent > 0 {
// Only when reported. A driver too old to give it would
// otherwise show every card at a flat zero, which reads as an
// idle fleet rather than a missing measurement.
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
float64(g.UtilizationPercent), labels...)
}
if g.Health.TemperatureC > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
float64(g.Health.TemperatureC), labels...)
}
healthy := 0.0
if g.Health.OK() {
healthy = 1
}
ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue,
float64(len(g.NVLinkPeers)), labels...)
ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue,
float64(g.Health.ECCErrors), labels...)
}
for class, n2 := range byClass {
ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue,
float64(n2), n.ID, class)
}
}
// Replicas by model and state, so a model stuck starting is visible as a
// shape on a graph rather than only in a log line nobody tails.
counts := map[[2]string]int{}
ready := map[string]int{}
for _, rep := range c.src.Replicas() {
// Each engine's share of its node, labelled with what it serves, so
// "which model is eating this machine" is one query.
for _, sh := range rep.Shards {
if l := loads[sh.NodeID]; l != nil {
if use, ok := l.Shards[sh.ID]; ok {
labels := []string{sh.NodeID, rep.Model, rep.ID, sh.ID}
ch <- prometheus.MustNewConstMetric(c.shardCPU, prometheus.GaugeValue, use.CPUCores, labels...)
ch <- prometheus.MustNewConstMetric(c.shardMemory, prometheus.GaugeValue, float64(use.RSSBytes), labels...)
}
}
}
counts[[2]string{rep.Model, rep.State.String()}]++
if rep.State == reconciler.Ready {
ready[rep.Model]++
}
if !rep.LastUsedAt.IsZero() {
// Only when known. A replica nothing has reported would otherwise
// look like the most idle thing in the cluster, and after a router
// restart that would be all of them at once.
ch <- prometheus.MustNewConstMetric(c.replicaIdle, prometheus.GaugeValue,
now.Sub(rep.LastUsedAt).Seconds(), rep.Model, rep.ID)
}
}
for k, v := range counts {
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
float64(v), k[0], k[1])
}
for model, v := range ready {
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
float64(v), model)
}
}