CI / test (push) Successful in 1m28s
The router published twenty-two metric families about requests, endpoints, admission and scoring — everything about traffic and nothing about the machines carrying it. There was no way to graph GPU usage, watch a node go unhealthy, see a fabric fill up, or answer "was the cluster full at the time", which is the first question anyone asks after an incident. The agents exported nothing at all. Exported by the control plane rather than by each agent. It already holds every number: hardware arrives at registration and state on each heartbeat, so this is a map read away. One scrape target for a fleet of hundreds, no Prometheus configuration per node, no firewall rule per machine, and nothing new running on the nodes. On the same /metrics as the traffic figures, because correlating "the cluster was full" with "requests were queueing" should not need two systems. A collector rather than gauges kept current by a ticker: the numbers are derived from state that already exists, so computing them when asked is simpler than maintaining a copy that can drift, and nothing runs between scrapes. GPU utilisation is new, and is the number people actually watch. Memory used says a model is loaded; this says whether it is doing anything, and a fleet at ninety percent occupancy and two percent utilisation is a fleet holding models nobody is asking for. Adding it caught two things. utilization.gpu goes last in the nvidia-smi query and is optional, because a driver too old to know it returns a shorter row — requiring it made an llmgw upgrade silently stop discovering GPUs on an older node, which is far worse than not knowing how busy a card is. And I gave the new protobuf field number 14, which ecc_errors already had; buf lint is why that was a moment rather than a corrupted wire format. Absent is kept distinct from zero throughout, for the same reason as everywhere else here: an unreported utilisation shown as a flat zero reads as an idle fleet rather than a missing measurement, and a replica nothing has reported would be the most idle thing in the cluster. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
201 lines
7.8 KiB
Go
201 lines
7.8 KiB
Go
// Package fleetmetrics exports what the cluster is made of and how much of it
|
|
// is in use.
|
|
//
|
|
// The router already exported twenty-two metric families about requests,
|
|
// endpoints, admission and scoring — everything about traffic, and nothing
|
|
// about the machines serving it. There was no way to graph GPU usage, spot a
|
|
// node going unhealthy, watch a fabric fill up, or answer "was the cluster full
|
|
// at the time" after an incident, which is the first question anyone asks.
|
|
//
|
|
// Exported by the control plane rather than by each agent, deliberately. The
|
|
// control plane already holds all of it: agents report their hardware at
|
|
// registration and their state on every heartbeat, so the data is a map read
|
|
// away. One scrape target for a fleet of hundreds, no per-node Prometheus
|
|
// configuration, no firewall rule per machine, and nothing new on the nodes.
|
|
package fleetmetrics
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
// Source is the control plane state this reads.
|
|
type Source interface {
|
|
Nodes() []inventory.Node
|
|
FreeGPUs(nodeID string) []int
|
|
Replicas() []reconciler.Replica
|
|
}
|
|
|
|
// Collector reports the fleet on each scrape.
|
|
//
|
|
// A collector rather than gauges kept up to date by a ticker: the numbers are
|
|
// derived from state that already exists, so computing them when asked is both
|
|
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
|
|
// runs between scrapes.
|
|
type Collector struct {
|
|
src Source
|
|
|
|
nodeUp *prometheus.Desc
|
|
nodeGPUs *prometheus.Desc
|
|
nodeGPUsFree *prometheus.Desc
|
|
nodeMemory *prometheus.Desc
|
|
nodeSeen *prometheus.Desc
|
|
|
|
gpuMemoryTotal *prometheus.Desc
|
|
gpuMemoryUsed *prometheus.Desc
|
|
gpuUtil *prometheus.Desc
|
|
gpuTemp *prometheus.Desc
|
|
gpuHealthy *prometheus.Desc
|
|
gpuECC *prometheus.Desc
|
|
|
|
replicaState *prometheus.Desc
|
|
replicaIdle *prometheus.Desc
|
|
replicaReady *prometheus.Desc
|
|
}
|
|
|
|
// New builds the collector.
|
|
func New(src Source) *Collector {
|
|
node := []string{"node"}
|
|
gpu := []string{"node", "gpu", "class"}
|
|
return &Collector{
|
|
src: src,
|
|
nodeUp: prometheus.NewDesc("llmgw_node_up",
|
|
"1 when a node is ready to take work, 0 otherwise.",
|
|
append(node, "state", "fabric"), nil),
|
|
nodeGPUs: prometheus.NewDesc("llmgw_node_gpus_total",
|
|
"GPUs a node reports.", append(node, "class"), nil),
|
|
nodeGPUsFree: prometheus.NewDesc("llmgw_node_gpus_free",
|
|
"GPUs on a node that are healthy and unallocated.", node, nil),
|
|
nodeMemory: prometheus.NewDesc("llmgw_node_memory_bytes",
|
|
"A node's total RAM, 0 when it does not report one.", node, nil),
|
|
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
|
|
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
|
|
|
|
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
|
|
"A GPU's framebuffer size.", gpu, nil),
|
|
gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
|
|
"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
|
|
gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
|
|
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
|
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
|
"this says whether it is doing anything.", gpu, nil),
|
|
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
|
"A GPU's reported temperature.", gpu, nil),
|
|
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
|
"1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil),
|
|
gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors",
|
|
"Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil),
|
|
|
|
replicaState: prometheus.NewDesc("llmgw_replicas",
|
|
"Replicas by model and state.", []string{"model", "state"}, nil),
|
|
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
|
|
"Replicas of a model that are serving.", []string{"model"}, nil),
|
|
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
|
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
|
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
|
}
|
|
}
|
|
|
|
// Describe implements prometheus.Collector.
|
|
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
|
for _, d := range []*prometheus.Desc{
|
|
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
|
|
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuECC,
|
|
c.replicaState, c.replicaReady, c.replicaIdle,
|
|
} {
|
|
ch <- d
|
|
}
|
|
}
|
|
|
|
// Collect implements prometheus.Collector.
|
|
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|
now := time.Now()
|
|
|
|
for _, n := range c.src.Nodes() {
|
|
fabric := ""
|
|
if len(n.Fabrics) > 0 {
|
|
fabric = n.Fabrics[0].Fabric
|
|
}
|
|
up := 0.0
|
|
if n.State == inventory.Ready {
|
|
up = 1
|
|
}
|
|
// The state is a label rather than a number, because a cordoned node
|
|
// and an unreachable one need different responses and a single gauge
|
|
// counting "not ready" would hide which it is.
|
|
ch <- prometheus.MustNewConstMetric(c.nodeUp, prometheus.GaugeValue, up,
|
|
n.ID, n.State.String(), fabric)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeGPUsFree, prometheus.GaugeValue,
|
|
float64(len(c.src.FreeGPUs(n.ID))), n.ID)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeMemory, prometheus.GaugeValue,
|
|
float64(n.MemoryTotalMB)*1024*1024, n.ID)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
|
|
now.Sub(n.LastSeen).Seconds(), n.ID)
|
|
|
|
byClass := map[string]int{}
|
|
for _, g := range n.GPUs {
|
|
class := g.Class()
|
|
byClass[class]++
|
|
labels := []string{n.ID, strconv.Itoa(g.Index), class}
|
|
|
|
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
|
|
float64(g.MemoryTotalMB)*1024*1024, labels...)
|
|
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
|
|
float64(g.MemoryUsedMB)*1024*1024, labels...)
|
|
if g.UtilizationPercent > 0 {
|
|
// Only when reported. A driver too old to give it would
|
|
// otherwise show every card at a flat zero, which reads as an
|
|
// idle fleet rather than a missing measurement.
|
|
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
|
float64(g.UtilizationPercent), labels...)
|
|
}
|
|
if g.Health.TemperatureC > 0 {
|
|
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
|
float64(g.Health.TemperatureC), labels...)
|
|
}
|
|
healthy := 0.0
|
|
if g.Health.OK() {
|
|
healthy = 1
|
|
}
|
|
ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...)
|
|
ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue,
|
|
float64(g.Health.ECCErrors), labels...)
|
|
}
|
|
for class, n2 := range byClass {
|
|
ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue,
|
|
float64(n2), n.ID, class)
|
|
}
|
|
}
|
|
|
|
// Replicas by model and state, so a model stuck starting is visible as a
|
|
// shape on a graph rather than only in a log line nobody tails.
|
|
counts := map[[2]string]int{}
|
|
ready := map[string]int{}
|
|
for _, rep := range c.src.Replicas() {
|
|
counts[[2]string{rep.Model, rep.State.String()}]++
|
|
if rep.State == reconciler.Ready {
|
|
ready[rep.Model]++
|
|
}
|
|
if !rep.LastUsedAt.IsZero() {
|
|
// Only when known. A replica nothing has reported would otherwise
|
|
// look like the most idle thing in the cluster, and after a router
|
|
// restart that would be all of them at once.
|
|
ch <- prometheus.MustNewConstMetric(c.replicaIdle, prometheus.GaugeValue,
|
|
now.Sub(rep.LastUsedAt).Seconds(), rep.Model, rep.ID)
|
|
}
|
|
}
|
|
for k, v := range counts {
|
|
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
|
|
float64(v), k[0], k[1])
|
|
}
|
|
for model, v := range ready {
|
|
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
|
float64(v), model)
|
|
}
|
|
}
|