Files
llmgw/internal/control/fleetmetrics/fleetmetrics.go
T
Ilker ManapandClaude Opus 5 753f4e3265
CI / test (push) Failing after 1m31s
Let a node say how much room it has left
Disk was the one resource nothing reported. `statfs` was already being called
on every model directory, but only to ask whether the filesystem was a network
one — the free space it returns in the same struct was thrown away. So the
only way to find out why a node was filling up was to ssh to it, which is what
this system exists to avoid.

Reported per filesystem rather than per configured directory. Free space
belongs to the disk, not the path: two model directories on one disk are one
pool of bytes, and counting them separately would let the control plane add
24 GB to 24 GB and place a 40 GB model on a node that has 24.

Shared filesystems are summed separately from local ones and left out of a
node's own total. Space on a filer belongs to everyone mounting it, so twenty
nodes would otherwise each report the same free terabyte, and a scheduler
would place twenty models into room for one.

Nothing reported means nothing is known, everywhere: an agent too old to send
storage and a node with no model directories both send nothing, and both must
read as unknown rather than as a full disk. `ctl nodes` and the dashboard show
a dash, the metric is absent rather than zero, and a missing directory — which
is what an unmounted filer looks like — is skipped rather than reported as
having no space.

Shown as free out of total, because the first is useless without the second:
8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root
partition, and only the second stops a model being placed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-18 09:38:57 +02:00

231 lines
9.5 KiB
Go

// Package fleetmetrics exports what the cluster is made of and how much of it
// is in use.
//
// The router already exported twenty-two metric families about requests,
// endpoints, admission and scoring — everything about traffic, and nothing
// about the machines serving it. There was no way to graph GPU usage, spot a
// node going unhealthy, watch a fabric fill up, or answer "was the cluster full
// at the time" after an incident, which is the first question anyone asks.
//
// Exported by the control plane rather than by each agent, deliberately. The
// control plane already holds all of it: agents report their hardware at
// registration and their state on every heartbeat, so the data is a map read
// away. One scrape target for a fleet of hundreds, no per-node Prometheus
// configuration, no firewall rule per machine, and nothing new on the nodes.
package fleetmetrics
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// Source is the control plane state this reads.
type Source interface {
Nodes() []inventory.Node
FreeGPUs(nodeID string) []int
Replicas() []reconciler.Replica
}
// Collector reports the fleet on each scrape.
//
// A collector rather than gauges kept up to date by a ticker: the numbers are
// derived from state that already exists, so computing them when asked is both
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
// runs between scrapes.
type Collector struct {
src Source
nodeUp *prometheus.Desc
nodeGPUs *prometheus.Desc
nodeGPUsFree *prometheus.Desc
nodeMemory *prometheus.Desc
nodeSeen *prometheus.Desc
nodeDiskFree *prometheus.Desc
nodeDiskSize *prometheus.Desc
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuTemp *prometheus.Desc
gpuHealthy *prometheus.Desc
gpuNVLink *prometheus.Desc
gpuECC *prometheus.Desc
replicaState *prometheus.Desc
replicaIdle *prometheus.Desc
replicaReady *prometheus.Desc
}
// New builds the collector.
func New(src Source) *Collector {
node := []string{"node"}
gpu := []string{"node", "gpu", "class"}
return &Collector{
src: src,
nodeUp: prometheus.NewDesc("llmgw_node_up",
"1 when a node is ready to take work, 0 otherwise.",
append(node, "state", "fabric"), nil),
nodeGPUs: prometheus.NewDesc("llmgw_node_gpus_total",
"GPUs a node reports.", append(node, "class"), nil),
nodeGPUsFree: prometheus.NewDesc("llmgw_node_gpus_free",
"GPUs on a node that are healthy and unallocated.", node, nil),
nodeMemory: prometheus.NewDesc("llmgw_node_memory_bytes",
"A node's total RAM, 0 when it does not report one.", node, nil),
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
// Only the disk a node fills itself: space on a shared filer is not
// this node's to spend, and summing it per node would have a fleet of
// twenty each report the same free terabyte. Absent rather than zero
// when nothing was reported, so an alert on "free is low" does not
// fire for every node that was never asked about its disk.
nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes",
"Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil),
nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes",
"Size of those filesystems, so free space can be read as a fraction.", node, nil),
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
"A GPU's framebuffer size.", gpu, nil),
gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
"A GPU's reported temperature.", gpu, nil),
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
"1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil),
gpuNVLink: prometheus.NewDesc("llmgw_gpu_nvlink_peers",
"GPUs reachable over NVLink from this one. Always 0 on an agent built without "+
"the nvml tag, which is every portable binary — so a flat zero across a fleet "+
"means the topology is unknown rather than absent, and tensor groups are being "+
"chosen by index.", gpu, nil),
gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors",
"Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil),
replicaState: prometheus.NewDesc("llmgw_replicas",
"Replicas by model and state.", []string{"model", "state"}, nil),
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
"Replicas of a model that are serving.", []string{"model"}, nil),
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
"which is not the same as idle.", []string{"model", "replica"}, nil),
}
}
// Describe implements prometheus.Collector.
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
} {
ch <- d
}
}
// Collect implements prometheus.Collector.
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
now := time.Now()
for _, n := range c.src.Nodes() {
fabric := ""
if len(n.Fabrics) > 0 {
fabric = n.Fabrics[0].Fabric
}
up := 0.0
if n.State == inventory.Ready {
up = 1
}
// The state is a label rather than a number, because a cordoned node
// and an unreachable one need different responses and a single gauge
// counting "not ready" would hide which it is.
ch <- prometheus.MustNewConstMetric(c.nodeUp, prometheus.GaugeValue, up,
n.ID, n.State.String(), fabric)
ch <- prometheus.MustNewConstMetric(c.nodeGPUsFree, prometheus.GaugeValue,
float64(len(c.src.FreeGPUs(n.ID))), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeMemory, prometheus.GaugeValue,
float64(n.MemoryTotalMB)*1024*1024, n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
now.Sub(n.LastSeen).Seconds(), n.ID)
// Nothing reported means nothing is known, so no sample at all. A zero
// here would read as a full disk and page someone about a node that
// was simply never asked.
if free, total := n.LocalDisk(); total > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue,
float64(free), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue,
float64(total), n.ID)
}
byClass := map[string]int{}
for _, g := range n.GPUs {
class := g.Class()
byClass[class]++
labels := []string{n.ID, strconv.Itoa(g.Index), class}
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
float64(g.MemoryTotalMB)*1024*1024, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
float64(g.MemoryUsedMB)*1024*1024, labels...)
if g.UtilizationPercent > 0 {
// Only when reported. A driver too old to give it would
// otherwise show every card at a flat zero, which reads as an
// idle fleet rather than a missing measurement.
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
float64(g.UtilizationPercent), labels...)
}
if g.Health.TemperatureC > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
float64(g.Health.TemperatureC), labels...)
}
healthy := 0.0
if g.Health.OK() {
healthy = 1
}
ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...)
ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue,
float64(len(g.NVLinkPeers)), labels...)
ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue,
float64(g.Health.ECCErrors), labels...)
}
for class, n2 := range byClass {
ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue,
float64(n2), n.ID, class)
}
}
// Replicas by model and state, so a model stuck starting is visible as a
// shape on a graph rather than only in a log line nobody tails.
counts := map[[2]string]int{}
ready := map[string]int{}
for _, rep := range c.src.Replicas() {
counts[[2]string{rep.Model, rep.State.String()}]++
if rep.State == reconciler.Ready {
ready[rep.Model]++
}
if !rep.LastUsedAt.IsZero() {
// Only when known. A replica nothing has reported would otherwise
// look like the most idle thing in the cluster, and after a router
// restart that would be all of them at once.
ch <- prometheus.MustNewConstMetric(c.replicaIdle, prometheus.GaugeValue,
now.Sub(rep.LastUsedAt).Seconds(), rep.Model, rep.ID)
}
}
for k, v := range counts {
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
float64(v), k[0], k[1])
}
for model, v := range ready {
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
float64(v), model)
}
}