CI / test (push) Successful in 1m32s
Three places where the system took the infrastructure's word for something and would have failed silently on real hardware. A declared fabric adapter that is not on the node. Fabric membership has to be configuration — OpenSM's default subnet prefix is identical on every fabric, so a matching prefix proves nothing — but taking it entirely on trust was a different mistake. A typo, or a card that enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter that does not exist. An adapter with no ACTIVE port. Physically linked with no subnet manager sits in INIT: it looks healthy to anything counting adapters, carries no data, survives a reboot, and looks like a software problem. Discovery had recorded port state all along and placement only ever asked whether a fabric had any adapters listed. Both end the same way — the replica starts, passes its health checks, and hangs on its first all-reduce, which is the failure the fabric invariant exists to prevent arriving through a door nobody was watching. The agent now compares declared against discovered, where both are visible for the only time, and does not advertise a fabric that cannot carry traffic. The reason is logged on the machine that knows it, because the operator who typed the adapter name is the only person who can fix it. The fabric's rate is the slowest active link, since a collective runs at the speed of its slowest. And NVLink topology, which no published binary could see. NVML is its only source, the bindings need cgo, and CGO_ENABLED=0 is deliberate so the portable binary runs on an older libc. The cost was silent: tensor groups fall back to the lowest free indices, which is right on an SXM board with NVSwitch and wrong on a PCIe box where bridges pair specific cards — a group straddling two of them runs several times slower and nothing says so. Releases now also publish an nvml build for those nodes, and llmgw_gpu_nvlink_peers makes the difference visible: a flat zero across a fleet means the topology is unknown rather than absent. A test that declared mlx5_0 and mlx5_1 on a node whose sysfs held neither had been passing, because nothing compared the two. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
210 lines
8.3 KiB
Go
210 lines
8.3 KiB
Go
// Package fleetmetrics exports what the cluster is made of and how much of it
|
|
// is in use.
|
|
//
|
|
// The router already exported twenty-two metric families about requests,
|
|
// endpoints, admission and scoring — everything about traffic, and nothing
|
|
// about the machines serving it. There was no way to graph GPU usage, spot a
|
|
// node going unhealthy, watch a fabric fill up, or answer "was the cluster full
|
|
// at the time" after an incident, which is the first question anyone asks.
|
|
//
|
|
// Exported by the control plane rather than by each agent, deliberately. The
|
|
// control plane already holds all of it: agents report their hardware at
|
|
// registration and their state on every heartbeat, so the data is a map read
|
|
// away. One scrape target for a fleet of hundreds, no per-node Prometheus
|
|
// configuration, no firewall rule per machine, and nothing new on the nodes.
|
|
package fleetmetrics
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
// Source is the control plane state this reads.
|
|
type Source interface {
|
|
Nodes() []inventory.Node
|
|
FreeGPUs(nodeID string) []int
|
|
Replicas() []reconciler.Replica
|
|
}
|
|
|
|
// Collector reports the fleet on each scrape.
|
|
//
|
|
// A collector rather than gauges kept up to date by a ticker: the numbers are
|
|
// derived from state that already exists, so computing them when asked is both
|
|
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
|
|
// runs between scrapes.
|
|
type Collector struct {
|
|
src Source
|
|
|
|
nodeUp *prometheus.Desc
|
|
nodeGPUs *prometheus.Desc
|
|
nodeGPUsFree *prometheus.Desc
|
|
nodeMemory *prometheus.Desc
|
|
nodeSeen *prometheus.Desc
|
|
|
|
gpuMemoryTotal *prometheus.Desc
|
|
gpuMemoryUsed *prometheus.Desc
|
|
gpuUtil *prometheus.Desc
|
|
gpuTemp *prometheus.Desc
|
|
gpuHealthy *prometheus.Desc
|
|
gpuNVLink *prometheus.Desc
|
|
gpuECC *prometheus.Desc
|
|
|
|
replicaState *prometheus.Desc
|
|
replicaIdle *prometheus.Desc
|
|
replicaReady *prometheus.Desc
|
|
}
|
|
|
|
// New builds the collector.
|
|
func New(src Source) *Collector {
|
|
node := []string{"node"}
|
|
gpu := []string{"node", "gpu", "class"}
|
|
return &Collector{
|
|
src: src,
|
|
nodeUp: prometheus.NewDesc("llmgw_node_up",
|
|
"1 when a node is ready to take work, 0 otherwise.",
|
|
append(node, "state", "fabric"), nil),
|
|
nodeGPUs: prometheus.NewDesc("llmgw_node_gpus_total",
|
|
"GPUs a node reports.", append(node, "class"), nil),
|
|
nodeGPUsFree: prometheus.NewDesc("llmgw_node_gpus_free",
|
|
"GPUs on a node that are healthy and unallocated.", node, nil),
|
|
nodeMemory: prometheus.NewDesc("llmgw_node_memory_bytes",
|
|
"A node's total RAM, 0 when it does not report one.", node, nil),
|
|
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
|
|
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
|
|
|
|
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
|
|
"A GPU's framebuffer size.", gpu, nil),
|
|
gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
|
|
"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
|
|
gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
|
|
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
|
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
|
"this says whether it is doing anything.", gpu, nil),
|
|
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
|
"A GPU's reported temperature.", gpu, nil),
|
|
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
|
"1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil),
|
|
gpuNVLink: prometheus.NewDesc("llmgw_gpu_nvlink_peers",
|
|
"GPUs reachable over NVLink from this one. Always 0 on an agent built without "+
|
|
"the nvml tag, which is every portable binary — so a flat zero across a fleet "+
|
|
"means the topology is unknown rather than absent, and tensor groups are being "+
|
|
"chosen by index.", gpu, nil),
|
|
gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors",
|
|
"Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil),
|
|
|
|
replicaState: prometheus.NewDesc("llmgw_replicas",
|
|
"Replicas by model and state.", []string{"model", "state"}, nil),
|
|
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
|
|
"Replicas of a model that are serving.", []string{"model"}, nil),
|
|
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
|
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
|
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
|
}
|
|
}
|
|
|
|
// Describe implements prometheus.Collector.
|
|
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
|
for _, d := range []*prometheus.Desc{
|
|
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
|
|
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
|
c.gpuNVLink, c.gpuECC,
|
|
c.replicaState, c.replicaReady, c.replicaIdle,
|
|
} {
|
|
ch <- d
|
|
}
|
|
}
|
|
|
|
// Collect implements prometheus.Collector.
|
|
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|
now := time.Now()
|
|
|
|
for _, n := range c.src.Nodes() {
|
|
fabric := ""
|
|
if len(n.Fabrics) > 0 {
|
|
fabric = n.Fabrics[0].Fabric
|
|
}
|
|
up := 0.0
|
|
if n.State == inventory.Ready {
|
|
up = 1
|
|
}
|
|
// The state is a label rather than a number, because a cordoned node
|
|
// and an unreachable one need different responses and a single gauge
|
|
// counting "not ready" would hide which it is.
|
|
ch <- prometheus.MustNewConstMetric(c.nodeUp, prometheus.GaugeValue, up,
|
|
n.ID, n.State.String(), fabric)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeGPUsFree, prometheus.GaugeValue,
|
|
float64(len(c.src.FreeGPUs(n.ID))), n.ID)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeMemory, prometheus.GaugeValue,
|
|
float64(n.MemoryTotalMB)*1024*1024, n.ID)
|
|
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
|
|
now.Sub(n.LastSeen).Seconds(), n.ID)
|
|
|
|
byClass := map[string]int{}
|
|
for _, g := range n.GPUs {
|
|
class := g.Class()
|
|
byClass[class]++
|
|
labels := []string{n.ID, strconv.Itoa(g.Index), class}
|
|
|
|
ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue,
|
|
float64(g.MemoryTotalMB)*1024*1024, labels...)
|
|
ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue,
|
|
float64(g.MemoryUsedMB)*1024*1024, labels...)
|
|
if g.UtilizationPercent > 0 {
|
|
// Only when reported. A driver too old to give it would
|
|
// otherwise show every card at a flat zero, which reads as an
|
|
// idle fleet rather than a missing measurement.
|
|
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
|
float64(g.UtilizationPercent), labels...)
|
|
}
|
|
if g.Health.TemperatureC > 0 {
|
|
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
|
float64(g.Health.TemperatureC), labels...)
|
|
}
|
|
healthy := 0.0
|
|
if g.Health.OK() {
|
|
healthy = 1
|
|
}
|
|
ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...)
|
|
ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue,
|
|
float64(len(g.NVLinkPeers)), labels...)
|
|
ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue,
|
|
float64(g.Health.ECCErrors), labels...)
|
|
}
|
|
for class, n2 := range byClass {
|
|
ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue,
|
|
float64(n2), n.ID, class)
|
|
}
|
|
}
|
|
|
|
// Replicas by model and state, so a model stuck starting is visible as a
|
|
// shape on a graph rather than only in a log line nobody tails.
|
|
counts := map[[2]string]int{}
|
|
ready := map[string]int{}
|
|
for _, rep := range c.src.Replicas() {
|
|
counts[[2]string{rep.Model, rep.State.String()}]++
|
|
if rep.State == reconciler.Ready {
|
|
ready[rep.Model]++
|
|
}
|
|
if !rep.LastUsedAt.IsZero() {
|
|
// Only when known. A replica nothing has reported would otherwise
|
|
// look like the most idle thing in the cluster, and after a router
|
|
// restart that would be all of them at once.
|
|
ch <- prometheus.MustNewConstMetric(c.replicaIdle, prometheus.GaugeValue,
|
|
now.Sub(rep.LastUsedAt).Seconds(), rep.Model, rep.ID)
|
|
}
|
|
}
|
|
for k, v := range counts {
|
|
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
|
|
float64(v), k[0], k[1])
|
|
}
|
|
for model, v := range ready {
|
|
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
|
float64(v), model)
|
|
}
|
|
}
|