// Package fleetmetrics exports what the cluster is made of and how much of it // is in use. // // The router already exported twenty-two metric families about requests, // endpoints, admission and scoring — everything about traffic, and nothing // about the machines serving it. There was no way to graph GPU usage, spot a // node going unhealthy, watch a fabric fill up, or answer "was the cluster full // at the time" after an incident, which is the first question anyone asks. // // Exported by the control plane rather than by each agent, deliberately. The // control plane already holds all of it: agents report their hardware at // registration and their state on every heartbeat, so the data is a map read // away. One scrape target for a fleet of hundreds, no per-node Prometheus // configuration, no firewall rule per machine, and nothing new on the nodes. package fleetmetrics import ( "strconv" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) // Source is the control plane state this reads. type Source interface { Nodes() []inventory.Node FreeGPUs(nodeID string) []int Replicas() []reconciler.Replica } // Collector reports the fleet on each scrape. // // A collector rather than gauges kept up to date by a ticker: the numbers are // derived from state that already exists, so computing them when asked is both // simpler and fresher than maintaining a parallel copy that can drift. Nothing // runs between scrapes. type Collector struct { src Source nodeUp *prometheus.Desc nodeGPUs *prometheus.Desc nodeGPUsFree *prometheus.Desc nodeMemory *prometheus.Desc nodeSeen *prometheus.Desc nodeDiskFree *prometheus.Desc nodeDiskSize *prometheus.Desc gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc gpuTemp *prometheus.Desc gpuHealthy *prometheus.Desc gpuNVLink *prometheus.Desc gpuECC *prometheus.Desc replicaState *prometheus.Desc replicaIdle *prometheus.Desc replicaReady *prometheus.Desc nodeCPU *prometheus.Desc nodeCPUs *prometheus.Desc nodeLoad *prometheus.Desc nodeMemUsed *prometheus.Desc nodeMemAvail *prometheus.Desc shardCPU *prometheus.Desc shardMemory *prometheus.Desc } // loadFresh is how old a node's load may be and still be reported. A node // that stopped sending it has an unknown load, and repeating its last figure // would draw a busy machine long after it went away. const loadFresh = time.Minute // New builds the collector. func New(src Source) *Collector { node := []string{"node"} gpu := []string{"node", "gpu", "class"} return &Collector{ src: src, nodeUp: prometheus.NewDesc("llmgw_node_up", "1 when a node is ready to take work, 0 otherwise.", append(node, "state", "fabric"), nil), nodeGPUs: prometheus.NewDesc("llmgw_node_gpus_total", "GPUs a node reports.", append(node, "class"), nil), nodeGPUsFree: prometheus.NewDesc("llmgw_node_gpus_free", "GPUs on a node that are healthy and unallocated.", node, nil), nodeMemory: prometheus.NewDesc("llmgw_node_memory_bytes", "A node's total RAM, 0 when it does not report one.", node, nil), nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds", "Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil), // Only the disk a node fills itself: space on a shared filer is not // this node's to spend, and summing it per node would have a fleet of // twenty each report the same free terabyte. Absent rather than zero // when nothing was reported, so an alert on "free is low" does not // fire for every node that was never asked about its disk. nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes", "Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil), nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes", "Size of those filesystems, so free space can be read as a fraction.", node, nil), gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes", "A GPU's framebuffer size.", gpu, nil), gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes", "A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil), gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent", "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius", "A GPU's reported temperature.", gpu, nil), gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy", "1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil), gpuNVLink: prometheus.NewDesc("llmgw_gpu_nvlink_peers", "GPUs reachable over NVLink from this one. Always 0 on an agent built without "+ "the nvml tag, which is every portable binary — so a flat zero across a fleet "+ "means the topology is unknown rather than absent, and tensor groups are being "+ "chosen by index.", gpu, nil), gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors", "Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil), replicaState: prometheus.NewDesc("llmgw_replicas", "Replicas by model and state.", []string{"model", "state"}, nil), replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready", "Replicas of a model that are serving.", []string{"model"}, nil), replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds", "Seconds since a replica was last sent a request. Absent when nothing has been reported, "+ "which is not the same as idle.", []string{"model", "replica"}, nil), // A node's own use, from its heartbeat. Absent when the agent has // not measured it lately, which is not the same as idle. nodeCPU: prometheus.NewDesc("llmgw_node_cpu_utilization_percent", "How busy a node's CPUs were between its last two heartbeats, 0 to 100 across all cores.", node, nil), nodeCPUs: prometheus.NewDesc("llmgw_node_cpus", "CPUs a node has, so a load average can be read against it.", node, nil), nodeLoad: prometheus.NewDesc("llmgw_node_load", "The kernel's load average over 1, 5 and 15 minutes.", append(node, "window"), nil), nodeMemUsed: prometheus.NewDesc("llmgw_node_memory_used_bytes", "RAM in use: total less what the kernel could hand to a new process.", node, nil), nodeMemAvail: prometheus.NewDesc("llmgw_node_memory_available_bytes", "RAM the kernel could hand to a new process without swapping.", node, nil), shardCPU: prometheus.NewDesc("llmgw_shard_cpu_cores", "Cores an engine process group kept busy, the way top counts: 2.5 is two and a half.", []string{"node", "model", "replica", "shard"}, nil), shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes", "Resident memory of an engine process group.", []string{"node", "model", "replica", "shard"}, nil), } } // Describe implements prometheus.Collector. func (c *Collector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, c.nodeDiskFree, c.nodeDiskSize, c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, } { ch <- d } } // Collect implements prometheus.Collector. func (c *Collector) Collect(ch chan<- prometheus.Metric) { now := time.Now() loads := map[string]*inventory.Load{} for _, n := range c.src.Nodes() { fabric := "" if len(n.Fabrics) > 0 { fabric = n.Fabrics[0].Fabric } up := 0.0 if n.State == inventory.Ready { up = 1 } // The state is a label rather than a number, because a cordoned node // and an unreachable one need different responses and a single gauge // counting "not ready" would hide which it is. ch <- prometheus.MustNewConstMetric(c.nodeUp, prometheus.GaugeValue, up, n.ID, n.State.String(), fabric) ch <- prometheus.MustNewConstMetric(c.nodeGPUsFree, prometheus.GaugeValue, float64(len(c.src.FreeGPUs(n.ID))), n.ID) ch <- prometheus.MustNewConstMetric(c.nodeMemory, prometheus.GaugeValue, float64(n.MemoryTotalMB)*1024*1024, n.ID) ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue, now.Sub(n.LastSeen).Seconds(), n.ID) // Nothing reported means nothing is known, so no sample at all. A zero // here would read as a full disk and page someone about a node that // was simply never asked. if free, total := n.LocalDisk(); total > 0 { ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue, float64(free), n.ID) ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue, float64(total), n.ID) } if l := n.Load; l != nil && now.Sub(l.At) < loadFresh { ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID) if l.CPUCount > 0 { ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID) } for w, v := range map[string]float64{"1m": l.Load1, "5m": l.Load5, "15m": l.Load15} { ch <- prometheus.MustNewConstMetric(c.nodeLoad, prometheus.GaugeValue, v, n.ID, w) } if l.MemUsedBytes > 0 || l.MemAvailBytes > 0 { ch <- prometheus.MustNewConstMetric(c.nodeMemUsed, prometheus.GaugeValue, float64(l.MemUsedBytes), n.ID) ch <- prometheus.MustNewConstMetric(c.nodeMemAvail, prometheus.GaugeValue, float64(l.MemAvailBytes), n.ID) } loads[n.ID] = l } byClass := map[string]int{} for _, g := range n.GPUs { class := g.Class() byClass[class]++ labels := []string{n.ID, strconv.Itoa(g.Index), class} ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, float64(g.MemoryTotalMB)*1024*1024, labels...) ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, float64(g.MemoryUsedMB)*1024*1024, labels...) if g.UtilizationPercent > 0 { // Only when reported. A driver too old to give it would // otherwise show every card at a flat zero, which reads as an // idle fleet rather than a missing measurement. ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, float64(g.UtilizationPercent), labels...) } if g.Health.TemperatureC > 0 { ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue, float64(g.Health.TemperatureC), labels...) } healthy := 0.0 if g.Health.OK() { healthy = 1 } ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...) ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue, float64(len(g.NVLinkPeers)), labels...) ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, float64(g.Health.ECCErrors), labels...) } for class, n2 := range byClass { ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue, float64(n2), n.ID, class) } } // Replicas by model and state, so a model stuck starting is visible as a // shape on a graph rather than only in a log line nobody tails. counts := map[[2]string]int{} ready := map[string]int{} for _, rep := range c.src.Replicas() { // Each engine's share of its node, labelled with what it serves, so // "which model is eating this machine" is one query. for _, sh := range rep.Shards { if l := loads[sh.NodeID]; l != nil { if use, ok := l.Shards[sh.ID]; ok { labels := []string{sh.NodeID, rep.Model, rep.ID, sh.ID} ch <- prometheus.MustNewConstMetric(c.shardCPU, prometheus.GaugeValue, use.CPUCores, labels...) ch <- prometheus.MustNewConstMetric(c.shardMemory, prometheus.GaugeValue, float64(use.RSSBytes), labels...) } } } counts[[2]string{rep.Model, rep.State.String()}]++ if rep.State == reconciler.Ready { ready[rep.Model]++ } if !rep.LastUsedAt.IsZero() { // Only when known. A replica nothing has reported would otherwise // look like the most idle thing in the cluster, and after a router // restart that would be all of them at once. ch <- prometheus.MustNewConstMetric(c.replicaIdle, prometheus.GaugeValue, now.Sub(rep.LastUsedAt).Seconds(), rep.Model, rep.ID) } } for k, v := range counts { ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue, float64(v), k[0], k[1]) } for model, v := range ready { ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue, float64(v), model) } }