Map the fleet's GPUs, and name the cards held and idle

The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 15:54:58 +02:00
1 parent bd36798b15
commit b6a802c03b
9 files changed
+403 -1

No files matched your search

+14
View File
@@ -417,3 +417,17 @@ func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
// EngineChoices are the engine builds chosen per node, or nil.
func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
// ModelOwners maps each model to the part of the tenancy tree its capacity
// counts against; a model owned by nobody is not in it.
func (c *Controller) ModelOwners() map[string]string {
out := map[string]string{}
if cat := c.Catalog(); cat != nil {
for _, m := range cat.Models() {
if m.Owner != "" {
out[m.Name] = m.Owner
}
}
}
return out
}
+32 -1
View File
@@ -73,6 +73,7 @@ type Collector struct {
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuAssigned *prometheus.Desc
gpuMemUtil *prometheus.Desc
gpuSMClock *prometheus.Desc
gpuTemp *prometheus.Desc
@@ -151,6 +152,10 @@ func New(src Source) *Collector {
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned",
"1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+
"Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.",
[]string{"node", "gpu", "model", "replica", "owner"}, nil),
gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent",
"How much of the sample period a GPU's memory was being read or written: "+
"decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil),
@@ -278,7 +283,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -294,6 +299,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
now := time.Now()
c.collectQuotas(ch)
c.collectAssignments(ch)
loads := map[string]*inventory.Load{}
for _, n := range c.src.Nodes() {
@@ -539,6 +545,12 @@ func b2f(b bool) float64 {
return 0
}
// OwnerSource says whose quota each model counts against, for labelling
// what its replicas hold. Asked for by type assertion.
type OwnerSource interface {
ModelOwners() map[string]string
}
// QuotaSource adds the tenancy tree's capacity quotas. Asked for by type
// assertion, like HealthSource.
type QuotaSource interface {
@@ -617,3 +629,22 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
}
}
// collectAssignments says which replica holds each GPU.
func (c *Collector) collectAssignments(ch chan<- prometheus.Metric) {
var owners map[string]string
if o, ok := c.src.(OwnerSource); ok {
owners = o.ModelOwners()
}
for _, r := range c.src.Replicas() {
if !r.State.Counts() {
continue
}
for _, sh := range r.Shards {
for _, g := range sh.GPUIndices {
ch <- prometheus.MustNewConstMetric(c.gpuAssigned, prometheus.GaugeValue, 1,
sh.NodeID, strconv.Itoa(g), r.Model, r.ID, owners[r.Model])
}
}
}
}
@@ -7,6 +7,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
@@ -77,3 +78,27 @@ func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
t.Errorf("a utilization the driver did not report was published:\n%s", out)
}
}
// ownedSource adds owners to fakeSource.
type ownedSource struct{ fakeSource }
func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
src := ownedSource{fakeSource{replicas: []reconciler.Replica{
{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
}}}
out := gather(t, src)
for _, want := range []string{
"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
if strings.Contains(out, "replica=old-1") {
t.Error("a stopped replica's GPU was counted as held")
}
}