Map the fleet's GPUs, and name the cards held and idle

The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -48,6 +48,14 @@ and a figure the driver does not give is left out rather than shown as zero.
   limit, clock, and what holds cards back.
 - **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat),
   `GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes).
+- **The cluster page's GPU map**: every card a square, a row per node,
+  coloured by how busy, how full or how hot it is (`?gpumap=util|memory|temp`),
+  with what holds it and its figures under the pointer; free cards dashed.
+- **Cards held and doing nothing**: the replicas whose every card reads under
+  5% busy, with when they were last asked — the first place to find room.
+- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
+  each card and whose quota it counts against, for joining the GPU series to
+  models and tenants.
 
 ### A GPU placement no longer runs on the CPU unnoticed
 

internal/control/control.go

@@ -417,3 +417,17 @@ func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
 
 // EngineChoices are the engine builds chosen per node, or nil.
 func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
+
+// ModelOwners maps each model to the part of the tenancy tree its capacity
+// counts against; a model owned by nobody is not in it.
+func (c *Controller) ModelOwners() map[string]string {
+	out := map[string]string{}
+	if cat := c.Catalog(); cat != nil {
+		for _, m := range cat.Models() {
+			if m.Owner != "" {
+				out[m.Name] = m.Owner
+			}
+		}
+	}
+	return out
+}

internal/control/fleetmetrics/fleetmetrics.go

@@ -73,6 +73,7 @@ type Collector struct {
 	gpuMemoryTotal *prometheus.Desc
 	gpuMemoryUsed  *prometheus.Desc
 	gpuUtil        *prometheus.Desc
+	gpuAssigned    *prometheus.Desc
 	gpuMemUtil     *prometheus.Desc
 	gpuSMClock     *prometheus.Desc
 	gpuTemp        *prometheus.Desc
@@ -151,6 +152,10 @@ func New(src Source) *Collector {
 			"How busy a GPU was over the last sample period. Absent when the driver does not "+
 				"report it, which is not the same as idle. Memory used says a model is loaded; "+
 				"this says whether it is doing anything.", gpu, nil),
+		gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned",
+			"1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+
+				"Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.",
+			[]string{"node", "gpu", "model", "replica", "owner"}, nil),
 		gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent",
 			"How much of the sample period a GPU's memory was being read or written: "+
 				"decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil),
@@ -278,7 +283,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
 		c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
 		c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
 		c.gpuNVLJoined, c.imexUp,
-		c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy,
+		c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
 		c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
 		c.gpuNVLink, c.gpuECC,
 		c.replicaState, c.replicaReady, c.replicaIdle,
@@ -294,6 +299,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
 func (c *Collector) Collect(ch chan<- prometheus.Metric) {
 	now := time.Now()
 	c.collectQuotas(ch)
+	c.collectAssignments(ch)
 
 	loads := map[string]*inventory.Load{}
 	for _, n := range c.src.Nodes() {
@@ -539,6 +545,12 @@ func b2f(b bool) float64 {
 	return 0
 }
 
+// OwnerSource says whose quota each model counts against, for labelling
+// what its replicas hold. Asked for by type assertion.
+type OwnerSource interface {
+	ModelOwners() map[string]string
+}
+
 // QuotaSource adds the tenancy tree's capacity quotas. Asked for by type
 // assertion, like HealthSource.
 type QuotaSource interface {
@@ -617,3 +629,22 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor
 		ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
 	}
 }
+
+// collectAssignments says which replica holds each GPU.
+func (c *Collector) collectAssignments(ch chan<- prometheus.Metric) {
+	var owners map[string]string
+	if o, ok := c.src.(OwnerSource); ok {
+		owners = o.ModelOwners()
+	}
+	for _, r := range c.src.Replicas() {
+		if !r.State.Counts() {
+			continue
+		}
+		for _, sh := range r.Shards {
+			for _, g := range sh.GPUIndices {
+				ch <- prometheus.MustNewConstMetric(c.gpuAssigned, prometheus.GaugeValue, 1,
+					sh.NodeID, strconv.Itoa(g), r.Model, r.ID, owners[r.Model])
+			}
+		}
+	}
+}

internal/control/fleetmetrics/sensors_test.go

@@ -7,6 +7,7 @@ import (
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
 )
 
 func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
@@ -77,3 +78,27 @@ func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
 		t.Errorf("a utilization the driver did not report was published:\n%s", out)
 	}
 }
+
+// ownedSource adds owners to fakeSource.
+type ownedSource struct{ fakeSource }
+
+func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
+
+func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
+	src := ownedSource{fakeSource{replicas: []reconciler.Replica{
+		{ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
+		{ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
+	}}}
+	out := gather(t, src)
+	for _, want := range []string{
+		"llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
+		"llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("missing %s in:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "replica=old-1") {
+		t.Error("a stopped replica's GPU was counted as held")
+	}
+}

internal/web/dashboard.go

@@ -20,6 +20,10 @@ type DashboardData struct {
 	CPU cpuCapacity
 	// Menu is the shared top menu; Nav below is what the cluster page adds.
 	Menu Nav
+	// GPUMap is every card coloured by one figure, and IdleGPUs the replicas
+	// holding cards that are doing nothing; both from the last heartbeats.
+	GPUMap   *gpuMap
+	IdleGPUs []idleHolding
 	// Notice and Problem are what the last change made from this page did,
 	// carried over the redirect back to it.
 	Notice, Problem string
@@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
 				d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
 			}
 			d.NodePrefix = sharedPrefix(ids)
+			if d.AnyGPUs {
+				d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
+				d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix)
+			}
 			for _, m := range d.Models {
 				d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
 			}

internal/web/gpumap.go

@@ -0,0 +1,232 @@
+package web
+
+import (
+	"fmt"
+	"html/template"
+	"sort"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+// The GPU map is every card in the fleet, a square each, a row per node,
+// coloured by one figure: how busy, how hot, or how full its memory is. A
+// hundred cards in a table are a hundred rows to read; on the map the one
+// that is not like the others is seen before anything is read.
+
+// gpuMapModes are what the map can be coloured by.
+var gpuMapModes = []struct{ Key, Label string }{
+	{"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
+}
+
+type gpuMap struct {
+	By    string
+	Modes []struct{ Key, Label string }
+	Rows  []gpuMapRow
+	// Legend says what the colours mean in this mode.
+	Legend string
+}
+
+type gpuMapRow struct {
+	Node  string
+	Cells []gpuCell
+}
+
+type gpuCell struct {
+	Index int
+	// Model and Replica hold the card; empty is a free card.
+	Model, Replica string
+	// Style is the cell's colour, built here from numbers alone, which is
+	// what makes it safe to put in a style attribute; Title says everything
+	// known about the card.
+	Style template.CSS
+	Title string
+	// Label is the figure written in the cell.
+	Label string
+}
+
+// idleHolding is a replica holding GPUs that are doing nothing.
+type idleHolding struct {
+	Model, Replica, Node string
+	GPUs                 []int
+	// MaxUtil is the busiest of its cards, now.
+	MaxUtil float64
+	// Idle is how long since a request was routed to it, or "" when not
+	// known.
+	Idle string
+}
+
+// buildGPUMap draws the map from what the nodes last reported and where
+// replicas are. Nil for a fleet without GPUs.
+func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
+	held := heldBy(reps)
+	m := &gpuMap{By: by, Modes: gpuMapModes}
+	if m.By == "" {
+		m.By = "util"
+	}
+	switch m.By {
+	case "temp":
+		m.Legend = "Blue is cool, red is 90 °C and over."
+	case "memory":
+		m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
+	default:
+		m.By = "util"
+		m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
+	}
+	for _, n := range nodes {
+		if n.GPUTotal == 0 {
+			continue
+		}
+		row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
+		for i := range n.GPUTotal {
+			c := gpuCell{Index: i}
+			if h, ok := held[gpuKey(n.ID, i)]; ok {
+				c.Model, c.Replica = h.model, h.replica
+			}
+			var title []string
+			title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
+			if c.Model != "" {
+				title = append(title, c.Model+" ("+c.Replica+")")
+			} else {
+				title = append(title, "free")
+			}
+			s, sampled := sensorOf(n, i)
+			if sampled {
+				if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
+					title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
+				}
+				if s.MemoryTotalMB > 0 {
+					title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
+				}
+				if s.TempC > 0 {
+					title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
+				}
+				if s.PowerW > 0 {
+					title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
+				}
+				if len(s.Throttle) > 0 {
+					title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
+				}
+			}
+			c.Title = strings.Join(title, " · ")
+			style, label := cellColour(m.By, s, sampled)
+			if c.Model == "" {
+				style += ";border-style:dashed"
+			}
+			c.Style, c.Label = template.CSS(style), label
+			row.Cells = append(row.Cells, c)
+		}
+		m.Rows = append(m.Rows, row)
+	}
+	if len(m.Rows) == 0 {
+		return nil
+	}
+	return m
+}
+
+// cellColour is a cell's background, and the figure written in it.
+func cellColour(by string, s sensorView, sampled bool) (string, string) {
+	const unknown = "background:var(--panel-2,#eee)"
+	switch by {
+	case "temp":
+		if !sampled || s.TempC <= 0 {
+			return unknown, ""
+		}
+		// 30 °C blue to 90 °C red.
+		f := clamp01((s.TempC - 30) / 60)
+		return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
+	case "memory":
+		if !sampled || s.MemoryTotalMB <= 0 {
+			return unknown, ""
+		}
+		f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
+		return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
+	}
+	if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
+		return unknown, ""
+	}
+	f := clamp01(s.UtilPercent / 100)
+	return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
+}
+
+// idleHoldings are the replicas whose GPUs all read under 5% busy now:
+// capacity held and not asked for, the first place to find room.
+func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
+	byID := map[string]adminapi.NodeView{}
+	for _, n := range nodes {
+		byID[n.ID] = n
+	}
+	var out []idleHolding
+	for _, r := range reps {
+		if r.State != "ready" {
+			continue
+		}
+		for _, sh := range r.Shards {
+			if len(sh.GPUs) == 0 {
+				continue
+			}
+			h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
+			idle, known := true, true
+			for _, g := range sh.GPUs {
+				s, ok := sensorOf(byID[sh.Node], g)
+				if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
+					known = false
+					break
+				}
+				h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
+				if s.UtilPercent >= 5 {
+					idle = false
+				}
+			}
+			if !known || !idle {
+				continue
+			}
+			if r.IdleSec >= 0 {
+				h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
+			}
+			out = append(out, h)
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
+	return out
+}
+
+type holder struct{ model, replica string }
+
+func heldBy(reps []adminapi.ReplicaView) map[string]holder {
+	out := map[string]holder{}
+	for _, r := range reps {
+		switch r.State {
+		case "stopped", "failed":
+			continue
+		}
+		for _, sh := range r.Shards {
+			for _, g := range sh.GPUs {
+				out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
+			}
+		}
+	}
+	return out
+}
+
+func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
+
+// sensorView is a card's last sample.
+type sensorView = struct {
+	TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
+	Throttle                                                []string
+}
+
+func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
+	if n.Load == nil {
+		return sensorView{}, false
+	}
+	for _, s := range n.Load.GPUs {
+		if s.Index == gpu {
+			return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
+				MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
+		}
+	}
+	return sensorView{}, false
+}

internal/web/gpumap_test.go

@@ -0,0 +1,58 @@
+package web
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+)
+
+// mapFleet is a node of four cards: chat holds two and is idle, code holds
+// one and is busy, one is free.
+type mapFleet struct{ fakeSource }
+
+func (mapFleet) Nodes() []adminapi.NodeView {
+	busy := func(i int, util, temp float64) inventory.GPUSensor {
+		return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
+	}
+	return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
+		Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
+}
+
+func (mapFleet) Replicas(string) []adminapi.ReplicaView {
+	return []adminapi.ReplicaView{
+		{ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
+		{ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
+	}
+}
+
+func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
+	_, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
+	if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
+		t.Errorf("%d cards on the map, want 4", n)
+	}
+	for _, want := range []string{
+		"gpu07 GPU 2 · code (code-1) · 93% busy",
+		"gpu07 GPU 3 · free",
+		"border-style:dashed",
+		"Cards held and doing nothing",
+		"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("the page lacks %q", want)
+		}
+	}
+	if strings.Contains(body, "<code>code-1</code></td><td><code>gpu07</code>") {
+		t.Error("a busy replica was listed as idle")
+	}
+	_, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
+	if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
+		t.Error("the map coloured by temperature does not say so, or show it")
+	}
+	_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
+	if strings.Contains(cpu, "gpu-map") {
+		t.Error("a fleet without GPUs has a GPU map")
+	}
+}

internal/web/static/base.css

@@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; }
 .held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; }
 .held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; }
 .held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; }
+
+/* The GPU map: a square per card, a row per node. */
+.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; }
+.gpu-row { display: flex; align-items: center; gap: 3px; }
+.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px;
+  font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; }

internal/web/templates/dashboard.html

@@ -49,6 +49,25 @@
 </div>
 {{else}}<p class="empty">No nodes have registered.</p>{{end}}
 
+{{with .GPUMap}}
+<h2>GPUs <span class="opt">by
+  {{range .Modes}}{{if eq .Key $.GPUMap.By}}<strong>{{.Label}}</strong>{{else}}<a href="?gpumap={{.Key}}">{{.Label}}</a>{{end}} {{end}}</span></h2>
+<div class="gpu-map">
+{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
+{{end}}</div>
+<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
+{{end}}
+{{with .IdleGPUs}}
+<h3>Cards held and doing nothing</h3>
+{{/* The first place to find room: replicas whose every card reads under 5%
+     busy now. A model kept warm on purpose is here too; whether it should
+     be is the operator's call. */}}
+<table class="rows"><tr><th>Model</th><th>Replica</th><th>Node</th><th class="num">GPUs</th><th class="num">Busiest card</th><th>Last request</th></tr>
+{{range .}}<tr><td>{{.Model}}</td><td class="clip"><code>{{.Replica}}</code></td><td><code>{{.Node}}</code></td>
+  <td class="num">{{len .GPUs}}</td><td class="num">{{printf "%.0f" .MaxUtil}}%</td><td>{{if .Idle}}{{.Idle}} ago{{else}}<span class="note">not known</span>{{end}}</td></tr>
+{{end}}</table>
+{{end}}
+
 <h2>Nodes</h2>
 {{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
 {{if .Nodes}}