From b6a802c03b5c8f2b90eb0e36ad44cb5528005c25 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 15:54:58 +0200 Subject: [PATCH] Map the fleet's GPUs, and name the cards held and idle The cluster page draws every card coloured by utilisation, memory or temperature, and lists replicas whose every card is under 5% busy. llmgw_gpu_assigned says which replica, model and owner hold each card. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 + internal/control/control.go | 14 ++ internal/control/fleetmetrics/fleetmetrics.go | 33 ++- internal/control/fleetmetrics/sensors_test.go | 25 ++ internal/web/dashboard.go | 8 + internal/web/gpumap.go | 232 ++++++++++++++++++ internal/web/gpumap_test.go | 58 +++++ internal/web/static/base.css | 7 + internal/web/templates/dashboard.html | 19 ++ 9 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 internal/web/gpumap.go create mode 100644 internal/web/gpumap_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c00c4ad..dc0aa06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,14 @@ and a figure the driver does not give is left out rather than shown as zero. limit, clock, and what holds cards back. - **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat), `GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes). +- **The cluster page's GPU map**: every card a square, a row per node, + coloured by how busy, how full or how hot it is (`?gpumap=util|memory|temp`), + with what holds it and its figures under the pointer; free cards dashed. +- **Cards held and doing nothing**: the replicas whose every card reads under + 5% busy, with when they were last asked — the first place to find room. +- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds + each card and whose quota it counts against, for joining the GPU series to + models and tenants. ### A GPU placement no longer runs on the CPU unnoticed diff --git a/internal/control/control.go b/internal/control/control.go index bf67ad8..72d663b 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -417,3 +417,17 @@ func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e } // EngineChoices are the engine builds chosen per node, or nil. func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices } + +// ModelOwners maps each model to the part of the tenancy tree its capacity +// counts against; a model owned by nobody is not in it. +func (c *Controller) ModelOwners() map[string]string { + out := map[string]string{} + if cat := c.Catalog(); cat != nil { + for _, m := range cat.Models() { + if m.Owner != "" { + out[m.Name] = m.Owner + } + } + } + return out +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 09144e4..7cbf5e5 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -73,6 +73,7 @@ type Collector struct { gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc + gpuAssigned *prometheus.Desc gpuMemUtil *prometheus.Desc gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc @@ -151,6 +152,10 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned", + "1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+ + "Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.", + []string{"node", "gpu", "model", "replica", "owner"}, nil), gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent", "How much of the sample period a GPU's memory was being read or written: "+ "decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil), @@ -278,7 +283,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -294,6 +299,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { func (c *Collector) Collect(ch chan<- prometheus.Metric) { now := time.Now() c.collectQuotas(ch) + c.collectAssignments(ch) loads := map[string]*inventory.Load{} for _, n := range c.src.Nodes() { @@ -539,6 +545,12 @@ func b2f(b bool) float64 { return 0 } +// OwnerSource says whose quota each model counts against, for labelling +// what its replicas hold. Asked for by type assertion. +type OwnerSource interface { + ModelOwners() map[string]string +} + // QuotaSource adds the tenancy tree's capacity quotas. Asked for by type // assertion, like HealthSource. type QuotaSource interface { @@ -617,3 +629,22 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...) } } + +// collectAssignments says which replica holds each GPU. +func (c *Collector) collectAssignments(ch chan<- prometheus.Metric) { + var owners map[string]string + if o, ok := c.src.(OwnerSource); ok { + owners = o.ModelOwners() + } + for _, r := range c.src.Replicas() { + if !r.State.Counts() { + continue + } + for _, sh := range r.Shards { + for _, g := range sh.GPUIndices { + ch <- prometheus.MustNewConstMetric(c.gpuAssigned, prometheus.GaugeValue, 1, + sh.NodeID, strconv.Itoa(g), r.Model, r.ID, owners[r.Model]) + } + } + } +} diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index 3e5358a..b761de8 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -7,6 +7,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { @@ -77,3 +78,27 @@ func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) { t.Errorf("a utilization the driver did not report was published:\n%s", out) } } + +// ownedSource adds owners to fakeSource. +type ownedSource struct{ fakeSource } + +func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} } + +func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) { + src := ownedSource{fakeSource{replicas: []reconciler.Replica{ + {ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}}, + {ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}}, + }}} + out := gather(t, src) + for _, want := range []string{ + "llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", + "llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + if strings.Contains(out, "replica=old-1") { + t.Error("a stopped replica's GPU was counted as held") + } +} diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index ba0f7cc..3b7ba3b 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -20,6 +20,10 @@ type DashboardData struct { CPU cpuCapacity // Menu is the shared top menu; Nav below is what the cluster page adds. Menu Nav + // GPUMap is every card coloured by one figure, and IdleGPUs the replicas + // holding cards that are doing nothing; both from the last heartbeats. + GPUMap *gpuMap + IdleGPUs []idleHolding // Notice and Problem are what the last change made from this page did, // carried over the redirect back to it. Notice, Problem string @@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0 } d.NodePrefix = sharedPrefix(ids) + if d.AnyGPUs { + d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix) + d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix) + } for _, m := range d.Models { d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0 } diff --git a/internal/web/gpumap.go b/internal/web/gpumap.go new file mode 100644 index 0000000..652e202 --- /dev/null +++ b/internal/web/gpumap.go @@ -0,0 +1,232 @@ +package web + +import ( + "fmt" + "html/template" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// The GPU map is every card in the fleet, a square each, a row per node, +// coloured by one figure: how busy, how hot, or how full its memory is. A +// hundred cards in a table are a hundred rows to read; on the map the one +// that is not like the others is seen before anything is read. + +// gpuMapModes are what the map can be coloured by. +var gpuMapModes = []struct{ Key, Label string }{ + {"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"}, +} + +type gpuMap struct { + By string + Modes []struct{ Key, Label string } + Rows []gpuMapRow + // Legend says what the colours mean in this mode. + Legend string +} + +type gpuMapRow struct { + Node string + Cells []gpuCell +} + +type gpuCell struct { + Index int + // Model and Replica hold the card; empty is a free card. + Model, Replica string + // Style is the cell's colour, built here from numbers alone, which is + // what makes it safe to put in a style attribute; Title says everything + // known about the card. + Style template.CSS + Title string + // Label is the figure written in the cell. + Label string +} + +// idleHolding is a replica holding GPUs that are doing nothing. +type idleHolding struct { + Model, Replica, Node string + GPUs []int + // MaxUtil is the busiest of its cards, now. + MaxUtil float64 + // Idle is how long since a request was routed to it, or "" when not + // known. + Idle string +} + +// buildGPUMap draws the map from what the nodes last reported and where +// replicas are. Nil for a fleet without GPUs. +func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap { + held := heldBy(reps) + m := &gpuMap{By: by, Modes: gpuMapModes} + if m.By == "" { + m.By = "util" + } + switch m.By { + case "temp": + m.Legend = "Blue is cool, red is 90 °C and over." + case "memory": + m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start." + default: + m.By = "util" + m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds." + } + for _, n := range nodes { + if n.GPUTotal == 0 { + continue + } + row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)} + for i := range n.GPUTotal { + c := gpuCell{Index: i} + if h, ok := held[gpuKey(n.ID, i)]; ok { + c.Model, c.Replica = h.model, h.replica + } + var title []string + title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i)) + if c.Model != "" { + title = append(title, c.Model+" ("+c.Replica+")") + } else { + title = append(title, "free") + } + s, sampled := sensorOf(n, i) + if sampled { + if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 { + title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent)) + } + if s.MemoryTotalMB > 0 { + title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)) + } + if s.TempC > 0 { + title = append(title, fmt.Sprintf("%.0f °C", s.TempC)) + } + if s.PowerW > 0 { + title = append(title, fmt.Sprintf("%.0f W", s.PowerW)) + } + if len(s.Throttle) > 0 { + title = append(title, "held back by "+strings.Join(s.Throttle, ", ")) + } + } + c.Title = strings.Join(title, " · ") + style, label := cellColour(m.By, s, sampled) + if c.Model == "" { + style += ";border-style:dashed" + } + c.Style, c.Label = template.CSS(style), label + row.Cells = append(row.Cells, c) + } + m.Rows = append(m.Rows, row) + } + if len(m.Rows) == 0 { + return nil + } + return m +} + +// cellColour is a cell's background, and the figure written in it. +func cellColour(by string, s sensorView, sampled bool) (string, string) { + const unknown = "background:var(--panel-2,#eee)" + switch by { + case "temp": + if !sampled || s.TempC <= 0 { + return unknown, "" + } + // 30 °C blue to 90 °C red. + f := clamp01((s.TempC - 30) / 60) + return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC) + case "memory": + if !sampled || s.MemoryTotalMB <= 0 { + return unknown, "" + } + f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB) + return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f) + } + if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 { + return unknown, "" + } + f := clamp01(s.UtilPercent / 100) + return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent) +} + +// idleHoldings are the replicas whose GPUs all read under 5% busy now: +// capacity held and not asked for, the first place to find room. +func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding { + byID := map[string]adminapi.NodeView{} + for _, n := range nodes { + byID[n.ID] = n + } + var out []idleHolding + for _, r := range reps { + if r.State != "ready" { + continue + } + for _, sh := range r.Shards { + if len(sh.GPUs) == 0 { + continue + } + h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs} + idle, known := true, true + for _, g := range sh.GPUs { + s, ok := sensorOf(byID[sh.Node], g) + if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 { + known = false + break + } + h.MaxUtil = max(h.MaxUtil, s.UtilPercent) + if s.UtilPercent >= 5 { + idle = false + } + } + if !known || !idle { + continue + } + if r.IdleSec >= 0 { + h.Idle = (time.Duration(r.IdleSec) * time.Second).String() + } + out = append(out, h) + } + } + sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) }) + return out +} + +type holder struct{ model, replica string } + +func heldBy(reps []adminapi.ReplicaView) map[string]holder { + out := map[string]holder{} + for _, r := range reps { + switch r.State { + case "stopped", "failed": + continue + } + for _, sh := range r.Shards { + for _, g := range sh.GPUs { + out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID} + } + } + } + return out +} + +func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) } + +// sensorView is a card's last sample. +type sensorView = struct { + TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64 + Throttle []string +} + +func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { + if n.Load == nil { + return sensorView{}, false + } + for _, s := range n.Load.GPUs { + if s.Index == gpu { + return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent, + MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true + } + } + return sensorView{}, false +} diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go new file mode 100644 index 0000000..e704dde --- /dev/null +++ b/internal/web/gpumap_test.go @@ -0,0 +1,58 @@ +package web + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// mapFleet is a node of four cards: chat holds two and is idle, code holds +// one and is busy, one is free. +type mapFleet struct{ fakeSource } + +func (mapFleet) Nodes() []adminapi.NodeView { + busy := func(i int, util, temp float64) inventory.GPUSensor { + return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300} + } + return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"}, + Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}} +} + +func (mapFleet) Replicas(string) []adminapi.ReplicaView { + return []adminapi.ReplicaView{ + {ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}}, + {ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}}, + } +} + +func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { + _, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + if n := strings.Count(body, `class="gpu-cell"`); n != 4 { + t.Errorf("%d cards on the map, want 4", n) + } + for _, want := range []string{ + "gpu07 GPU 2 · code (code-1) · 93% busy", + "gpu07 GPU 3 · free", + "border-style:dashed", + "Cards held and doing nothing", + "chat-1gpu07", "30m0s ago", + } { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + if strings.Contains(body, "code-1gpu07") { + t.Error("a busy replica was listed as idle") + } + _, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil) + if !strings.Contains(temp, "temperature") || !strings.Contains(temp, ">81") { + t.Error("the map coloured by temperature does not say so, or show it") + } + _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + if strings.Contains(cpu, "gpu-map") { + t.Error("a fleet without GPUs has a GPU map") + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index be58733..3dbae5c 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; } .held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; } .held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; } .held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; } + +/* The GPU map: a square per card, a row per node. */ +.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; } +.gpu-row { display: flex; align-items: center; gap: 3px; } +.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px; + font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 20e6742..ff3513d 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -49,6 +49,25 @@ {{else}}

No nodes have registered.

{{end}} +{{with .GPUMap}} +

GPUs by + {{range .Modes}}{{if eq .Key $.GPUMap.By}}{{.Label}}{{else}}{{.Label}}{{end}} {{end}}

+
+{{range .Rows}}
{{.Node}}{{range .Cells}}{{.Label}}{{end}}
+{{end}}
+

{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.

+{{end}} +{{with .IdleGPUs}} +

Cards held and doing nothing

+{{/* The first place to find room: replicas whose every card reads under 5% + busy now. A model kept warm on purpose is here too; whether it should + be is the operator's call. */}} + +{{range .}} + +{{end}}
ModelReplicaNodeGPUsBusiest cardLast request
{{.Model}}{{.Replica}}{{.Node}}{{len .GPUs}}{{printf "%.0f" .MaxUtil}}%{{if .Idle}}{{.Idle}} ago{{else}}not known{{end}}
+{{end}} +

Nodes

{{with .Notice}}

{{.}}

{{end}}{{with .Problem}}

{{.}}

{{end}} {{if .Nodes}}