Map the fleet's GPUs, and name the cards held and idle

The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 15:54:58 +02:00
1 parent bd36798b15
commit b6a802c03b
9 files changed
+403 -1

No files matched your search

+8
View File
@@ -20,6 +20,10 @@ type DashboardData struct {
CPU cpuCapacity
// Menu is the shared top menu; Nav below is what the cluster page adds.
Menu Nav
// GPUMap is every card coloured by one figure, and IdleGPUs the replicas
// holding cards that are doing nothing; both from the last heartbeats.
GPUMap *gpuMap
IdleGPUs []idleHolding
// Notice and Problem are what the last change made from this page did,
// carried over the redirect back to it.
Notice, Problem string
@@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
}
d.NodePrefix = sharedPrefix(ids)
if d.AnyGPUs {
d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix)
}
for _, m := range d.Models {
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
}
+232
View File
@@ -0,0 +1,232 @@
package web
import (
"fmt"
"html/template"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// The GPU map is every card in the fleet, a square each, a row per node,
// coloured by one figure: how busy, how hot, or how full its memory is. A
// hundred cards in a table are a hundred rows to read; on the map the one
// that is not like the others is seen before anything is read.
// gpuMapModes are what the map can be coloured by.
var gpuMapModes = []struct{ Key, Label string }{
{"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
}
type gpuMap struct {
By string
Modes []struct{ Key, Label string }
Rows []gpuMapRow
// Legend says what the colours mean in this mode.
Legend string
}
type gpuMapRow struct {
Node string
Cells []gpuCell
}
type gpuCell struct {
Index int
// Model and Replica hold the card; empty is a free card.
Model, Replica string
// Style is the cell's colour, built here from numbers alone, which is
// what makes it safe to put in a style attribute; Title says everything
// known about the card.
Style template.CSS
Title string
// Label is the figure written in the cell.
Label string
}
// idleHolding is a replica holding GPUs that are doing nothing.
type idleHolding struct {
Model, Replica, Node string
GPUs []int
// MaxUtil is the busiest of its cards, now.
MaxUtil float64
// Idle is how long since a request was routed to it, or "" when not
// known.
Idle string
}
// buildGPUMap draws the map from what the nodes last reported and where
// replicas are. Nil for a fleet without GPUs.
func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
held := heldBy(reps)
m := &gpuMap{By: by, Modes: gpuMapModes}
if m.By == "" {
m.By = "util"
}
switch m.By {
case "temp":
m.Legend = "Blue is cool, red is 90 °C and over."
case "memory":
m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
default:
m.By = "util"
m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
}
for _, n := range nodes {
if n.GPUTotal == 0 {
continue
}
row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
for i := range n.GPUTotal {
c := gpuCell{Index: i}
if h, ok := held[gpuKey(n.ID, i)]; ok {
c.Model, c.Replica = h.model, h.replica
}
var title []string
title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
if c.Model != "" {
title = append(title, c.Model+" ("+c.Replica+")")
} else {
title = append(title, "free")
}
s, sampled := sensorOf(n, i)
if sampled {
if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
}
if s.MemoryTotalMB > 0 {
title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
}
if s.TempC > 0 {
title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
}
if s.PowerW > 0 {
title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
}
if len(s.Throttle) > 0 {
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
}
}
c.Title = strings.Join(title, " · ")
style, label := cellColour(m.By, s, sampled)
if c.Model == "" {
style += ";border-style:dashed"
}
c.Style, c.Label = template.CSS(style), label
row.Cells = append(row.Cells, c)
}
m.Rows = append(m.Rows, row)
}
if len(m.Rows) == 0 {
return nil
}
return m
}
// cellColour is a cell's background, and the figure written in it.
func cellColour(by string, s sensorView, sampled bool) (string, string) {
const unknown = "background:var(--panel-2,#eee)"
switch by {
case "temp":
if !sampled || s.TempC <= 0 {
return unknown, ""
}
// 30 °C blue to 90 °C red.
f := clamp01((s.TempC - 30) / 60)
return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
case "memory":
if !sampled || s.MemoryTotalMB <= 0 {
return unknown, ""
}
f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
}
if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
return unknown, ""
}
f := clamp01(s.UtilPercent / 100)
return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
}
// idleHoldings are the replicas whose GPUs all read under 5% busy now:
// capacity held and not asked for, the first place to find room.
func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
byID := map[string]adminapi.NodeView{}
for _, n := range nodes {
byID[n.ID] = n
}
var out []idleHolding
for _, r := range reps {
if r.State != "ready" {
continue
}
for _, sh := range r.Shards {
if len(sh.GPUs) == 0 {
continue
}
h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
idle, known := true, true
for _, g := range sh.GPUs {
s, ok := sensorOf(byID[sh.Node], g)
if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
known = false
break
}
h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
if s.UtilPercent >= 5 {
idle = false
}
}
if !known || !idle {
continue
}
if r.IdleSec >= 0 {
h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
}
out = append(out, h)
}
}
sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
return out
}
type holder struct{ model, replica string }
func heldBy(reps []adminapi.ReplicaView) map[string]holder {
out := map[string]holder{}
for _, r := range reps {
switch r.State {
case "stopped", "failed":
continue
}
for _, sh := range r.Shards {
for _, g := range sh.GPUs {
out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
}
}
}
return out
}
func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
// sensorView is a card's last sample.
type sensorView = struct {
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
}
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
if n.Load == nil {
return sensorView{}, false
}
for _, s := range n.Load.GPUs {
if s.Index == gpu {
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
}
}
return sensorView{}, false
}
+58
View File
@@ -0,0 +1,58 @@
package web
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// mapFleet is a node of four cards: chat holds two and is idle, code holds
// one and is busy, one is free.
type mapFleet struct{ fakeSource }
func (mapFleet) Nodes() []adminapi.NodeView {
busy := func(i int, util, temp float64) inventory.GPUSensor {
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
}
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
}
func (mapFleet) Replicas(string) []adminapi.ReplicaView {
return []adminapi.ReplicaView{
{ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
{ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
}
}
func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
_, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
t.Errorf("%d cards on the map, want 4", n)
}
for _, want := range []string{
"gpu07 GPU 2 · code (code-1) · 93% busy",
"gpu07 GPU 3 · free",
"border-style:dashed",
"Cards held and doing nothing",
"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
if strings.Contains(body, "<code>code-1</code></td><td><code>gpu07</code>") {
t.Error("a busy replica was listed as idle")
}
_, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
t.Error("the map coloured by temperature does not say so, or show it")
}
_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
if strings.Contains(cpu, "gpu-map") {
t.Error("a fleet without GPUs has a GPU map")
}
}
+7
View File
@@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; }
.held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; }
.held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; }
.held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; }
/* The GPU map: a square per card, a row per node. */
.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; }
.gpu-row { display: flex; align-items: center; gap: 3px; }
.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px;
font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; }
+19
View File
@@ -49,6 +49,25 @@
</div>
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
{{with .GPUMap}}
<h2>GPUs <span class="opt">by
{{range .Modes}}{{if eq .Key $.GPUMap.By}}<strong>{{.Label}}</strong>{{else}}<a href="?gpumap={{.Key}}">{{.Label}}</a>{{end}} {{end}}</span></h2>
<div class="gpu-map">
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
{{end}}</div>
<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
{{end}}
{{with .IdleGPUs}}
<h3>Cards held and doing nothing</h3>
{{/* The first place to find room: replicas whose every card reads under 5%
busy now. A model kept warm on purpose is here too; whether it should
be is the operator's call. */}}
<table class="rows"><tr><th>Model</th><th>Replica</th><th>Node</th><th class="num">GPUs</th><th class="num">Busiest card</th><th>Last request</th></tr>
{{range .}}<tr><td>{{.Model}}</td><td class="clip"><code>{{.Replica}}</code></td><td><code>{{.Node}}</code></td>
<td class="num">{{len .GPUs}}</td><td class="num">{{printf "%.0f" .MaxUtil}}%</td><td>{{if .Idle}}{{.Idle}} ago{{else}}<span class="note">not known</span>{{end}}</td></tr>
{{end}}</table>
{{end}}
<h2>Nodes</h2>
{{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
{{if .Nodes}}