diff --git a/CHANGELOG.md b/CHANGELOG.md
index c00c4ada..dc0aa063 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,14 @@ and a figure the driver does not give is left out rather than shown as zero.
limit, clock, and what holds cards back.
- **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat),
`GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes).
+- **The cluster page's GPU map**: every card a square, a row per node,
+ coloured by how busy, how full or how hot it is (`?gpumap=util|memory|temp`),
+ with what holds it and its figures under the pointer; free cards dashed.
+- **Cards held and doing nothing**: the replicas whose every card reads under
+ 5% busy, with when they were last asked — the first place to find room.
+- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
+ each card and whose quota it counts against, for joining the GPU series to
+ models and tenants.
### A GPU placement no longer runs on the CPU unnoticed
diff --git a/internal/control/control.go b/internal/control/control.go
index bf67ad82..72d663b4 100644
--- a/internal/control/control.go
+++ b/internal/control/control.go
@@ -417,3 +417,17 @@ func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
// EngineChoices are the engine builds chosen per node, or nil.
func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
+
+// ModelOwners maps each model to the part of the tenancy tree its capacity
+// counts against; a model owned by nobody is not in it.
+func (c *Controller) ModelOwners() map[string]string {
+ out := map[string]string{}
+ if cat := c.Catalog(); cat != nil {
+ for _, m := range cat.Models() {
+ if m.Owner != "" {
+ out[m.Name] = m.Owner
+ }
+ }
+ }
+ return out
+}
diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go
index 09144e4d..7cbf5e59 100644
--- a/internal/control/fleetmetrics/fleetmetrics.go
+++ b/internal/control/fleetmetrics/fleetmetrics.go
@@ -73,6 +73,7 @@ type Collector struct {
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
+ gpuAssigned *prometheus.Desc
gpuMemUtil *prometheus.Desc
gpuSMClock *prometheus.Desc
gpuTemp *prometheus.Desc
@@ -151,6 +152,10 @@ func New(src Source) *Collector {
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
+ gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned",
+ "1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+
+ "Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.",
+ []string{"node", "gpu", "model", "replica", "owner"}, nil),
gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent",
"How much of the sample period a GPU's memory was being read or written: "+
"decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil),
@@ -278,7 +283,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
- c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy,
+ c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -294,6 +299,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
now := time.Now()
c.collectQuotas(ch)
+ c.collectAssignments(ch)
loads := map[string]*inventory.Load{}
for _, n := range c.src.Nodes() {
@@ -539,6 +545,12 @@ func b2f(b bool) float64 {
return 0
}
+// OwnerSource says whose quota each model counts against, for labelling
+// what its replicas hold. Asked for by type assertion.
+type OwnerSource interface {
+ ModelOwners() map[string]string
+}
+
// QuotaSource adds the tenancy tree's capacity quotas. Asked for by type
// assertion, like HealthSource.
type QuotaSource interface {
@@ -617,3 +629,22 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
}
}
+
+// collectAssignments says which replica holds each GPU.
+func (c *Collector) collectAssignments(ch chan<- prometheus.Metric) {
+ var owners map[string]string
+ if o, ok := c.src.(OwnerSource); ok {
+ owners = o.ModelOwners()
+ }
+ for _, r := range c.src.Replicas() {
+ if !r.State.Counts() {
+ continue
+ }
+ for _, sh := range r.Shards {
+ for _, g := range sh.GPUIndices {
+ ch <- prometheus.MustNewConstMetric(c.gpuAssigned, prometheus.GaugeValue, 1,
+ sh.NodeID, strconv.Itoa(g), r.Model, r.ID, owners[r.Model])
+ }
+ }
+ }
+}
diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go
index 3e5358a5..b761de81 100644
--- a/internal/control/fleetmetrics/sensors_test.go
+++ b/internal/control/fleetmetrics/sensors_test.go
@@ -7,6 +7,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
@@ -77,3 +78,27 @@ func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) {
t.Errorf("a utilization the driver did not report was published:\n%s", out)
}
}
+
+// ownedSource adds owners to fakeSource.
+type ownedSource struct{ fakeSource }
+
+func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} }
+
+func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
+ src := ownedSource{fakeSource{replicas: []reconciler.Replica{
+ {ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}},
+ {ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}},
+ }}}
+ out := gather(t, src)
+ for _, want := range []string{
+ "llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
+ "llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1",
+ } {
+ if !strings.Contains(out, want) {
+ t.Errorf("missing %s in:\n%s", want, out)
+ }
+ }
+ if strings.Contains(out, "replica=old-1") {
+ t.Error("a stopped replica's GPU was counted as held")
+ }
+}
diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go
index ba0f7cc2..3b7ba3bf 100644
--- a/internal/web/dashboard.go
+++ b/internal/web/dashboard.go
@@ -20,6 +20,10 @@ type DashboardData struct {
CPU cpuCapacity
// Menu is the shared top menu; Nav below is what the cluster page adds.
Menu Nav
+ // GPUMap is every card coloured by one figure, and IdleGPUs the replicas
+ // holding cards that are doing nothing; both from the last heartbeats.
+ GPUMap *gpuMap
+ IdleGPUs []idleHolding
// Notice and Problem are what the last change made from this page did,
// carried over the redirect back to it.
Notice, Problem string
@@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
}
d.NodePrefix = sharedPrefix(ids)
+ if d.AnyGPUs {
+ d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
+ d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix)
+ }
for _, m := range d.Models {
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
}
diff --git a/internal/web/gpumap.go b/internal/web/gpumap.go
new file mode 100644
index 00000000..652e202c
--- /dev/null
+++ b/internal/web/gpumap.go
@@ -0,0 +1,232 @@
+package web
+
+import (
+ "fmt"
+ "html/template"
+ "sort"
+ "strings"
+ "time"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+// The GPU map is every card in the fleet, a square each, a row per node,
+// coloured by one figure: how busy, how hot, or how full its memory is. A
+// hundred cards in a table are a hundred rows to read; on the map the one
+// that is not like the others is seen before anything is read.
+
+// gpuMapModes are what the map can be coloured by.
+var gpuMapModes = []struct{ Key, Label string }{
+ {"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
+}
+
+type gpuMap struct {
+ By string
+ Modes []struct{ Key, Label string }
+ Rows []gpuMapRow
+ // Legend says what the colours mean in this mode.
+ Legend string
+}
+
+type gpuMapRow struct {
+ Node string
+ Cells []gpuCell
+}
+
+type gpuCell struct {
+ Index int
+ // Model and Replica hold the card; empty is a free card.
+ Model, Replica string
+ // Style is the cell's colour, built here from numbers alone, which is
+ // what makes it safe to put in a style attribute; Title says everything
+ // known about the card.
+ Style template.CSS
+ Title string
+ // Label is the figure written in the cell.
+ Label string
+}
+
+// idleHolding is a replica holding GPUs that are doing nothing.
+type idleHolding struct {
+ Model, Replica, Node string
+ GPUs []int
+ // MaxUtil is the busiest of its cards, now.
+ MaxUtil float64
+ // Idle is how long since a request was routed to it, or "" when not
+ // known.
+ Idle string
+}
+
+// buildGPUMap draws the map from what the nodes last reported and where
+// replicas are. Nil for a fleet without GPUs.
+func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
+ held := heldBy(reps)
+ m := &gpuMap{By: by, Modes: gpuMapModes}
+ if m.By == "" {
+ m.By = "util"
+ }
+ switch m.By {
+ case "temp":
+ m.Legend = "Blue is cool, red is 90 °C and over."
+ case "memory":
+ m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
+ default:
+ m.By = "util"
+ m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
+ }
+ for _, n := range nodes {
+ if n.GPUTotal == 0 {
+ continue
+ }
+ row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
+ for i := range n.GPUTotal {
+ c := gpuCell{Index: i}
+ if h, ok := held[gpuKey(n.ID, i)]; ok {
+ c.Model, c.Replica = h.model, h.replica
+ }
+ var title []string
+ title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
+ if c.Model != "" {
+ title = append(title, c.Model+" ("+c.Replica+")")
+ } else {
+ title = append(title, "free")
+ }
+ s, sampled := sensorOf(n, i)
+ if sampled {
+ if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
+ title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
+ }
+ if s.MemoryTotalMB > 0 {
+ title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
+ }
+ if s.TempC > 0 {
+ title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
+ }
+ if s.PowerW > 0 {
+ title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
+ }
+ if len(s.Throttle) > 0 {
+ title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
+ }
+ }
+ c.Title = strings.Join(title, " · ")
+ style, label := cellColour(m.By, s, sampled)
+ if c.Model == "" {
+ style += ";border-style:dashed"
+ }
+ c.Style, c.Label = template.CSS(style), label
+ row.Cells = append(row.Cells, c)
+ }
+ m.Rows = append(m.Rows, row)
+ }
+ if len(m.Rows) == 0 {
+ return nil
+ }
+ return m
+}
+
+// cellColour is a cell's background, and the figure written in it.
+func cellColour(by string, s sensorView, sampled bool) (string, string) {
+ const unknown = "background:var(--panel-2,#eee)"
+ switch by {
+ case "temp":
+ if !sampled || s.TempC <= 0 {
+ return unknown, ""
+ }
+ // 30 °C blue to 90 °C red.
+ f := clamp01((s.TempC - 30) / 60)
+ return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
+ case "memory":
+ if !sampled || s.MemoryTotalMB <= 0 {
+ return unknown, ""
+ }
+ f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
+ return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
+ }
+ if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
+ return unknown, ""
+ }
+ f := clamp01(s.UtilPercent / 100)
+ return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
+}
+
+// idleHoldings are the replicas whose GPUs all read under 5% busy now:
+// capacity held and not asked for, the first place to find room.
+func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
+ byID := map[string]adminapi.NodeView{}
+ for _, n := range nodes {
+ byID[n.ID] = n
+ }
+ var out []idleHolding
+ for _, r := range reps {
+ if r.State != "ready" {
+ continue
+ }
+ for _, sh := range r.Shards {
+ if len(sh.GPUs) == 0 {
+ continue
+ }
+ h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
+ idle, known := true, true
+ for _, g := range sh.GPUs {
+ s, ok := sensorOf(byID[sh.Node], g)
+ if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
+ known = false
+ break
+ }
+ h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
+ if s.UtilPercent >= 5 {
+ idle = false
+ }
+ }
+ if !known || !idle {
+ continue
+ }
+ if r.IdleSec >= 0 {
+ h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
+ }
+ out = append(out, h)
+ }
+ }
+ sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
+ return out
+}
+
+type holder struct{ model, replica string }
+
+func heldBy(reps []adminapi.ReplicaView) map[string]holder {
+ out := map[string]holder{}
+ for _, r := range reps {
+ switch r.State {
+ case "stopped", "failed":
+ continue
+ }
+ for _, sh := range r.Shards {
+ for _, g := range sh.GPUs {
+ out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
+ }
+ }
+ }
+ return out
+}
+
+func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
+
+// sensorView is a card's last sample.
+type sensorView = struct {
+ TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
+ Throttle []string
+}
+
+func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
+ if n.Load == nil {
+ return sensorView{}, false
+ }
+ for _, s := range n.Load.GPUs {
+ if s.Index == gpu {
+ return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
+ MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
+ }
+ }
+ return sensorView{}, false
+}
diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go
new file mode 100644
index 00000000..e704ddea
--- /dev/null
+++ b/internal/web/gpumap_test.go
@@ -0,0 +1,58 @@
+package web
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+)
+
+// mapFleet is a node of four cards: chat holds two and is idle, code holds
+// one and is busy, one is free.
+type mapFleet struct{ fakeSource }
+
+func (mapFleet) Nodes() []adminapi.NodeView {
+ busy := func(i int, util, temp float64) inventory.GPUSensor {
+ return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
+ }
+ return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
+ Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
+}
+
+func (mapFleet) Replicas(string) []adminapi.ReplicaView {
+ return []adminapi.ReplicaView{
+ {ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
+ {ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
+ }
+}
+
+func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
+ _, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
+ if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
+ t.Errorf("%d cards on the map, want 4", n)
+ }
+ for _, want := range []string{
+ "gpu07 GPU 2 · code (code-1) · 93% busy",
+ "gpu07 GPU 3 · free",
+ "border-style:dashed",
+ "Cards held and doing nothing",
+ "chat-1
gpu07", "30m0s ago",
+ } {
+ if !strings.Contains(body, want) {
+ t.Errorf("the page lacks %q", want)
+ }
+ }
+ if strings.Contains(body, "code-1gpu07") {
+ t.Error("a busy replica was listed as idle")
+ }
+ _, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
+ if !strings.Contains(temp, "temperature") || !strings.Contains(temp, ">81") {
+ t.Error("the map coloured by temperature does not say so, or show it")
+ }
+ _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
+ if strings.Contains(cpu, "gpu-map") {
+ t.Error("a fleet without GPUs has a GPU map")
+ }
+}
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index be58733c..3dbae5cb 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; }
.held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; }
.held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; }
.held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; }
+
+/* The GPU map: a square per card, a row per node. */
+.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; }
+.gpu-row { display: flex; align-items: center; gap: 3px; }
+.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px;
+ font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; }
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html
index 20e67429..ff3513df 100644
--- a/internal/web/templates/dashboard.html
+++ b/internal/web/templates/dashboard.html
@@ -49,6 +49,25 @@
{{else}}No nodes have registered.
{{end}} +{{with .GPUMap}} +{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.
+{{end}} +{{with .IdleGPUs}} +| Model | Replica | Node | GPUs | Busiest card | Last request |
|---|---|---|---|---|---|
| {{.Model}} | {{.Replica}} | {{.Node}} |
+ {{len .GPUs}} | {{printf "%.0f" .MaxUtil}}% | {{if .Idle}}{{.Idle}} ago{{else}}not known{{end}} |
{{.}}
{{end}}{{with .Problem}}{{.}}
{{end}} {{if .Nodes}}