diff --git a/CHANGELOG.md b/CHANGELOG.md index c00c4ada..dc0aa063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,14 @@ and a figure the driver does not give is left out rather than shown as zero. limit, clock, and what holds cards back. - **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat), `GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes). +- **The cluster page's GPU map**: every card a square, a row per node, + coloured by how busy, how full or how hot it is (`?gpumap=util|memory|temp`), + with what holds it and its figures under the pointer; free cards dashed. +- **Cards held and doing nothing**: the replicas whose every card reads under + 5% busy, with when they were last asked — the first place to find room. +- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds + each card and whose quota it counts against, for joining the GPU series to + models and tenants. ### A GPU placement no longer runs on the CPU unnoticed diff --git a/internal/control/control.go b/internal/control/control.go index bf67ad82..72d663b4 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -417,3 +417,17 @@ func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e } // EngineChoices are the engine builds chosen per node, or nil. func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices } + +// ModelOwners maps each model to the part of the tenancy tree its capacity +// counts against; a model owned by nobody is not in it. +func (c *Controller) ModelOwners() map[string]string { + out := map[string]string{} + if cat := c.Catalog(); cat != nil { + for _, m := range cat.Models() { + if m.Owner != "" { + out[m.Name] = m.Owner + } + } + } + return out +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 09144e4d..7cbf5e59 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -73,6 +73,7 @@ type Collector struct { gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc + gpuAssigned *prometheus.Desc gpuMemUtil *prometheus.Desc gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc @@ -151,6 +152,10 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned", + "1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+ + "Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.", + []string{"node", "gpu", "model", "replica", "owner"}, nil), gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent", "How much of the sample period a GPU's memory was being read or written: "+ "decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil), @@ -278,7 +283,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -294,6 +299,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { func (c *Collector) Collect(ch chan<- prometheus.Metric) { now := time.Now() c.collectQuotas(ch) + c.collectAssignments(ch) loads := map[string]*inventory.Load{} for _, n := range c.src.Nodes() { @@ -539,6 +545,12 @@ func b2f(b bool) float64 { return 0 } +// OwnerSource says whose quota each model counts against, for labelling +// what its replicas hold. Asked for by type assertion. +type OwnerSource interface { + ModelOwners() map[string]string +} + // QuotaSource adds the tenancy tree's capacity quotas. Asked for by type // assertion, like HealthSource. type QuotaSource interface { @@ -617,3 +629,22 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...) } } + +// collectAssignments says which replica holds each GPU. +func (c *Collector) collectAssignments(ch chan<- prometheus.Metric) { + var owners map[string]string + if o, ok := c.src.(OwnerSource); ok { + owners = o.ModelOwners() + } + for _, r := range c.src.Replicas() { + if !r.State.Counts() { + continue + } + for _, sh := range r.Shards { + for _, g := range sh.GPUIndices { + ch <- prometheus.MustNewConstMetric(c.gpuAssigned, prometheus.GaugeValue, 1, + sh.NodeID, strconv.Itoa(g), r.Model, r.ID, owners[r.Model]) + } + } + } +} diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index 3e5358a5..b761de81 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -7,6 +7,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { @@ -77,3 +78,27 @@ func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) { t.Errorf("a utilization the driver did not report was published:\n%s", out) } } + +// ownedSource adds owners to fakeSource. +type ownedSource struct{ fakeSource } + +func (ownedSource) ModelOwners() map[string]string { return map[string]string{"chat": "acme/ml"} } + +func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) { + src := ownedSource{fakeSource{replicas: []reconciler.Replica{ + {ID: "chat-1", Model: "chat", State: reconciler.Ready, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{0, 1}}}}, + {ID: "old-1", Model: "old", State: reconciler.Stopped, Shards: []reconciler.Shard{{NodeID: "g1", GPUIndices: []int{2}}}}, + }}} + out := gather(t, src) + for _, want := range []string{ + "llmgw_gpu_assigned gpu=0 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", + "llmgw_gpu_assigned gpu=1 model=chat node=g1 owner=acme/ml replica=chat-1 -> 1", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + if strings.Contains(out, "replica=old-1") { + t.Error("a stopped replica's GPU was counted as held") + } +} diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index ba0f7cc2..3b7ba3bf 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -20,6 +20,10 @@ type DashboardData struct { CPU cpuCapacity // Menu is the shared top menu; Nav below is what the cluster page adds. Menu Nav + // GPUMap is every card coloured by one figure, and IdleGPUs the replicas + // holding cards that are doing nothing; both from the last heartbeats. + GPUMap *gpuMap + IdleGPUs []idleHolding // Notice and Problem are what the last change made from this page did, // carried over the redirect back to it. Notice, Problem string @@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0 } d.NodePrefix = sharedPrefix(ids) + if d.AnyGPUs { + d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix) + d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix) + } for _, m := range d.Models { d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0 } diff --git a/internal/web/gpumap.go b/internal/web/gpumap.go new file mode 100644 index 00000000..652e202c --- /dev/null +++ b/internal/web/gpumap.go @@ -0,0 +1,232 @@ +package web + +import ( + "fmt" + "html/template" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// The GPU map is every card in the fleet, a square each, a row per node, +// coloured by one figure: how busy, how hot, or how full its memory is. A +// hundred cards in a table are a hundred rows to read; on the map the one +// that is not like the others is seen before anything is read. + +// gpuMapModes are what the map can be coloured by. +var gpuMapModes = []struct{ Key, Label string }{ + {"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"}, +} + +type gpuMap struct { + By string + Modes []struct{ Key, Label string } + Rows []gpuMapRow + // Legend says what the colours mean in this mode. + Legend string +} + +type gpuMapRow struct { + Node string + Cells []gpuCell +} + +type gpuCell struct { + Index int + // Model and Replica hold the card; empty is a free card. + Model, Replica string + // Style is the cell's colour, built here from numbers alone, which is + // what makes it safe to put in a style attribute; Title says everything + // known about the card. + Style template.CSS + Title string + // Label is the figure written in the cell. + Label string +} + +// idleHolding is a replica holding GPUs that are doing nothing. +type idleHolding struct { + Model, Replica, Node string + GPUs []int + // MaxUtil is the busiest of its cards, now. + MaxUtil float64 + // Idle is how long since a request was routed to it, or "" when not + // known. + Idle string +} + +// buildGPUMap draws the map from what the nodes last reported and where +// replicas are. Nil for a fleet without GPUs. +func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap { + held := heldBy(reps) + m := &gpuMap{By: by, Modes: gpuMapModes} + if m.By == "" { + m.By = "util" + } + switch m.By { + case "temp": + m.Legend = "Blue is cool, red is 90 °C and over." + case "memory": + m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start." + default: + m.By = "util" + m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds." + } + for _, n := range nodes { + if n.GPUTotal == 0 { + continue + } + row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)} + for i := range n.GPUTotal { + c := gpuCell{Index: i} + if h, ok := held[gpuKey(n.ID, i)]; ok { + c.Model, c.Replica = h.model, h.replica + } + var title []string + title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i)) + if c.Model != "" { + title = append(title, c.Model+" ("+c.Replica+")") + } else { + title = append(title, "free") + } + s, sampled := sensorOf(n, i) + if sampled { + if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 { + title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent)) + } + if s.MemoryTotalMB > 0 { + title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)) + } + if s.TempC > 0 { + title = append(title, fmt.Sprintf("%.0f °C", s.TempC)) + } + if s.PowerW > 0 { + title = append(title, fmt.Sprintf("%.0f W", s.PowerW)) + } + if len(s.Throttle) > 0 { + title = append(title, "held back by "+strings.Join(s.Throttle, ", ")) + } + } + c.Title = strings.Join(title, " · ") + style, label := cellColour(m.By, s, sampled) + if c.Model == "" { + style += ";border-style:dashed" + } + c.Style, c.Label = template.CSS(style), label + row.Cells = append(row.Cells, c) + } + m.Rows = append(m.Rows, row) + } + if len(m.Rows) == 0 { + return nil + } + return m +} + +// cellColour is a cell's background, and the figure written in it. +func cellColour(by string, s sensorView, sampled bool) (string, string) { + const unknown = "background:var(--panel-2,#eee)" + switch by { + case "temp": + if !sampled || s.TempC <= 0 { + return unknown, "" + } + // 30 °C blue to 90 °C red. + f := clamp01((s.TempC - 30) / 60) + return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC) + case "memory": + if !sampled || s.MemoryTotalMB <= 0 { + return unknown, "" + } + f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB) + return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f) + } + if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 { + return unknown, "" + } + f := clamp01(s.UtilPercent / 100) + return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent) +} + +// idleHoldings are the replicas whose GPUs all read under 5% busy now: +// capacity held and not asked for, the first place to find room. +func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding { + byID := map[string]adminapi.NodeView{} + for _, n := range nodes { + byID[n.ID] = n + } + var out []idleHolding + for _, r := range reps { + if r.State != "ready" { + continue + } + for _, sh := range r.Shards { + if len(sh.GPUs) == 0 { + continue + } + h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs} + idle, known := true, true + for _, g := range sh.GPUs { + s, ok := sensorOf(byID[sh.Node], g) + if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 { + known = false + break + } + h.MaxUtil = max(h.MaxUtil, s.UtilPercent) + if s.UtilPercent >= 5 { + idle = false + } + } + if !known || !idle { + continue + } + if r.IdleSec >= 0 { + h.Idle = (time.Duration(r.IdleSec) * time.Second).String() + } + out = append(out, h) + } + } + sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) }) + return out +} + +type holder struct{ model, replica string } + +func heldBy(reps []adminapi.ReplicaView) map[string]holder { + out := map[string]holder{} + for _, r := range reps { + switch r.State { + case "stopped", "failed": + continue + } + for _, sh := range r.Shards { + for _, g := range sh.GPUs { + out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID} + } + } + } + return out +} + +func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) } + +// sensorView is a card's last sample. +type sensorView = struct { + TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64 + Throttle []string +} + +func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { + if n.Load == nil { + return sensorView{}, false + } + for _, s := range n.Load.GPUs { + if s.Index == gpu { + return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent, + MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true + } + } + return sensorView{}, false +} diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go new file mode 100644 index 00000000..e704ddea --- /dev/null +++ b/internal/web/gpumap_test.go @@ -0,0 +1,58 @@ +package web + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// mapFleet is a node of four cards: chat holds two and is idle, code holds +// one and is busy, one is free. +type mapFleet struct{ fakeSource } + +func (mapFleet) Nodes() []adminapi.NodeView { + busy := func(i int, util, temp float64) inventory.GPUSensor { + return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300} + } + return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"}, + Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}} +} + +func (mapFleet) Replicas(string) []adminapi.ReplicaView { + return []adminapi.ReplicaView{ + {ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}}, + {ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}}, + } +} + +func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { + _, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + if n := strings.Count(body, `class="gpu-cell"`); n != 4 { + t.Errorf("%d cards on the map, want 4", n) + } + for _, want := range []string{ + "gpu07 GPU 2 · code (code-1) · 93% busy", + "gpu07 GPU 3 · free", + "border-style:dashed", + "Cards held and doing nothing", + "chat-1gpu07", "30m0s ago", + } { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + if strings.Contains(body, "code-1gpu07") { + t.Error("a busy replica was listed as idle") + } + _, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil) + if !strings.Contains(temp, "temperature") || !strings.Contains(temp, ">81") { + t.Error("the map coloured by temperature does not say so, or show it") + } + _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + if strings.Contains(cpu, "gpu-map") { + t.Error("a fleet without GPUs has a GPU map") + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index be58733c..3dbae5cb 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; } .held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; } .held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; } .held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; } + +/* The GPU map: a square per card, a row per node. */ +.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; } +.gpu-row { display: flex; align-items: center; gap: 3px; } +.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px; + font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 20e67429..ff3513df 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -49,6 +49,25 @@ {{else}}

No nodes have registered.

{{end}} +{{with .GPUMap}} +

GPUs by + {{range .Modes}}{{if eq .Key $.GPUMap.By}}{{.Label}}{{else}}{{.Label}}{{end}} {{end}}

+
+{{range .Rows}}
{{.Node}}{{range .Cells}}{{.Label}}{{end}}
+{{end}}
+

{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.

+{{end}} +{{with .IdleGPUs}} +

Cards held and doing nothing

+{{/* The first place to find room: replicas whose every card reads under 5% + busy now. A model kept warm on purpose is here too; whether it should + be is the operator's call. */}} + +{{range .}} + +{{end}}
ModelReplicaNodeGPUsBusiest cardLast request
{{.Model}}{{.Replica}}{{.Node}}{{len .GPUs}}{{printf "%.0f" .MaxUtil}}%{{if .Idle}}{{.Idle}} ago{{else}}not known{{end}}
+{{end}} +

Nodes

{{with .Notice}}

{{.}}

{{end}}{{with .Problem}}

{{.}}

{{end}} {{if .Nodes}}