Map the fleet's GPUs, and name the cards held and idle
The cluster page draws every card coloured by utilisation, memory or temperature, and lists replicas whose every card is under 5% busy. llmgw_gpu_assigned says which replica, model and owner hold each card. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bd36798b15
commit
b6a802c03b
9 files changed
+403
-1
No files matched your search
@@ -20,6 +20,10 @@ type DashboardData struct {
|
||||
CPU cpuCapacity
|
||||
// Menu is the shared top menu; Nav below is what the cluster page adds.
|
||||
Menu Nav
|
||||
// GPUMap is every card coloured by one figure, and IdleGPUs the replicas
|
||||
// holding cards that are doing nothing; both from the last heartbeats.
|
||||
GPUMap *gpuMap
|
||||
IdleGPUs []idleHolding
|
||||
// Notice and Problem are what the last change made from this page did,
|
||||
// carried over the redirect back to it.
|
||||
Notice, Problem string
|
||||
@@ -198,6 +202,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
|
||||
}
|
||||
d.NodePrefix = sharedPrefix(ids)
|
||||
if d.AnyGPUs {
|
||||
d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
|
||||
d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix)
|
||||
}
|
||||
for _, m := range d.Models {
|
||||
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// The GPU map is every card in the fleet, a square each, a row per node,
|
||||
// coloured by one figure: how busy, how hot, or how full its memory is. A
|
||||
// hundred cards in a table are a hundred rows to read; on the map the one
|
||||
// that is not like the others is seen before anything is read.
|
||||
|
||||
// gpuMapModes are what the map can be coloured by.
|
||||
var gpuMapModes = []struct{ Key, Label string }{
|
||||
{"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
|
||||
}
|
||||
|
||||
type gpuMap struct {
|
||||
By string
|
||||
Modes []struct{ Key, Label string }
|
||||
Rows []gpuMapRow
|
||||
// Legend says what the colours mean in this mode.
|
||||
Legend string
|
||||
}
|
||||
|
||||
type gpuMapRow struct {
|
||||
Node string
|
||||
Cells []gpuCell
|
||||
}
|
||||
|
||||
type gpuCell struct {
|
||||
Index int
|
||||
// Model and Replica hold the card; empty is a free card.
|
||||
Model, Replica string
|
||||
// Style is the cell's colour, built here from numbers alone, which is
|
||||
// what makes it safe to put in a style attribute; Title says everything
|
||||
// known about the card.
|
||||
Style template.CSS
|
||||
Title string
|
||||
// Label is the figure written in the cell.
|
||||
Label string
|
||||
}
|
||||
|
||||
// idleHolding is a replica holding GPUs that are doing nothing.
|
||||
type idleHolding struct {
|
||||
Model, Replica, Node string
|
||||
GPUs []int
|
||||
// MaxUtil is the busiest of its cards, now.
|
||||
MaxUtil float64
|
||||
// Idle is how long since a request was routed to it, or "" when not
|
||||
// known.
|
||||
Idle string
|
||||
}
|
||||
|
||||
// buildGPUMap draws the map from what the nodes last reported and where
|
||||
// replicas are. Nil for a fleet without GPUs.
|
||||
func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
|
||||
held := heldBy(reps)
|
||||
m := &gpuMap{By: by, Modes: gpuMapModes}
|
||||
if m.By == "" {
|
||||
m.By = "util"
|
||||
}
|
||||
switch m.By {
|
||||
case "temp":
|
||||
m.Legend = "Blue is cool, red is 90 °C and over."
|
||||
case "memory":
|
||||
m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
|
||||
default:
|
||||
m.By = "util"
|
||||
m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.GPUTotal == 0 {
|
||||
continue
|
||||
}
|
||||
row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
|
||||
for i := range n.GPUTotal {
|
||||
c := gpuCell{Index: i}
|
||||
if h, ok := held[gpuKey(n.ID, i)]; ok {
|
||||
c.Model, c.Replica = h.model, h.replica
|
||||
}
|
||||
var title []string
|
||||
title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
|
||||
if c.Model != "" {
|
||||
title = append(title, c.Model+" ("+c.Replica+")")
|
||||
} else {
|
||||
title = append(title, "free")
|
||||
}
|
||||
s, sampled := sensorOf(n, i)
|
||||
if sampled {
|
||||
if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
|
||||
title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
|
||||
}
|
||||
if s.MemoryTotalMB > 0 {
|
||||
title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
|
||||
}
|
||||
if s.TempC > 0 {
|
||||
title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
|
||||
}
|
||||
if s.PowerW > 0 {
|
||||
title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
|
||||
}
|
||||
if len(s.Throttle) > 0 {
|
||||
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
|
||||
}
|
||||
}
|
||||
c.Title = strings.Join(title, " · ")
|
||||
style, label := cellColour(m.By, s, sampled)
|
||||
if c.Model == "" {
|
||||
style += ";border-style:dashed"
|
||||
}
|
||||
c.Style, c.Label = template.CSS(style), label
|
||||
row.Cells = append(row.Cells, c)
|
||||
}
|
||||
m.Rows = append(m.Rows, row)
|
||||
}
|
||||
if len(m.Rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// cellColour is a cell's background, and the figure written in it.
|
||||
func cellColour(by string, s sensorView, sampled bool) (string, string) {
|
||||
const unknown = "background:var(--panel-2,#eee)"
|
||||
switch by {
|
||||
case "temp":
|
||||
if !sampled || s.TempC <= 0 {
|
||||
return unknown, ""
|
||||
}
|
||||
// 30 °C blue to 90 °C red.
|
||||
f := clamp01((s.TempC - 30) / 60)
|
||||
return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
|
||||
case "memory":
|
||||
if !sampled || s.MemoryTotalMB <= 0 {
|
||||
return unknown, ""
|
||||
}
|
||||
f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
|
||||
return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
|
||||
}
|
||||
if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
|
||||
return unknown, ""
|
||||
}
|
||||
f := clamp01(s.UtilPercent / 100)
|
||||
return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
|
||||
}
|
||||
|
||||
// idleHoldings are the replicas whose GPUs all read under 5% busy now:
|
||||
// capacity held and not asked for, the first place to find room.
|
||||
func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
|
||||
byID := map[string]adminapi.NodeView{}
|
||||
for _, n := range nodes {
|
||||
byID[n.ID] = n
|
||||
}
|
||||
var out []idleHolding
|
||||
for _, r := range reps {
|
||||
if r.State != "ready" {
|
||||
continue
|
||||
}
|
||||
for _, sh := range r.Shards {
|
||||
if len(sh.GPUs) == 0 {
|
||||
continue
|
||||
}
|
||||
h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
|
||||
idle, known := true, true
|
||||
for _, g := range sh.GPUs {
|
||||
s, ok := sensorOf(byID[sh.Node], g)
|
||||
if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
|
||||
known = false
|
||||
break
|
||||
}
|
||||
h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
|
||||
if s.UtilPercent >= 5 {
|
||||
idle = false
|
||||
}
|
||||
}
|
||||
if !known || !idle {
|
||||
continue
|
||||
}
|
||||
if r.IdleSec >= 0 {
|
||||
h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
|
||||
return out
|
||||
}
|
||||
|
||||
type holder struct{ model, replica string }
|
||||
|
||||
func heldBy(reps []adminapi.ReplicaView) map[string]holder {
|
||||
out := map[string]holder{}
|
||||
for _, r := range reps {
|
||||
switch r.State {
|
||||
case "stopped", "failed":
|
||||
continue
|
||||
}
|
||||
for _, sh := range r.Shards {
|
||||
for _, g := range sh.GPUs {
|
||||
out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
|
||||
|
||||
// sensorView is a card's last sample.
|
||||
type sensorView = struct {
|
||||
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
|
||||
Throttle []string
|
||||
}
|
||||
|
||||
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
|
||||
if n.Load == nil {
|
||||
return sensorView{}, false
|
||||
}
|
||||
for _, s := range n.Load.GPUs {
|
||||
if s.Index == gpu {
|
||||
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
|
||||
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
|
||||
}
|
||||
}
|
||||
return sensorView{}, false
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// mapFleet is a node of four cards: chat holds two and is idle, code holds
|
||||
// one and is busy, one is free.
|
||||
type mapFleet struct{ fakeSource }
|
||||
|
||||
func (mapFleet) Nodes() []adminapi.NodeView {
|
||||
busy := func(i int, util, temp float64) inventory.GPUSensor {
|
||||
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
|
||||
}
|
||||
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
|
||||
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
|
||||
}
|
||||
|
||||
func (mapFleet) Replicas(string) []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{
|
||||
{ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
|
||||
{ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
|
||||
_, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
|
||||
t.Errorf("%d cards on the map, want 4", n)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"gpu07 GPU 2 · code (code-1) · 93% busy",
|
||||
"gpu07 GPU 3 · free",
|
||||
"border-style:dashed",
|
||||
"Cards held and doing nothing",
|
||||
"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page lacks %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, "<code>code-1</code></td><td><code>gpu07</code>") {
|
||||
t.Error("a busy replica was listed as idle")
|
||||
}
|
||||
_, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
|
||||
if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
|
||||
t.Error("the map coloured by temperature does not say so, or show it")
|
||||
}
|
||||
_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
if strings.Contains(cpu, "gpu-map") {
|
||||
t.Error("a fleet without GPUs has a GPU map")
|
||||
}
|
||||
}
|
||||
@@ -862,3 +862,10 @@ table.rows td.warn { color: var(--warn); cursor: help; }
|
||||
.held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; }
|
||||
.held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; }
|
||||
.held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; }
|
||||
|
||||
/* The GPU map: a square per card, a row per node. */
|
||||
.gpu-map { display: flex; flex-direction: column; gap: 3px; margin: 6px 0; overflow-x: auto; }
|
||||
.gpu-row { display: flex; align-items: center; gap: 3px; }
|
||||
.gpu-node { width: 7em; flex: none; font-size: 12px; color: var(--muted, #666); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.gpu-cell { width: 26px; height: 26px; flex: none; border: 1px solid rgba(0,0,0,.18); border-radius: 4px;
|
||||
font-size: 10px; line-height: 26px; text-align: center; color: rgba(0,0,0,.7); cursor: default; }
|
||||
@@ -49,6 +49,25 @@
|
||||
</div>
|
||||
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
|
||||
|
||||
{{with .GPUMap}}
|
||||
<h2>GPUs <span class="opt">by
|
||||
{{range .Modes}}{{if eq .Key $.GPUMap.By}}<strong>{{.Label}}</strong>{{else}}<a href="?gpumap={{.Key}}">{{.Label}}</a>{{end}} {{end}}</span></h2>
|
||||
<div class="gpu-map">
|
||||
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
|
||||
{{end}}</div>
|
||||
<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
|
||||
{{end}}
|
||||
{{with .IdleGPUs}}
|
||||
<h3>Cards held and doing nothing</h3>
|
||||
{{/* The first place to find room: replicas whose every card reads under 5%
|
||||
busy now. A model kept warm on purpose is here too; whether it should
|
||||
be is the operator's call. */}}
|
||||
<table class="rows"><tr><th>Model</th><th>Replica</th><th>Node</th><th class="num">GPUs</th><th class="num">Busiest card</th><th>Last request</th></tr>
|
||||
{{range .}}<tr><td>{{.Model}}</td><td class="clip"><code>{{.Replica}}</code></td><td><code>{{.Node}}</code></td>
|
||||
<td class="num">{{len .GPUs}}</td><td class="num">{{printf "%.0f" .MaxUtil}}%</td><td>{{if .Idle}}{{.Idle}} ago{{else}}<span class="note">not known</span>{{end}}</td></tr>
|
||||
{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
|
||||
{{if .Nodes}}
|
||||
|
||||
Reference in new issue
Block a user