Files
llmgw/internal/web/gpumap.go
T
Ilker ManapandClaude Opus 5.5 b6a802c03b Map the fleet's GPUs, and name the cards held and idle
The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 15:54:58 +02:00

233 lines
6.5 KiB
Go

package web
import (
"fmt"
"html/template"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// The GPU map is every card in the fleet, a square each, a row per node,
// coloured by one figure: how busy, how hot, or how full its memory is. A
// hundred cards in a table are a hundred rows to read; on the map the one
// that is not like the others is seen before anything is read.
// gpuMapModes are what the map can be coloured by.
var gpuMapModes = []struct{ Key, Label string }{
{"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
}
type gpuMap struct {
By string
Modes []struct{ Key, Label string }
Rows []gpuMapRow
// Legend says what the colours mean in this mode.
Legend string
}
type gpuMapRow struct {
Node string
Cells []gpuCell
}
type gpuCell struct {
Index int
// Model and Replica hold the card; empty is a free card.
Model, Replica string
// Style is the cell's colour, built here from numbers alone, which is
// what makes it safe to put in a style attribute; Title says everything
// known about the card.
Style template.CSS
Title string
// Label is the figure written in the cell.
Label string
}
// idleHolding is a replica holding GPUs that are doing nothing.
type idleHolding struct {
Model, Replica, Node string
GPUs []int
// MaxUtil is the busiest of its cards, now.
MaxUtil float64
// Idle is how long since a request was routed to it, or "" when not
// known.
Idle string
}
// buildGPUMap draws the map from what the nodes last reported and where
// replicas are. Nil for a fleet without GPUs.
func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
held := heldBy(reps)
m := &gpuMap{By: by, Modes: gpuMapModes}
if m.By == "" {
m.By = "util"
}
switch m.By {
case "temp":
m.Legend = "Blue is cool, red is 90 °C and over."
case "memory":
m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
default:
m.By = "util"
m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
}
for _, n := range nodes {
if n.GPUTotal == 0 {
continue
}
row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
for i := range n.GPUTotal {
c := gpuCell{Index: i}
if h, ok := held[gpuKey(n.ID, i)]; ok {
c.Model, c.Replica = h.model, h.replica
}
var title []string
title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
if c.Model != "" {
title = append(title, c.Model+" ("+c.Replica+")")
} else {
title = append(title, "free")
}
s, sampled := sensorOf(n, i)
if sampled {
if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
}
if s.MemoryTotalMB > 0 {
title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
}
if s.TempC > 0 {
title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
}
if s.PowerW > 0 {
title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
}
if len(s.Throttle) > 0 {
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
}
}
c.Title = strings.Join(title, " · ")
style, label := cellColour(m.By, s, sampled)
if c.Model == "" {
style += ";border-style:dashed"
}
c.Style, c.Label = template.CSS(style), label
row.Cells = append(row.Cells, c)
}
m.Rows = append(m.Rows, row)
}
if len(m.Rows) == 0 {
return nil
}
return m
}
// cellColour is a cell's background, and the figure written in it.
func cellColour(by string, s sensorView, sampled bool) (string, string) {
const unknown = "background:var(--panel-2,#eee)"
switch by {
case "temp":
if !sampled || s.TempC <= 0 {
return unknown, ""
}
// 30 °C blue to 90 °C red.
f := clamp01((s.TempC - 30) / 60)
return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
case "memory":
if !sampled || s.MemoryTotalMB <= 0 {
return unknown, ""
}
f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
}
if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
return unknown, ""
}
f := clamp01(s.UtilPercent / 100)
return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
}
// idleHoldings are the replicas whose GPUs all read under 5% busy now:
// capacity held and not asked for, the first place to find room.
func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
byID := map[string]adminapi.NodeView{}
for _, n := range nodes {
byID[n.ID] = n
}
var out []idleHolding
for _, r := range reps {
if r.State != "ready" {
continue
}
for _, sh := range r.Shards {
if len(sh.GPUs) == 0 {
continue
}
h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
idle, known := true, true
for _, g := range sh.GPUs {
s, ok := sensorOf(byID[sh.Node], g)
if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
known = false
break
}
h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
if s.UtilPercent >= 5 {
idle = false
}
}
if !known || !idle {
continue
}
if r.IdleSec >= 0 {
h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
}
out = append(out, h)
}
}
sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
return out
}
type holder struct{ model, replica string }
func heldBy(reps []adminapi.ReplicaView) map[string]holder {
out := map[string]holder{}
for _, r := range reps {
switch r.State {
case "stopped", "failed":
continue
}
for _, sh := range r.Shards {
for _, g := range sh.GPUs {
out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
}
}
}
return out
}
func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
// sensorView is a card's last sample.
type sensorView = struct {
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
}
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
if n.Load == nil {
return sensorView{}, false
}
for _, s := range n.Load.GPUs {
if s.Index == gpu {
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
}
}
return sensorView{}, false
}