package web
import (
"fmt"
"html/template"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// The GPU map is every card in the fleet, a square each, a row per node,
// coloured by one figure: how busy, how hot, or how full its memory is. A
// hundred cards in a table are a hundred rows to read; on the map the one
// that is not like the others is seen before anything is read.
// gpuMapModes are what the map can be coloured by.
var gpuMapModes = []struct{ Key, Label string }{
{"util", "busy"}, {"memory", "memory"}, {"temp", "temperature"},
}
type gpuMap struct {
By string
Modes []struct{ Key, Label string }
Rows []gpuMapRow
// Legend says what the colours mean in this mode.
Legend string
}
type gpuMapRow struct {
Node string
Cells []gpuCell
}
type gpuCell struct {
Index int
// Model and Replica hold the card; empty is a free card.
Model, Replica string
// Style is the cell's colour, built here from numbers alone, which is
// what makes it safe to put in a style attribute; Title says everything
// known about the card.
Style template.CSS
Title string
// Label is the figure written in the cell.
Label string
// Faulty is a card whose wear needs seeing to.
Faulty bool
}
// idleHolding is a replica holding GPUs that are doing nothing.
type idleHolding struct {
Model, Replica, Node string
GPUs []int
// MaxUtil is the busiest of its cards, now.
MaxUtil float64
// Idle is how long since a request was routed to it, or "" when not
// known.
Idle string
}
// buildGPUMap draws the map from what the nodes last reported and where
// replicas are. Nil for a fleet without GPUs.
func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
held := heldBy(reps)
m := &gpuMap{By: by, Modes: gpuMapModes}
if m.By == "" {
m.By = "util"
}
switch m.By {
case "temp":
m.Legend = "Blue is cool, red is 90 °C and over."
case "memory":
m.Legend = "Pale is empty, dark is full. Engines reserve most of a card's memory when they start."
default:
m.By = "util"
m.Legend = "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds."
}
for _, n := range nodes {
if n.GPUTotal == 0 {
continue
}
row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
for i := range n.GPUTotal {
c := gpuCell{Index: i}
if h, ok := held[gpuKey(n.ID, i)]; ok {
c.Model, c.Replica = h.model, h.replica
}
var title []string
title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
if c.Model != "" {
title = append(title, c.Model+" ("+c.Replica+")")
} else {
title = append(title, "free")
}
s, sampled := sensorOf(n, i)
if sampled {
if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
title = append(title, fmt.Sprintf("%.0f%% busy", s.UtilPercent))
}
if s.MemoryTotalMB > 0 {
title = append(title, fmt.Sprintf("%.1f of %.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024))
}
if s.TempC > 0 {
title = append(title, fmt.Sprintf("%.0f °C", s.TempC))
}
if s.PowerW > 0 {
title = append(title, fmt.Sprintf("%.0f W", s.PowerW))
}
if len(s.Throttle) > 0 {
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
}
title = append(title, s.Problems...)
}
c.Title = strings.Join(title, " · ")
style, label := cellColour(m.By, s, sampled)
if c.Model == "" {
style += ";border-style:dashed"
}
// A card wearing out is outlined whatever the map is coloured
// by: it is the one thing on the map that needs doing.
if len(s.Problems) > 0 {
style += ";border:2px solid #c0392b"
c.Faulty = true
}
c.Style, c.Label = template.CSS(style), label
row.Cells = append(row.Cells, c)
}
m.Rows = append(m.Rows, row)
}
if len(m.Rows) == 0 {
return nil
}
return m
}
// cellColour is a cell's background, and the figure written in it.
func cellColour(by string, s sensorView, sampled bool) (string, string) {
const unknown = "background:var(--panel-2,#eee)"
switch by {
case "temp":
if !sampled || s.TempC <= 0 {
return unknown, ""
}
// 30 °C blue to 90 °C red.
f := clamp01((s.TempC - 30) / 60)
return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
case "memory":
if !sampled || s.MemoryTotalMB <= 0 {
return unknown, ""
}
f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
}
if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
return unknown, ""
}
f := clamp01(s.UtilPercent / 100)
return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
}
// idleHoldings are the replicas whose GPUs all read under 5% busy now:
// capacity held and not asked for, the first place to find room.
func idleHoldings(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
byID := map[string]adminapi.NodeView{}
for _, n := range nodes {
byID[n.ID] = n
}
var out []idleHolding
for _, r := range reps {
if r.State != "ready" {
continue
}
for _, sh := range r.Shards {
if len(sh.GPUs) == 0 {
continue
}
h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
idle, known := true, true
for _, g := range sh.GPUs {
s, ok := sensorOf(byID[sh.Node], g)
if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
known = false
break
}
h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
if s.UtilPercent >= 5 {
idle = false
}
}
if !known || !idle {
continue
}
if r.IdleSec >= 0 {
h.Idle = (time.Duration(r.IdleSec) * time.Second).String()
}
out = append(out, h)
}
}
sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
return out
}
type holder struct{ model, replica string }
func heldBy(reps []adminapi.ReplicaView) map[string]holder {
out := map[string]holder{}
for _, r := range reps {
switch r.State {
case "stopped", "failed":
continue
}
for _, sh := range r.Shards {
for _, g := range sh.GPUs {
out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
}
}
}
return out
}
func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }
// sensorView is a card's last sample.
type sensorView = struct {
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
Problems []string
}
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
if n.Load == nil {
return sensorView{}, false
}
for _, s := range n.Load.GPUs {
if s.Index == gpu {
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
Problems: s.Wear.Problems()}, true
}
}
return sensorView{}, false
}