llmgw/internal/web/gpumap.go

package web

import (
	"fmt"
	"html/template"
	"sort"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)

// The GPU map is every card in the fleet, a square each, a row per node,
// coloured by one figure: how much of the chip is at work, how busy, how
// hot, or how full its memory is. A
// hundred cards in a table are a hundred rows to read; on the map the one
// that is not like the others is seen before anything is read.

// gpuMapModes are what the map can be coloured by; each is named by the key
// dash.gpumap.mode.<mode>.
//
// work is the default. util, the driver's busy figure, is the share of time
// any kernel ran, and a kernel on one SM of a hundred reads 100%: a map of
// it is green wherever a model is loaded and asked anything. work is the
// SMs' share of time with work where the card's counters can be read, and
// its power draw against its limit elsewhere, which rises with how much of
// the chip runs rather than with whether anything does.
var gpuMapModes = []string{"work", "util", "memory", "temp"}

func init() {
	for _, m := range gpuMapModes {
		registerKeys("dash.gpumap.mode." + m)
	}
}

type gpuMap struct {
	By    string
	Modes []struct{ Key, Label string }
	Rows  []gpuMapRow
	// Legend says what the colours mean in this mode.
	Legend string
}

type gpuMapRow struct {
	Node  string
	Cells []gpuCell
}

type gpuCell struct {
	Index int
	// Model and Replica hold the card; empty is a free card.
	Model, Replica string
	// Style is the cell's colour, built here from numbers alone, which is
	// what makes it safe to put in a style attribute; Title says everything
	// known about the card.
	Style template.CSS
	Title string
	// Label is the figure written in the cell.
	Label string
	// Faulty is a card whose wear needs seeing to.
	Faulty bool
}

// idleHolding is a replica holding GPUs that are doing nothing.
type idleHolding struct {
	Model, Replica, Node string
	GPUs                 []int
	// MaxUtil is the busiest of its cards, now.
	MaxUtil float64
	// Idle is how long since a request was routed to it, or "" when not
	// known.
	Idle string
}

// buildGPUMap draws the map from what the nodes last reported and where
// replicas are. Nil for a fleet without GPUs.
func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, prefix string) *gpuMap {
	held := heldBy(reps)
	m := &gpuMap{By: by}
	for _, k := range gpuMapModes {
		m.Modes = append(m.Modes, struct{ Key, Label string }{k, l.T("dash.gpumap.mode." + k)})
	}
	switch m.By {
	case "util":
		m.Legend = l.T("dash.gpumap.legend.util")
	case "temp":
		m.Legend = l.T("dash.gpumap.legend.temp")
	case "memory":
		m.Legend = l.T("dash.gpumap.legend.memory")
	default:
		m.By = "work"
		m.Legend = l.T("dash.gpumap.legend.work")
	}
	for _, n := range nodes {
		if n.GPUTotal == 0 {
			continue
		}
		row := gpuMapRow{Node: strings.TrimPrefix(n.ID, prefix)}
		for i := range n.GPUTotal {
			c := gpuCell{Index: i}
			if h, ok := held[gpuKey(n.ID, i)]; ok {
				c.Model, c.Replica = h.model, h.replica
			}
			var title []string
			title = append(title, fmt.Sprintf("%s GPU %d", n.ID, i))
			if c.Model != "" {
				title = append(title, c.Model+" ("+c.Replica+")")
			} else {
				title = append(title, l.T("dash.gpumap.free"))
			}
			s, sampled := sensorOf(n, i)
			if sampled {
				if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
					title = append(title, l.T("dash.gpumap.busy", l.Num(s.UtilPercent, 0)))
				}
				if p := s.Profile; p != nil && p.SMActive >= 0 {
					title = append(title, l.T("dash.gpumap.sm_active", l.Num(p.SMActive, 0)))
					if p.TensorActive >= 0 {
						title = append(title, l.T("dash.gpumap.tensor_active", l.Num(p.TensorActive, 0)))
					}
				}
				if s.MemoryTotalMB > 0 {
					title = append(title, l.T("dash.gpumap.memory", l.Num(s.MemoryUsedMB/1024, 1), l.Num(s.MemoryTotalMB/1024, 0)))
				}
				if s.TempC > 0 {
					title = append(title, l.Num(s.TempC, 0)+" °C")
				}
				if s.PowerW > 0 && s.PowerLimitW > 0 {
					title = append(title, l.T("dash.gpumap.power_of", l.Num(s.PowerW, 0), l.Num(s.PowerLimitW, 0)))
				} else if s.PowerW > 0 {
					title = append(title, l.Num(s.PowerW, 0)+" W")
				}
				if len(s.Throttle) > 0 {
					// The driver's own names for the reasons, which are
					// the same in every language.
					title = append(title, l.T("dash.gpumap.held_back", strings.Join(s.Throttle, ", ")))
				}
				// What wears a card out is said by the inventory, in English.
				title = append(title, s.Problems...)
			}
			c.Title = strings.Join(title, " · ")
			style, label := cellColour(m.By, s, sampled)
			if c.Model == "" {
				style += ";border-style:dashed"
			}
			// A card wearing out is outlined whatever the map is coloured
			// by: it is the one thing on the map that needs doing.
			if len(s.Problems) > 0 {
				style += ";border:2px solid #c0392b"
				c.Faulty = true
			}
			c.Style, c.Label = template.CSS(style), label
			row.Cells = append(row.Cells, c)
		}
		m.Rows = append(m.Rows, row)
	}
	if len(m.Rows) == 0 {
		return nil
	}
	return m
}

// cellColour is a cell's background, and the figure written in it.
func cellColour(by string, s sensorView, sampled bool) (string, string) {
	const unknown = "background:var(--panel-2,#eee)"
	switch by {
	case "temp":
		if !sampled || s.TempC <= 0 {
			return unknown, ""
		}
		// 30 °C blue to 90 °C red.
		f := clamp01((s.TempC - 30) / 60)
		return fmt.Sprintf("background:hsl(%.0f,70%%,55%%)", 220-220*f), fmt.Sprintf("%.0f", s.TempC)
	case "memory":
		if !sampled || s.MemoryTotalMB <= 0 {
			return unknown, ""
		}
		f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
		return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
	case "work":
		if !sampled {
			return unknown, ""
		}
		var f float64
		switch {
		case s.Profile != nil && s.Profile.SMActive >= 0:
			f = clamp01(s.Profile.SMActive / 100)
		case s.PowerW > 0 && s.PowerLimitW > 0:
			f = clamp01(s.PowerW / s.PowerLimitW)
		default:
			return unknown, ""
		}
		return fmt.Sprintf("background:hsl(200,%.0f%%,%.0f%%)", 15+55*f, 88-50*f), fmt.Sprintf("%.0f", 100*f)
	}
	if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
		return unknown, ""
	}
	f := clamp01(s.UtilPercent / 100)
	return fmt.Sprintf("background:hsl(140,%.0f%%,%.0f%%)", 15+50*f, 88-50*f), fmt.Sprintf("%.0f", s.UtilPercent)
}

// idleHoldings are the replicas whose GPUs all read under 5% busy now:
// capacity held and not asked for, the first place to find room.
func idleHoldings(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView, prefix string) []idleHolding {
	byID := map[string]adminapi.NodeView{}
	for _, n := range nodes {
		byID[n.ID] = n
	}
	var out []idleHolding
	for _, r := range reps {
		if r.State != "ready" {
			continue
		}
		for _, sh := range r.Shards {
			if len(sh.GPUs) == 0 {
				continue
			}
			h := idleHolding{Model: r.Model, Replica: r.ID, Node: strings.TrimPrefix(sh.Node, prefix), GPUs: sh.GPUs}
			idle, known := true, true
			for _, g := range sh.GPUs {
				s, ok := sensorOf(byID[sh.Node], g)
				if !ok || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
					known = false
					break
				}
				h.MaxUtil = max(h.MaxUtil, s.UtilPercent)
				if s.UtilPercent >= 5 {
					idle = false
				}
			}
			if !known || !idle {
				continue
			}
			if r.IdleSec >= 0 {
				h.Idle = l.clock(time.Duration(r.IdleSec) * time.Second)
			}
			out = append(out, h)
		}
	}
	sort.SliceStable(out, func(i, j int) bool { return len(out[i].GPUs) > len(out[j].GPUs) })
	return out
}

// clock is a duration the way Go writes one, 30m0s, with the units in this
// language's letters: 30dk0sn. Exact to the second, unlike Age, because
// how long a replica has sat unused is read against the idle threshold.
func (l *Lang) clock(d time.Duration) string {
	s := d.String()
	if l == english {
		return s
	}
	// Runs of digits are kept, with the decimal point this language
	// writes; each run of letters between them is a unit.
	var b strings.Builder
	unit := func(u string) {
		switch u {
		case "h", "m", "s", "ms":
			b.WriteString(l.T("dash.clock." + u))
		default:
			b.WriteString(u)
		}
	}
	start := 0
	for i, r := range s {
		if (r >= '0' && r <= '9') || r == '.' || r == '-' {
			if start < i {
				unit(s[start:i])
			}
			start = i + 1
			if r == '.' {
				b.WriteString(l.decimal)
			} else {
				b.WriteRune(r)
			}
		}
	}
	unit(s[start:])
	return b.String()
}

func init() {
	registerKeys("dash.clock.h", "dash.clock.m", "dash.clock.s", "dash.clock.ms")
}

// clockText is clock for a duration the control plane has already written
// out, such as a node's last-seen age. What is not a duration is left as it
// is.
func (l *Lang) clockText(s string) string {
	d, err := time.ParseDuration(s)
	if err != nil {
		return s
	}
	return l.clock(d)
}

type holder struct{ model, replica string }

func heldBy(reps []adminapi.ReplicaView) map[string]holder {
	out := map[string]holder{}
	for _, r := range reps {
		switch r.State {
		case "stopped", "failed":
			continue
		}
		for _, sh := range r.Shards {
			for _, g := range sh.GPUs {
				out[gpuKey(sh.Node, g)] = holder{r.Model, r.ID}
			}
		}
	}
	return out
}

func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu) }

// sensorView is a card's last sample.
type sensorView = struct {
	TempC, PowerW, PowerLimitW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
	Throttle                                                             []string
	Problems                                                             []string
	Profile                                                              *inventory.GPUProfile
}

func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
	if n.Load == nil {
		return sensorView{}, false
	}
	for _, s := range n.Load.GPUs {
		if s.Index == gpu {
			return sensorView{TempC: s.TempC, PowerW: s.PowerW, PowerLimitW: s.PowerLimitW, UtilPercent: s.UtilPercent,
				MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
				Problems: s.Wear.Problems(), Profile: s.Profile}, true
		}
	}
	return sensorView{}, false
}