Files
llmgw/internal/web/dashboard.go
T
Ilker ManapandClaude Opus 5.5 b6a802c03b Map the fleet's GPUs, and name the cards held and idle
The cluster page draws every card coloured by utilisation, memory or
temperature, and lists replicas whose every card is under 5% busy.
llmgw_gpu_assigned says which replica, model and owner hold each card.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 15:54:58 +02:00

355 lines
12 KiB
Go

package web
import (
"context"
"html/template"
"log/slog"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// DashboardData is the whole page.
type DashboardData struct {
// CPU is the fleet's capacity when it has no GPUs, which is shown instead
// of a GPU table that would only say there are none.
CPU cpuCapacity
// Menu is the shared top menu; Nav below is what the cluster page adds.
Menu Nav
// GPUMap is every card coloured by one figure, and IdleGPUs the replicas
// holding cards that are doing nothing; both from the last heartbeats.
GPUMap *gpuMap
IdleGPUs []idleHolding
// Notice and Problem are what the last change made from this page did,
// carried over the redirect back to it.
Notice, Problem string
Nodes []adminapi.NodeView
Models []adminapi.ModelView
Replicas []adminapi.ReplicaView
Unplaced []adminapi.ExplainView
Usage []store.KeyUsage
Summary adminapi.Summary
// Fabrics is the physical fabrics and what disagrees about them, from a
// source that can say.
Fabrics *fabrics.Report
// NVLink is the multi-node NVLink domains, from a source that can say.
NVLink *nvlink.Report
Users []store.UserUsage
Models2 []store.ModelUsage
// Tiles are the headline figures, which are numbers rather than charts.
Requests24h int64
Tokens24h int64
Errors24h int64
Spend24h int64
RequestSpark template.HTML
TokenSpark template.HTML
HasPeople bool
// NodePrefix is what every node's name starts with, left off in tables
// so a row fits on a line; the full name is on hover.
NodePrefix string
// Columns shown only when some row has something in them: a CPU-only
// cluster's nodes table was three columns of dashes wide.
AnyGPUs, AnyClasses, AnyFabrics, AnyFallbacks bool
// Charts are pre-rendered rather than assembled in the template: building
// SVG out of template fragments is how a chart ends up unescaped.
RequestChart template.HTML
TokenChart template.HTML
LatencyChart template.HTML
HasCharts bool
NodeCount int
ModelCount int
ReadyReplicas int
TotalReplicas int
AuthEnabled bool
// OnAdminListener decides which links the page offers.
//
// The same page is served twice, and the two have different neighbours:
// metrics, pprof and the management API exist only on the admin listener,
// while account administration and the profile exist only on the
// client-facing one. A nav that assumes one of them sends every visitor to
// the other into a 404 — or, on the public port, into "no api key was
// provided", which is a far more confusing way to say the same thing.
OnAdminListener bool
// Fleet is the operational charts, nil when no metrics store is configured.
Fleet *FleetCharts
// Nav says which optional admin pages are mounted. Asked rather than
// assumed: a link to a page that was never registered lands on whatever the
// catch-all route serves, which on the client-facing port is the landing
// page — an admin clicking "mirror" and getting the marketing copy has no
// way to tell that from a bug.
Nav NavExtras
RefreshSec int
Version string
Now string
}
// Source is what the dashboard reads. An interface so the page can be rendered
// in a test without a control plane.
type Source interface {
Nodes() []adminapi.NodeView
Models() []adminapi.ModelView
Replicas(model string) []adminapi.ReplicaView
Explain(model string) adminapi.ExplainView
// Summary is the fleet in aggregate: capacity per GPU class, free whole
// nodes per fabric, what is idle.
//
// The tables below say what exists; this says how much of it is in use,
// which is a different question and the one asked first. Deriving it in the
// page would mean two implementations of the same arithmetic drifting
// apart, so it comes from where ctl status gets it.
Summary() adminapi.Summary
}
// FabricSource is a Source that can say which physical fabrics the adapters
// are on. Optional, so a source without one still draws the page.
type FabricSource interface {
Fabrics() fabrics.Report
}
// UsageSource is the accounting half, absent when authentication is off.
type UsageSource interface {
UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
UsageBuckets(ctx context.Context, since time.Time, width time.Duration) ([]store.Bucket, error)
// UsageByUser and UsageByModel answer the two questions a key-by-key table
// answers badly: who is using this cluster, and what are they running. One
// person holds a laptop key, a CI key and a notebook key, and adding three
// rows up by eye is a chore that gets skipped.
UsageByUser(ctx context.Context, since time.Time) ([]store.UserUsage, error)
UsageByModel(ctx context.Context, since time.Time) ([]store.ModelUsage, error)
}
// Dashboard serves the read-only cluster view.
//
// It belongs on the admin listener and nowhere else. That listener is bound to
// loopback by default and also serves pprof, which can dump the process heap —
// where API keys live in plaintext. Exposing this page publicly would mean
// exposing that, so reaching it from elsewhere is an SSH tunnel's job:
//
// ssh -L 9090:127.0.0.1:9090 llmgw-ctl
//
// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`,
// where it needs a shell on the machine rather than a browser session this
// project has no way to authenticate.
// NavExtras says which optional admin pages exist.
//
// A struct rather than another bool parameter: Dashboard already takes one, and
// two adjacent booleans at a call site are swapped silently and discovered by
// somebody clicking the wrong link.
type NavExtras struct {
// Mirror is true when /admin/mirror is mounted.
Mirror bool
// History reads the cluster's past for the operational charts. Nil leaves
// them off the page rather than drawing empty frames.
History History
// Window is how far back those charts look.
Window time.Duration
}
func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
onAdminListener bool, nav NavExtras, log *slog.Logger,
) http.HandlerFunc {
if log == nil {
log = slog.Default()
}
if refresh <= 0 {
refresh = 5 * time.Second
}
return func(w http.ResponseWriter, r *http.Request) {
d := DashboardData{
Menu: navFrom(r, "cluster"),
OnAdminListener: onAdminListener,
Nav: nav,
Fleet: LoadFleetCharts(r.Context(), nav.History, nav.Window, log),
RefreshSec: int(refresh.Seconds()),
Version: version,
Now: time.Now().UTC().Format(time.RFC3339),
AuthEnabled: usage != nil,
}
if d.Menu.Admin {
d.Notice, d.Problem = r.URL.Query().Get("notice"), r.URL.Query().Get("problem")
}
if src != nil {
d.Nodes = src.Nodes()
d.CPU = cpuCapacityOf(d.Nodes)
d.Models = src.Models()
d.Replicas = src.Replicas("")
d.NodeCount = len(d.Nodes)
d.ModelCount = len(d.Models)
d.TotalReplicas = len(d.Replicas)
for _, rep := range d.Replicas {
if rep.State == "ready" {
d.ReadyReplicas++
}
}
d.Unplaced = unplaced(src, d.Models)
ids := make([]string, 0, len(d.Nodes))
for _, n := range d.Nodes {
ids = append(ids, n.ID)
d.AnyGPUs = d.AnyGPUs || n.GPUTotal > 0
d.AnyClasses = d.AnyClasses || len(n.Classes) > 0
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
}
d.NodePrefix = sharedPrefix(ids)
if d.AnyGPUs {
d.GPUMap = buildGPUMap(d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
d.IdleGPUs = idleHoldings(d.Nodes, d.Replicas, d.NodePrefix)
}
for _, m := range d.Models {
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
}
if d.AnyGPUs {
d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window)
}
d.Summary = src.Summary()
if fs, ok := src.(FabricSource); ok {
r := fs.Fabrics()
d.Fabrics = &r
}
if ns, ok := src.(NVLinkSource); ok {
if r := ns.NVLink(); len(r.Domains) > 0 {
d.NVLink = &r
}
}
}
if usage != nil {
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
if err != nil {
// A reporting query failing must not take the page with it: the
// cluster state above is what someone is most likely here for.
log.Warn("could not read usage for the dashboard", "err", err)
}
d.Usage = rows
d.addCharts(r.Context(), usage, log)
d.addPeopleAndModels(r.Context(), usage, log)
}
// Only ever shown to an administrator, whatever wrapped it.
d.Menu.Admin = true
d.Menu.Mirror = d.Menu.Mirror || nav.Mirror
render(w, "dashboard", d)
}
}
// addPeopleAndModels answers who is using the cluster and what they are running.
//
// Both are ranked rather than plotted over time: the question is which person
// and which model, and a reader comparing names wants them beside each other
// rather than threaded through a time axis they then have to untangle.
//
// A reporting query failing leaves the section out rather than the page: the
// cluster state above it is what someone is most likely here for.
func (d *DashboardData) addPeopleAndModels(ctx context.Context, usage UsageSource, log *slog.Logger) {
since := time.Now().Add(-24 * time.Hour)
users, err := usage.UsageByUser(ctx, since)
if err != nil {
log.Warn("could not read per-user usage", "err", err)
} else {
d.Users = users
d.HasPeople = len(users) > 0
}
models, err := usage.UsageByModel(ctx, since)
if err != nil {
log.Warn("could not read per-model usage", "err", err)
return
}
d.Models2 = models
}
// addCharts draws the last day of traffic.
//
// Hourly, over 24 hours: fine enough to show a working day and coarse enough
// that one slow minute does not look like an incident.
func (d *DashboardData) addCharts(ctx context.Context, usage UsageSource, log *slog.Logger) {
buckets, err := usage.UsageBuckets(ctx, time.Now().Add(-24*time.Hour), time.Hour)
if err != nil {
log.Warn("could not read usage buckets for the dashboard", "err", err)
return
}
if len(buckets) == 0 {
return
}
var requests, tokens, latency Series
requests.Colour, tokens.Colour, latency.Colour = "--accent", "--ok", "--warn"
for _, b := range buckets {
requests.Values = append(requests.Values, float64(b.Requests))
requests.Labels = append(requests.Labels, hourLabel(b.Start, float64(b.Requests), "requests"))
tok := float64(b.Prompt + b.Completion)
tokens.Values = append(tokens.Values, tok)
tokens.Labels = append(tokens.Labels, hourLabel(b.Start, tok, "tokens"))
latency.Values = append(latency.Values, float64(b.LatencyMs))
latency.Labels = append(latency.Labels, hourLabel(b.Start, float64(b.LatencyMs), "ms mean"))
}
d.RequestChart = BarChart(requests, "requests/h")
d.TokenChart = BarChart(tokens, "tokens/h")
d.LatencyChart = BarChart(latency, "ms")
d.HasCharts = true
// The headline figures. Numbers rather than charts, because a single
// current value is a number — plotting one is how a chart ends up with one
// bar in it. The sparkline beside each says only whether it is rising,
// falling or flat, which is the one thing the number cannot say.
for _, b := range buckets {
d.Requests24h += b.Requests
d.Tokens24h += b.Prompt + b.Completion
d.Errors24h += b.Errors
d.Spend24h += b.CostMicros
}
d.RequestSpark = Sparkline(requests.Values, "--viz-1")
d.TokenSpark = Sparkline(tokens.Values, "--viz-3")
}
// unplaced collects the placement explanations for models short of replicas.
//
// Only for those: asking why a satisfied model was not placed produces an empty
// answer, and a page full of empty answers hides the one that matters.
func unplaced(src Source, models []adminapi.ModelView) []adminapi.ExplainView {
var out []adminapi.ExplainView
for _, m := range models {
if m.Ready >= m.Min {
continue
}
if e := src.Explain(m.Name); len(e.Rejections) > 0 {
out = append(out, e)
}
}
return out
}
// cpuCapacity is what a fleet without GPUs has to offer.
type cpuCapacity struct {
Nodes, Ready int
MemoryGB int
DiskFreeGB, DiskTotal int
}
func cpuCapacityOf(nodes []adminapi.NodeView) cpuCapacity {
var c cpuCapacity
for _, n := range nodes {
c.Nodes++
if n.State == "ready" {
c.Ready++
}
c.MemoryGB += n.MemoryGB
c.DiskFreeGB += n.DiskFreeGB
c.DiskTotal += n.DiskTotalGB
}
return c
}