llmgw/internal/web/dashboard.go

package web

import (
	"context"
	"html/template"
	"log/slog"
	"net/http"
	"slices"
	"strconv"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// DashboardData is the whole page.
type DashboardData struct {
	// CPU is the fleet's capacity when it has no GPUs, which is shown instead
	// of a GPU table that would only say there are none.
	CPU cpuCapacity
	// Menu is the shared top menu; Nav below is what the cluster page adds.
	Menu Nav
	// GPUMap is every card coloured by one figure, and IdleGPUs the replicas
	// holding cards that are doing nothing; both from the last heartbeats.
	GPUMap   *gpuMap
	IdleGPUs []idleHolding
	// Efficiency is what each model gets from its cards, from the metrics
	// store; nil without one.
	Efficiency []modelEfficiency
	// Notice and Problem are what the last change made from this page did,
	// carried over the redirect back to it.
	Notice, Problem string
	Nodes           []adminapi.NodeView
	Models          []adminapi.ModelView
	Replicas        []adminapi.ReplicaView
	Unplaced        []adminapi.ExplainView
	Usage           []store.KeyUsage
	Summary         adminapi.Summary
	// Fabrics is the physical fabrics and what disagrees about them, from a
	// source that can say.
	Fabrics *fabrics.Report
	// NVLink is the multi-node NVLink domains, from a source that can say.
	NVLink *nvlink.Report

	Users   []store.UserUsage
	Models2 []store.ModelUsage

	// Tiles are the headline figures, which are numbers rather than charts.
	Requests24h  int64
	Tokens24h    int64
	Errors24h    int64
	Spend24h     int64
	RequestSpark template.HTML
	TokenSpark   template.HTML

	HasPeople bool

	// NodePrefix is what every node's name starts with, left off in tables
	// so a row fits on a line; the full name is on hover.
	NodePrefix string
	// Columns shown only when some row has something in them: a CPU-only
	// cluster's nodes table was three columns of dashes wide.
	AnyGPUs, AnyClasses, AnyFabrics, AnyFallbacks bool

	// Charts are pre-rendered rather than assembled in the template: building
	// SVG out of template fragments is how a chart ends up unescaped.
	RequestChart template.HTML
	TokenChart   template.HTML
	LatencyChart template.HTML
	HasCharts    bool

	NodeCount     int
	ModelCount    int
	ReadyReplicas int
	TotalReplicas int

	AuthEnabled bool
	// OnAdminListener decides which links the page offers.
	//
	// The same page is served twice, and the two have different neighbours:
	// metrics, pprof and the management API exist only on the admin listener,
	// while account administration and the profile exist only on the
	// client-facing one. A nav that assumes one of them sends every visitor to
	// the other into a 404 — or, on the public port, into "no api key was
	// provided", which is a far more confusing way to say the same thing.
	OnAdminListener bool
	// Fleet is the operational charts, nil when no metrics store is configured.
	Fleet *FleetCharts

	// Nav says which optional admin pages are mounted. Asked rather than
	// assumed: a link to a page that was never registered lands on whatever the
	// catch-all route serves, which on the client-facing port is the landing
	// page — an admin clicking "mirror" and getting the marketing copy has no
	// way to tell that from a bug.
	Nav        NavExtras
	RefreshSec int
	// RefreshMeta is the reload for a browser without script.
	RefreshMeta template.HTML
	Version     string
	Now         string
}

// stateWords are every state a node, a replica, a shard or an engine's
// installation can be in. Each is a CSS class and a word; the class stays as
// it is and the word is looked up, as dash.state.<state>. A state not listed
// here shows its key, which the catalogue test is there to stop.
var stateWords = []string{
	"ready", "cordoned", "draining", "unreachable", "lost", "unknown",
	"placing", "pending", "starting", "loading", "stopped", "failed",
	"installing", "unsupported", "skipped",
}

func init() {
	for _, s := range stateWords {
		registerKeys("dash.state." + s)
	}
	// How bad a fabric's or an NVLink domain's problem is.
	registerKeys("fabric.level.fail", "fabric.level.warn", "fabric.level.info")
}

// refreshMeta is the meta element that reloads a page every sec seconds,
// for the <noscript> in its head.
//
// Built here rather than written in the template because the inside of a
// <noscript> is text to an HTML tokenizer, and the catalogue check would
// read the element's attribute names there as English left on the page.
func refreshMeta(sec int) template.HTML {
	return template.HTML(`<meta http-equiv="refresh" content="` + strconv.Itoa(sec) + `">`) //nolint:gosec // a number
}

// Source is what the dashboard reads. An interface so the page can be rendered
// in a test without a control plane.
type Source interface {
	Nodes() []adminapi.NodeView
	Models() []adminapi.ModelView
	Replicas(model string) []adminapi.ReplicaView
	Explain(model string) adminapi.ExplainView
	// Summary is the fleet in aggregate: capacity per GPU class, free whole
	// nodes per fabric, what is idle.
	//
	// The tables below say what exists; this says how much of it is in use,
	// which is a different question and the one asked first. Deriving it in the
	// page would mean two implementations of the same arithmetic drifting
	// apart, so it comes from where ctl status gets it.
	Summary() adminapi.Summary
}

// FabricSource is a Source that can say which physical fabrics the adapters
// are on. Optional, so a source without one still draws the page.
type FabricSource interface {
	Fabrics() fabrics.Report
}

// UsageSource is the accounting half, absent when authentication is off.
type UsageSource interface {
	UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
	UsageBuckets(ctx context.Context, since time.Time, width time.Duration) ([]store.Bucket, error)
	// UsageByUser and UsageByModel answer the two questions a key-by-key table
	// answers badly: who is using this cluster, and what are they running. One
	// person holds a laptop key, a CI key and a notebook key, and adding three
	// rows up by eye is a chore that gets skipped.
	UsageByUser(ctx context.Context, since time.Time) ([]store.UserUsage, error)
	UsageByModel(ctx context.Context, since time.Time) ([]store.ModelUsage, error)
}

// Dashboard serves the read-only cluster view.
//
// It belongs on the admin listener and nowhere else. That listener is bound to
// loopback by default and also serves pprof, which can dump the process heap —
// where API keys live in plaintext. Exposing this page publicly would mean
// exposing that, so reaching it from elsewhere is an SSH tunnel's job:
//
//	ssh -L 9090:127.0.0.1:9090 llmgw-ctl
//
// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`,
// where it needs a shell on the machine rather than a browser session this
// project has no way to authenticate.
// NavExtras says which optional admin pages exist.
//
// A struct rather than another bool parameter: Dashboard already takes one, and
// two adjacent booleans at a call site are swapped silently and discovered by
// somebody clicking the wrong link.
type NavExtras struct {
	// Mirror is true when /admin/mirror is mounted.
	Mirror bool
	// History reads the cluster's past for the operational charts. Nil leaves
	// them off the page rather than drawing empty frames.
	History History
	// Window is how far back those charts look.
	Window time.Duration
}

func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
	onAdminListener bool, nav NavExtras, log *slog.Logger,
) http.HandlerFunc {
	if log == nil {
		log = slog.Default()
	}
	if refresh <= 0 {
		refresh = 5 * time.Second
	}
	return func(w http.ResponseWriter, r *http.Request) {
		l := langOf(r)
		d := DashboardData{
			Menu:            navFrom(r, "cluster"),
			OnAdminListener: onAdminListener,
			Nav:             nav,
			Fleet:           l.loadFleetCharts(r.Context(), nav.History, nav.Window, log),
			RefreshSec:      int(refresh.Seconds()),
			RefreshMeta:     refreshMeta(int(refresh.Seconds())),
			Version:         version,
			Now:             time.Now().UTC().Format(time.RFC3339),
			AuthEnabled:     usage != nil,
		}
		if d.Menu.Admin {
			d.Notice, d.Problem = r.URL.Query().Get("notice"), r.URL.Query().Get("problem")
		}
		if src != nil {
			// A copy, since the ages are rewritten in the reader's units
			// and the source's slice may be what it hands everybody.
			d.Nodes = slices.Clone(src.Nodes())
			for i := range d.Nodes {
				d.Nodes[i].LastSeenAgo = l.clockText(d.Nodes[i].LastSeenAgo)
			}
			d.CPU = cpuCapacityOf(d.Nodes)
			d.Models = src.Models()
			d.Replicas = src.Replicas("")
			d.NodeCount = len(d.Nodes)
			d.ModelCount = len(d.Models)
			d.TotalReplicas = len(d.Replicas)
			for _, rep := range d.Replicas {
				if rep.State == "ready" {
					d.ReadyReplicas++
				}
			}
			d.Unplaced = unplaced(src, d.Models)
			ids := make([]string, 0, len(d.Nodes))
			for _, n := range d.Nodes {
				ids = append(ids, n.ID)
				d.AnyGPUs = d.AnyGPUs || n.GPUTotal > 0
				d.AnyClasses = d.AnyClasses || len(n.Classes) > 0
				d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
			}
			d.NodePrefix = sharedPrefix(ids)
			if d.AnyGPUs {
				d.GPUMap = buildGPUMap(l, d.Nodes, d.Replicas, r.URL.Query().Get("gpumap"), d.NodePrefix)
				d.IdleGPUs = idleHoldings(l, d.Nodes, d.Replicas, d.NodePrefix)
			}
			for _, m := range d.Models {
				d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
			}
			if d.AnyGPUs {
				d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window)
				d.Efficiency = loadEfficiency(r.Context(), nav.History)
			}
			d.Summary = src.Summary()
			if fs, ok := src.(FabricSource); ok {
				r := fs.Fabrics()
				d.Fabrics = &r
			}
			if ns, ok := src.(NVLinkSource); ok {
				if r := ns.NVLink(); len(r.Domains) > 0 {
					d.NVLink = &r
				}
			}
		}
		if usage != nil {
			rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
			if err != nil {
				// A reporting query failing must not take the page with it: the
				// cluster state above is what someone is most likely here for.
				log.Warn("could not read usage for the dashboard", "err", err)
			}
			d.Usage = rows
			d.addCharts(r.Context(), l, usage, log)
			d.addPeopleAndModels(r.Context(), usage, log)
		}
		// Only ever shown to an administrator, whatever wrapped it.
		d.Menu.Admin = true
		d.Menu.Mirror = d.Menu.Mirror || nav.Mirror
		render(w, r, "dashboard", d)
	}
}

// addPeopleAndModels answers who is using the cluster and what they are running.
//
// Both are ranked rather than plotted over time: the question is which person
// and which model, and a reader comparing names wants them beside each other
// rather than threaded through a time axis they then have to untangle.
//
// A reporting query failing leaves the section out rather than the page: the
// cluster state above it is what someone is most likely here for.
func (d *DashboardData) addPeopleAndModels(ctx context.Context, usage UsageSource, log *slog.Logger) {
	since := time.Now().Add(-24 * time.Hour)

	users, err := usage.UsageByUser(ctx, since)
	if err != nil {
		log.Warn("could not read per-user usage", "err", err)
	} else {
		d.Users = users
		d.HasPeople = len(users) > 0
	}

	models, err := usage.UsageByModel(ctx, since)
	if err != nil {
		log.Warn("could not read per-model usage", "err", err)
		return
	}
	d.Models2 = models
}

// addCharts draws the last day of traffic.
//
// Hourly, over 24 hours: fine enough to show a working day and coarse enough
// that one slow minute does not look like an incident.
func (d *DashboardData) addCharts(ctx context.Context, l *Lang, usage UsageSource, log *slog.Logger) {
	buckets, err := usage.UsageBuckets(ctx, time.Now().Add(-24*time.Hour), time.Hour)
	if err != nil {
		log.Warn("could not read usage buckets for the dashboard", "err", err)
		return
	}
	if len(buckets) == 0 {
		return
	}

	var requests, tokens, latency Series
	requests.Colour, tokens.Colour, latency.Colour = "--accent", "--ok", "--warn"
	for _, b := range buckets {
		requests.Values = append(requests.Values, float64(b.Requests))
		requests.Labels = append(requests.Labels, l.hourLabel(b.Start, float64(b.Requests), l.T("dash.unit.requests")))

		tok := float64(b.Prompt + b.Completion)
		tokens.Values = append(tokens.Values, tok)
		tokens.Labels = append(tokens.Labels, l.hourLabel(b.Start, tok, l.T("dash.unit.tokens")))

		latency.Values = append(latency.Values, float64(b.LatencyMs))
		latency.Labels = append(latency.Labels, l.hourLabel(b.Start, float64(b.LatencyMs), l.T("dash.unit.ms_mean")))
	}

	d.RequestChart = l.BarChart(requests, l.T("dash.unit.requests_h"))
	d.TokenChart = l.BarChart(tokens, l.T("dash.unit.tokens_h"))
	d.LatencyChart = l.BarChart(latency, "ms")
	d.HasCharts = true

	// The headline figures. Numbers rather than charts, because a single
	// current value is a number — plotting one is how a chart ends up with one
	// bar in it. The sparkline beside each says only whether it is rising,
	// falling or flat, which is the one thing the number cannot say.
	for _, b := range buckets {
		d.Requests24h += b.Requests
		d.Tokens24h += b.Prompt + b.Completion
		d.Errors24h += b.Errors
		d.Spend24h += b.CostMicros
	}
	d.RequestSpark = l.Sparkline(requests.Values, "--viz-1")
	d.TokenSpark = l.Sparkline(tokens.Values, "--viz-3")
}

// unplaced collects the placement explanations for models short of replicas.
//
// Only for those: asking why a satisfied model was not placed produces an empty
// answer, and a page full of empty answers hides the one that matters.
func unplaced(src Source, models []adminapi.ModelView) []adminapi.ExplainView {
	var out []adminapi.ExplainView
	for _, m := range models {
		if m.Ready >= m.Min {
			continue
		}
		if e := src.Explain(m.Name); len(e.Rejections) > 0 {
			out = append(out, e)
		}
	}
	return out
}

// cpuCapacity is what a fleet without GPUs has to offer.
type cpuCapacity struct {
	Nodes, Ready          int
	MemoryGB              int
	DiskFreeGB, DiskTotal int
}

func cpuCapacityOf(nodes []adminapi.NodeView) cpuCapacity {
	var c cpuCapacity
	for _, n := range nodes {
		c.Nodes++
		if n.State == "ready" {
			c.Ready++
		}
		c.MemoryGB += n.MemoryGB
		c.DiskFreeGB += n.DiskFreeGB
		c.DiskTotal += n.DiskTotalGB
	}
	return c
}