CI / test (push) Successful in 1m32s
The metrics were exported and the only way to look at them was to stand up Prometheus beside the binary. For a project whose whole shape is one static binary with no dependencies, that is the wrong default: the common case is somebody wanting to know how full their cluster is, and that should not cost a monitoring stack. /admin now shows capacity per GPU class with a usage bar, free whole nodes per fabric — the number that decides whether a replica spanning nodes can land, since free cards on the wrong fabric are not capacity for a model that needs this one — and which replicas are holding hardware nobody is asking for. Nodes gain their memory and how many model files they can see. Taken from the same Summary that ctl status uses rather than derived in the page, because two implementations of the same arithmetic drift apart and the one on the screen is the one people would believe. A faulty card counts as neither used nor free, matching the scheduler. Counting it free is how a cluster looks like it has room it does not, and a page that disagreed with placement about that would be worse than no page. Prometheus stays supported and is now the answer to a narrower question: history, alerting, several clusters at once. The dashboard shows what is true now and keeps nothing, which is the honest division between them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
181 lines
6.0 KiB
Go
181 lines
6.0 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"html/template"
|
|
"log/slog"
|
|
"net/http"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// DashboardData is the whole page.
|
|
type DashboardData struct {
|
|
Nodes []adminapi.NodeView
|
|
Models []adminapi.ModelView
|
|
Replicas []adminapi.ReplicaView
|
|
Unplaced []adminapi.ExplainView
|
|
Usage []store.KeyUsage
|
|
Summary adminapi.Summary
|
|
|
|
// Charts are pre-rendered rather than assembled in the template: building
|
|
// SVG out of template fragments is how a chart ends up unescaped.
|
|
RequestChart template.HTML
|
|
TokenChart template.HTML
|
|
LatencyChart template.HTML
|
|
HasCharts bool
|
|
|
|
NodeCount int
|
|
ModelCount int
|
|
ReadyReplicas int
|
|
TotalReplicas int
|
|
|
|
AuthEnabled bool
|
|
// OnAdminListener decides which links the page offers.
|
|
//
|
|
// The same page is served twice, and the two have different neighbours:
|
|
// metrics, pprof and the management API exist only on the admin listener,
|
|
// while account administration and the profile exist only on the
|
|
// client-facing one. A nav that assumes one of them sends every visitor to
|
|
// the other into a 404 — or, on the public port, into "no api key was
|
|
// provided", which is a far more confusing way to say the same thing.
|
|
OnAdminListener bool
|
|
RefreshSec int
|
|
Version string
|
|
Now string
|
|
}
|
|
|
|
// Source is what the dashboard reads. An interface so the page can be rendered
|
|
// in a test without a control plane.
|
|
type Source interface {
|
|
Nodes() []adminapi.NodeView
|
|
Models() []adminapi.ModelView
|
|
Replicas(model string) []adminapi.ReplicaView
|
|
Explain(model string) adminapi.ExplainView
|
|
// Summary is the fleet in aggregate: capacity per GPU class, free whole
|
|
// nodes per fabric, what is idle.
|
|
//
|
|
// The tables below say what exists; this says how much of it is in use,
|
|
// which is a different question and the one asked first. Deriving it in the
|
|
// page would mean two implementations of the same arithmetic drifting
|
|
// apart, so it comes from where ctl status gets it.
|
|
Summary() adminapi.Summary
|
|
}
|
|
|
|
// UsageSource is the accounting half, absent when authentication is off.
|
|
type UsageSource interface {
|
|
UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
|
|
UsageBuckets(ctx context.Context, since time.Time, width time.Duration) ([]store.Bucket, error)
|
|
}
|
|
|
|
// Dashboard serves the read-only cluster view.
|
|
//
|
|
// It belongs on the admin listener and nowhere else. That listener is bound to
|
|
// loopback by default and also serves pprof, which can dump the process heap —
|
|
// where API keys live in plaintext. Exposing this page publicly would mean
|
|
// exposing that, so reaching it from elsewhere is an SSH tunnel's job:
|
|
//
|
|
// ssh -L 9090:127.0.0.1:9090 llmgw-ctl
|
|
//
|
|
// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`,
|
|
// where it needs a shell on the machine rather than a browser session this
|
|
// project has no way to authenticate.
|
|
func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
|
|
onAdminListener bool, log *slog.Logger,
|
|
) http.HandlerFunc {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if refresh <= 0 {
|
|
refresh = 5 * time.Second
|
|
}
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
d := DashboardData{
|
|
OnAdminListener: onAdminListener,
|
|
RefreshSec: int(refresh.Seconds()),
|
|
Version: version,
|
|
Now: time.Now().UTC().Format(time.RFC3339),
|
|
AuthEnabled: usage != nil,
|
|
}
|
|
if src != nil {
|
|
d.Nodes = src.Nodes()
|
|
d.Models = src.Models()
|
|
d.Replicas = src.Replicas("")
|
|
d.NodeCount = len(d.Nodes)
|
|
d.ModelCount = len(d.Models)
|
|
d.TotalReplicas = len(d.Replicas)
|
|
for _, rep := range d.Replicas {
|
|
if rep.State == "ready" {
|
|
d.ReadyReplicas++
|
|
}
|
|
}
|
|
d.Unplaced = unplaced(src, d.Models)
|
|
d.Summary = src.Summary()
|
|
}
|
|
if usage != nil {
|
|
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
|
|
if err != nil {
|
|
// A reporting query failing must not take the page with it: the
|
|
// cluster state above is what someone is most likely here for.
|
|
log.Warn("could not read usage for the dashboard", "err", err)
|
|
}
|
|
d.Usage = rows
|
|
d.addCharts(r.Context(), usage, log)
|
|
}
|
|
render(w, "dashboard", d)
|
|
}
|
|
}
|
|
|
|
// addCharts draws the last day of traffic.
|
|
//
|
|
// Hourly, over 24 hours: fine enough to show a working day and coarse enough
|
|
// that one slow minute does not look like an incident.
|
|
func (d *DashboardData) addCharts(ctx context.Context, usage UsageSource, log *slog.Logger) {
|
|
buckets, err := usage.UsageBuckets(ctx, time.Now().Add(-24*time.Hour), time.Hour)
|
|
if err != nil {
|
|
log.Warn("could not read usage buckets for the dashboard", "err", err)
|
|
return
|
|
}
|
|
if len(buckets) == 0 {
|
|
return
|
|
}
|
|
|
|
var requests, tokens, latency Series
|
|
requests.Colour, tokens.Colour, latency.Colour = "--accent", "--ok", "--warn"
|
|
for _, b := range buckets {
|
|
requests.Values = append(requests.Values, float64(b.Requests))
|
|
requests.Labels = append(requests.Labels, hourLabel(b.Start, float64(b.Requests), "requests"))
|
|
|
|
tok := float64(b.Prompt + b.Completion)
|
|
tokens.Values = append(tokens.Values, tok)
|
|
tokens.Labels = append(tokens.Labels, hourLabel(b.Start, tok, "tokens"))
|
|
|
|
latency.Values = append(latency.Values, float64(b.LatencyMs))
|
|
latency.Labels = append(latency.Labels, hourLabel(b.Start, float64(b.LatencyMs), "ms mean"))
|
|
}
|
|
|
|
d.RequestChart = BarChart(requests, "requests/h")
|
|
d.TokenChart = BarChart(tokens, "tokens/h")
|
|
d.LatencyChart = BarChart(latency, "ms")
|
|
d.HasCharts = true
|
|
}
|
|
|
|
// unplaced collects the placement explanations for models short of replicas.
|
|
//
|
|
// Only for those: asking why a satisfied model was not placed produces an empty
|
|
// answer, and a page full of empty answers hides the one that matters.
|
|
func unplaced(src Source, models []adminapi.ModelView) []adminapi.ExplainView {
|
|
var out []adminapi.ExplainView
|
|
for _, m := range models {
|
|
if m.Ready >= m.Min {
|
|
continue
|
|
}
|
|
if e := src.Explain(m.Name); len(e.Rejections) > 0 {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
return out
|
|
}
|