Lead the cluster page with its health and refresh it in place
Capacity, nodes, models and replicas come first, then a rule, then traffic, people, what is being run and usage. The people and per-model bar charts, which repeated the tables under them, are gone. The page swaps its content every few seconds with htmx instead of reloading, so a reader keeps their place, and it stops while the tab is hidden. Without script the meta refresh still reloads it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1f27935570
commit
b06fb9f51f
4 files changed
+113
-131
No files matched your search
@@ -2,7 +2,6 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -37,9 +36,7 @@ type DashboardData struct {
|
||||
RequestSpark template.HTML
|
||||
TokenSpark template.HTML
|
||||
|
||||
UserChart template.HTML
|
||||
ModelChart template.HTML
|
||||
HasPeople bool
|
||||
HasPeople bool
|
||||
|
||||
// Charts are pre-rendered rather than assembled in the template: building
|
||||
// SVG out of template fragments is how a chart ends up unescaped.
|
||||
@@ -203,16 +200,7 @@ func (d *DashboardData) addPeopleAndModels(ctx context.Context, usage UsageSourc
|
||||
log.Warn("could not read per-user usage", "err", err)
|
||||
} else {
|
||||
d.Users = users
|
||||
rows := make([]Row, 0, len(users))
|
||||
for _, u := range topUsers(users, 8) {
|
||||
rows = append(rows, Row{
|
||||
Label: userLabel(u),
|
||||
Value: float64(u.Tokens()),
|
||||
Note: fmt.Sprintf("%d req", u.Requests),
|
||||
})
|
||||
}
|
||||
d.UserChart = HBarChart(rows, "--viz-1", "tok")
|
||||
d.HasPeople = len(rows) > 0
|
||||
d.HasPeople = len(users) > 0
|
||||
}
|
||||
|
||||
models, err := usage.UsageByModel(ctx, since)
|
||||
@@ -221,45 +209,6 @@ func (d *DashboardData) addPeopleAndModels(ctx context.Context, usage UsageSourc
|
||||
return
|
||||
}
|
||||
d.Models2 = models
|
||||
rows := make([]Row, 0, len(models))
|
||||
for i, m := range models {
|
||||
if i >= 8 {
|
||||
break
|
||||
}
|
||||
rows = append(rows, Row{
|
||||
Label: m.Model,
|
||||
Value: float64(m.Requests),
|
||||
Note: humanLatency(m.AvgLatency),
|
||||
})
|
||||
}
|
||||
d.ModelChart = HBarChart(rows, "--viz-3", "req")
|
||||
}
|
||||
|
||||
// userLabel names a person the way they would recognise themselves.
|
||||
//
|
||||
// A key with no owner predates accounts, and calling it "unknown" would imply
|
||||
// something is wrong; it is an operator-issued key doing its job.
|
||||
func userLabel(u store.UserUsage) string {
|
||||
switch {
|
||||
case u.Email != "":
|
||||
return u.Email
|
||||
case u.Name != "":
|
||||
return u.Name
|
||||
default:
|
||||
return "operator-issued keys"
|
||||
}
|
||||
}
|
||||
|
||||
// topUsers keeps the chart readable.
|
||||
//
|
||||
// Eight, because a ranked bar chart stops being a comparison somewhere around
|
||||
// there and becomes a table drawn slowly. The full list is in the table below
|
||||
// it, which is the right form for twenty rows.
|
||||
func topUsers(users []store.UserUsage, n int) []store.UserUsage {
|
||||
if len(users) > n {
|
||||
return users[:n]
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
// addCharts draws the last day of traffic.
|
||||
|
||||
@@ -532,3 +532,5 @@ table.repo-files td:last-child .opt { display: block; margin-bottom: .2rem; }
|
||||
.budget.over .budget-bar > span { background: var(--bad); }
|
||||
.budget.over .budget-head b { color: var(--bad); }
|
||||
.own-use .tile { min-width: 9rem; }
|
||||
/* Between what the cluster is and how it is being used, on the cluster page. */
|
||||
hr.part { border: 0; border-top: 1px solid var(--line); margin: 2.25rem 0 .5rem; }
|
||||
@@ -1,7 +1,8 @@
|
||||
{{define "dashboard"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="refresh" content="{{.RefreshSec}}">
|
||||
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
|
||||
<script src="/_static/htmx-2.0.11.min.js"></script>
|
||||
<title>llmgw — cluster</title><link rel="stylesheet" href="{{stylesheet}}">
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
@@ -14,6 +15,11 @@
|
||||
</nav>{{else}}{{template "nav" .Menu}}{{end}}
|
||||
</header>
|
||||
|
||||
{{/* The page refreshes this part in place rather than reloading: a reload
|
||||
every few seconds threw away the scroll position of anyone reading the
|
||||
bottom of it. Paused while the tab is hidden. Without script, the meta
|
||||
refresh in the head does the old thing. */}}
|
||||
<div id="live" hx-get="{{if .OnAdminListener}}/{{else}}/admin{{end}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
|
||||
{{if .HasCharts}}
|
||||
<div class="tiles">
|
||||
<div class="tile"><div class="v">{{.Requests24h}}</div><div class="k">requests · 24h</div>{{.RequestSpark}}</div>
|
||||
@@ -43,61 +49,6 @@
|
||||
</div>
|
||||
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
|
||||
|
||||
{{if .Summary.Fabrics}}
|
||||
<h2>Fabrics</h2>
|
||||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}<tr>
|
||||
<td><code>{{.Fabric}}</code></td>
|
||||
<td class="num">{{.Nodes}}</td>
|
||||
<td class="num">{{.LargestFreeBlock}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||||
for a model that needs this one.</p>
|
||||
{{end}}
|
||||
|
||||
{{if .Summary.IdleReplicas}}
|
||||
<h2>Holding hardware nothing is asking for</h2>
|
||||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||||
{{range .Summary.IdleReplicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
<td class="num">{{.GPUs}}</td>
|
||||
<td class="num">{{duration .IdleSec}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .HasPeople}}
|
||||
<h2>People</h2>
|
||||
{{.UserChart}}
|
||||
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
|
||||
{{range .Users}}<tr>
|
||||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
|
||||
<td class="num">{{.Keys}}</td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{units .CostMicros}}</td>
|
||||
<td>{{ago .LastSeen}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Models2}}
|
||||
<h2>What is being run</h2>
|
||||
{{.ModelChart}}
|
||||
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
|
||||
{{range .Models2}}<tr>
|
||||
<td><code>{{.Model}}</code></td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{latency .AvgLatency}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Counted by the model that actually answered, which differs from the one
|
||||
requested whenever a fallback served it.</p>
|
||||
{{end}}
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{if .Nodes}}
|
||||
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||||
@@ -123,28 +74,6 @@ requested whenever a fallback served it.</p>
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||||
{{if .Err}}
|
||||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||||
happening, which is the opposite of not knowing. */}}
|
||||
<p class="note error-row">No history: {{.Err}}</p>
|
||||
{{else}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Saturation "title" "Pool saturation, by model"
|
||||
"note" "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure."}}
|
||||
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model"
|
||||
"note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}}
|
||||
</div>
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica"
|
||||
"note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}}
|
||||
{{template "tschart" dict "t" .KV "title" "KV cache occupancy, by replica"
|
||||
"note" "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both."}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<h2>Models</h2>
|
||||
|
||||
{{if .Models}}
|
||||
@@ -187,6 +116,55 @@ requested whenever a fallback served it.</p>
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||||
|
||||
{{if .Summary.IdleReplicas}}
|
||||
<h2>Holding hardware nothing is asking for</h2>
|
||||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||||
{{range .Summary.IdleReplicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
<td class="num">{{.GPUs}}</td>
|
||||
<td class="num">{{duration .IdleSec}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Summary.Fabrics}}
|
||||
<h2>Fabrics</h2>
|
||||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}<tr>
|
||||
<td><code>{{.Fabric}}</code></td>
|
||||
<td class="num">{{.Nodes}}</td>
|
||||
<td class="num">{{.LargestFreeBlock}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||||
for a model that needs this one.</p>
|
||||
{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||||
{{if .Err}}
|
||||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||||
happening, which is the opposite of not knowing. */}}
|
||||
<p class="note error-row">No history: {{.Err}}</p>
|
||||
{{else}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Saturation "title" "Pool saturation, by model"
|
||||
"note" "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure."}}
|
||||
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model"
|
||||
"note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}}
|
||||
</div>
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica"
|
||||
"note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}}
|
||||
{{template "tschart" dict "t" .KV "title" "KV cache occupancy, by replica"
|
||||
"note" "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both."}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<hr class="part">
|
||||
|
||||
<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
|
||||
{{if .HasCharts}}
|
||||
<p class="note">Requests</p>{{.RequestChart}}
|
||||
@@ -198,6 +176,34 @@ requested whenever a fallback served it.</p>
|
||||
<p class="empty">No traffic recorded yet.</p>
|
||||
{{end}}
|
||||
|
||||
{{if .HasPeople}}
|
||||
<h2>People</h2>
|
||||
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
|
||||
{{range .Users}}<tr>
|
||||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
|
||||
<td class="num">{{.Keys}}</td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{units .CostMicros}}</td>
|
||||
<td>{{ago .LastSeen}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Models2}}
|
||||
<h2>What is being run</h2>
|
||||
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
|
||||
{{range .Models2}}<tr>
|
||||
<td><code>{{.Model}}</code></td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{latency .AvgLatency}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Counted by the model that actually answered, which differs from the one
|
||||
requested whenever a fallback served it.</p>
|
||||
{{end}}
|
||||
|
||||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||||
{{if .AuthEnabled}}
|
||||
{{if .Usage}}
|
||||
@@ -228,6 +234,8 @@ exposed:
|
||||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||||
{{end}}
|
||||
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||||
<span>{{.Now}}</span>
|
||||
|
||||
@@ -626,3 +626,26 @@ func TestTheStylesheetIsNamedByItsContent(t *testing.T) {
|
||||
t.Errorf("rendered as %s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageLeadsWithHealthAndRefreshesInPlace(t *testing.T) {
|
||||
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil)
|
||||
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
|
||||
// What the cluster is, before how it is being used.
|
||||
nodes, replicas := strings.Index(body, "<h2>Nodes</h2>"), strings.Index(body, "<h2>Replicas</h2>")
|
||||
people, usage := strings.Index(body, "<h2>People</h2>"), strings.Index(body, "<h2>Usage")
|
||||
if nodes < 0 || replicas < 0 || people < 0 || usage < 0 {
|
||||
t.Fatalf("a section is missing: nodes %d replicas %d people %d usage %d", nodes, replicas, people, usage)
|
||||
}
|
||||
if !(nodes < replicas && replicas < people && people < usage) {
|
||||
t.Errorf("sections out of order: nodes %d replicas %d people %d usage %d", nodes, replicas, people, usage)
|
||||
}
|
||||
// Swapped in place, so a reader keeps their place; reloaded only without
|
||||
// script.
|
||||
if !strings.Contains(body, `id="live" hx-get="/"`) || !strings.Contains(body, `hx-select="#live"`) {
|
||||
t.Error("the page does not refresh its content in place")
|
||||
}
|
||||
if !strings.Contains(body, `<noscript><meta http-equiv="refresh"`) {
|
||||
t.Error("without script the page should still refresh")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user