diff --git a/docs/install.md b/docs/install.md
index 54ebf91c..c095a165 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -564,8 +564,17 @@ serving — requests, latency, time to first token, tokens, admission, per
endpoint — and what it is made of, per node and per GPU, including memory,
temperature, ECC faults and utilisation.
-Metrics get their own listener, because the admin one also carries pprof and
-pprof can dump the process heap, where API keys sit in plaintext:
+**The dashboard is the answer for most clusters.** `/admin` shows capacity per
+GPU class, free whole nodes per fabric, which replicas are holding hardware
+nothing is asking for, and the same node and model tables `ctl` prints — with
+no monitoring stack to stand up beside it. One binary, no dependencies, which
+is the point.
+
+Prometheus is worth adding when you want history, alerts, or one view across
+several clusters. The dashboard shows what is true now; it keeps nothing.
+
+Metrics then get their own listener, because the admin one also carries pprof
+and pprof can dump the process heap, where API keys sit in plaintext:
```yaml
admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep local
diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go
index 46aac1d5..16a5c54c 100644
--- a/internal/web/dashboard.go
+++ b/internal/web/dashboard.go
@@ -18,6 +18,7 @@ type DashboardData struct {
Replicas []adminapi.ReplicaView
Unplaced []adminapi.ExplainView
Usage []store.KeyUsage
+ Summary adminapi.Summary
// Charts are pre-rendered rather than assembled in the template: building
// SVG out of template fragments is how a chart ends up unescaped.
@@ -53,6 +54,14 @@ type Source interface {
Models() []adminapi.ModelView
Replicas(model string) []adminapi.ReplicaView
Explain(model string) adminapi.ExplainView
+ // Summary is the fleet in aggregate: capacity per GPU class, free whole
+ // nodes per fabric, what is idle.
+ //
+ // The tables below say what exists; this says how much of it is in use,
+ // which is a different question and the one asked first. Deriving it in the
+ // page would mean two implementations of the same arithmetic drifting
+ // apart, so it comes from where ctl status gets it.
+ Summary() adminapi.Summary
}
// UsageSource is the accounting half, absent when authentication is off.
@@ -103,6 +112,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
}
}
d.Unplaced = unplaced(src, d.Models)
+ d.Summary = src.Summary()
}
if usage != nil {
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index a6fd2a36..8df9831e 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -110,3 +110,13 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line);
.note.truncated { color: var(--warn, #a15c00); margin: .2rem 0 .8rem; }
/* Offered under a reply that contains files worth keeping. */
.link.download { display: inline-block; margin: .3rem 0 .8rem; }
+
+/* A usage bar inside a table cell. Drawn with two elements rather than a
+ gradient so the number stays selectable and readable by a screen reader. */
+.meter { position: relative; min-width: 9rem; }
+.meter > span {
+ display: block; height: .75rem; border-radius: 2px;
+ background: var(--accent); opacity: .55;
+}
+.meter > em { font-style: normal; font-size: .8rem; color: var(--muted); }
+.bad { color: var(--bad); }
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html
index 568dcb7d..9ec393b6 100644
--- a/internal/web/templates/dashboard.html
+++ b/internal/web/templates/dashboard.html
@@ -20,19 +20,57 @@
+
Capacity
+{{if .Summary.GPUs}}
+| Class | In use | Free | Faulty | Usage |
+{{range .Summary.GPUs}}
+ {{.Class}} |
+ {{.Used}}/{{.Total}} |
+ {{.Free}} |
+ {{if .Unhealthy}}{{.Unhealthy}}{{else}}—{{end}} |
+ {{pct .Used .Total}}% |
+
{{end}}
+{{else}}No GPUs have been reported.
{{end}}
+
+{{if .Summary.Fabrics}}
+Fabrics
+| Fabric | Nodes | Fully free | Weights |
+{{range .Summary.Fabrics}}
+ {{.Fabric}} |
+ {{.Nodes}} |
+ {{.LargestFreeBlock}} |
+ {{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}not reported{{end}} |
+
{{end}}
+Fully free counts whole unused nodes, which is what decides whether a
+replica spanning nodes can land: free cards on the wrong fabric are not capacity
+for a model that needs this one.
+{{end}}
+
+{{if .Summary.IdleReplicas}}
+Holding hardware nothing is asking for
+| Replica | Model | GPUs | Unused for |
+{{range .Summary.IdleReplicas}}
+ {{.ID}} |
+ {{.Model}} |
+ {{.GPUs}} |
+ {{duration .IdleSec}} |
+
{{end}}
+{{end}}
+
Nodes
{{if .Nodes}}
-| Node | State | GPUs | Classes | Fabrics | Replicas | Last seen | Agent |
+| Node | State | GPUs | RAM | Classes | Fabrics | Models | Replicas | Last seen |
{{range .Nodes}}
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} |
{{.State}} |
{{.GPUFree}}/{{.GPUTotal}} |
+ {{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}} |
{{join .Classes}} |
{{join .Fabrics}} |
+ {{if .Models}}{{len .Models}}{{else}}—{{end}} |
{{.Replicas}} |
{{or .LastSeenAgo "—"}} |
- {{or .AgentVersion "—"}} |
-
{{if .Unhealthy}}| {{join .Unhealthy}} |
{{end}}
+{{if .Unhealthy}}| {{join .Unhealthy}} |
{{end}}
{{end}}
{{else}}No agents have registered.
{{end}}
diff --git a/internal/web/web.go b/internal/web/web.go
index 0508bb12..4b63bec4 100644
--- a/internal/web/web.go
+++ b/internal/web/web.go
@@ -45,6 +45,22 @@ var funcs = template.FuncMap{
"duration": func(sec int64) string {
return time.Duration(sec * int64(time.Second)).Round(time.Second).String()
},
+ // pct is for the usage meters, and is clamped because the two counts it
+ // divides can disagree for an instant — a GPU freed between one read and
+ // the next — and a bar wider than its column wrecks the table it is in.
+ "pct": func(used, total int) int {
+ if total <= 0 {
+ return 0
+ }
+ p := used * 100 / total
+ if p < 0 {
+ return 0
+ }
+ if p > 100 {
+ return 100
+ }
+ return p
+ },
"ago": func(t time.Time) string {
if t.IsZero() {
return "never"
diff --git a/internal/web/web_test.go b/internal/web/web_test.go
index 75fa6cb0..33945363 100644
--- a/internal/web/web_test.go
+++ b/internal/web/web_test.go
@@ -107,6 +107,21 @@ func (fakeSource) Nodes() []adminapi.NodeView {
}}
}
+func (fakeSource) Summary() adminapi.Summary {
+ return adminapi.Summary{
+ Nodes: adminapi.NodeTotals{Total: 1, Ready: 1},
+ GPUs: []adminapi.ClassUse{
+ {Class: "h100-80g", Total: 8, Used: 3, Unhealthy: 1},
+ },
+ Fabrics: []adminapi.FabricUse{
+ {Fabric: "ib-a", Nodes: 1, LargestFreeBlock: 0, NodesWithWeights: 1, Models: 2},
+ },
+ IdleReplicas: []adminapi.IdleReplica{
+ {ID: "served-abc", Model: "served", IdleSec: 3600, GPUs: 4},
+ },
+ }
+}
+
func (fakeSource) Models() []adminapi.ModelView {
return []adminapi.ModelView{
{Name: "served", Ready: 1, Min: 1, Max: 2, Profiles: []string{"tp4"}},
@@ -225,6 +240,7 @@ func (emptySource) Nodes() []adminapi.NodeView { return nil }
func (emptySource) Models() []adminapi.ModelView { return nil }
func (emptySource) Replicas(string) []adminapi.ReplicaView { return nil }
func (emptySource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
+func (emptySource) Summary() adminapi.Summary { return adminapi.Summary{} }
func TestStaticServesAssetsAndNothingElse(t *testing.T) {
// The assets live in their own embed. A file server over the tree that also
@@ -409,3 +425,56 @@ func TestTheDashboardOffersOnlyLinksThatExistWhereItIsServed(t *testing.T) {
t.Error("the public page does not say where metrics and pprof live")
}
}
+
+func TestTheDashboardShowsCapacityWithoutPrometheus(t *testing.T) {
+ // The whole point of putting this on the page: knowing how full a cluster
+ // is should not require standing up a monitoring stack beside it. The
+ // tables say what exists; this says how much of it is in use, which is the
+ // question asked first.
+ h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
+ code, page := get(t, h, "http://127.0.0.1:9090/", nil)
+ if code != http.StatusOK {
+ t.Fatalf("status %d", code)
+ }
+ for _, want := range []string{
+ "Capacity", "h100-80g", "3/8", // used of total
+ "Fabrics", "ib-a",
+ "Holding hardware nothing is asking for", "served-abc",
+ } {
+ if !strings.Contains(page, want) {
+ t.Errorf("the dashboard does not show %q", want)
+ }
+ }
+}
+
+func TestAFaultyCardIsShownAsNeitherUsedNorFree(t *testing.T) {
+ // Counting a faulty card as free is how a cluster looks like it has room it
+ // does not, and the page must not disagree with the scheduler about that.
+ h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
+ code, page := get(t, h, "http://127.0.0.1:9090/", nil)
+ if code != http.StatusOK {
+ t.Fatalf("status %d", code)
+ }
+ if !strings.Contains(page, "Faulty") {
+ t.Error("faulty cards are not shown")
+ }
+ // 8 total, 3 used, 1 faulty, so 4 free.
+ if !strings.Contains(page, ">4<") {
+ t.Errorf("free count is wrong or missing:\n%s", page)
+ }
+}
+
+func TestAUsageMeterCannotOverrunItsColumn(t *testing.T) {
+ // The two counts it divides can disagree for an instant — a GPU freed
+ // between one read and the next — and a bar wider than its cell wrecks the
+ // table.
+ pct, ok := funcs["pct"].(func(int, int) int)
+ if !ok {
+ t.Fatal("the pct helper is missing")
+ }
+ for _, tc := range [][2]int{{0, 0}, {9, 8}, {-1, 8}, {100, 8}} {
+ if got := pct(tc[0], tc[1]); got < 0 || got > 100 {
+ t.Errorf("pct(%d,%d) = %d", tc[0], tc[1], got)
+ }
+ }
+}
diff --git a/test/e2e/signup_test.go b/test/e2e/signup_test.go
index 8ff34e5e..890200e2 100644
--- a/test/e2e/signup_test.go
+++ b/test/e2e/signup_test.go
@@ -471,6 +471,7 @@ func (fakeCluster) Nodes() []adminapi.NodeView {
}
func (fakeCluster) Models() []adminapi.ModelView { return nil }
func (fakeCluster) Replicas(string) []adminapi.ReplicaView { return nil }
+func (fakeCluster) Summary() adminapi.Summary { return adminapi.Summary{} }
func (fakeCluster) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
func TestTheChatPageTalksToTheModelWithoutTheBrowserHoldingAKey(t *testing.T) {