diff --git a/docs/install.md b/docs/install.md index 54ebf91..c095a16 100644 --- a/docs/install.md +++ b/docs/install.md @@ -564,8 +564,17 @@ serving — requests, latency, time to first token, tokens, admission, per endpoint — and what it is made of, per node and per GPU, including memory, temperature, ECC faults and utilisation. -Metrics get their own listener, because the admin one also carries pprof and -pprof can dump the process heap, where API keys sit in plaintext: +**The dashboard is the answer for most clusters.** `/admin` shows capacity per +GPU class, free whole nodes per fabric, which replicas are holding hardware +nothing is asking for, and the same node and model tables `ctl` prints — with +no monitoring stack to stand up beside it. One binary, no dependencies, which +is the point. + +Prometheus is worth adding when you want history, alerts, or one view across +several clusters. The dashboard shows what is true now; it keeps nothing. + +Metrics then get their own listener, because the admin one also carries pprof +and pprof can dump the process heap, where API keys sit in plaintext: ```yaml admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep local diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 46aac1d..16a5c54 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -18,6 +18,7 @@ type DashboardData struct { Replicas []adminapi.ReplicaView Unplaced []adminapi.ExplainView Usage []store.KeyUsage + Summary adminapi.Summary // Charts are pre-rendered rather than assembled in the template: building // SVG out of template fragments is how a chart ends up unescaped. @@ -53,6 +54,14 @@ type Source interface { Models() []adminapi.ModelView Replicas(model string) []adminapi.ReplicaView Explain(model string) adminapi.ExplainView + // Summary is the fleet in aggregate: capacity per GPU class, free whole + // nodes per fabric, what is idle. + // + // The tables below say what exists; this says how much of it is in use, + // which is a different question and the one asked first. Deriving it in the + // page would mean two implementations of the same arithmetic drifting + // apart, so it comes from where ctl status gets it. + Summary() adminapi.Summary } // UsageSource is the accounting half, absent when authentication is off. @@ -103,6 +112,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat } } d.Unplaced = unplaced(src, d.Models) + d.Summary = src.Summary() } if usage != nil { rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour)) diff --git a/internal/web/static/base.css b/internal/web/static/base.css index a6fd2a3..8df9831 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -110,3 +110,13 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line); .note.truncated { color: var(--warn, #a15c00); margin: .2rem 0 .8rem; } /* Offered under a reply that contains files worth keeping. */ .link.download { display: inline-block; margin: .3rem 0 .8rem; } + +/* A usage bar inside a table cell. Drawn with two elements rather than a + gradient so the number stays selectable and readable by a screen reader. */ +.meter { position: relative; min-width: 9rem; } +.meter > span { + display: block; height: .75rem; border-radius: 2px; + background: var(--accent); opacity: .55; +} +.meter > em { font-style: normal; font-size: .8rem; color: var(--muted); } +.bad { color: var(--bad); } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 568dcb7..9ec393b 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -20,19 +20,57 @@ +

Capacity

+{{if .Summary.GPUs}} + +{{range .Summary.GPUs}} + + + + + +{{end}}
ClassIn useFreeFaultyUsage
{{.Class}}{{.Used}}/{{.Total}}{{.Free}}{{if .Unhealthy}}{{.Unhealthy}}{{else}}—{{end}}{{pct .Used .Total}}%
+{{else}}

No GPUs have been reported.

{{end}} + +{{if .Summary.Fabrics}} +

Fabrics

+ +{{range .Summary.Fabrics}} + + + + +{{end}}
FabricNodesFully freeWeights
{{.Fabric}}{{.Nodes}}{{.LargestFreeBlock}}{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}not reported{{end}}
+

Fully free counts whole unused nodes, which is what decides whether a +replica spanning nodes can land: free cards on the wrong fabric are not capacity +for a model that needs this one.

+{{end}} + +{{if .Summary.IdleReplicas}} +

Holding hardware nothing is asking for

+ +{{range .Summary.IdleReplicas}} + + + + +{{end}}
ReplicaModelGPUsUnused for
{{.ID}}{{.Model}}{{.GPUs}}{{duration .IdleSec}}
+{{end}} +

Nodes

{{if .Nodes}} - +
NodeStateGPUsClassesFabricsReplicasLast seenAgent
{{range .Nodes}} + + - -{{if .Unhealthy}}{{end}} +{{if .Unhealthy}}{{end}} {{end}}
NodeStateGPUsRAMClassesFabricsModelsReplicasLast seen
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} {{.State}} {{.GPUFree}}/{{.GPUTotal}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}} {{join .Classes}} {{join .Fabrics}}{{if .Models}}{{len .Models}}{{else}}—{{end}} {{.Replicas}} {{or .LastSeenAgo "—"}}{{or .AgentVersion "—"}}
{{join .Unhealthy}}
{{join .Unhealthy}}
{{else}}

No agents have registered.

{{end}} diff --git a/internal/web/web.go b/internal/web/web.go index 0508bb1..4b63bec 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -45,6 +45,22 @@ var funcs = template.FuncMap{ "duration": func(sec int64) string { return time.Duration(sec * int64(time.Second)).Round(time.Second).String() }, + // pct is for the usage meters, and is clamped because the two counts it + // divides can disagree for an instant — a GPU freed between one read and + // the next — and a bar wider than its column wrecks the table it is in. + "pct": func(used, total int) int { + if total <= 0 { + return 0 + } + p := used * 100 / total + if p < 0 { + return 0 + } + if p > 100 { + return 100 + } + return p + }, "ago": func(t time.Time) string { if t.IsZero() { return "never" diff --git a/internal/web/web_test.go b/internal/web/web_test.go index 75fa6cb..3394536 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -107,6 +107,21 @@ func (fakeSource) Nodes() []adminapi.NodeView { }} } +func (fakeSource) Summary() adminapi.Summary { + return adminapi.Summary{ + Nodes: adminapi.NodeTotals{Total: 1, Ready: 1}, + GPUs: []adminapi.ClassUse{ + {Class: "h100-80g", Total: 8, Used: 3, Unhealthy: 1}, + }, + Fabrics: []adminapi.FabricUse{ + {Fabric: "ib-a", Nodes: 1, LargestFreeBlock: 0, NodesWithWeights: 1, Models: 2}, + }, + IdleReplicas: []adminapi.IdleReplica{ + {ID: "served-abc", Model: "served", IdleSec: 3600, GPUs: 4}, + }, + } +} + func (fakeSource) Models() []adminapi.ModelView { return []adminapi.ModelView{ {Name: "served", Ready: 1, Min: 1, Max: 2, Profiles: []string{"tp4"}}, @@ -225,6 +240,7 @@ func (emptySource) Nodes() []adminapi.NodeView { return nil } func (emptySource) Models() []adminapi.ModelView { return nil } func (emptySource) Replicas(string) []adminapi.ReplicaView { return nil } func (emptySource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } +func (emptySource) Summary() adminapi.Summary { return adminapi.Summary{} } func TestStaticServesAssetsAndNothingElse(t *testing.T) { // The assets live in their own embed. A file server over the tree that also @@ -409,3 +425,56 @@ func TestTheDashboardOffersOnlyLinksThatExistWhereItIsServed(t *testing.T) { t.Error("the public page does not say where metrics and pprof live") } } + +func TestTheDashboardShowsCapacityWithoutPrometheus(t *testing.T) { + // The whole point of putting this on the page: knowing how full a cluster + // is should not require standing up a monitoring stack beside it. The + // tables say what exists; this says how much of it is in use, which is the + // question asked first. + h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil) + code, page := get(t, h, "http://127.0.0.1:9090/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + for _, want := range []string{ + "Capacity", "h100-80g", "3/8", // used of total + "Fabrics", "ib-a", + "Holding hardware nothing is asking for", "served-abc", + } { + if !strings.Contains(page, want) { + t.Errorf("the dashboard does not show %q", want) + } + } +} + +func TestAFaultyCardIsShownAsNeitherUsedNorFree(t *testing.T) { + // Counting a faulty card as free is how a cluster looks like it has room it + // does not, and the page must not disagree with the scheduler about that. + h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil) + code, page := get(t, h, "http://127.0.0.1:9090/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + if !strings.Contains(page, "Faulty") { + t.Error("faulty cards are not shown") + } + // 8 total, 3 used, 1 faulty, so 4 free. + if !strings.Contains(page, ">4<") { + t.Errorf("free count is wrong or missing:\n%s", page) + } +} + +func TestAUsageMeterCannotOverrunItsColumn(t *testing.T) { + // The two counts it divides can disagree for an instant — a GPU freed + // between one read and the next — and a bar wider than its cell wrecks the + // table. + pct, ok := funcs["pct"].(func(int, int) int) + if !ok { + t.Fatal("the pct helper is missing") + } + for _, tc := range [][2]int{{0, 0}, {9, 8}, {-1, 8}, {100, 8}} { + if got := pct(tc[0], tc[1]); got < 0 || got > 100 { + t.Errorf("pct(%d,%d) = %d", tc[0], tc[1], got) + } + } +} diff --git a/test/e2e/signup_test.go b/test/e2e/signup_test.go index 8ff34e5..890200e 100644 --- a/test/e2e/signup_test.go +++ b/test/e2e/signup_test.go @@ -471,6 +471,7 @@ func (fakeCluster) Nodes() []adminapi.NodeView { } func (fakeCluster) Models() []adminapi.ModelView { return nil } func (fakeCluster) Replicas(string) []adminapi.ReplicaView { return nil } +func (fakeCluster) Summary() adminapi.Summary { return adminapi.Summary{} } func (fakeCluster) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } func TestTheChatPageTalksToTheModelWithoutTheBrowserHoldingAKey(t *testing.T) {