The metrics were exported and the only way to look at them was to stand up Prometheus beside the binary. For a project whose whole shape is one static binary with no dependencies, that is the wrong default: the common case is somebody wanting to know how full their cluster is, and that should not cost a monitoring stack. /admin now shows capacity per GPU class with a usage bar, free whole nodes per fabric — the number that decides whether a replica spanning nodes can land, since free cards on the wrong fabric are not capacity for a model that needs this one — and which replicas are holding hardware nobody is asking for. Nodes gain their memory and how many model files they can see. Taken from the same Summary that ctl status uses rather than derived in the page, because two implementations of the same arithmetic drift apart and the one on the screen is the one people would believe. A faulty card counts as neither used nor free, matching the scheduler. Counting it free is how a cluster looks like it has room it does not, and a page that disagreed with placement about that would be worse than no page. Prometheus stays supported and is now the answer to a narrower question: history, alerting, several clusters at once. The dashboard shows what is true now and keeps nothing, which is the honest division between them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
159 lines
6.5 KiB
HTML
159 lines
6.5 KiB
HTML
{{define "dashboard"}}<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<meta http-equiv="refresh" content="{{.RefreshSec}}">
|
||
<title>llmgw — cluster</title><link rel="stylesheet" href="/_static/base.css">
|
||
</head><body><div class="wrap">
|
||
<header>
|
||
<h1>Cluster</h1>
|
||
<p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
|
||
<nav>
|
||
{{if .OnAdminListener}}
|
||
<a href="/metrics">metrics</a>
|
||
<a href="/debug/pprof/">pprof</a>
|
||
<a href="/api/v1/nodes">api</a>
|
||
{{else}}
|
||
<a href="/admin/users">accounts</a>
|
||
<a href="/profile">your keys</a>
|
||
<a href="/chat">chat</a>
|
||
{{end}}
|
||
</nav>
|
||
</header>
|
||
|
||
<h2>Capacity</h2>
|
||
{{if .Summary.GPUs}}
|
||
<table><tr><th>Class</th><th class="num">In use</th><th class="num">Free</th><th class="num">Faulty</th><th>Usage</th></tr>
|
||
{{range .Summary.GPUs}}<tr>
|
||
<td><code>{{.Class}}</code></td>
|
||
<td class="num">{{.Used}}/{{.Total}}</td>
|
||
<td class="num">{{.Free}}</td>
|
||
<td class="num">{{if .Unhealthy}}<span class="bad">{{.Unhealthy}}</span>{{else}}—{{end}}</td>
|
||
<td class="meter"><span style="width:{{pct .Used .Total}}%"></span><em>{{pct .Used .Total}}%</em></td>
|
||
</tr>{{end}}</table>
|
||
{{else}}<p class="empty">No GPUs have been reported.</p>{{end}}
|
||
|
||
{{if .Summary.Fabrics}}
|
||
<h2>Fabrics</h2>
|
||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||
{{range .Summary.Fabrics}}<tr>
|
||
<td><code>{{.Fabric}}</code></td>
|
||
<td class="num">{{.Nodes}}</td>
|
||
<td class="num">{{.LargestFreeBlock}}</td>
|
||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||
for a model that needs this one.</p>
|
||
{{end}}
|
||
|
||
{{if .Summary.IdleReplicas}}
|
||
<h2>Holding hardware nothing is asking for</h2>
|
||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||
{{range .Summary.IdleReplicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="num">{{.GPUs}}</td>
|
||
<td class="num">{{duration .IdleSec}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
<h2>Nodes</h2>
|
||
{{if .Nodes}}
|
||
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||
{{range .Nodes}}<tr>
|
||
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}</td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
<td class="num">{{.GPUFree}}/{{.GPUTotal}}</td>
|
||
<td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
|
||
<td>{{join .Classes}}</td>
|
||
<td>{{join .Fabrics}}</td>
|
||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||
<td class="num">{{.Replicas}}</td>
|
||
<td>{{or .LastSeenAgo "—"}}</td>
|
||
</tr>{{if .Unhealthy}}<tr><td colspan="9" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||
|
||
<h2>Models</h2>
|
||
{{if .Models}}
|
||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||
{{range .Models}}<tr>
|
||
<td><code>{{.Name}}</code></td>
|
||
<td>{{join .Aliases}}</td>
|
||
<td class="num">{{.Ready}}</td>
|
||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||
<td>{{join .Profiles}}</td>
|
||
<td>{{join .Fallbacks}}</td>
|
||
</tr>{{end}}</table>
|
||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||
|
||
{{if .Unplaced}}
|
||
<h2>Why nothing was placed</h2>
|
||
{{range .Unplaced}}
|
||
<p><code>{{.Model}}</code></p>
|
||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||
{{end}}</pre>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<h2>Replicas</h2>
|
||
{{if .Replicas}}
|
||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||
{{range .Replicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
<td>{{or .Pool "—"}}</td>
|
||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||
<td class="num">{{duration .AgeSec}}</td>
|
||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||
|
||
<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
|
||
{{if .HasCharts}}
|
||
<p class="note">Requests</p>{{.RequestChart}}
|
||
<div class="chart-row">
|
||
<div><p class="note">Tokens</p>{{.TokenChart}}</div>
|
||
<div><p class="note">Mean latency</p>{{.LatencyChart}}</div>
|
||
</div>
|
||
{{else if .AuthEnabled}}
|
||
<p class="empty">No traffic recorded yet.</p>
|
||
{{end}}
|
||
|
||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||
{{if .AuthEnabled}}
|
||
{{if .Usage}}
|
||
<table><tr><th>Key</th><th>Project</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
|
||
{{range .Usage}}<tr>
|
||
<td>{{if .KeyName}}{{.KeyName}} <span class="note">{{.KeyPrefix}}</span>{{else}}<code>{{.KeyID}}</code>{{end}}</td>
|
||
<td>{{.ProjectID}}</td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
|
||
<td class="num">{{.Prompt}}</td>
|
||
<td class="num">{{.Completion}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
|
||
written in batches, so the last few seconds may be missing.</p>
|
||
{{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
|
||
{{else}}
|
||
<p class="empty">Authentication is off, so there is no identity to account against.</p>
|
||
{{end}}
|
||
|
||
{{if not .OnAdminListener}}
|
||
<p class="note">Metrics, pprof and the management API are on the admin listener,
|
||
which is bound to loopback and not reachable from here. pprof can dump the
|
||
process heap, where API keys sit in plaintext, which is why that port is not
|
||
exposed:
|
||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||
{{end}}
|
||
|
||
<footer>
|
||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||
<span>{{.Now}}</span>
|
||
</footer>
|
||
</div></body></html>{{end}}
|