Ilker ManapandClaude Opus 5
753f4e3265
CI / test (push) Failing after 1m31s
Disk was the one resource nothing reported. `statfs` was already being called on every model directory, but only to ask whether the filesystem was a network one — the free space it returns in the same struct was thrown away. So the only way to find out why a node was filling up was to ssh to it, which is what this system exists to avoid. Reported per filesystem rather than per configured directory. Free space belongs to the disk, not the path: two model directories on one disk are one pool of bytes, and counting them separately would let the control plane add 24 GB to 24 GB and place a 40 GB model on a node that has 24. Shared filesystems are summed separately from local ones and left out of a node's own total. Space on a filer belongs to everyone mounting it, so twenty nodes would otherwise each report the same free terabyte, and a scheduler would place twenty models into room for one. Nothing reported means nothing is known, everywhere: an agent too old to send storage and a node with no model directories both send nothing, and both must read as unknown rather than as a full disk. `ctl nodes` and the dashboard show a dash, the metric is absent rather than zero, and a missing directory — which is what an unmounted filer looks like — is skipped rather than reported as having no space. Shown as free out of total, because the first is useless without the second: 8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root partition, and only the second stops a model being placed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
202 lines
8.5 KiB
HTML
202 lines
8.5 KiB
HTML
{{define "dashboard"}}<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<meta http-equiv="refresh" content="{{.RefreshSec}}">
|
||
<title>llmgw — cluster</title><link rel="stylesheet" href="/_static/base.css">
|
||
</head><body><div class="wrap">
|
||
<header>
|
||
<h1>Cluster</h1>
|
||
<p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
|
||
<nav>
|
||
{{if .OnAdminListener}}
|
||
<a href="/metrics">metrics</a>
|
||
<a href="/debug/pprof/">pprof</a>
|
||
<a href="/api/v1/nodes">api</a>
|
||
{{else}}
|
||
<a href="/admin/users">accounts</a>
|
||
<a href="/profile">your keys</a>
|
||
<a href="/chat">chat</a>
|
||
{{end}}
|
||
</nav>
|
||
</header>
|
||
|
||
{{if .HasCharts}}
|
||
<div class="tiles">
|
||
<div class="tile"><div class="v">{{.Requests24h}}</div><div class="k">requests · 24h</div>{{.RequestSpark}}</div>
|
||
<div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">tokens · 24h</div>{{.TokenSpark}}</div>
|
||
<div class="tile"><div class="v">{{.Errors24h}}</div><div class="k">errors · 24h</div></div>
|
||
<div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">spend · 24h</div></div>
|
||
</div>
|
||
{{end}}
|
||
|
||
<h2>Capacity</h2>
|
||
{{if .Summary.GPUs}}
|
||
<table><tr><th>Class</th><th class="num">In use</th><th class="num">Free</th><th class="num">Faulty</th><th>Usage</th></tr>
|
||
{{range .Summary.GPUs}}<tr>
|
||
<td><code>{{.Class}}</code></td>
|
||
<td class="num">{{.Used}}/{{.Total}}</td>
|
||
<td class="num">{{.Free}}</td>
|
||
<td class="num">{{if .Unhealthy}}<span class="bad">{{.Unhealthy}}</span>{{else}}—{{end}}</td>
|
||
<td class="meter"><span style="width:{{pct .Used .Total}}%"></span><em>{{pct .Used .Total}}%</em></td>
|
||
</tr>{{end}}</table>
|
||
{{else}}<p class="empty">No GPUs have been reported.</p>{{end}}
|
||
|
||
{{if .Summary.Fabrics}}
|
||
<h2>Fabrics</h2>
|
||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||
{{range .Summary.Fabrics}}<tr>
|
||
<td><code>{{.Fabric}}</code></td>
|
||
<td class="num">{{.Nodes}}</td>
|
||
<td class="num">{{.LargestFreeBlock}}</td>
|
||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||
for a model that needs this one.</p>
|
||
{{end}}
|
||
|
||
{{if .Summary.IdleReplicas}}
|
||
<h2>Holding hardware nothing is asking for</h2>
|
||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||
{{range .Summary.IdleReplicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="num">{{.GPUs}}</td>
|
||
<td class="num">{{duration .IdleSec}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if .HasPeople}}
|
||
<h2>People</h2>
|
||
{{.UserChart}}
|
||
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
|
||
{{range .Users}}<tr>
|
||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
|
||
<td class="num">{{.Keys}}</td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td>{{ago .LastSeen}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if .Models2}}
|
||
<h2>What is being run</h2>
|
||
{{.ModelChart}}
|
||
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
|
||
{{range .Models2}}<tr>
|
||
<td><code>{{.Model}}</code></td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{printf "%.0f" .AvgLatency}} ms</td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Counted by the model that actually answered, which differs from the one
|
||
requested whenever a fallback served it.</p>
|
||
{{end}}
|
||
|
||
<h2>Nodes</h2>
|
||
{{if .Nodes}}
|
||
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||
{{range .Nodes}}<tr>
|
||
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}</td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
<td class="num">{{.GPUFree}}/{{.GPUTotal}}</td>
|
||
<td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
|
||
{{/* Free out of total, because the first is meaningless without the
|
||
second, and an em dash where nothing was reported: an agent that
|
||
was never asked about its disk must not read as a full one. */}}
|
||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||
<td>{{join .Classes}}</td>
|
||
<td>{{join .Fabrics}}</td>
|
||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||
<td class="num">{{.Replicas}}</td>
|
||
<td>{{or .LastSeenAgo "—"}}</td>
|
||
</tr>{{if .Unhealthy}}<tr><td colspan="9" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||
|
||
<h2>Models</h2>
|
||
{{if .Models}}
|
||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||
{{range .Models}}<tr>
|
||
<td><code>{{.Name}}</code></td>
|
||
<td>{{join .Aliases}}</td>
|
||
<td class="num">{{.Ready}}</td>
|
||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||
<td>{{join .Profiles}}</td>
|
||
<td>{{join .Fallbacks}}</td>
|
||
</tr>{{end}}</table>
|
||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||
|
||
{{if .Unplaced}}
|
||
<h2>Why nothing was placed</h2>
|
||
{{range .Unplaced}}
|
||
<p><code>{{.Model}}</code></p>
|
||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||
{{end}}</pre>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<h2>Replicas</h2>
|
||
{{if .Replicas}}
|
||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||
{{range .Replicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
<td>{{or .Pool "—"}}</td>
|
||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||
<td class="num">{{duration .AgeSec}}</td>
|
||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||
|
||
<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
|
||
{{if .HasCharts}}
|
||
<p class="note">Requests</p>{{.RequestChart}}
|
||
<div class="chart-row">
|
||
<div><p class="note">Tokens</p>{{.TokenChart}}</div>
|
||
<div><p class="note">Mean latency</p>{{.LatencyChart}}</div>
|
||
</div>
|
||
{{else if .AuthEnabled}}
|
||
<p class="empty">No traffic recorded yet.</p>
|
||
{{end}}
|
||
|
||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||
{{if .AuthEnabled}}
|
||
{{if .Usage}}
|
||
<table><tr><th>Key</th><th>Project</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
|
||
{{range .Usage}}<tr>
|
||
<td>{{if .KeyName}}{{.KeyName}} <span class="note">{{.KeyPrefix}}</span>{{else}}<code>{{.KeyID}}</code>{{end}}</td>
|
||
<td>{{.ProjectID}}</td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
|
||
<td class="num">{{.Prompt}}</td>
|
||
<td class="num">{{.Completion}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
|
||
written in batches, so the last few seconds may be missing.</p>
|
||
{{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
|
||
{{else}}
|
||
<p class="empty">Authentication is off, so there is no identity to account against.</p>
|
||
{{end}}
|
||
|
||
{{if not .OnAdminListener}}
|
||
<p class="note">Metrics, pprof and the management API are on the admin listener,
|
||
which is bound to loopback and not reachable from here. pprof can dump the
|
||
process heap, where API keys sit in plaintext, which is why that port is not
|
||
exposed:
|
||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||
{{end}}
|
||
|
||
<footer>
|
||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||
<span>{{.Now}}</span>
|
||
</footer>
|
||
</div></body></html>{{end}}
|