The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.
A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.
Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.
Static assets are named by their content, as the stylesheet already
was.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
293 lines
15 KiB
HTML
293 lines
15 KiB
HTML
{{define "dashboard"}}<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
|
||
<script src="/_static/htmx-2.0.11.min.js"></script>
|
||
<title>llmgw — cluster</title><link rel="stylesheet" href="{{stylesheet}}">
|
||
</head><body><div class="wrap">
|
||
<header>
|
||
<h1>Cluster</h1>
|
||
<p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
|
||
{{if .OnAdminListener}}<nav>
|
||
<a href="/metrics">metrics</a>
|
||
<a href="/debug/pprof/">pprof</a>
|
||
<a href="/api/v1/nodes">api</a>
|
||
</nav>{{else}}{{template "nav" .Menu}}{{end}}
|
||
</header>
|
||
|
||
{{/* The page refreshes this part in place rather than reloading: a reload
|
||
every few seconds threw away the scroll position of anyone reading the
|
||
bottom of it. Paused while the tab is hidden. Without script, the meta
|
||
refresh in the head does the old thing. */}}
|
||
<div id="live" hx-get="{{if .OnAdminListener}}/{{else}}/admin{{end}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
|
||
{{if .HasCharts}}
|
||
<div class="tiles">
|
||
<div class="tile"><div class="v">{{.Requests24h}}</div><div class="k">requests · 24h</div>{{.RequestSpark}}</div>
|
||
<div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">tokens · 24h</div>{{.TokenSpark}}</div>
|
||
<div class="tile"><div class="v">{{.Errors24h}}</div><div class="k">errors · 24h</div></div>
|
||
<div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">spend · 24h</div></div>
|
||
</div>
|
||
{{end}}
|
||
|
||
<h2>Capacity</h2>
|
||
{{if .Summary.GPUs}}
|
||
<table><tr><th>Class</th><th class="num">In use</th><th class="num">Free</th><th class="num">Faulty</th><th>Usage</th></tr>
|
||
{{range .Summary.GPUs}}<tr>
|
||
<td><code>{{.Class}}</code></td>
|
||
<td class="num">{{.Used}}/{{.Total}}</td>
|
||
<td class="num">{{.Free}}</td>
|
||
<td class="num">{{if .Unhealthy}}<span class="bad">{{.Unhealthy}}</span>{{else}}—{{end}}</td>
|
||
<td class="meter"><span style="width:{{pct .Used .Total}}%"></span><em>{{pct .Used .Total}}%</em></td>
|
||
</tr>{{end}}</table>
|
||
{{else if .CPU.Nodes}}
|
||
{{/* No GPUs is not no capacity: a CPU fleet still has memory and disk,
|
||
which is what decides what it can hold. */}}
|
||
<div class="capacity-cpu">
|
||
<div><b>{{.CPU.Ready}}/{{.CPU.Nodes}}</b> <span>nodes ready · CPU only</span></div>
|
||
<div><b>{{.CPU.MemoryGB}} GB</b> <span>memory</span></div>
|
||
{{if .CPU.DiskTotal}}<div><b>{{.CPU.DiskFreeGB}} of {{.CPU.DiskTotal}} GB</b> <span>free for weights</span></div>{{end}}
|
||
</div>
|
||
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
|
||
|
||
<h2>Nodes</h2>
|
||
{{if .Nodes}}
|
||
<table class="nodes"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th><th class="num">GPUs</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||
{{range .Nodes}}<tr>
|
||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}"><code>{{.ID}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||
unknown is not idle. */}}
|
||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||
<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>
|
||
{{/* Free out of total, because the first is meaningless without the
|
||
second, and an em dash where nothing was reported: an agent that
|
||
was never asked about its disk must not read as a full one. */}}
|
||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">none yet</span>{{end}}</td>
|
||
<td>{{join .Classes}}</td>
|
||
<td>{{join .Fabrics}}</td>
|
||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||
<td class="num">{{.Replicas}}</td>
|
||
<td>{{or .LastSeenAgo "—"}}</td>
|
||
</tr>{{if .Unhealthy}}<tr><td colspan="12" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||
{{range .Provision}}{{if ne .State "ready"}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||
|
||
<h2>Models</h2>
|
||
|
||
{{if .Models}}
|
||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||
{{range .Models}}<tr>
|
||
<td><code>{{.Name}}</code></td>
|
||
<td>{{join .Aliases}}</td>
|
||
<td class="num">{{.Ready}}</td>
|
||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||
<td>{{join .Profiles}}</td>
|
||
<td>{{join .Fallbacks}}</td>
|
||
</tr>
|
||
{{/* Under the row rather than in a column: a sentence in a table cell either
|
||
wraps every other column into a stripe or gets cut off. */}}
|
||
{{if .Description}}<tr><td colspan="6" class="url-row">{{.Description}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||
|
||
{{if .Unplaced}}
|
||
<h2>Why nothing was placed</h2>
|
||
{{range .Unplaced}}
|
||
<p><code>{{.Model}}</code></p>
|
||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||
{{end}}</pre>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<h2>Replicas</h2>
|
||
{{if .Replicas}}
|
||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||
{{range .Replicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="state state-{{.State}}">{{.State}}</td>
|
||
<td>{{or .Pool "—"}}</td>
|
||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||
<td class="num">{{duration .AgeSec}}</td>
|
||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||
|
||
{{if .Summary.IdleReplicas}}
|
||
<h2>Holding hardware nothing is asking for</h2>
|
||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||
{{range .Summary.IdleReplicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="num">{{.GPUs}}</td>
|
||
<td class="num">{{duration .IdleSec}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if or .Summary.Fabrics .Fabrics}}
|
||
<h2>Fabrics</h2>
|
||
{{with .Fabrics}}{{if .Problems}}
|
||
<ul class="fabric-problems">{{range .Problems}}
|
||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}
|
||
</ul>{{end}}{{end}}
|
||
{{if .Summary.Fabrics}}
|
||
<table><tr><th>Fabric</th><th>Is physically</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||
{{range .Summary.Fabrics}}{{$name := .Fabric}}<tr>
|
||
<td><code>{{.Fabric}}</code></td>
|
||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||
<td class="num">{{.Nodes}}</td>
|
||
<td class="num">{{.LargestFreeBlock}}</td>
|
||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
{{with .Fabrics}}{{if .Physical}}
|
||
<p><a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">See each fabric's map →</a></p>
|
||
<h3>Subnet managers</h3>
|
||
<table><tr><th>Fabric</th><th>Is physically</th><th>Master</th><th>Standby</th><th class="num">Nodes</th><th>Last change of master</th></tr>
|
||
{{range .Physical}}<tr>
|
||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">undeclared</span>{{end}}</td>
|
||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</td>
|
||
<td>{{if eq .Kind "roce"}}<span class="note">none: RoCE</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">LID {{.LID}}, priority {{.Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">none</span>{{end}}{{else}}<span class="note">not listed</span>{{end}}</td>
|
||
<td class="num">{{len .Members}}</td>
|
||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">({{len .}} in a day)</span>{{end}}{{else}}<span class="note">none in a day</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}{{end}}
|
||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||
unused nodes, which is what decides whether such a replica can land.</p>
|
||
{{end}}
|
||
|
||
{{with .Fleet}}
|
||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||
{{if .Err}}
|
||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||
happening, which is the opposite of not knowing. */}}
|
||
<p class="note error-row">No history: {{.Err}}</p>
|
||
{{else}}
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .Saturation "title" "Pool saturation, by model"
|
||
"note" "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure."}}
|
||
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model"
|
||
"note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}}
|
||
</div>
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica"
|
||
"note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}}
|
||
{{template "tschart" dict "t" .KV "title" "KV cache occupancy, by replica"
|
||
"note" "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both."}}
|
||
</div>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<hr class="part">
|
||
|
||
<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
|
||
{{if .HasCharts}}
|
||
<p class="note">Requests</p>{{.RequestChart}}
|
||
<div class="chart-row">
|
||
<div><p class="note">Tokens</p>{{.TokenChart}}</div>
|
||
<div><p class="note">Mean latency</p>{{.LatencyChart}}</div>
|
||
</div>
|
||
{{else if .AuthEnabled}}
|
||
<p class="empty">No traffic recorded yet.</p>
|
||
{{end}}
|
||
|
||
{{if .HasPeople}}
|
||
<h2>People</h2>
|
||
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
|
||
{{range .Users}}<tr>
|
||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
|
||
<td class="num">{{.Keys}}</td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td>{{ago .LastSeen}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if .Models2}}
|
||
<h2>What is being run</h2>
|
||
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
|
||
{{range .Models2}}<tr>
|
||
<td><code>{{.Model}}</code></td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{latency .AvgLatency}}</td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Counted by the model that actually answered, which differs from the one
|
||
requested whenever a fallback served it.</p>
|
||
{{end}}
|
||
|
||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||
{{if .AuthEnabled}}
|
||
{{if .Usage}}
|
||
<table><tr><th>Person</th><th>Through</th><th>Spends against</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
|
||
{{range .Usage}}<tr>
|
||
<td>{{or .Email "—"}}</td>
|
||
<td>{{if .Chat}}chat page{{else if .KeyName}}{{.KeyName}} <code class="note">{{.KeyPrefix}}</code>{{else}}<code title="{{.KeyID}}">{{.KeyID}}</code>{{end}}</td>
|
||
<td>{{if .ProjectPath}}{{.ProjectPath}}{{else if .ProjectID}}<code>{{.ProjectID}}</code>{{else}}personal{{end}}</td>
|
||
<td class="num">{{.Requests}}</td>
|
||
<td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
|
||
<td class="num">{{compact .Prompt}}</td>
|
||
<td class="num">{{compact .Completion}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
|
||
written in batches, so the last few seconds may be missing.</p>
|
||
{{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
|
||
{{else}}
|
||
<p class="empty">Authentication is off, so there is no identity to account against.</p>
|
||
{{end}}
|
||
|
||
{{if not .OnAdminListener}}
|
||
<p class="note">Metrics, pprof and the management API are on the admin listener,
|
||
which is bound to loopback and not reachable from here. pprof can dump the
|
||
process heap, where API keys sit in plaintext, which is why that port is not
|
||
exposed:
|
||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||
{{end}}
|
||
|
||
</div>
|
||
|
||
<footer>
|
||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||
<span>{{.Now}}</span>
|
||
</footer>
|
||
</div></body></html>{{end}}
|
||
|
||
{{/* One chart: the SVG, its legend, and the sentence that says how to read it.
|
||
A legend is always present for two lines or more — colour alone is not an
|
||
encoding a reader can rely on — and a single line needs none, because the
|
||
title already names it. */}}
|
||
{{define "tschart"}}
|
||
<div class="chart-wrap">
|
||
<h3 class="chart-title">{{.title}}</h3>
|
||
{{if .t.Empty}}
|
||
<p class="empty">Nothing recorded yet.</p>
|
||
{{else}}
|
||
{{.t.SVG}}
|
||
<span class="chart-peak">peak {{.t.Peak}}</span>
|
||
{{if gt (len .t.Legend) 1}}
|
||
<div class="legend">
|
||
{{range .t.Legend}}<span><i style="background:var({{cssvar .Colour}})"></i>{{.Name}} <b>{{.Last}}</b></span>{{end}}
|
||
</div>
|
||
{{end}}
|
||
{{end}}
|
||
<p class="note">{{.note}}</p>
|
||
</div>
|
||
{{end}}
|
||
|
||
{{/* A small horizontal bar for a percentage, coloured as it fills. */}}
|
||
{{define "minibar"}}<span class="minibar{{if ge .P 90}} hot{{else if ge .P 75}} warm{{end}}"><span style="width: {{.P}}%"></span></span>{{end}}
|