llmgw/internal/web/templates/dashboard.html

{{define "dashboard"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<meta http-equiv="refresh" content="{{.RefreshSec}}">
<title>llmgw — cluster</title><link rel="stylesheet" href="/_static/base.css">
</head><body><div class="wrap">
<header>
  <h1>Cluster</h1>
  <p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
  <nav>
{{if .OnAdminListener}}
    <a href="/metrics">metrics</a>
    <a href="/debug/pprof/">pprof</a>
    <a href="/api/v1/nodes">api</a>
{{else}}
    <a href="/admin/users">accounts</a>
{{if .Nav.Mirror}}    <a href="/admin/mirror">mirror</a>
{{end}}
    <a href="/profile">your keys</a>
    <a href="/chat">chat</a>
{{end}}
  </nav>
</header>

{{if .HasCharts}}
<div class="tiles">
  <div class="tile"><div class="v">{{.Requests24h}}</div><div class="k">requests · 24h</div>{{.RequestSpark}}</div>
  <div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">tokens · 24h</div>{{.TokenSpark}}</div>
  <div class="tile"><div class="v">{{.Errors24h}}</div><div class="k">errors · 24h</div></div>
  <div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">spend · 24h</div></div>
</div>
{{end}}

<h2>Capacity</h2>
{{if .Summary.GPUs}}
<table><tr><th>Class</th><th class="num">In use</th><th class="num">Free</th><th class="num">Faulty</th><th>Usage</th></tr>
{{range .Summary.GPUs}}<tr>
  <td><code>{{.Class}}</code></td>
  <td class="num">{{.Used}}/{{.Total}}</td>
  <td class="num">{{.Free}}</td>
  <td class="num">{{if .Unhealthy}}<span class="bad">{{.Unhealthy}}</span>{{else}}—{{end}}</td>
  <td class="meter"><span style="width:{{pct .Used .Total}}%"></span><em>{{pct .Used .Total}}%</em></td>
</tr>{{end}}</table>
{{else}}<p class="empty">No GPUs have been reported.</p>{{end}}

{{if .Summary.Fabrics}}
<h2>Fabrics</h2>
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
{{range .Summary.Fabrics}}<tr>
  <td><code>{{.Fabric}}</code></td>
  <td class="num">{{.Nodes}}</td>
  <td class="num">{{.LargestFreeBlock}}</td>
  <td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
</tr>{{end}}</table>
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
replica spanning nodes can land: free cards on the wrong fabric are not capacity
for a model that needs this one.</p>
{{end}}

{{if .Summary.IdleReplicas}}
<h2>Holding hardware nothing is asking for</h2>
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
{{range .Summary.IdleReplicas}}<tr>
  <td><code>{{.ID}}</code></td>
  <td>{{.Model}}</td>
  <td class="num">{{.GPUs}}</td>
  <td class="num">{{duration .IdleSec}}</td>
</tr>{{end}}</table>
{{end}}

{{if .HasPeople}}
<h2>People</h2>
{{.UserChart}}
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
{{range .Users}}<tr>
  <td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
  <td class="num">{{.Keys}}</td>
  <td class="num">{{.Requests}}</td>
  <td class="num">{{compact .Tokens}}</td>
  <td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
  <td class="num">{{units .CostMicros}}</td>
  <td>{{ago .LastSeen}}</td>
</tr>{{end}}</table>
{{end}}

{{if .Models2}}
<h2>What is being run</h2>
{{.ModelChart}}
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
{{range .Models2}}<tr>
  <td><code>{{.Model}}</code></td>
  <td class="num">{{.Requests}}</td>
  <td class="num">{{compact .Tokens}}</td>
  <td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
  <td class="num">{{printf "%.0f" .AvgLatency}} ms</td>
</tr>{{end}}</table>
<p class="note">Counted by the model that actually answered, which differs from the one
requested whenever a fallback served it.</p>
{{end}}

<h2>Nodes</h2>
{{if .Nodes}}
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
{{range .Nodes}}<tr>
  <td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}</td>
  <td class="state state-{{.State}}">{{.State}}</td>
  <td class="num">{{.GPUFree}}/{{.GPUTotal}}</td>
  <td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
  {{/* Free out of total, because the first is meaningless without the
       second, and an em dash where nothing was reported: an agent that
       was never asked about its disk must not read as a full one. */}}
  <td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
  <td>{{join .Classes}}</td>
  <td>{{join .Fabrics}}</td>
  <td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
  <td class="num">{{.Replicas}}</td>
  <td>{{or .LastSeenAgo "—"}}</td>
</tr>{{if .Unhealthy}}<tr><td colspan="9" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
{{end}}</table>
{{else}}<p class="empty">No agents have registered.</p>{{end}}

<h2>Models</h2>
{{if .Models}}
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
{{range .Models}}<tr>
  <td><code>{{.Name}}</code></td>
  <td>{{join .Aliases}}</td>
  <td class="num">{{.Ready}}</td>
  <td class="num">{{.Min}}–{{.Max}}</td>
  <td>{{join .Profiles}}</td>
  <td>{{join .Fallbacks}}</td>
</tr>
{{/* Under the row rather than in a column: a sentence in a table cell either
     wraps every other column into a stripe or gets cut off. */}}
{{if .Description}}<tr><td colspan="6" class="url-row">{{.Description}}</td></tr>{{end}}
{{end}}</table>
{{else}}<p class="empty">The catalog is empty.</p>{{end}}

{{if .Unplaced}}
<h2>Why nothing was placed</h2>
{{range .Unplaced}}
<p><code>{{.Model}}</code></p>
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
{{end}}</pre>
{{end}}
{{end}}

<h2>Replicas</h2>
{{if .Replicas}}
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
{{range .Replicas}}<tr>
  <td><code>{{.ID}}</code></td>
  <td>{{.Model}}</td>
  <td class="state state-{{.State}}">{{.State}}</td>
  <td>{{or .Pool "—"}}</td>
  <td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
  <td class="num">{{duration .AgeSec}}</td>
  <td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
{{end}}</table>
{{else}}<p class="empty">No replicas.</p>{{end}}

<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
{{if .HasCharts}}
<p class="note">Requests</p>{{.RequestChart}}
<div class="chart-row">
  <div><p class="note">Tokens</p>{{.TokenChart}}</div>
  <div><p class="note">Mean latency</p>{{.LatencyChart}}</div>
</div>
{{else if .AuthEnabled}}
<p class="empty">No traffic recorded yet.</p>
{{end}}

<h2>Usage <span class="note">— last 24 hours</span></h2>
{{if .AuthEnabled}}
  {{if .Usage}}
<table><tr><th>Key</th><th>Project</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
{{range .Usage}}<tr>
  <td>{{if .KeyName}}{{.KeyName}} <span class="note">{{.KeyPrefix}}</span>{{else}}<code>{{.KeyID}}</code>{{end}}</td>
  <td>{{.ProjectID}}</td>
  <td class="num">{{.Requests}}</td>
  <td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
  <td class="num">{{.Prompt}}</td>
  <td class="num">{{.Completion}}</td>
  <td class="num">{{units .CostMicros}}</td>
  <td><span class="note">{{ago .LastAt}}</span></td>
</tr>{{end}}</table>
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
written in batches, so the last few seconds may be missing.</p>
  {{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
{{else}}
<p class="empty">Authentication is off, so there is no identity to account against.</p>
{{end}}

{{if not .OnAdminListener}}
<p class="note">Metrics, pprof and the management API are on the admin listener,
which is bound to loopback and not reachable from here. pprof can dump the
process heap, where API keys sit in plaintext, which is why that port is not
exposed:
<code>ssh -L 9090:127.0.0.1:9090 &lt;control-plane&gt;</code></p>
{{end}}

<footer>
  <span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
  <span>{{.Now}}</span>
</footer>
</div></body></html>{{end}}