The portal in English and Turkish: a catalogue per language, a switcher at the foot of every page, landing.language, Turkish numbers and dates
Every template and handler message in internal/web goes through a JSON
catalogue under internal/web/locales/<code>/, with English as the fallback.
The language is the person's choice (a cookie set by /_lang/{code}), else
Accept-Language, else landing.language. Tests refuse a key some language
lacks, a translation with other arguments than English, and template text
left in English.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
11231d8c2b
113 files changed
+8091
-1981
No files matched your search
@@ -1,19 +1,19 @@
|
||||
{{define "dashboard"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<html lang="{{lang}}"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
|
||||
<noscript>{{.RefreshMeta}}</noscript>
|
||||
<meta name="htmx-config" content='{"allowEval":false}'>
|
||||
<script src="/_static/htmx-2.0.11.min.js"></script>
|
||||
<script defer src="/_static/pages.js"></script>
|
||||
<title>llmgw — cluster</title><link rel="stylesheet" href="{{stylesheet}}">
|
||||
<title>llmgw — {{t "dash.title"}}</title><link rel="stylesheet" href="{{stylesheet}}">
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
<h1>Cluster</h1>
|
||||
<p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
|
||||
<h1>{{t "dash.h1"}}</h1>
|
||||
<p class="sub">{{t "dash.sub" .NodeCount .ReadyReplicas .TotalReplicas .ModelCount}}</p>
|
||||
{{if .OnAdminListener}}<nav>
|
||||
<a href="/metrics">metrics</a>
|
||||
<a href="/debug/pprof/">pprof</a>
|
||||
<a href="/api/v1/nodes">api</a>
|
||||
<a href="/metrics">{{t "dash.nav.metrics"}}</a>
|
||||
<a href="/debug/pprof/" translate="no">pprof</a>
|
||||
<a href="/api/v1/nodes" translate="no">api</a>
|
||||
</nav>{{else}}{{template "nav" .Menu}}{{end}}
|
||||
</header>
|
||||
|
||||
@@ -24,16 +24,16 @@
|
||||
<div id="live" hx-get="{{if .OnAdminListener}}/{{else}}/admin{{end}}" hx-trigger="every {{.RefreshSec}}s" data-visible-only hx-select="#live" hx-swap="outerHTML">
|
||||
{{if .HasCharts}}
|
||||
<div class="tiles">
|
||||
<div class="tile"><div class="v">{{.Requests24h}}</div><div class="k">requests · 24h</div>{{.RequestSpark}}</div>
|
||||
<div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">tokens · 24h</div>{{.TokenSpark}}</div>
|
||||
<div class="tile"><div class="v">{{.Errors24h}}</div><div class="k">errors · 24h</div></div>
|
||||
<div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">spend · 24h</div></div>
|
||||
<div class="tile"><div class="v">{{count .Requests24h}}</div><div class="k">{{t "dash.tile.requests"}}</div>{{.RequestSpark}}</div>
|
||||
<div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">{{t "dash.tile.tokens"}}</div>{{.TokenSpark}}</div>
|
||||
<div class="tile"><div class="v">{{count .Errors24h}}</div><div class="k">{{t "dash.tile.errors"}}</div></div>
|
||||
<div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">{{t "dash.tile.spend"}}</div></div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<h2>Capacity</h2>
|
||||
<h2>{{t "dash.capacity"}}</h2>
|
||||
{{if .Summary.GPUs}}
|
||||
<table><tr><th>Class</th><th class="num">In use</th><th class="num">Free</th><th class="num">Faulty</th><th>Usage</th></tr>
|
||||
<table><tr><th>{{t "dash.cap.class"}}</th><th class="num">{{t "dash.cap.in_use"}}</th><th class="num">{{t "dash.cap.free"}}</th><th class="num">{{t "dash.cap.faulty"}}</th><th>{{t "dash.cap.usage"}}</th></tr>
|
||||
{{range .Summary.GPUs}}<tr>
|
||||
<td><code>{{.Class}}</code></td>
|
||||
<td class="num">{{.Used}}/{{.Total}}</td>
|
||||
@@ -45,74 +45,78 @@
|
||||
{{/* No GPUs is not no capacity: a CPU fleet still has memory and disk,
|
||||
which is what decides what it can hold. */}}
|
||||
<div class="capacity-cpu">
|
||||
<div><b>{{.CPU.Ready}}/{{.CPU.Nodes}}</b> <span>nodes ready · CPU only</span></div>
|
||||
<div><b>{{.CPU.MemoryGB}} GB</b> <span>memory</span></div>
|
||||
{{if .CPU.DiskTotal}}<div><b>{{.CPU.DiskFreeGB}} of {{.CPU.DiskTotal}} GB</b> <span>free for weights</span></div>{{end}}
|
||||
<div><b>{{.CPU.Ready}}/{{.CPU.Nodes}}</b> <span>{{t "dash.cpu.ready"}}</span></div>
|
||||
<div><b>{{count .CPU.MemoryGB}} GB</b> <span>{{t "dash.cpu.memory"}}</span></div>
|
||||
{{if .CPU.DiskTotal}}<div><b>{{t "dash.cpu.disk_of" (count .CPU.DiskFreeGB) (count .CPU.DiskTotal)}}</b> <span>{{t "dash.cpu.disk_free"}}</span></div>{{end}}
|
||||
</div>
|
||||
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
|
||||
{{else}}<p class="empty">{{t "dash.no_nodes"}}</p>{{end}}
|
||||
|
||||
{{with .GPUMap}}
|
||||
<h2>GPUs <span class="opt">by
|
||||
<h2>{{t "dash.gpumap.title"}} <span class="opt">{{t "dash.gpumap.by"}}
|
||||
{{range .Modes}}{{if eq .Key $.GPUMap.By}}<strong>{{.Label}}</strong>{{else}}<a href="?gpumap={{.Key}}">{{.Label}}</a>{{end}} {{end}}</span></h2>
|
||||
<div class="gpu-map">
|
||||
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
|
||||
{{end}}</div>
|
||||
<p class="note">{{.Legend}} A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.</p>
|
||||
<p class="note">{{.Legend}} {{t "dash.gpumap.note"}}</p>
|
||||
{{end}}
|
||||
{{with .Efficiency}}
|
||||
<h3>What each model gets from its cards <span class="opt">last 15 minutes</span></h3>
|
||||
<h3>{{t "dash.eff.title"}} <span class="opt">{{t "dash.eff.window"}}</span></h3>
|
||||
{{/* Per card, because a model on eight cards making twice the tokens of one
|
||||
on two is the worse of the two; per watt-hour, because that is what the
|
||||
room is paying. */}}
|
||||
<table class="rows"><tr><th>Model</th><th class="num">GPUs</th><th class="num">Output tokens/s</th><th class="num">Per GPU</th><th class="num">Busy</th><th class="num">Draw</th><th class="num">Tokens per Wh</th></tr>
|
||||
{{range .}}<tr><td>{{.Model}}</td><td class="num">{{printf "%.0f" .GPUs}}</td><td class="num">{{printf "%.1f" .TokensPerSec}}</td>
|
||||
<td class="num">{{printf "%.1f" .PerGPU}}</td><td class="num">{{printf "%.0f" .Util}}%</td>
|
||||
<td class="num">{{if .Watts}}{{printf "%.0f" .Watts}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{printf "%.0f" .PerWh}}{{else}}—{{end}}</td></tr>
|
||||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
|
||||
{{range .}}<tr><td>{{.Model}}</td><td class="num">{{num .GPUs 0}}</td><td class="num">{{num .TokensPerSec 1}}</td>
|
||||
<td class="num">{{num .PerGPU 1}}</td><td class="num">{{num .Util 0}}%</td>
|
||||
<td class="num">{{if .Watts}}{{num .Watts 0}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{num .PerWh 0}}{{else}}—{{end}}</td></tr>
|
||||
{{end}}</table>
|
||||
{{end}}
|
||||
{{with .IdleGPUs}}
|
||||
<h3>Cards held and doing nothing</h3>
|
||||
<h3>{{t "dash.idle.title"}}</h3>
|
||||
{{/* The first place to find room: replicas whose every card reads under 5%
|
||||
busy now. A model kept warm on purpose is here too; whether it should
|
||||
be is the operator's call. */}}
|
||||
<table class="rows"><tr><th>Model</th><th>Replica</th><th>Node</th><th class="num">GPUs</th><th class="num">Busiest card</th><th>Last request</th></tr>
|
||||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.node"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.idle.busiest"}}</th><th>{{t "dash.idle.last_request"}}</th></tr>
|
||||
{{range .}}<tr><td>{{.Model}}</td><td class="clip"><code>{{.Replica}}</code></td><td><code>{{.Node}}</code></td>
|
||||
<td class="num">{{len .GPUs}}</td><td class="num">{{printf "%.0f" .MaxUtil}}%</td><td>{{if .Idle}}{{.Idle}} ago{{else}}<span class="note">not known</span>{{end}}</td></tr>
|
||||
<td class="num">{{len .GPUs}}</td><td class="num">{{num .MaxUtil 0}}%</td><td>{{if .Idle}}{{t "dash.idle.ago" .Idle}}{{else}}<span class="note">{{t "dash.idle.unknown"}}</span>{{end}}</td></tr>
|
||||
{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
<h2>Nodes</h2>
|
||||
<h2>{{t "dash.nodes"}}</h2>
|
||||
{{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
|
||||
{{if .Nodes}}
|
||||
<table class="nodes rows"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">Disk</th><th>Engines</th>{{if .AnyGPUs}}<th>vLLM build</th>{{end}}{{if .AnyClasses}}<th>Classes</th>{{end}}{{if .AnyFabrics}}<th>Fabrics</th>{{end}}<th class="num">Models</th><th class="num">Replicas</th><th>Seen</th></tr>
|
||||
<table class="nodes rows"><tr><th>{{t "dash.col.node"}}</th><th>{{t "dash.col.state"}}</th><th>CPU</th><th>{{t "dash.nodes.ram"}}</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">{{t "dash.nodes.disk"}}</th><th>{{t "dash.nodes.engines"}}</th>{{if .AnyGPUs}}<th>{{t "dash.nodes.build"}}</th>{{end}}{{if .AnyClasses}}<th>{{t "dash.nodes.classes"}}</th>{{end}}{{if .AnyFabrics}}<th>{{t "dash.nodes.fabrics"}}</th>{{end}}<th class="num">{{t "dash.nodes.models"}}</th><th class="num">{{t "dash.nodes.replicas"}}</th><th>{{t "dash.nodes.seen"}}</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}" title="{{.ID}}"><code>{{shortNode .ID $.NodePrefix}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">{{t "dash.nodes.logs"}}</a></td>
|
||||
{{/* The state is a class as well as a word: the class stays, the word
|
||||
is the reader's. */}}
|
||||
<td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
|
||||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||||
unknown is not idle. */}}
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{num .CPUPercent 0}}%{{if .CPUs}} <span class="note">{{t "dash.nodes.cpus_of" .CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">{{t "dash.nodes.mem_of" (bytes .MemTotal)}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||||
{{if $.AnyGPUs}}<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>{{end}}
|
||||
{{/* Free out of total, because the first is meaningless without the
|
||||
second, and an em dash where nothing was reported: an agent that
|
||||
was never asked about its disk must not read as a full one. */}}
|
||||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">none yet</span>{{end}}</td>
|
||||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">{{t "dash.nodes.no_engines"}}</span>{{end}}</td>
|
||||
{{/* Which vLLM build the node installs: left to its hardware, or chosen.
|
||||
What is installed beside it, so a choice and its outcome are read
|
||||
together. Only a GPU node has one; changing it is for an
|
||||
administrator, on the page reached through a session. */}}
|
||||
{{if $.AnyGPUs}}<td class="clip">{{if .GPUTotal}}{{$node := .}}
|
||||
{{range .Provision}}{{if eq .Engine "vllm"}}<span class="note" title="{{.Detail}}">{{.Version}} {{with .Variant}}{{buildLabel .}} {{end}}{{.State}}</span>{{end}}{{end}}
|
||||
{{range .Provision}}{{if eq .Engine "vllm"}}<span class="note" title="{{.Detail}}">{{.Version}} {{with .Variant}}{{buildLabel .}} {{end}}{{t (print "dash.state." .State)}}</span>{{end}}{{end}}
|
||||
{{if and $.Menu.CSRF .EngineBuilds}}
|
||||
<form method="post" action="/admin/nodes/{{.ID}}/engines" class="inline build-choice">
|
||||
<input type="hidden" name="csrf" value="{{$.Menu.CSRF}}">
|
||||
<input type="hidden" name="engine" value="vllm">
|
||||
<select name="build" title="the vLLM build this node installs">
|
||||
<select name="build" title="{{t "dash.nodes.build_hint"}}">
|
||||
{{range engineBuilds "vllm"}}<option value="{{.}}"{{if eq . (index $node.EngineBuilds "vllm")}} selected{{end}}>{{buildLabel .}}</option>{{end}}
|
||||
</select>
|
||||
<noscript><button type="submit">Save</button></noscript>
|
||||
{{/* For a browser without script; the script below hides it. Not in a
|
||||
<noscript>, whose inside is text to the catalogue check. */}}
|
||||
<button type="submit">{{t "common.save"}}</button>
|
||||
</form>
|
||||
{{end}}
|
||||
{{else}}—{{end}}</td>{{end}}
|
||||
@@ -121,42 +125,47 @@
|
||||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{.Replicas}}</td>
|
||||
<td>{{or .LastSeenAgo "—"}}</td>
|
||||
{{/* What is unhealthy, what an install said and the node's warnings come
|
||||
from the control plane and the agent, in English. */}}
|
||||
</tr>{{if .Unhealthy}}<tr><td colspan="12" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||||
{{range .Provision}}{{if and (ne .State "ready") (ne .State "skipped")}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||||
{{range .Provision}}{{if and (ne .State "ready") (ne .State "skipped")}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{t (print "dash.state." .State)}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||||
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
<script>
|
||||
// A build chosen is saved as it is chosen, with the page reloaded on the
|
||||
// answer: one select, no separate button to forget.
|
||||
document.querySelectorAll("form.build-choice select").forEach(function (s) {
|
||||
s.form.querySelectorAll("button").forEach(function (b) { b.hidden = true; });
|
||||
s.addEventListener("change", function () { s.form.submit(); });
|
||||
});
|
||||
</script>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
{{else}}<p class="empty">{{t "dash.no_agents"}}</p>{{end}}
|
||||
|
||||
<h2>Models</h2>
|
||||
<h2>{{t "dash.models"}}</h2>
|
||||
|
||||
{{if .Models}}
|
||||
<table class="rows"><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Class</th><th>Reserves</th><th>Profiles</th>{{if .AnyFallbacks}}<th>Fallbacks</th>{{end}}<th>What it is for</th></tr>
|
||||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.models.aliases"}}</th><th class="num">{{t "dash.models.ready"}}</th><th class="num">{{t "dash.models.wanted"}}</th><th>{{t "dash.models.class"}}</th><th>{{t "dash.models.reserves"}}</th><th>{{t "dash.models.profiles"}}</th>{{if .AnyFallbacks}}<th>{{t "dash.models.fallbacks"}}</th>{{end}}<th>{{t "dash.models.for"}}</th></tr>
|
||||
{{range .Models}}<tr>
|
||||
<td><code>{{.Name}}</code></td>
|
||||
<td>{{join .Aliases}}</td>
|
||||
<td class="num">{{.Ready}}</td>
|
||||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||||
<td{{if .HeldFor}} class="warn" title="gave room to {{.HeldFor}}"{{end}}>{{.Class}}{{if .Preempted}} <span class="note" title="replicas lost to a higher class">−{{.Preempted}}</span>{{end}}</td>
|
||||
<td{{if .HeldFor}} class="warn" title="{{t "dash.models.held_for" .HeldFor}}"{{end}}>{{.Class}}{{if .Preempted}} <span class="note" title="{{t "dash.models.preempted"}}">−{{.Preempted}}</span>{{end}}</td>
|
||||
{{/* The warnings are the catalogue's own, in English. */}}
|
||||
<td{{with .Warnings}} class="warn" title="{{join .}}"{{end}}>{{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}}
|
||||
{{- if .MeasuredMemoryGB}} <span class="note">{{printf "%.1f" .MeasuredMemoryGB}}G used</span>{{end}}{{if .Warnings}} ⚠{{end}}</td>
|
||||
<td>{{if .External}}<span class="tag" title="served outside the cluster; its prompts leave it">external</span> {{.External}}{{else}}{{join .Profiles}}{{end}}</td>
|
||||
{{- if .MeasuredMemoryGB}} <span class="note">{{t "dash.models.used" (num .MeasuredMemoryGB 1)}}</span>{{end}}{{if .Warnings}} ⚠{{end}}</td>
|
||||
<td>{{if .External}}<span class="tag" title="{{t "dash.models.external_hint"}}">{{t "dash.models.external"}}</span> {{.External}}{{else}}{{join .Profiles}}{{end}}</td>
|
||||
{{if $.AnyFallbacks}}<td>{{join .Fallbacks}}</td>{{end}}
|
||||
{{/* Cut to the column, the whole sentence on hover: a row of its own for
|
||||
each description made the table twice as long as the catalog. */}}
|
||||
<td class="clip"{{with .Description}} title="{{.}}"{{end}}>{{or .Description "—"}}</td>
|
||||
</tr>
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||||
{{else}}<p class="empty">{{t "dash.models.empty"}}</p>{{end}}
|
||||
|
||||
{{if .Unplaced}}
|
||||
<h2>Why nothing was placed</h2>
|
||||
<h2>{{t "dash.unplaced"}}</h2>
|
||||
{{/* The scheduler's reasons, as it gives them. */}}
|
||||
{{range .Unplaced}}
|
||||
<p><code>{{.Model}}</code></p>
|
||||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||||
@@ -164,181 +173,167 @@ document.querySelectorAll("form.build-choice select").forEach(function (s) {
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<h2>Replicas</h2>
|
||||
<h2>{{t "dash.replicas"}}</h2>
|
||||
{{if .Replicas}}
|
||||
<table class="rows"><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
<table class="rows"><tr><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.state"}}</th><th>{{t "dash.replicas.pool"}}</th><th>{{t "dash.replicas.endpoint"}}</th><th class="num">{{t "dash.replicas.age"}}</th><th>{{t "dash.nodes"}}</th></tr>
|
||||
{{range .Replicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
<td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
|
||||
<td>{{or .Pool "—"}}</td>
|
||||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||||
<td class="num">{{duration .AgeSec}}</td>
|
||||
<td class="num">{{age .AgeSec}}</td>
|
||||
<td>{{range .Shards}}<code title="{{.Node}}">{{shortNode .Node $.NodePrefix}}</code> {{end}}</td>
|
||||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||||
{{else}}<p class="empty">{{t "dash.replicas.none"}}</p>{{end}}
|
||||
|
||||
{{if .Summary.IdleReplicas}}
|
||||
<h2>Holding hardware nothing is asking for</h2>
|
||||
<table><tr><th>Replica</th><th>Model</th><th class="num">GPUs</th><th class="num">Unused for</th></tr>
|
||||
<h2>{{t "dash.holding"}}</h2>
|
||||
<table><tr><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.holding.unused_for"}}</th></tr>
|
||||
{{range .Summary.IdleReplicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
<td class="num">{{.GPUs}}</td>
|
||||
<td class="num">{{duration .IdleSec}}</td>
|
||||
<td class="num">{{age .IdleSec}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if or .Summary.Fabrics (and .Fabrics (or .Fabrics.Physical .Fabrics.Problems))}}
|
||||
<h2>Fabrics</h2>
|
||||
<h2>{{t "dash.nodes.fabrics"}}</h2>
|
||||
{{with .Fabrics}}{{if .Problems}}
|
||||
<ul class="fabric-problems">{{range .Problems}}
|
||||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}
|
||||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{t (print "fabric.level." .Level)}}</span> {{.Message}}</li>{{end}}
|
||||
</ul>{{end}}{{end}}
|
||||
{{if .Summary.Fabrics}}
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
<table><tr><th>{{t "dash.fabrics.fabric"}}</th><th>{{t "dash.fabrics.physically"}}</th><th class="num">{{t "dash.nodes"}}</th><th class="num">{{t "dash.fabrics.fully_free"}}</th><th>{{t "dash.fabrics.weights"}}</th></tr>
|
||||
{{range .Summary.Fabrics}}{{$name := .Fabric}}<tr>
|
||||
<td><code>{{.Fabric}}</code></td>
|
||||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{t "fabric.gbps" .RateGbps}}</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||||
<td class="num">{{.Nodes}}</td>
|
||||
<td class="num">{{.LargestFreeBlock}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{t "dash.fabrics.weights_on" .Models .NodesWithWeights .Nodes}}{{else}}<span class="note">{{t "dash.fabrics.not_reported"}}</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
{{with .Fabrics}}{{if .Physical}}
|
||||
<p><a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">See each fabric's map →</a></p>
|
||||
<h3>Subnet managers</h3>
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th>Master</th><th>Standby</th><th class="num">Nodes</th><th>Last change of master</th></tr>
|
||||
<p><a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">{{t "dash.fabrics.see_map"}}</a></p>
|
||||
<h3>{{t "dash.fabrics.sms"}}</h3>
|
||||
<table><tr><th>{{t "dash.fabrics.fabric"}}</th><th>{{t "dash.fabrics.physically"}}</th><th>{{t "dash.fabrics.master"}}</th><th>{{t "dash.fabrics.standby"}}</th><th class="num">{{t "dash.nodes"}}</th><th>{{t "dash.fabrics.last_change"}}</th></tr>
|
||||
{{range .Physical}}<tr>
|
||||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">undeclared</span>{{end}}</td>
|
||||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}<span class="note">none: RoCE</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">LID {{.LID}}, priority {{.Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">none</span>{{end}}{{else}}<span class="note">not listed</span>{{end}}</td>
|
||||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">{{t "fabric.undeclared"}}</span>{{end}}</td>
|
||||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{t "fabric.gbps" .RateGbps}}</span>{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}<span class="note">{{t "dash.fabrics.roce"}}</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">{{t "dash.fabrics.lid_priority" .LID .Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">{{t "dash.fabrics.none"}}</span>{{end}}{{else}}<span class="note">{{t "dash.fabrics.not_listed"}}</span>{{end}}</td>
|
||||
<td class="num">{{len .Members}}</td>
|
||||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">({{len .}} in a day)</span>{{end}}{{else}}<span class="note">none in a day</span>{{end}}</td>
|
||||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">{{t "dash.fabrics.in_a_day" (len .)}}</span>{{end}}{{else}}<span class="note">{{t "dash.fabrics.none_in_a_day"}}</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||||
unused nodes, which is what decides whether such a replica can land.</p>
|
||||
<p class="note">{{t "dash.fabrics.note"}}</p>
|
||||
{{end}}
|
||||
|
||||
{{with .NVLink}}
|
||||
<h2>NVLink domains</h2>
|
||||
<p>{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}}{{range .Domains}} · <code>{{shortDomain .Key}}</code> {{.Usable}}/{{len .Members}} trays usable{{end}}
|
||||
{{if .Problems}} · <span class="bad">{{len .Problems}} problem{{if ne (len .Problems) 1}}s{{end}}</span>{{end}}
|
||||
· <a href="{{if $.OnAdminListener}}/nvlink{{else}}/admin/nvlink{{end}}">see the racks →</a></p>
|
||||
<h2>{{t "dash.nvlink.title"}}</h2>
|
||||
<p>{{tn "dash.nvlink.domains" (len .Domains)}}{{range .Domains}} · <code>{{shortDomain .Key}}</code> {{t "dash.nvlink.trays" .Usable (len .Members)}}{{end}}
|
||||
{{if .Problems}} · <span class="bad">{{tn "dash.nvlink.problems" (len .Problems)}}</span>{{end}}
|
||||
· <a href="{{if $.OnAdminListener}}/nvlink{{else}}/admin/nvlink{{end}}">{{t "dash.nvlink.see_racks"}}</a></p>
|
||||
{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||||
<h2>{{t "dash.fleet.title"}} <span class="opt">{{t "dash.fleet.last" .Window}}</span></h2>
|
||||
{{if .Err}}
|
||||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||||
happening, which is the opposite of not knowing. */}}
|
||||
<p class="note error-row">No history: {{.Err}}</p>
|
||||
<p class="note error-row">{{t "dash.no_history" .Err}}</p>
|
||||
{{else}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Saturation "title" "Pool saturation, by model"
|
||||
"note" "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure."}}
|
||||
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model"
|
||||
"note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}}
|
||||
{{template "tschart" dict "t" .Saturation "title" (t "dash.fleet.saturation") "note" (t "dash.fleet.saturation_note")}}
|
||||
{{template "tschart" dict "t" .Ready "title" (t "dash.fleet.ready") "note" (t "dash.fleet.ready_note")}}
|
||||
</div>
|
||||
{{if $.AnyGPUs}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .GPUBusy "title" "GPUs busy, by class"
|
||||
"note" "The average card of each class. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one."}}
|
||||
{{template "tschart" dict "t" .GPUMemory "title" "GPU memory in use, by class"
|
||||
"note" "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served."}}
|
||||
{{template "tschart" dict "t" .GPUBusy "title" (t "dash.fleet.gpu_busy") "note" (t "dash.fleet.gpu_busy_note")}}
|
||||
{{template "tschart" dict "t" .GPUMemory "title" (t "dash.fleet.gpu_memory") "note" (t "dash.fleet.gpu_memory_note")}}
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica"
|
||||
"note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}}
|
||||
{{template "tschart" dict "t" .KV "title" "KV cache occupancy, by replica"
|
||||
"note" "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both."}}
|
||||
{{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "dash.fleet.queue_note")}}
|
||||
{{template "tschart" dict "t" .KV "title" (t "dash.fleet.kv") "note" (t "dash.fleet.kv_note")}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<hr class="part">
|
||||
|
||||
<h2>Traffic <span class="note">— last 24 hours, hourly</span></h2>
|
||||
<h2>{{t "dash.traffic"}} <span class="note">{{t "dash.traffic.window"}}</span></h2>
|
||||
{{if .HasCharts}}
|
||||
<p class="note">Requests</p>{{.RequestChart}}
|
||||
<p class="note">{{t "dash.traffic.requests"}}</p>{{.RequestChart}}
|
||||
<div class="chart-row">
|
||||
<div><p class="note">Tokens</p>{{.TokenChart}}</div>
|
||||
<div><p class="note">Mean latency</p>{{.LatencyChart}}</div>
|
||||
<div><p class="note">{{t "dash.traffic.tokens"}}</p>{{.TokenChart}}</div>
|
||||
<div><p class="note">{{t "dash.traffic.latency"}}</p>{{.LatencyChart}}</div>
|
||||
</div>
|
||||
{{else if .AuthEnabled}}
|
||||
<p class="empty">No traffic recorded yet.</p>
|
||||
<p class="empty">{{t "dash.traffic.none"}}</p>
|
||||
{{end}}
|
||||
|
||||
{{if .HasPeople}}
|
||||
<h2>People</h2>
|
||||
<table><tr><th>Person</th><th class="num">Keys</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Spend</th><th>Last seen</th></tr>
|
||||
<h2>{{t "dash.people"}}</h2>
|
||||
<table><tr><th>{{t "dash.col.person"}}</th><th class="num">{{t "dash.people.keys"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.tokens"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.col.spend"}}</th><th>{{t "dash.people.last_seen"}}</th></tr>
|
||||
{{range .Users}}<tr>
|
||||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">operator-issued keys</span>{{end}}</td>
|
||||
<td class="num">{{.Keys}}</td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">{{t "dash.people.operator_keys"}}</span>{{end}}</td>
|
||||
<td class="num">{{count .Keys}}</td>
|
||||
<td class="num">{{count .Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{count .Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{units .CostMicros}}</td>
|
||||
<td>{{ago .LastSeen}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Models2}}
|
||||
<h2>What is being run</h2>
|
||||
<table><tr><th>Model</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Errors</th><th class="num">Mean latency</th></tr>
|
||||
<h2>{{t "dash.running"}}</h2>
|
||||
<table><tr><th>{{t "dash.col.model"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.tokens"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.traffic.latency"}}</th></tr>
|
||||
{{range .Models2}}<tr>
|
||||
<td><code>{{.Model}}</code></td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{count .Requests}}</td>
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{.Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="bad">{{count .Errors}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{latency .AvgLatency}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Counted by the model that actually answered, which differs from the one
|
||||
requested whenever a fallback served it.</p>
|
||||
<p class="note">{{t "dash.running.note"}}</p>
|
||||
{{end}}
|
||||
|
||||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||||
<h2>{{t "dash.usage"}} <span class="note">{{t "dash.usage.window"}}</span></h2>
|
||||
{{if .AuthEnabled}}
|
||||
{{if .Usage}}
|
||||
<table><tr><th>Person</th><th>Through</th><th>Spends against</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
|
||||
<table><tr><th>{{t "dash.col.person"}}</th><th>{{t "dash.usage.through"}}</th><th>{{t "dash.usage.against"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.usage.prompt"}}</th><th class="num">{{t "dash.usage.completion"}}</th><th class="num">{{t "dash.col.spend"}}</th><th>{{t "dash.usage.last"}}</th></tr>
|
||||
{{range .Usage}}<tr>
|
||||
<td>{{or .Email "—"}}</td>
|
||||
<td>{{if .Chat}}chat page{{else if .KeyName}}{{.KeyName}} <code class="note">{{.KeyPrefix}}</code>{{else}}<code title="{{.KeyID}}">{{.KeyID}}</code>{{end}}</td>
|
||||
<td>{{if .ProjectPath}}{{.ProjectPath}}{{else if .ProjectID}}<code>{{.ProjectID}}</code>{{else}}personal{{end}}</td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
|
||||
<td>{{if .Chat}}{{t "dash.usage.chat_page"}}{{else if .KeyName}}{{.KeyName}} <code class="note">{{.KeyPrefix}}</code>{{else}}<code title="{{.KeyID}}">{{.KeyID}}</code>{{end}}</td>
|
||||
<td>{{if .ProjectPath}}{{.ProjectPath}}{{else if .ProjectID}}<code>{{.ProjectID}}</code>{{else}}{{t "dash.usage.personal"}}{{end}}</td>
|
||||
<td class="num">{{count .Requests}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="state-failed">{{count .Errors}}</span>{{else}}0{{end}}</td>
|
||||
<td class="num">{{compact .Prompt}}</td>
|
||||
<td class="num">{{compact .Completion}}</td>
|
||||
<td class="num">{{units .CostMicros}}</td>
|
||||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
|
||||
written in batches, so the last few seconds may be missing.</p>
|
||||
{{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
|
||||
<p class="note">{{t "dash.usage.note"}}</p>
|
||||
{{else}}<p class="empty">{{t "dash.usage.none"}}</p>{{end}}
|
||||
{{else}}
|
||||
<p class="empty">Authentication is off, so there is no identity to account against.</p>
|
||||
<p class="empty">{{t "dash.usage.auth_off"}}</p>
|
||||
{{end}}
|
||||
|
||||
{{if not .OnAdminListener}}
|
||||
<p class="note">Metrics, pprof and the management API are on the admin listener,
|
||||
which is bound to loopback and not reachable from here. pprof can dump the
|
||||
process heap, where API keys sit in plaintext, which is why that port is not
|
||||
exposed:
|
||||
<p class="note">{{t "dash.admin_listener"}}
|
||||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||||
{{end}}
|
||||
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||||
<span>llmgw {{.Version}} · {{t "common.refreshes_every" .RefreshSec}}</span>
|
||||
<span>{{.Now}}</span>
|
||||
</footer>
|
||||
{{template "langs"}}</footer>
|
||||
</div></body></html>{{end}}
|
||||
|
||||
{{/* One chart: the SVG, its legend, and the sentence that says how to read it.
|
||||
@@ -349,10 +344,10 @@ exposed:
|
||||
<div class="chart-wrap">
|
||||
<h3 class="chart-title">{{.title}}</h3>
|
||||
{{if .t.Empty}}
|
||||
<p class="empty">Nothing recorded yet.</p>
|
||||
<p class="empty">{{t "chart.nothing_yet"}}</p>
|
||||
{{else}}
|
||||
{{.t.SVG}}
|
||||
<span class="chart-peak">peak {{.t.Peak}}</span>
|
||||
<span class="chart-peak">{{t "chart.peak" .t.Peak}}</span>
|
||||
{{if gt (len .t.Legend) 1}}
|
||||
<div class="legend">
|
||||
{{range .t.Legend}}<span><i style="background:var({{cssvar .Colour}})"></i>{{.Name}} <b>{{.Last}}</b></span>{{end}}
|
||||
|
||||
Reference in new issue
Block a user