CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
397 lines
24 KiB
HTML
397 lines
24 KiB
HTML
{{define "dashboard"}}<!doctype html>
|
||
<html lang="{{lang}}"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<noscript>{{.RefreshMeta}}</noscript>
|
||
<meta name="htmx-config" content='{"allowEval":false}'>
|
||
<script src="/_static/htmx-2.0.11.min.js"></script>
|
||
<script defer src="{{asset "pages.js"}}"></script>
|
||
<title>llmgw — {{t "dash.title"}}</title><link rel="stylesheet" href="{{stylesheet}}">
|
||
</head><body><div class="wrap">
|
||
<header>
|
||
<h1>{{t "dash.h1"}}</h1>
|
||
<p class="sub">{{t "dash.sub" .NodeCount .ReadyReplicas .TotalReplicas .ModelCount}}</p>
|
||
{{if .OnAdminListener}}<nav>
|
||
<a href="/metrics">{{t "dash.nav.metrics"}}</a>
|
||
<a href="/debug/pprof/" translate="no">pprof</a>
|
||
<a href="/api/v1/nodes" translate="no">api</a>
|
||
</nav>{{else}}{{template "nav" .Menu}}{{end}}
|
||
</header>
|
||
|
||
{{/* The page refreshes this part in place rather than reloading: a reload
|
||
every few seconds threw away the scroll position of anyone reading the
|
||
bottom of it. Paused while the tab is hidden. Without script, the meta
|
||
refresh in the head does the old thing. */}}
|
||
<div id="live" hx-get="{{if .OnAdminListener}}/{{else}}/admin{{end}}" hx-trigger="every {{.RefreshSec}}s" data-visible-only hx-select="#live" hx-swap="outerHTML">
|
||
{{if .HasCharts}}
|
||
<div class="tiles">
|
||
<div class="tile"><div class="v">{{count .Requests24h}}</div><div class="k">{{t "dash.tile.requests"}}</div>{{.RequestSpark}}</div>
|
||
<div class="tile"><div class="v">{{compact .Tokens24h}}</div><div class="k">{{t "dash.tile.tokens"}}</div>{{.TokenSpark}}</div>
|
||
<div class="tile"><div class="v">{{count .Errors24h}}</div><div class="k">{{t "dash.tile.errors"}}</div></div>
|
||
<div class="tile"><div class="v">{{units .Spend24h}}</div><div class="k">{{t "dash.tile.spend"}}</div></div>
|
||
</div>
|
||
{{end}}
|
||
|
||
<h2>{{t "dash.capacity"}}</h2>
|
||
{{if .Summary.GPUs}}
|
||
<table><tr><th>{{t "dash.cap.class"}}</th><th class="num">{{t "dash.cap.in_use"}}</th><th class="num">{{t "dash.cap.free"}}</th><th class="num">{{t "dash.cap.faulty"}}</th><th>{{t "dash.cap.usage"}}</th></tr>
|
||
{{range .Summary.GPUs}}<tr>
|
||
<td><code>{{.Class}}</code></td>
|
||
<td class="num">{{.Used}}/{{.Total}}</td>
|
||
<td class="num">{{.Free}}</td>
|
||
<td class="num">{{if .Unhealthy}}<span class="bad">{{.Unhealthy}}</span>{{else}}—{{end}}</td>
|
||
<td class="meter"><span style="width:{{pct .Used .Total}}%"></span><em>{{pct .Used .Total}}%</em></td>
|
||
</tr>{{end}}</table>
|
||
{{else if .CPU.Nodes}}
|
||
{{/* No GPUs is not no capacity: a CPU fleet still has memory and disk,
|
||
which is what decides what it can hold. */}}
|
||
<div class="capacity-cpu">
|
||
<div><b>{{.CPU.Ready}}/{{.CPU.Nodes}}</b> <span>{{t "dash.cpu.ready"}}</span></div>
|
||
<div><b>{{count .CPU.MemoryGB}} GB</b> <span>{{t "dash.cpu.memory"}}</span></div>
|
||
{{if .CPU.DiskTotal}}<div><b>{{t "dash.cpu.disk_of" (count .CPU.DiskFreeGB) (count .CPU.DiskTotal)}}</b> <span>{{t "dash.cpu.disk_free"}}</span></div>{{end}}
|
||
</div>
|
||
{{else}}<p class="empty">{{t "dash.no_nodes"}}</p>{{end}}
|
||
|
||
{{with .GPUMap}}
|
||
<h2>{{t "dash.gpumap.title"}} <span class="opt">{{t "dash.gpumap.by"}}
|
||
{{range .Modes}}{{if eq .Key $.GPUMap.By}}<strong>{{.Label}}</strong>{{else}}<a href="?gpumap={{.Key}}">{{.Label}}</a>{{end}} {{end}}</span></h2>
|
||
<div class="gpu-map">
|
||
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
|
||
{{end}}</div>
|
||
<p class="note">{{.Legend}} {{t "dash.gpumap.note"}}</p>
|
||
{{end}}
|
||
{{with .Efficiency}}
|
||
<h3>{{t "dash.eff.title"}} <span class="opt">{{t "dash.eff.window"}}</span></h3>
|
||
{{/* Per card, because a model on eight cards making twice the tokens of one
|
||
on two is the worse of the two; per watt-hour, because that is what the
|
||
room is paying. */}}
|
||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.power_load"}}</th><th class="num">{{t "dash.eff.sm_active"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
|
||
{{range .}}<tr><td>{{.Model}}</td><td class="num">{{num .GPUs 0}}</td><td class="num">{{num .TokensPerSec 1}}</td>
|
||
<td class="num">{{num .PerGPU 1}}</td><td class="num">{{pc .Util 0}}</td>
|
||
<td class="num">{{if ge .PowerLoad 0.0}}{{pc .PowerLoad 0}}{{else}}—{{end}}</td><td class="num">{{if ge .SMActive 0.0}}{{pc .SMActive 0}}{{else}}—{{end}}</td>
|
||
<td class="num">{{if .Watts}}{{num .Watts 0}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{num .PerWh 0}}{{else}}—{{end}}</td></tr>
|
||
{{end}}</table>
|
||
<p class="note">{{t "dash.eff.note"}}</p>
|
||
{{end}}
|
||
{{with .IdleGPUs}}
|
||
<h3>{{t "dash.idle.title"}}</h3>
|
||
{{/* The first place to find room: replicas whose every card reads under 5%
|
||
busy now. A model kept warm on purpose is here too; whether it should
|
||
be is the operator's call. */}}
|
||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.node"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.idle.busiest"}}</th><th>{{t "dash.idle.last_request"}}</th></tr>
|
||
{{range .}}<tr><td>{{.Model}}</td><td class="clip"><code>{{.Replica}}</code></td><td><code>{{.Node}}</code></td>
|
||
<td class="num">{{len .GPUs}}</td><td class="num">{{pc .MaxUtil 0}}</td><td>{{if .Idle}}{{t "dash.idle.ago" .Idle}}{{else}}<span class="note">{{t "dash.idle.unknown"}}</span>{{end}}</td></tr>
|
||
{{end}}</table>
|
||
{{end}}
|
||
|
||
<h2>{{t "dash.nodes"}}</h2>
|
||
{{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
|
||
{{if .Nodes}}
|
||
<table class="nodes rows"><tr><th>{{t "dash.col.node"}}</th><th>{{t "dash.col.state"}}</th><th>CPU</th><th>{{t "dash.nodes.ram"}}</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">{{t "dash.nodes.disk"}}</th><th>{{t "dash.nodes.engines"}}</th>{{if .AnyGPUs}}<th>{{t "dash.nodes.build"}}</th>{{end}}{{if .AnyClasses}}<th>{{t "dash.nodes.classes"}}</th>{{end}}{{if .AnyFabrics}}<th>{{t "dash.nodes.fabrics"}}</th>{{end}}<th class="num">{{t "dash.nodes.models"}}</th><th class="num">{{t "dash.nodes.replicas"}}</th><th>{{t "dash.nodes.seen"}}</th></tr>
|
||
{{range .Nodes}}<tr>
|
||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}" title="{{.ID}}"><code>{{shortNode .ID $.NodePrefix}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">{{t "dash.nodes.logs"}}</a></td>
|
||
{{/* The state is a class as well as a word: the class stays, the word
|
||
is the reader's. */}}
|
||
<td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
|
||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||
unknown is not idle. */}}
|
||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{pc .CPUPercent 0}}{{if .CPUs}} <span class="note">{{t "dash.nodes.cpus_of" .CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">{{t "dash.nodes.mem_of" (bytes .MemTotal)}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||
{{if $.AnyGPUs}}<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>{{end}}
|
||
{{/* Free out of total, because the first is meaningless without the
|
||
second, and an em dash where nothing was reported: an agent that
|
||
was never asked about its disk must not read as a full one. */}}
|
||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">{{t "dash.nodes.no_engines"}}</span>{{end}}</td>
|
||
{{/* Which vLLM build the node installs: left to its hardware, or chosen.
|
||
What is installed beside it, so a choice and its outcome are read
|
||
together. Only a GPU node has one; changing it is for an
|
||
administrator, on the page reached through a session. */}}
|
||
{{if $.AnyGPUs}}<td class="clip">{{if .GPUTotal}}{{$node := .}}
|
||
{{range .Provision}}{{if eq .Engine "vllm"}}<span class="note" title="{{.Detail}}">{{.Version}} {{with .Variant}}{{buildLabel .}} {{end}}{{t (print "dash.state." .State)}}</span>{{end}}{{end}}
|
||
{{if and $.Menu.CSRF .EngineBuilds}}
|
||
<form method="post" action="/admin/nodes/{{.ID}}/engines" class="inline build-choice">
|
||
<input type="hidden" name="csrf" value="{{$.Menu.CSRF}}">
|
||
<input type="hidden" name="engine" value="vllm">
|
||
<select name="build" title="{{t "dash.nodes.build_hint"}}">
|
||
{{range engineBuilds "vllm"}}<option value="{{.}}"{{if eq . (index $node.EngineBuilds "vllm")}} selected{{end}}>{{buildLabel .}}</option>{{end}}
|
||
</select>
|
||
{{/* For a browser without script; the script below hides it. Not in a
|
||
<noscript>, whose inside is text to the catalogue check. */}}
|
||
<button type="submit">{{t "common.save"}}</button>
|
||
</form>
|
||
{{end}}
|
||
{{else}}—{{end}}</td>{{end}}
|
||
{{if $.AnyClasses}}<td>{{join .Classes}}</td>{{end}}
|
||
{{if $.AnyFabrics}}<td>{{join .Fabrics}}</td>{{end}}
|
||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||
<td class="num">{{.Replicas}}</td>
|
||
<td>{{or .LastSeenAgo "—"}}</td>
|
||
{{/* What is unhealthy, what an install said and the node's warnings come
|
||
from the control plane and the agent, in English. */}}
|
||
</tr>{{if .Unhealthy}}<tr><td colspan="12" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||
{{range .Provision}}{{if and (ne .State "ready") (ne .State "skipped")}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{t (print "dash.state." .State)}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
<script>
|
||
// A build chosen is saved as it is chosen, with the page reloaded on the
|
||
// answer: one select, no separate button to forget.
|
||
document.querySelectorAll("form.build-choice select").forEach(function (s) {
|
||
s.form.querySelectorAll("button").forEach(function (b) { b.hidden = true; });
|
||
s.addEventListener("change", function () { s.form.submit(); });
|
||
});
|
||
</script>
|
||
{{else}}<p class="empty">{{t "dash.no_agents"}}</p>{{end}}
|
||
|
||
<h2>{{t "dash.models"}}</h2>
|
||
|
||
{{if .Models}}
|
||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.models.aliases"}}</th><th class="num">{{t "dash.models.ready"}}</th><th class="num">{{t "dash.models.wanted"}}</th><th>{{t "dash.models.class"}}</th><th>{{t "dash.models.reserves"}}</th><th>{{t "dash.models.profiles"}}</th>{{if .AnyFallbacks}}<th>{{t "dash.models.fallbacks"}}</th>{{end}}<th>{{t "dash.models.for"}}</th></tr>
|
||
{{range .Models}}<tr>
|
||
<td><code>{{.Name}}</code></td>
|
||
<td>{{join .Aliases}}</td>
|
||
<td class="num">{{.Ready}}</td>
|
||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||
<td{{if .HeldFor}} class="warn" title="{{t "dash.models.held_for" .HeldFor}}"{{end}}>{{.Class}}{{if .Preempted}} <span class="note" title="{{t "dash.models.preempted"}}">−{{.Preempted}}</span>{{end}}</td>
|
||
{{/* The warnings are the catalogue's own, in English. */}}
|
||
<td{{with .Warnings}} class="warn" title="{{join .}}"{{end}}>{{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}}
|
||
{{- if .MeasuredMemoryGB}} <span class="note">{{t "dash.models.used" (num .MeasuredMemoryGB 1)}}</span>{{end}}{{if .Warnings}} ⚠{{end}}</td>
|
||
<td>{{if .External}}<span class="tag" title="{{t "dash.models.external_hint"}}">{{t "dash.models.external"}}</span> {{.External}}{{else}}{{join .Profiles}}{{end}}</td>
|
||
{{if $.AnyFallbacks}}<td>{{join .Fallbacks}}</td>{{end}}
|
||
{{/* Cut to the column, the whole sentence on hover: a row of its own for
|
||
each description made the table twice as long as the catalog. */}}
|
||
<td class="clip"{{with .Description}} title="{{.}}"{{end}}>{{or .Description "—"}}</td>
|
||
</tr>
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">{{t "dash.models.empty"}}</p>{{end}}
|
||
|
||
{{with .Presets}}
|
||
<h3 id="presets">{{t "dash.presets.title"}}</h3>
|
||
<p class="note">{{t "dash.presets.about"}}</p>
|
||
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.presets.makes"}}</th><th>{{t "dash.presets.card"}}</th><th>{{t "dash.presets.licence"}}</th><th>{{t "dash.presets.fits"}}</th><th>{{t "dash.presets.state"}}</th><th></th></tr>
|
||
{{range .Rows}}<tr>
|
||
<td><code>{{.Name}}</code></td>
|
||
<td title="{{.Description}}">{{.Makes}}</td>
|
||
<td>{{.Card}}</td>
|
||
<td>{{.Licence}}</td>
|
||
<td>{{or .Fits (t "dash.presets.fits_none")}}</td>
|
||
{{/* Why the leader does not add it itself is the control plane's, in English. */}}
|
||
<td{{if .Declined}} class="note"{{end}}>{{or .State "—"}}{{with .Blocked}} <span class="note">({{.}})</span>{{end}}</td>
|
||
<td>{{if and $.Menu.CSRF (not .InCatalog)}}
|
||
<form method="post" action="/admin/presets/{{.Name}}" class="inline">
|
||
<input type="hidden" name="csrf" value="{{$.Menu.CSRF}}">
|
||
<button type="submit"{{if .AddsComfyUI}} title="{{t "dash.presets.adds_comfyui"}}"{{end}}>{{if .Declined}}{{t "dash.presets.add_again"}}{{else}}{{t "dash.presets.add"}}{{end}}</button>
|
||
</form>{{if .AddsComfyUI}} <span class="warn">{{t "dash.presets.adds_comfyui"}}</span>{{end}}{{end}}</td>
|
||
</tr>
|
||
{{end}}</table>
|
||
<p>{{if .Auto}}{{t "dash.presets.auto_on"}}{{else}}{{t "dash.presets.auto_off"}}{{end}}
|
||
{{if $.Menu.CSRF}}
|
||
<form method="post" action="/admin/presets/auto" class="inline">
|
||
<input type="hidden" name="csrf" value="{{$.Menu.CSRF}}">
|
||
<input type="hidden" name="on" value="{{if .Auto}}false{{else}}true{{end}}">
|
||
<button type="submit">{{if .Auto}}{{t "dash.presets.turn_off"}}{{else}}{{t "dash.presets.turn_on"}}{{end}}</button>
|
||
</form>{{end}}</p>
|
||
{{end}}
|
||
|
||
{{if .Unplaced}}
|
||
<h2>{{t "dash.unplaced"}}</h2>
|
||
{{/* The scheduler's reasons, as it gives them. */}}
|
||
{{range .Unplaced}}
|
||
<p><code>{{.Model}}</code></p>
|
||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||
{{end}}</pre>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<h2>{{t "dash.replicas"}}</h2>
|
||
{{if .Replicas}}
|
||
<table class="rows"><tr><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.state"}}</th><th>{{t "dash.replicas.pool"}}</th><th>{{t "dash.replicas.endpoint"}}</th><th class="num">{{t "dash.replicas.age"}}</th><th>{{t "dash.nodes"}}</th></tr>
|
||
{{range .Replicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
|
||
<td>{{or .Pool "—"}}</td>
|
||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||
<td class="num">{{age .AgeSec}}</td>
|
||
<td>{{range .Shards}}<code title="{{.Node}}">{{shortNode .Node $.NodePrefix}}</code> {{end}}</td>
|
||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||
{{end}}</table>
|
||
{{else}}<p class="empty">{{t "dash.replicas.none"}}</p>{{end}}
|
||
|
||
{{if .Summary.IdleReplicas}}
|
||
<h2>{{t "dash.holding"}}</h2>
|
||
<table><tr><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.holding.unused_for"}}</th></tr>
|
||
{{range .Summary.IdleReplicas}}<tr>
|
||
<td><code>{{.ID}}</code></td>
|
||
<td>{{.Model}}</td>
|
||
<td class="num">{{.GPUs}}</td>
|
||
<td class="num">{{age .IdleSec}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if or .Summary.Fabrics (and .Fabrics (or .Fabrics.Physical .Fabrics.Problems))}}
|
||
<h2>{{t "dash.nodes.fabrics"}}</h2>
|
||
{{with .Fabrics}}{{if .Problems}}
|
||
<ul class="fabric-problems">{{range .Problems}}
|
||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{t (print "fabric.level." .Level)}}</span> {{.Message}}</li>{{end}}
|
||
</ul>{{end}}{{end}}
|
||
{{if .Summary.Fabrics}}
|
||
<table><tr><th>{{t "dash.fabrics.fabric"}}</th><th>{{t "dash.fabrics.physically"}}</th><th class="num">{{t "dash.nodes"}}</th><th class="num">{{t "dash.fabrics.fully_free"}}</th><th>{{t "dash.fabrics.weights"}}</th></tr>
|
||
{{range .Summary.Fabrics}}{{$name := .Fabric}}<tr>
|
||
<td><code>{{.Fabric}}</code></td>
|
||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{t "fabric.gbps" .RateGbps}}</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||
<td class="num">{{.Nodes}}</td>
|
||
<td class="num">{{.LargestFreeBlock}}</td>
|
||
<td>{{if .NodesWithWeights}}{{t "dash.fabrics.weights_on" .Models .NodesWithWeights .Nodes}}{{else}}<span class="note">{{t "dash.fabrics.not_reported"}}</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
{{with .Fabrics}}{{if .Physical}}
|
||
<p><a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">{{t "dash.fabrics.see_map"}}</a></p>
|
||
<h3>{{t "dash.fabrics.sms"}}</h3>
|
||
<table><tr><th>{{t "dash.fabrics.fabric"}}</th><th>{{t "dash.fabrics.physically"}}</th><th>{{t "dash.fabrics.master"}}</th><th>{{t "dash.fabrics.standby"}}</th><th class="num">{{t "dash.nodes"}}</th><th>{{t "dash.fabrics.last_change"}}</th></tr>
|
||
{{range .Physical}}<tr>
|
||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">{{t "fabric.undeclared"}}</span>{{end}}</td>
|
||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{t "fabric.gbps" .RateGbps}}</span>{{end}}</td>
|
||
<td>{{if eq .Kind "roce"}}<span class="note">{{t "dash.fabrics.roce"}}</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">{{t "dash.fabrics.lid_priority" .LID .Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">{{t "dash.fabrics.none"}}</span>{{end}}{{else}}<span class="note">{{t "dash.fabrics.not_listed"}}</span>{{end}}</td>
|
||
<td class="num">{{len .Members}}</td>
|
||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">{{t "dash.fabrics.in_a_day" (len .)}}</span>{{end}}{{else}}<span class="note">{{t "dash.fabrics.none_in_a_day"}}</span>{{end}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}{{end}}
|
||
<p class="note">{{t "dash.fabrics.note"}}</p>
|
||
{{end}}
|
||
|
||
{{with .NVLink}}
|
||
<h2>{{t "dash.nvlink.title"}}</h2>
|
||
<p>{{tn "dash.nvlink.domains" (len .Domains)}}{{range .Domains}} · <code>{{shortDomain .Key}}</code> {{t "dash.nvlink.trays" .Usable (len .Members)}}{{end}}
|
||
{{if .Problems}} · <span class="bad">{{tn "dash.nvlink.problems" (len .Problems)}}</span>{{end}}
|
||
· <a href="{{if $.OnAdminListener}}/nvlink{{else}}/admin/nvlink{{end}}">{{t "dash.nvlink.see_racks"}}</a></p>
|
||
{{end}}
|
||
|
||
{{with .Fleet}}
|
||
<h2>{{t "dash.fleet.title"}} <span class="opt">{{t "dash.fleet.last" .Window}}</span></h2>
|
||
{{if .Err}}
|
||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||
happening, which is the opposite of not knowing. */}}
|
||
<p class="note error-row">{{t "dash.no_history" .Err}}</p>
|
||
{{else}}
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .Saturation "title" (t "dash.fleet.saturation") "note" (t "dash.fleet.saturation_note")}}
|
||
{{template "tschart" dict "t" .Ready "title" (t "dash.fleet.ready") "note" (t "dash.fleet.ready_note")}}
|
||
</div>
|
||
{{if $.AnyGPUs}}
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .GPUBusy "title" (t "dash.fleet.gpu_busy") "note" (t "dash.fleet.gpu_busy_note")}}
|
||
{{template "tschart" dict "t" .GPUMemory "title" (t "dash.fleet.gpu_memory") "note" (t "dash.fleet.gpu_memory_note")}}
|
||
</div>
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .GPUPowerLoad "title" (t "dash.fleet.gpu_power_load") "note" (t "dash.fleet.gpu_power_load_note")}}
|
||
{{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "dash.fleet.gpu_sm_active") "note" (t "dash.fleet.gpu_sm_active_note")}}{{end}}
|
||
</div>
|
||
{{end}}
|
||
<div class="chart-row">
|
||
{{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "dash.fleet.queue_note")}}
|
||
{{template "tschart" dict "t" .KV "title" (t "dash.fleet.kv") "note" (t "dash.fleet.kv_note")}}
|
||
</div>
|
||
{{end}}
|
||
{{end}}
|
||
|
||
<hr class="part">
|
||
|
||
<h2>{{t "dash.traffic"}} <span class="note">{{t "dash.traffic.window"}}</span></h2>
|
||
{{if .HasCharts}}
|
||
<p class="note">{{t "dash.traffic.requests"}}</p>{{.RequestChart}}
|
||
<div class="chart-row">
|
||
<div><p class="note">{{t "dash.traffic.tokens"}}</p>{{.TokenChart}}</div>
|
||
<div><p class="note">{{t "dash.traffic.latency"}}</p>{{.LatencyChart}}</div>
|
||
</div>
|
||
{{else if .AuthEnabled}}
|
||
<p class="empty">{{t "dash.traffic.none"}}</p>
|
||
{{end}}
|
||
|
||
{{if .HasPeople}}
|
||
<h2>{{t "dash.people"}}</h2>
|
||
<table><tr><th>{{t "dash.col.person"}}</th><th class="num">{{t "dash.people.keys"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.tokens"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.col.spend"}}</th><th>{{t "dash.people.last_seen"}}</th></tr>
|
||
{{range .Users}}<tr>
|
||
<td>{{if .Email}}{{.Email}}{{else}}<span class="note">{{t "dash.people.operator_keys"}}</span>{{end}}</td>
|
||
<td class="num">{{count .Keys}}</td>
|
||
<td class="num">{{count .Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{count .Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td>{{ago .LastSeen}}</td>
|
||
</tr>{{end}}</table>
|
||
{{end}}
|
||
|
||
{{if .Models2}}
|
||
<h2>{{t "dash.running"}}</h2>
|
||
<table><tr><th>{{t "dash.col.model"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.tokens"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.traffic.latency"}}</th></tr>
|
||
{{range .Models2}}<tr>
|
||
<td><code>{{.Model}}</code></td>
|
||
<td class="num">{{count .Requests}}</td>
|
||
<td class="num">{{compact .Tokens}}</td>
|
||
<td class="num">{{if .Errors}}<span class="bad">{{count .Errors}}</span>{{else}}—{{end}}</td>
|
||
<td class="num">{{latency .AvgLatency}}</td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">{{t "dash.running.note"}}</p>
|
||
{{end}}
|
||
|
||
<h2>{{t "dash.usage"}} <span class="note">{{t "dash.usage.window"}}</span></h2>
|
||
{{if .AuthEnabled}}
|
||
{{if .Usage}}
|
||
<table><tr><th>{{t "dash.col.person"}}</th><th>{{t "dash.usage.through"}}</th><th>{{t "dash.usage.against"}}</th><th class="num">{{t "dash.col.requests"}}</th><th class="num">{{t "dash.col.errors"}}</th><th class="num">{{t "dash.usage.prompt"}}</th><th class="num">{{t "dash.usage.completion"}}</th><th class="num">{{t "dash.col.spend"}}</th><th>{{t "dash.usage.last"}}</th></tr>
|
||
{{range .Usage}}<tr>
|
||
<td>{{or .Email "—"}}</td>
|
||
<td>{{if .Chat}}{{t "dash.usage.chat_page"}}{{else if .KeyName}}{{.KeyName}} <code class="note">{{.KeyPrefix}}</code>{{else}}<code title="{{.KeyID}}">{{.KeyID}}</code>{{end}}</td>
|
||
<td>{{if .ProjectPath}}{{.ProjectPath}}{{else if .ProjectID}}<code>{{.ProjectID}}</code>{{else}}{{t "dash.usage.personal"}}{{end}}</td>
|
||
<td class="num">{{count .Requests}}</td>
|
||
<td class="num">{{if .Errors}}<span class="state-failed">{{count .Errors}}</span>{{else}}0{{end}}</td>
|
||
<td class="num">{{compact .Prompt}}</td>
|
||
<td class="num">{{compact .Completion}}</td>
|
||
<td class="num">{{units .CostMicros}}</td>
|
||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||
</tr>{{end}}</table>
|
||
<p class="note">{{t "dash.usage.note"}}</p>
|
||
{{else}}<p class="empty">{{t "dash.usage.none"}}</p>{{end}}
|
||
{{else}}
|
||
<p class="empty">{{t "dash.usage.auth_off"}}</p>
|
||
{{end}}
|
||
|
||
{{if not .OnAdminListener}}
|
||
<p class="note">{{t "dash.admin_listener"}}
|
||
<code>ssh -L 9090:127.0.0.1:9090 <control-plane></code></p>
|
||
{{end}}
|
||
|
||
</div>
|
||
|
||
<footer>
|
||
<span>llmgw {{.Version}} · {{t "common.refreshes_every" .RefreshSec}}</span>
|
||
<span>{{stamp .Now}}</span>
|
||
{{template "langs"}}</footer>
|
||
</div></body></html>{{end}}
|
||
|
||
{{/* One chart: the SVG, its legend, and the sentence that says how to read it.
|
||
A legend is always present for two lines or more — colour alone is not an
|
||
encoding a reader can rely on — and a single line needs none, because the
|
||
title already names it. */}}
|
||
{{define "tschart"}}
|
||
<div class="chart-wrap">
|
||
<h3 class="chart-title">{{.title}}</h3>
|
||
{{if .t.Empty}}
|
||
<p class="empty">{{t "chart.nothing_yet"}}</p>
|
||
{{else}}
|
||
{{.t.SVG}}
|
||
<span class="chart-peak">{{t "chart.peak" .t.Peak}}</span>
|
||
{{if gt (len .t.Legend) 1}}
|
||
<div class="legend">
|
||
{{range .t.Legend}}<span><i style="background:var({{cssvar .Colour}})"></i>{{.Name}} <b>{{.Last}}</b></span>{{end}}
|
||
</div>
|
||
{{end}}
|
||
{{end}}
|
||
<p class="note">{{.note}}</p>
|
||
</div>
|
||
{{end}}
|
||
|
||
{{/* A small horizontal bar for a percentage, coloured as it fills. */}}
|
||
{{define "minibar"}}<span class="minibar{{if ge .P 90}} hot{{else if ge .P 75}} warm{{end}}"><span style="width: {{.P}}%"></span></span>{{end}}
|