llmgw/internal/web/templates/node.html

{{define "node"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
<script src="/_static/htmx-2.0.11.min.js"></script>
<title>llmgw — {{.Node.ID}}</title><link rel="stylesheet" href="{{stylesheet}}">
</head><body><div class="wrap">
<header>
  <h1>{{.Node.ID}}</h1>
  <p class="sub"><span class="state state-{{.Node.State}}">{{.Node.State}}</span>
    {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
    · last seen {{or .Node.LastSeenAgo "never"}} ago · <a href="{{.Logs}}">logs</a></p>
  {{if .OnAdminListener}}<nav><a href="{{.Back}}">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header>

{{/* Refreshed in place, like the cluster page. */}}
<div id="live" hx-get="{{.Self}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
{{with .Node}}
<div class="tiles">
  {{with .Load}}
  <div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
  <div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
  <div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
  {{else}}
  <div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
  {{end}}
  {{if .DiskTotalGB}}<div class="tile"><div class="v">{{.DiskFreeGB}}G</div><div class="k">disk free · of {{.DiskTotalGB}}G</div></div>{{end}}
  {{if .GPUTotal}}<div class="tile"><div class="v">{{.GPUFree}}/{{.GPUTotal}}</div><div class="k">GPUs free</div></div>{{end}}
</div>
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{end}}

{{if or .Node.IBPorts .Node.Fabrics}}
<h2>Fabric</h2>
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}} · <a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">the map →</a>{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
{{if .Node.IBPorts}}
<table class="rows"><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
{{range .Node.IBPorts}}<tr>
  <td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
  <td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
  <td class="num">{{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}}</td>
  <td>{{or .Fabric "—"}}</td>
  <td>{{if .Physical}}{{.Physical}}{{else if .SMError}}<span class="note" title="{{.SMError}}">unknown</span>{{else}}—{{end}}</td>
  <td>{{if .Errors}}{{range $k, $v := .Errors}}<span class="tag bad">{{$k}} {{$v}}</span> {{end}}{{else}}<span class="note">none</span>{{end}}</td>
  <td class="num">{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}</td>
  <td class="num">{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
<p class="note">Re-read on every heartbeat. A port that goes down takes its fabric off
this node within seconds; one that comes back returns it. Errors are cumulative since
the adapter came up: a number that grows is the one to look at.</p>
{{end}}
{{end}}

{{if .Node.GPUs}}
<h2>GPUs and NVLink</h2>
<p class="sub">
  {{with .Node.NVLinkDomain}}In NVLink domain <code>{{.}}</code>: tensor parallelism can span this node and the rest of its domain.
  {{else}}NVLink ends at this machine.{{end}}
  {{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}<span class="state-ready">active</span>, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}<span class="bad">{{or .Node.IMEX "not installed"}}</span>{{end}}{{end}}
</p>
<div class="gpu-wiring">
<table class="rows nvl-matrix" aria-label="NVLink between GPUs">
  <tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
  {{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
    {{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over NVLink">NV</td>{{else}}<td class="pcie" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over PCIe">·</td>{{end}}{{end}}
  </tr>{{end}}
</table>
<table class="rows compact gpu-list">
  <tr><th>GPU</th><th>Name</th><th class="num">NUMA</th><th>Nearest adapter</th><th>Multi-node NVLink</th></tr>
  {{range .Node.GPUs}}<tr>
    <td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.NUMA}}</td>
    <td>{{if .NearestNIC}}<code>{{.NearestNIC}}</code> <span class="note">{{.NICPath}}</span>{{else}}—{{end}}</td>
    <td>{{if .Domain}}<span class="state-ready">joined</span>{{if and .Bandwidth (ne .Bandwidth "Full")}} <span class="bad">{{.Bandwidth}} bandwidth</span>{{end}}{{else if .FabricState}}<span class="bad">{{.FabricState}}</span>{{else}}<span class="note">—</span>{{end}}</td>
  </tr>{{end}}
</table>
</div>
<p class="note">NV is a pair of cards joined by NVLink; a tensor-parallel group kept to NV pairs runs its
all-reduces at NVLink speed, and placement picks it that way. The nearest adapter is where a card's share of
traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS is across the processors.</p>
{{end}}

<h2>What runs here</h2>
{{if .Engines}}
<table class="rows"><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
{{range .Engines}}<tr>
  <td>{{.Model}}</td>
  <td><code>{{.Replica}}</code></td>
  <td class="state state-{{.State}}">{{.State}}</td>
  <td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{printf "%.1f" .CPUCores}} cores</span></td>
  <td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
{{else}}<p class="empty">No engines are placed on this node.</p>{{end}}
<p class="note">Engines: {{if .Node.Engines}}{{join .Node.Engines}}{{else}}none installed{{end}}{{if .Node.Models}} · {{len .Node.Models}} model file(s) on disk{{end}}.</p>
</div>

{{if .HasHistory}}{{with .Charts}}
<h2>The last {{.Window}}</h2>
{{if .Err}}<p class="note error-row">No history: {{.Err}}</p>{{else}}
<div class="chart-row">
  {{template "tschart" dict "t" .CPU "title" "CPU" "note" "All cores together, from the node's heartbeat."}}
  {{template "tschart" dict "t" .Memory "title" "RAM in use" "note" "Total less what the kernel could hand to a new process."}}
</div>
<div class="chart-row">
  {{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}}
  {{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use."}}
</div>
<div class="chart-row">
  {{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}}
  {{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}}
</div>
{{end}}
{{end}}{{else}}
<p class="note">No metrics store is configured, so there is no history here; <code>monitoring.prometheus</code> in the server configuration adds it.</p>
{{end}}

<footer><span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}