llmgw/internal/web/templates/node.html

{{define "node"}}<!doctype html>
<html lang="{{lang}}"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<noscript>{{.RefreshMeta}}</noscript>
<meta name="htmx-config" content='{"allowEval":false}'>
<script src="/_static/htmx-2.0.11.min.js"></script>
<script defer src="{{asset "pages.js"}}"></script>
<title>llmgw — {{.Node.ID}}</title><link rel="stylesheet" href="{{stylesheet}}">
</head><body><div class="wrap">
<header>
  <h1>{{.Node.ID}}</h1>
  <p class="sub"><span class="state state-{{.Node.State}}">{{t (print "dash.state." .Node.State)}}</span>
    {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
    {{with .Node.AgentVersion}} · <span{{if ne . $.Version}} class="bad" title="{{t "node.version_differs" $.Version}}"{{end}}>llmgw {{.}}</span>{{end}}
    · {{if .Node.LastSeenAgo}}{{t "node.last_seen" .Node.LastSeenAgo}}{{else}}{{t "node.never_seen"}}{{end}} · <a href="{{.Logs}}">{{t "dash.nodes.logs"}}</a></p>
  {{if .OnAdminListener}}<nav><a href="{{.Back}}">{{t "node.nav.cluster"}}</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header>

{{/* Refreshed in place, like the cluster page. */}}
<div id="live" hx-get="{{.Self}}" hx-trigger="every {{.RefreshSec}}s" data-visible-only hx-select="#live" hx-swap="outerHTML">
{{with .Node}}
<div class="tiles">
  {{with .Load}}
  <div class="tile"><div class="v">{{pc .CPUPercent 0}}</div><div class="k">CPU{{if .CPUs}} · {{t "node.cores" .CPUs}}{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
  <div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">{{t "node.ram_in_use_of" (bytes .MemTotal)}}</div>{{template "minibar" pctbar .MemPercent}}</div>
  <div class="tile"><div class="v">{{num .Load1 2}}</div><div class="k">{{t "node.load" (num .Load5 2) (num .Load15 2)}}</div></div>
  {{/* Only what this machine has sensors for: a VM has none, and a tile
       saying 0 °C would be a reading nobody took. */}}
  {{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{num .CPUTempC 0}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{num .CPUPowerW 0}} W{{end}}</div></div>{{end}}
  {{if .InletTempC}}<div class="tile"><div class="v">{{num .InletTempC 1}} °C</div><div class="k">{{t "node.inlet"}}</div></div>{{end}}
  {{else}}
  <div class="tile"><div class="v">—</div><div class="k">{{t "node.not_measured"}}</div></div>
  {{end}}
  {{if .DiskTotalGB}}<div class="tile"><div class="v">{{.DiskFreeGB}}G</div><div class="k">{{t "node.disk_free" .DiskTotalGB}}</div></div>{{end}}
  {{if .GPUTotal}}<div class="tile"><div class="v">{{.GPUFree}}/{{.GPUTotal}}</div><div class="k">{{t "node.gpus_free"}}</div></div>{{end}}
</div>
{{/* The node's warnings and what is unhealthy come from the control plane
     and the agent, in English. */}}
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}}
<h2>{{t "node.gpus"}}</h2>
<table class="rows"><tr><th class="num">GPU</th><th>{{t "node.gpu.busy"}}</th><th>{{t "node.gpu.memory_used"}}</th><th class="num">{{t "node.gpu.temp"}}</th><th class="num">{{t "node.gpu.memory_temp"}}</th><th>{{t "node.gpu.power"}}</th><th class="num">{{t "node.gpu.clock"}}</th><th>{{t "node.gpu.models"}}</th><th>{{t "node.gpu.held_back"}}</th></tr>
{{range .GPUs}}<tr>
  <td class="num">{{.Index}}</td>
  {{/* From the last sample; a dash where the driver or an older agent did
       not say, rather than a zero nobody measured. */}}
  <td class="load">{{if and .MemoryTotalMB (ge .UtilPercent 0.0)}}{{template "minibar" pctbar .UtilPercent}}<span class="load-v">{{pc .UtilPercent 0}}{{if ge .MemUtilPercent 0.0}} <span class="note">{{t "node.gpu.memory_util" (num .MemUtilPercent 0)}}</span>{{end}}</span>{{else}}—{{end}}</td>
  <td class="load">{{if .MemoryTotalMB}}{{template "minibar" pctbar (percent .MemoryUsedMB .MemoryTotalMB)}}<span class="load-v">{{num (gib .MemoryUsedMB) 1}} <span class="note">{{t "node.of_gb" (num (gib .MemoryTotalMB) 0)}}</span></span>{{else}}—{{end}}</td>
  <td class="num">{{if .TempC}}{{num .TempC 0}} °C{{else}}—{{end}}</td>
  <td class="num">{{if .MemoryTempC}}{{num .MemoryTempC 0}} °C{{else}}—{{end}}</td>
  <td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{num .PowerW 0}} W <span class="note">{{t "node.of" (num .PowerLimitW 0)}}</span></span>{{else if .PowerW}}{{num .PowerW 0}} W{{else}}—{{end}}</td>
  <td class="num">{{if .SMClockMHz}}{{num .SMClockMHz 0}} MHz{{else}}—{{end}}</td>
  {{/* The driver's reasons and the wear are named as the driver and the
       inventory name them, in English. */}}
  <td>{{with index $.GPUModels .Index}}{{join .}}{{else}}<span class="note">—</span>{{end}}</td>
  <td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">{{t "node.gpu.nothing"}}</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{t "node.gpu.ecc" (num .ECCUncorrected 0)}}</span>{{end}}{{with .Wear}}{{with .Problems}}<br><span class="bad">{{join .}}</span>{{end}}{{if gt .PCIeReplays 0}} <span class="note">{{t "node.gpu.pcie" (count .PCIeReplays)}}</span>{{end}}{{end}}</td>
</tr>{{end}}</table>
{{end}}{{end}}
{{end}}

{{if or .Node.IBPorts .Node.Fabrics}}
<h2>{{t "node.fabric"}}</h2>
<p class="sub">{{if .Node.Fabrics}}{{t "node.fabric.on" (join .Node.Fabrics)}} · <a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">{{t "node.fabric.map"}}</a>{{else}}<span class="bad">{{t "node.fabric.none"}}</span>{{end}}</p>
{{if .Node.IBPorts}}
<table class="rows"><tr><th>{{t "node.ib.adapter"}}</th><th>{{t "dash.col.state"}}</th><th class="num">{{t "node.ib.rate"}}</th><th>{{t "node.fabric"}}</th><th>{{t "node.ib.physically"}}</th><th>{{t "node.ib.errors"}}</th><th class="num">{{t "node.ib.sent"}}</th><th class="num">{{t "node.ib.received"}}</th></tr>
{{range .Node.IBPorts}}<tr>
  <td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag" translate="no">RoCE</span>{{end}}</td>
  {{/* The port's state and its counters are the adapter's own words. */}}
  <td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
  <td class="num">{{if .RateGbps}}{{t "fabric.gbps" .RateGbps}}{{else}}—{{end}}</td>
  <td>{{or .Fabric "—"}}</td>
  <td>{{if .Physical}}{{.Physical}}{{else if .SMError}}<span class="note" title="{{.SMError}}">{{t "node.unknown"}}</span>{{else}}—{{end}}</td>
  <td>{{if .Errors}}{{range $k, $v := .Errors}}<span class="tag bad">{{$k}} {{$v}}</span> {{end}}{{else}}<span class="note">{{t "node.none"}}</span>{{end}}</td>
  <td class="num">{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}</td>
  <td class="num">{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
<p class="note">{{t "node.ib.note"}}</p>
{{end}}
{{end}}

{{if .Node.GPUs}}
<h2>{{t "node.nvl.title"}}</h2>
<p class="sub">
  {{with .Node.NVLinkDomain}}{{th "node.nvl.domain" .}}
  {{else}}{{t "node.nvl.ends_here"}}{{end}}
  {{if or .Node.NVLinkDomain .Node.IMEX}} · <span translate="no">IMEX</span> {{if eq .Node.IMEX "active"}}<span class="state-ready">{{t "node.imex.active"}}</span>, {{tn "node.imex.channels" .Node.IMEXChannels}}{{else}}<span class="bad">{{with .Node.IMEX}}{{.}}{{else}}{{t "node.imex.not_installed"}}{{end}}</span>{{end}}{{end}}
</p>
<div class="gpu-wiring">
<table class="rows nvl-matrix" aria-label="{{t "node.nvl.matrix"}}">
  <tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
  {{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
    {{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="{{t "node.nvl.over_nvlink" $a.Index $b.Index}}" translate="no">NV</td>{{else}}<td class="pcie" title="{{t "node.nvl.over_pcie" $a.Index $b.Index}}">·</td>{{end}}{{end}}
  </tr>{{end}}
</table>
<table class="rows compact gpu-list">
  <tr><th>GPU</th><th>{{t "node.gpu.name"}}</th><th class="num">{{t "node.gpu.compute"}}</th><th class="num">{{t "node.gpu.numa"}}</th><th>{{t "node.gpu.nearest"}}</th><th>{{t "node.gpu.multinode"}}</th></tr>
  {{range .Node.GPUs}}<tr>
    <td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.ComputeCapability}}</td><td class="num">{{.NUMA}}</td>
    <td>{{if .NearestNIC}}<code>{{.NearestNIC}}</code> <span class="note">{{.NICPath}}</span>{{else}}—{{end}}</td>
    {{/* The bandwidth and fabric state are the driver's words. */}}
    <td>{{if .Domain}}<span class="state-ready">{{t "node.gpu.joined"}}</span>{{if and .Bandwidth (ne .Bandwidth "Full")}} <span class="bad">{{t "node.gpu.bandwidth" .Bandwidth}}</span>{{end}}{{else if .FabricState}}<span class="bad">{{.FabricState}}</span>{{else}}<span class="note">—</span>{{end}}</td>
  </tr>{{end}}
</table>
</div>
<p class="note">{{t "node.nvl.note"}}</p>
{{end}}

<h2>{{t "node.runs"}}</h2>
{{if .Engines}}
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.state"}}</th><th>GPU</th><th>CPU</th><th class="num">{{t "node.memory"}}</th></tr>
{{range .Engines}}<tr>
  <td>{{.Model}}</td>
  <td><code>{{.Replica}}</code></td>
  <td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
  <td>{{if .GPUs}}{{range $i, $g := .GPUs}}{{if $i}}, {{end}}{{$g}}{{end}}{{else}}—{{end}}</td>
  <td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{t "node.cores_f" (num .CPUCores 1)}}</span></td>
  <td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
{{else}}<p class="empty">{{t "node.no_engines"}}</p>{{end}}
<p class="note">{{if .EngineVersions}}{{t "node.engines" (join .EngineVersions)}}{{else if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.</p>
</div>

{{if .HasHistory}}{{with .Charts}}
<h2>{{t "node.last" .Window}}</h2>
{{if .Err}}<p class="note error-row">{{t "dash.no_history" .Err}}</p>{{else}}
<div class="chart-row">
  {{template "tschart" dict "t" .CPU "title" "CPU" "note" (t "node.chart.cpu_note")}}
  {{template "tschart" dict "t" .Memory "title" (t "node.chart.ram") "note" (t "node.chart.ram_note")}}
</div>
<div class="chart-row">
  {{template "tschart" dict "t" .EngineCPU "title" (t "node.chart.cores") "note" (t "node.chart.cores_note")}}
  {{template "tschart" dict "t" .EngineMemory "title" (t "node.chart.engine_memory") "note" (t "node.chart.engine_memory_note")}}
</div>
{{if $.Node.GPUTotal}}
<div class="chart-row">
  {{template "tschart" dict "t" .GPUUtil "title" (t "node.chart.gpu_busy") "note" (t "node.chart.gpu_busy_note")}}
  {{template "tschart" dict "t" .GPUMemory "title" (t "node.chart.gpu_memory") "note" (t "node.chart.gpu_memory_note")}}
</div>
<div class="chart-row">
  {{template "tschart" dict "t" .GPUTemp "title" (t "node.chart.gpu_temp") "note" (t "node.chart.gpu_temp_note")}}
  {{template "tschart" dict "t" .GPUPower "title" (t "node.chart.gpu_power") "note" (t "node.chart.gpu_power_note")}}
</div>
<div class="chart-row">
  {{template "tschart" dict "t" .GPUPowerLoad "title" (t "node.chart.gpu_power_load") "note" (t "node.chart.gpu_power_load_note")}}
  {{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "node.chart.gpu_sm_active") "note" (t "node.chart.gpu_sm_active_note")}}{{end}}
</div>
{{end}}
<div class="chart-row">
  {{template "tschart" dict "t" .Load "title" (t "node.chart.load") "note" (t "node.chart.load_note")}}
  {{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "node.chart.queue_note")}}
</div>
{{end}}
{{end}}{{else}}
<p class="note">{{th "node.no_store"}}</p>
{{end}}

<footer><span>llmgw {{.Version}} · {{t "common.refreshes_every" .RefreshSec}}</span><span>{{stamp .Now}}</span>{{template "langs"}}</footer>
</div></body></html>{{end}}