The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
70 lines
4.1 KiB
HTML
70 lines
4.1 KiB
HTML
{{define "node"}}<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
|
|
<script src="/_static/htmx-2.0.11.min.js"></script>
|
|
<title>llmgw — {{.Node.ID}}</title><link rel="stylesheet" href="{{stylesheet}}">
|
|
</head><body><div class="wrap">
|
|
<header>
|
|
<h1>{{.Node.ID}}</h1>
|
|
<p class="sub"><span class="state state-{{.Node.State}}">{{.Node.State}}</span>
|
|
{{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
|
|
· last seen {{or .Node.LastSeenAgo "never"}} ago · <a href="{{.Logs}}">logs</a> · <a href="{{.Back}}">cluster</a></p>
|
|
{{if .OnAdminListener}}<nav><a href="{{.Back}}">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
|
|
</header>
|
|
|
|
{{/* Refreshed in place, like the cluster page. */}}
|
|
<div id="live" hx-get="{{.Self}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
|
|
{{with .Node}}
|
|
<div class="tiles">
|
|
{{with .Load}}
|
|
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
|
|
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
|
|
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
|
|
{{else}}
|
|
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
|
|
{{end}}
|
|
{{if .DiskTotalGB}}<div class="tile"><div class="v">{{.DiskFreeGB}}G</div><div class="k">disk free · of {{.DiskTotalGB}}G</div></div>{{end}}
|
|
{{if .GPUTotal}}<div class="tile"><div class="v">{{.GPUFree}}/{{.GPUTotal}}</div><div class="k">GPUs free</div></div>{{end}}
|
|
</div>
|
|
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
|
|
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
|
|
{{end}}
|
|
|
|
<h2>What runs here</h2>
|
|
{{if .Engines}}
|
|
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
|
{{range .Engines}}<tr>
|
|
<td>{{.Model}}</td>
|
|
<td><code>{{.Replica}}</code></td>
|
|
<td class="state state-{{.State}}">{{.State}}</td>
|
|
<td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{printf "%.1f" .CPUCores}} cores</span></td>
|
|
<td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
|
|
</tr>{{end}}</table>
|
|
{{else}}<p class="empty">No engines are placed on this node.</p>{{end}}
|
|
<p class="note">Engines: {{if .Node.Engines}}{{join .Node.Engines}}{{else}}none installed{{end}}{{if .Node.Models}} · {{len .Node.Models}} model file(s) on disk{{end}}.</p>
|
|
</div>
|
|
|
|
{{if .HasHistory}}{{with .Charts}}
|
|
<h2>The last {{.Window}}</h2>
|
|
{{if .Err}}<p class="note error-row">No history: {{.Err}}</p>{{else}}
|
|
<div class="chart-row">
|
|
{{template "tschart" dict "t" .CPU "title" "CPU" "note" "All cores together, from the node's heartbeat."}}
|
|
{{template "tschart" dict "t" .Memory "title" "RAM in use" "note" "Total less what the kernel could hand to a new process."}}
|
|
</div>
|
|
<div class="chart-row">
|
|
{{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}}
|
|
{{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory: the weights and the KV cache, mostly."}}
|
|
</div>
|
|
<div class="chart-row">
|
|
{{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}}
|
|
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}}
|
|
</div>
|
|
{{end}}
|
|
{{end}}{{else}}
|
|
<p class="note">No metrics store is configured, so there is no history here; <code>monitoring.prometheus</code> in the server configuration adds it.</p>
|
|
{{end}}
|
|
|
|
<footer><span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span><span>{{.Now}}</span></footer>
|
|
</div></body></html>{{end}}
|