A node's page says which llmgw it runs and which model is on each GPU
CI / test (push) Successful in 4m13s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 11:31:52 +02:00
1 parent 178b168bfd
commit 4b6b3eeddb
6 files changed
+83 -7

No files matched your search

+6
View File
@@ -26,6 +26,12 @@ APIs change between minors without a deprecation period.
## Unreleased ## Unreleased
### Added
- **A node's page says what it runs.** It now shows the agent's version, marked when the control plane runs another,
and the engines with their versions. Each GPU row now names the models loaded on that card, and each engine row its
GPUs.
### Fixed ### Fixed
- **A broken `auth.ldap` no longer stops every control plane.** It is the cluster's setting. A CA file that one - **A broken `auth.ldap` no longer stops every control plane.** It is the cluster's setting. A CA file that one
+2
View File
@@ -62,6 +62,8 @@
"node.memory": "Memory", "node.memory": "Memory",
"node.cores_f": "%s cores", "node.cores_f": "%s cores",
"node.no_engines": "No engines are placed on this node.", "node.no_engines": "No engines are placed on this node.",
"node.version_differs": "the control plane runs %s",
"node.gpu.models": "Models",
"node.engines": "Engines: %s", "node.engines": "Engines: %s",
"node.engines_none": "Engines: none installed", "node.engines_none": "Engines: none installed",
"node.model_files": "%d model file(s) on disk", "node.model_files": "%d model file(s) on disk",
+2
View File
@@ -61,6 +61,8 @@
"node.memory": "Bellek", "node.memory": "Bellek",
"node.cores_f": "%s çekirdek", "node.cores_f": "%s çekirdek",
"node.no_engines": "Bu düğüme yerleştirilmiş motor yok.", "node.no_engines": "Bu düğüme yerleştirilmiş motor yok.",
"node.version_differs": "control plane %s çalıştırıyor",
"node.gpu.models": "Modeller",
"node.engines": "Motorlar: %s", "node.engines": "Motorlar: %s",
"node.engines_none": "Motorlar: hiçbiri kurulu değil", "node.engines_none": "Motorlar: hiçbiri kurulu değil",
"node.model_files": "diskte %d model dosyası", "node.model_files": "diskte %d model dosyası",
+22 -4
View File
@@ -6,6 +6,7 @@ import (
"html/template" "html/template"
"log/slog" "log/slog"
"net/http" "net/http"
"slices"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -20,9 +21,13 @@ type nodePageData struct {
OnAdminListener bool OnAdminListener bool
Node adminapi.NodeView Node adminapi.NodeView
Engines []nodeEngine Engines []nodeEngine
Charts *nodeCharts // GPUModels are the models on each of the node's GPUs, by index.
HasHistory bool GPUModels map[int][]string
Back, Logs string // EngineVersions are the installed engines and their versions, sorted.
EngineVersions []string
Charts *nodeCharts
HasHistory bool
Back, Logs string
// Self is this page's address, for the in-place refresh. // Self is this page's address, for the in-place refresh.
Self string Self string
Version, Now string Version, Now string
@@ -34,6 +39,7 @@ type nodePageData struct {
// nodeEngine is one engine running on the node, and its share of it. // nodeEngine is one engine running on the node, and its share of it.
type nodeEngine struct { type nodeEngine struct {
Replica, Model, Shard, State string Replica, Model, Shard, State string
GPUs []int
CPUCores float64 CPUCores float64
MemoryBytes int64 MemoryBytes int64
// CPUShare is its cores as a percentage of the machine, for the bar. // CPUShare is its cores as a percentage of the machine, for the bar.
@@ -128,7 +134,15 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener
continue continue
} }
e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State, e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State,
CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes} GPUs: sh.GPUs, CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes}
for _, g := range sh.GPUs {
if d.GPUModels == nil {
d.GPUModels = map[int][]string{}
}
if !slices.Contains(d.GPUModels[g], rep.Model) {
d.GPUModels[g] = append(d.GPUModels[g], rep.Model)
}
}
if l := d.Node.Load; l != nil && l.CPUs > 0 { if l := d.Node.Load; l != nil && l.CPUs > 0 {
e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs) e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs)
} }
@@ -137,6 +151,10 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener
} }
} }
sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores }) sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores })
for e, v := range d.Node.EngineVersions {
d.EngineVersions = append(d.EngineVersions, e+" "+v)
}
sort.Strings(d.EngineVersions)
if history != nil { if history != nil {
d.Charts = loadNodeCharts(r.Context(), l, history, id, replicas, d.Node.GPUTotal, window, log) d.Charts = loadNodeCharts(r.Context(), l, history, id, replicas, d.Node.GPUTotal, window, log)
+45
View File
@@ -270,3 +270,48 @@ func TestTheClusterPageDrawsItsGPUsByClass(t *testing.T) {
} }
} }
} }
// versionSource is hotSource with an agent a release behind, a model on its
// second GPU, and engines with their versions.
type versionSource struct{ hotSource }
func (versionSource) Nodes() []adminapi.NodeView {
n := hotSource{}.Nodes()[0]
n.AgentVersion = "v0"
n.EngineVersions = map[string]string{"llamacpp": "b10991", "comfyui": "0.39.2"}
return []adminapi.NodeView{n}
}
func (versionSource) Replicas(string) []adminapi.ReplicaView {
r := fakeSource{}.Replicas("")
r[0].Shards[0].GPUs = []int{1}
return r
}
func TestANodesPageSaysWhatItRunsAndWhichModelIsOnEachGPU(t *testing.T) {
_, body := get(t, nodeMux(versionSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
model := versionSource{}.Replicas("")[0].Model
for _, want := range []string{
// The page's control plane runs v1: an agent on v0 is marked.
`class="bad" title="the control plane runs v1">llmgw v0</span>`,
"Engines: comfyui 0.39.2, llamacpp b10991",
} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
// GPU 1 holds the model, GPU 0 nothing.
rows := strings.Split(body, "<tr>")
var gpu0, gpu1 string
for _, row := range rows {
switch {
case strings.HasPrefix(row, "\n <td class=\"num\">0</td>"):
gpu0 = row
case strings.HasPrefix(row, "\n <td class=\"num\">1</td>"):
gpu1 = row
}
}
if !strings.Contains(gpu1, "<td>"+model+"</td>") || strings.Contains(gpu0, model) {
t.Errorf("GPU rows:\n0: %.300s\n1: %.300s", gpu0, gpu1)
}
}
+6 -3
View File
@@ -11,6 +11,7 @@
<h1>{{.Node.ID}}</h1> <h1>{{.Node.ID}}</h1>
<p class="sub"><span class="state state-{{.Node.State}}">{{t (print "dash.state." .Node.State)}}</span> <p class="sub"><span class="state state-{{.Node.State}}">{{t (print "dash.state." .Node.State)}}</span>
{{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}} {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
{{with .Node.AgentVersion}} · <span{{if ne . $.Version}} class="bad" title="{{t "node.version_differs" $.Version}}"{{end}}>llmgw {{.}}</span>{{end}}
· {{if .Node.LastSeenAgo}}{{t "node.last_seen" .Node.LastSeenAgo}}{{else}}{{t "node.never_seen"}}{{end}} · <a href="{{.Logs}}">{{t "dash.nodes.logs"}}</a></p> · {{if .Node.LastSeenAgo}}{{t "node.last_seen" .Node.LastSeenAgo}}{{else}}{{t "node.never_seen"}}{{end}} · <a href="{{.Logs}}">{{t "dash.nodes.logs"}}</a></p>
{{if .OnAdminListener}}<nav><a href="{{.Back}}">{{t "node.nav.cluster"}}</a></nav>{{else}}{{template "nav" .Nav}}{{end}} {{if .OnAdminListener}}<nav><a href="{{.Back}}">{{t "node.nav.cluster"}}</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header> </header>
@@ -39,7 +40,7 @@
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}} {{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}} {{with .Load}}{{if .GPUs}}
<h2>{{t "node.gpus"}}</h2> <h2>{{t "node.gpus"}}</h2>
<table class="rows"><tr><th class="num">GPU</th><th>{{t "node.gpu.busy"}}</th><th>{{t "node.gpu.memory_used"}}</th><th class="num">{{t "node.gpu.temp"}}</th><th class="num">{{t "node.gpu.memory_temp"}}</th><th>{{t "node.gpu.power"}}</th><th class="num">{{t "node.gpu.clock"}}</th><th>{{t "node.gpu.held_back"}}</th></tr> <table class="rows"><tr><th class="num">GPU</th><th>{{t "node.gpu.busy"}}</th><th>{{t "node.gpu.memory_used"}}</th><th class="num">{{t "node.gpu.temp"}}</th><th class="num">{{t "node.gpu.memory_temp"}}</th><th>{{t "node.gpu.power"}}</th><th class="num">{{t "node.gpu.clock"}}</th><th>{{t "node.gpu.models"}}</th><th>{{t "node.gpu.held_back"}}</th></tr>
{{range .GPUs}}<tr> {{range .GPUs}}<tr>
<td class="num">{{.Index}}</td> <td class="num">{{.Index}}</td>
{{/* From the last sample; a dash where the driver or an older agent did {{/* From the last sample; a dash where the driver or an older agent did
@@ -52,6 +53,7 @@
<td class="num">{{if .SMClockMHz}}{{num .SMClockMHz 0}} MHz{{else}}—{{end}}</td> <td class="num">{{if .SMClockMHz}}{{num .SMClockMHz 0}} MHz{{else}}—{{end}}</td>
{{/* The driver's reasons and the wear are named as the driver and the {{/* The driver's reasons and the wear are named as the driver and the
inventory name them, in English. */}} inventory name them, in English. */}}
<td>{{with index $.GPUModels .Index}}{{join .}}{{else}}<span class="note">—</span>{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">{{t "node.gpu.nothing"}}</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{t "node.gpu.ecc" (num .ECCUncorrected 0)}}</span>{{end}}{{with .Wear}}{{with .Problems}}<br><span class="bad">{{join .}}</span>{{end}}{{if gt .PCIeReplays 0}} <span class="note">{{t "node.gpu.pcie" (count .PCIeReplays)}}</span>{{end}}{{end}}</td> <td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">{{t "node.gpu.nothing"}}</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{t "node.gpu.ecc" (num .ECCUncorrected 0)}}</span>{{end}}{{with .Wear}}{{with .Problems}}<br><span class="bad">{{join .}}</span>{{end}}{{if gt .PCIeReplays 0}} <span class="note">{{t "node.gpu.pcie" (count .PCIeReplays)}}</span>{{end}}{{end}}</td>
</tr>{{end}}</table> </tr>{{end}}</table>
{{end}}{{end}} {{end}}{{end}}
@@ -106,16 +108,17 @@
<h2>{{t "node.runs"}}</h2> <h2>{{t "node.runs"}}</h2>
{{if .Engines}} {{if .Engines}}
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.state"}}</th><th>CPU</th><th class="num">{{t "node.memory"}}</th></tr> <table class="rows"><tr><th>{{t "dash.col.model"}}</th><th>{{t "dash.col.replica"}}</th><th>{{t "dash.col.state"}}</th><th>GPU</th><th>CPU</th><th class="num">{{t "node.memory"}}</th></tr>
{{range .Engines}}<tr> {{range .Engines}}<tr>
<td>{{.Model}}</td> <td>{{.Model}}</td>
<td><code>{{.Replica}}</code></td> <td><code>{{.Replica}}</code></td>
<td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td> <td class="state state-{{.State}}">{{t (print "dash.state." .State)}}</td>
<td>{{if .GPUs}}{{range $i, $g := .GPUs}}{{if $i}}, {{end}}{{$g}}{{end}}{{else}}—{{end}}</td>
<td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{t "node.cores_f" (num .CPUCores 1)}}</span></td> <td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{t "node.cores_f" (num .CPUCores 1)}}</span></td>
<td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td> <td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
</tr>{{end}}</table> </tr>{{end}}</table>
{{else}}<p class="empty">{{t "node.no_engines"}}</p>{{end}} {{else}}<p class="empty">{{t "node.no_engines"}}</p>{{end}}
<p class="note">{{if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.</p> <p class="note">{{if .EngineVersions}}{{t "node.engines" (join .EngineVersions)}}{{else if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.</p>
</div> </div>
{{if .HasHistory}}{{with .Charts}} {{if .HasHistory}}{{with .Charts}}