diff --git a/CHANGELOG.md b/CHANGELOG.md index 5642436..92d6559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ APIs change between minors without a deprecation period. ## Unreleased +### Added + +- **A node's page says what it runs.** It now shows the agent's version, marked when the control plane runs another, + and the engines with their versions. Each GPU row now names the models loaded on that card, and each engine row its + GPUs. + ### Fixed - **A broken `auth.ldap` no longer stops every control plane.** It is the cluster's setting. A CA file that one diff --git a/internal/web/locales/en/node.json b/internal/web/locales/en/node.json index 69394af..be76b4f 100644 --- a/internal/web/locales/en/node.json +++ b/internal/web/locales/en/node.json @@ -62,6 +62,8 @@ "node.memory": "Memory", "node.cores_f": "%s cores", "node.no_engines": "No engines are placed on this node.", + "node.version_differs": "the control plane runs %s", + "node.gpu.models": "Models", "node.engines": "Engines: %s", "node.engines_none": "Engines: none installed", "node.model_files": "%d model file(s) on disk", diff --git a/internal/web/locales/tr/node.json b/internal/web/locales/tr/node.json index 3303da3..7df2668 100644 --- a/internal/web/locales/tr/node.json +++ b/internal/web/locales/tr/node.json @@ -61,6 +61,8 @@ "node.memory": "Bellek", "node.cores_f": "%s çekirdek", "node.no_engines": "Bu düğüme yerleştirilmiş motor yok.", + "node.version_differs": "control plane %s çalıştırıyor", + "node.gpu.models": "Modeller", "node.engines": "Motorlar: %s", "node.engines_none": "Motorlar: hiçbiri kurulu değil", "node.model_files": "diskte %d model dosyası", diff --git a/internal/web/node_page.go b/internal/web/node_page.go index 37538e7..81bd3cb 100644 --- a/internal/web/node_page.go +++ b/internal/web/node_page.go @@ -6,6 +6,7 @@ import ( "html/template" "log/slog" "net/http" + "slices" "sort" "strings" "time" @@ -20,9 +21,13 @@ type nodePageData struct { OnAdminListener bool Node adminapi.NodeView Engines []nodeEngine - Charts *nodeCharts - HasHistory bool - Back, Logs string + // GPUModels are the models on each of the node's GPUs, by index. + GPUModels map[int][]string + // EngineVersions are the installed engines and their versions, sorted. + EngineVersions []string + Charts *nodeCharts + HasHistory bool + Back, Logs string // Self is this page's address, for the in-place refresh. Self string Version, Now string @@ -34,6 +39,7 @@ type nodePageData struct { // nodeEngine is one engine running on the node, and its share of it. type nodeEngine struct { Replica, Model, Shard, State string + GPUs []int CPUCores float64 MemoryBytes int64 // CPUShare is its cores as a percentage of the machine, for the bar. @@ -128,7 +134,15 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener continue } e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State, - CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes} + GPUs: sh.GPUs, CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes} + for _, g := range sh.GPUs { + if d.GPUModels == nil { + d.GPUModels = map[int][]string{} + } + if !slices.Contains(d.GPUModels[g], rep.Model) { + d.GPUModels[g] = append(d.GPUModels[g], rep.Model) + } + } if l := d.Node.Load; l != nil && l.CPUs > 0 { e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs) } @@ -137,6 +151,10 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener } } sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores }) + for e, v := range d.Node.EngineVersions { + d.EngineVersions = append(d.EngineVersions, e+" "+v) + } + sort.Strings(d.EngineVersions) if history != nil { d.Charts = loadNodeCharts(r.Context(), l, history, id, replicas, d.Node.GPUTotal, window, log) diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index f947477..81318e7 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -270,3 +270,48 @@ func TestTheClusterPageDrawsItsGPUsByClass(t *testing.T) { } } } + +// versionSource is hotSource with an agent a release behind, a model on its +// second GPU, and engines with their versions. +type versionSource struct{ hotSource } + +func (versionSource) Nodes() []adminapi.NodeView { + n := hotSource{}.Nodes()[0] + n.AgentVersion = "v0" + n.EngineVersions = map[string]string{"llamacpp": "b10991", "comfyui": "0.39.2"} + return []adminapi.NodeView{n} +} + +func (versionSource) Replicas(string) []adminapi.ReplicaView { + r := fakeSource{}.Replicas("") + r[0].Shards[0].GPUs = []int{1} + return r +} + +func TestANodesPageSaysWhatItRunsAndWhichModelIsOnEachGPU(t *testing.T) { + _, body := get(t, nodeMux(versionSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) + model := versionSource{}.Replicas("")[0].Model + for _, want := range []string{ + // The page's control plane runs v1: an agent on v0 is marked. + `class="bad" title="the control plane runs v1">llmgw v0`, + "Engines: comfyui 0.39.2, llamacpp b10991", + } { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + // GPU 1 holds the model, GPU 0 nothing. + rows := strings.Split(body, "") + var gpu0, gpu1 string + for _, row := range rows { + switch { + case strings.HasPrefix(row, "\n 0"): + gpu0 = row + case strings.HasPrefix(row, "\n 1"): + gpu1 = row + } + } + if !strings.Contains(gpu1, ""+model+"") || strings.Contains(gpu0, model) { + t.Errorf("GPU rows:\n0: %.300s\n1: %.300s", gpu0, gpu1) + } +} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index 3b8439a..f242d27 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -11,6 +11,7 @@

{{.Node.ID}}

{{t (print "dash.state." .Node.State)}} {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}} + {{with .Node.AgentVersion}} · llmgw {{.}}{{end}} · {{if .Node.LastSeenAgo}}{{t "node.last_seen" .Node.LastSeenAgo}}{{else}}{{t "node.never_seen"}}{{end}} · {{t "dash.nodes.logs"}}

{{if .OnAdminListener}}{{else}}{{template "nav" .Nav}}{{end}} @@ -39,7 +40,7 @@ {{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} {{with .Load}}{{if .GPUs}}

{{t "node.gpus"}}

- +
GPU{{t "node.gpu.busy"}}{{t "node.gpu.memory_used"}}{{t "node.gpu.temp"}}{{t "node.gpu.memory_temp"}}{{t "node.gpu.power"}}{{t "node.gpu.clock"}}{{t "node.gpu.held_back"}}
{{range .GPUs}} {{/* From the last sample; a dash where the driver or an older agent did @@ -52,6 +53,7 @@ {{/* The driver's reasons and the wear are named as the driver and the inventory name them, in English. */}} + {{end}}
GPU{{t "node.gpu.busy"}}{{t "node.gpu.memory_used"}}{{t "node.gpu.temp"}}{{t "node.gpu.memory_temp"}}{{t "node.gpu.power"}}{{t "node.gpu.clock"}}{{t "node.gpu.models"}}{{t "node.gpu.held_back"}}
{{.Index}}{{if .SMClockMHz}}{{num .SMClockMHz 0}} MHz{{else}}—{{end}}{{with index $.GPUModels .Index}}{{join .}}{{else}}—{{end}} {{if .Throttle}}{{join .Throttle}}{{else}}{{t "node.gpu.nothing"}}{{end}}{{if gt .ECCUncorrected 0.0}} {{t "node.gpu.ecc" (num .ECCUncorrected 0)}}{{end}}{{with .Wear}}{{with .Problems}}
{{join .}}{{end}}{{if gt .PCIeReplays 0}} {{t "node.gpu.pcie" (count .PCIeReplays)}}{{end}}{{end}}
{{end}}{{end}} @@ -106,16 +108,17 @@

{{t "node.runs"}}

{{if .Engines}} - +
{{t "dash.col.model"}}{{t "dash.col.replica"}}{{t "dash.col.state"}}CPU{{t "node.memory"}}
{{range .Engines}} + {{end}}
{{t "dash.col.model"}}{{t "dash.col.replica"}}{{t "dash.col.state"}}GPUCPU{{t "node.memory"}}
{{.Model}} {{.Replica}} {{t (print "dash.state." .State)}}{{if .GPUs}}{{range $i, $g := .GPUs}}{{if $i}}, {{end}}{{$g}}{{end}}{{else}}—{{end}} {{template "minibar" pctbar .CPUShare}}{{t "node.cores_f" (num .CPUCores 1)}} {{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}
{{else}}

{{t "node.no_engines"}}

{{end}} -

{{if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.

+

{{if .EngineVersions}}{{t "node.engines" (join .EngineVersions)}}{{else if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.

{{if .HasHistory}}{{with .Charts}}