Find multi-node NVLink domains, place across them, and show them as racks

A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:07:59 +02:00
1 parent 5bbc619a31
commit 20479a2d5f
31 files changed
+1535 -131

No files matched your search

+8
View File
@@ -9,6 +9,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -28,6 +29,8 @@ type DashboardData struct {
// Fabrics is the physical fabrics and what disagrees about them, from a
// source that can say.
Fabrics *fabrics.Report
// NVLink is the multi-node NVLink domains, from a source that can say.
NVLink *nvlink.Report
Users []store.UserUsage
Models2 []store.ModelUsage
@@ -179,6 +182,11 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
r := fs.Fabrics()
d.Fabrics = &r
}
if ns, ok := src.(NVLinkSource); ok {
if r := ns.NVLink(); len(r.Domains) > 0 {
d.NVLink = &r
}
}
}
if usage != nil {
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
+29
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)
@@ -145,3 +146,31 @@ func TestAFabricsPageCarriesItsMapSafely(t *testing.T) {
t.Errorf("json: %d %s", code, body)
}
}
type nvlSource struct{ busySource }
func (nvlSource) NVLink() nvlink.Report {
return nvlink.Report{
Domains: []nvlink.Domain{{Key: "76b1ad76-7c24/7", ClusterUUID: "76b1ad76-7c24", CliqueID: "7", GPUs: 8, Joined: 7, Usable: 1,
Members: []nvlink.Member{
{Node: "t1", Usable: true, IMEX: "active", IMEXChannels: 1, GPUs: []nvlink.GPU{{Index: 0, Joined: true, State: "Completed"}}},
{Node: "t2", IMEX: "inactive", GPUs: []nvlink.GPU{{Index: 3, State: "In Progress"}}},
}}},
Problems: []nvlink.Problem{{Level: nvlink.Fail, Domain: "76b1ad76-7c24/7", Message: "t2: nvidia-imex is inactive"}},
}
}
func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
_, body := get(t, NVLinkPage(nvlSource{}, false, "v1"), "http://x/admin/nvlink", nil)
for _, want := range []string{`class="rack troubled"`, "76b1ad76·7", "1/2</b><span>trays usable", `class="chip joined"`,
`class="chip lost"`, `class="tray out"`, "t2: nvidia-imex is inactive", `class="imex up"`} {
if !strings.Contains(body, want) {
t.Errorf("missing %q", want)
}
}
// And the cluster page points at it.
_, body = get(t, Dashboard(nvlSource{}, fakeUsage{}, "v1", time.Second, false, NavExtras{}, nil), "http://x/admin", nil)
if !strings.Contains(body, "see the racks") {
t.Error("the cluster page does not lead to the racks")
}
}
+56
View File
@@ -0,0 +1,56 @@
package web
import (
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
)
// NVLinkSource is a Source that can report multi-node NVLink domains.
type NVLinkSource interface {
NVLink() nvlink.Report
}
type nvlinkPageData struct {
Nav Nav
OnAdminListener bool
Nodes string
Domains []nvlinkDomainView
Problems []nvlink.Problem
Version, Now string
}
type nvlinkDomainView struct {
nvlink.Domain
Short string
Problems []nvlink.Problem
}
// NVLinkPage shows every multi-node NVLink domain as a rack of trays.
func NVLinkPage(src Source, onAdminListener bool, version string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ns, ok := src.(NVLinkSource)
if !ok {
http.NotFound(w, r)
return
}
rep := ns.NVLink()
d := nvlinkPageData{Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, Nodes: "/admin/nodes",
Problems: rep.Problems, Version: version, Now: time.Now().UTC().Format(time.RFC3339)}
if onAdminListener {
d.Nodes = "/nodes"
}
for _, dom := range rep.Domains {
v := nvlinkDomainView{Domain: dom, Short: nvlink.Short(dom.Key)}
for _, p := range rep.Problems {
if p.Domain == dom.Key {
v.Problems = append(v.Problems, p)
}
}
d.Domains = append(d.Domains, v)
}
noStore(w)
render(w, "nvlink", d)
}
}
+68
View File
@@ -677,3 +677,71 @@ ul.fabric-problems li.info { color: var(--muted); }
}
.mode-rails .fm-host.down circle:not(.fm-halo) { fill: var(--fm-bad); stroke: #fecdd3; }
.fm-svg.focus .fm-warn { opacity: .25; }
/* A node's GPUs: which pairs NVLink joins. */
.gpu-wiring { display: flex; flex-wrap: wrap; gap: 1.25rem; align-items: flex-start; }
table.nvl-matrix { width: auto; border-collapse: separate; border-spacing: 3px; background: none; font-size: .75rem; }
table.nvl-matrix th { font-weight: 600; color: var(--muted); padding: 0 .3rem; text-align: center; border: 0; }
table.nvl-matrix td { width: 2rem; height: 2rem; text-align: center; border-radius: 6px; padding: 0; border: 0;
font-weight: 700; font-size: .7rem; }
table.nvl-matrix td.self { background: var(--line); }
table.nvl-matrix td.nv { color: #04211c; background: linear-gradient(135deg, #5eead4, #2dd4bf);
box-shadow: 0 0 10px rgba(45, 212, 191, .45); }
table.nvl-matrix td.pcie { color: var(--muted); background: var(--card); border: 1px solid var(--line); }
table.gpu-list { width: auto; flex: 1 1 22rem; }
/* ---------------------------------------------------------------------
NVLink domains, drawn as racks: the switch band on top, a backplane
carrying light down the side, trays of GPU chips that glow when joined.
--------------------------------------------------------------------- */
.racks { display: grid; grid-template-columns: repeat(auto-fill, minmax(26rem, 1fr)); gap: 1.25rem; margin: 1rem 0 1.5rem; }
.rack { border-radius: 16px; overflow: hidden; color: #dbe4ff;
background: radial-gradient(700px 260px at 50% -20%, #1a3b3b 0%, #0b1020 60%), #0b1020;
border: 1px solid #1c2f3f; box-shadow: 0 20px 50px -28px rgba(45, 212, 191, .45); }
.rack.troubled { border-color: #4a1d2c; box-shadow: 0 20px 50px -28px rgba(251, 113, 133, .45); }
.rack-head { display: flex; justify-content: space-between; align-items: flex-start; gap: 1rem; padding: .9rem 1rem .6rem; }
.rack-head h2 { margin: 0; color: #fff; font-size: 1.15rem; letter-spacing: .02em; }
.rack-head .note, .rack-head code { color: #7c89ad; background: none; font-size: .72rem; }
.rack-stats { display: flex; gap: 1rem; text-align: right; }
.rack-stats b { display: block; font-size: 1.1rem; color: #fff; font-variant-numeric: tabular-nums; }
.rack-stats span { font-size: .62rem; letter-spacing: .1em; text-transform: uppercase; color: #7c89ad; }
.rack-body { padding: .25rem 1rem 1rem; }
.nvswitch { position: relative; height: 1.9rem; border-radius: 8px; overflow: hidden; display: flex; align-items: center;
justify-content: center; font-size: .68rem; letter-spacing: .22em; text-transform: uppercase; color: #ccfbf1;
background: linear-gradient(90deg, #0f3b3a, #115e59 50%, #0f3b3a); border: 1px solid #2dd4bf55;
box-shadow: 0 0 22px rgba(45, 212, 191, .35), inset 0 0 12px rgba(45, 212, 191, .25); }
.nvswitch span { position: relative; z-index: 1; text-shadow: 0 0 8px rgba(94, 234, 212, .8); }
.nvswitch i { position: absolute; inset: 0; background: linear-gradient(100deg, transparent 30%, rgba(255,255,255,.28) 50%, transparent 70%);
transform: translateX(-100%); animation: nv-shimmer 3.2s ease-in-out infinite; }
@keyframes nv-shimmer { to { transform: translateX(100%); } }
.rack-trays { position: relative; margin-top: .6rem; padding-left: 1.1rem; display: flex; flex-direction: column; gap: .45rem; }
.backplane { position: absolute; left: .3rem; top: -.6rem; bottom: .3rem; width: 4px; border-radius: 4px;
background: linear-gradient(180deg, #2dd4bf, #0f766e 40%, #2dd4bf 60%, #0f766e); background-size: 100% 300%;
box-shadow: 0 0 12px rgba(45, 212, 191, .6); animation: nv-flow 2.4s linear infinite; }
@keyframes nv-flow { to { background-position: 0 300%; } }
.tray { position: relative; display: flex; align-items: center; gap: .75rem; padding: .45rem .6rem; border-radius: 10px;
background: linear-gradient(180deg, rgba(255,255,255,.05), rgba(255,255,255,.015)); border: 1px solid #1f2d4a; }
.tray::before { content: ""; position: absolute; left: -.8rem; top: 50%; width: .8rem; height: 2px;
background: linear-gradient(90deg, #2dd4bf, transparent); box-shadow: 0 0 6px #2dd4bf; }
.tray.out { border-color: #5b2333; background: linear-gradient(180deg, rgba(251,113,133,.08), rgba(251,113,133,.02)); }
.tray.out::before { background: linear-gradient(90deg, #fb7185, transparent); box-shadow: 0 0 6px #fb7185; }
.tray-name { min-width: 6.5rem; font-size: .8rem; font-weight: 700; color: #e0f2fe; text-decoration: none; }
.tray-name:hover { color: #5eead4; }
.chips { display: flex; flex-wrap: wrap; gap: .35rem; flex: 1; }
.chip { width: 1.7rem; height: 1.7rem; border-radius: 6px; display: inline-flex; align-items: center; justify-content: center;
font-size: .62rem; font-weight: 800; color: #04211c; cursor: default; }
.chip.joined { background: linear-gradient(135deg, #5eead4, #14b8a6); box-shadow: 0 0 12px rgba(45, 212, 191, .55);
animation: nv-breathe 3.6s ease-in-out infinite; }
.chip.joined.slow { background: linear-gradient(135deg, #fde68a, #f59e0b); box-shadow: 0 0 12px rgba(245, 158, 11, .6); }
.chip.lost { color: #fff; background: linear-gradient(135deg, #fb7185, #be123c); box-shadow: 0 0 14px rgba(251, 113, 133, .7);
animation: nv-alarm 1.2s ease-in-out infinite; }
@keyframes nv-breathe { 50% { box-shadow: 0 0 4px rgba(45, 212, 191, .25); } }
@keyframes nv-alarm { 50% { opacity: .55; } }
.imex { font-size: .6rem; font-weight: 800; letter-spacing: .08em; padding: .15rem .4rem; border-radius: 999px;
color: #fecdd3; border: 1px solid #fb7185; }
.imex.up { color: #99f6e4; border-color: #2dd4bf; box-shadow: 0 0 8px rgba(45, 212, 191, .35); }
.rack-problems { padding: 0 1rem .9rem; margin: 0; }
.rack-problems li { color: #dbe4ff; }
@media (prefers-reduced-motion: reduce) {
.nvswitch i, .backplane, .chip.joined, .chip.lost { animation: none; }
}
+7
View File
@@ -165,6 +165,13 @@ A replica spanning nodes is only ever formed on one of those. Fully free counts
unused nodes, which is what decides whether such a replica can land.</p>
{{end}}
{{with .NVLink}}
<h2>NVLink domains</h2>
<p>{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}}{{range .Domains}} · <code>{{shortDomain .Key}}</code> {{.Usable}}/{{len .Members}} trays usable{{end}}
{{if .Problems}} · <span class="bad">{{len .Problems}} problem{{if ne (len .Problems) 1}}s{{end}}</span>{{end}}
· <a href="{{if $.OnAdminListener}}/nvlink{{else}}/admin/nvlink{{end}}">see the racks →</a></p>
{{end}}
{{with .Fleet}}
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
{{if .Err}}
+28
View File
@@ -52,6 +52,34 @@ the adapter came up: a number that grows is the one to look at.</p>
{{end}}
{{end}}
{{if .Node.GPUs}}
<h2>GPUs and NVLink</h2>
<p class="sub">
{{with .Node.NVLinkDomain}}In NVLink domain <code>{{.}}</code>: tensor parallelism can span this node and the rest of its domain.
{{else}}NVLink ends at this machine.{{end}}
{{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}<span class="state-ready">active</span>, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}<span class="bad">{{or .Node.IMEX "not installed"}}</span>{{end}}{{end}}
</p>
<div class="gpu-wiring">
<table class="nvl-matrix" aria-label="NVLink between GPUs">
<tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
{{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
{{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over NVLink">NV</td>{{else}}<td class="pcie" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over PCIe">·</td>{{end}}{{end}}
</tr>{{end}}
</table>
<table class="compact gpu-list">
<tr><th>GPU</th><th>Name</th><th class="num">NUMA</th><th>Nearest adapter</th><th>Multi-node NVLink</th></tr>
{{range .Node.GPUs}}<tr>
<td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.NUMA}}</td>
<td>{{if .NearestNIC}}<code>{{.NearestNIC}}</code> <span class="note">{{.NICPath}}</span>{{else}}—{{end}}</td>
<td>{{if .Domain}}<span class="state-ready">joined</span>{{if and .Bandwidth (ne .Bandwidth "Full")}} <span class="bad">{{.Bandwidth}} bandwidth</span>{{end}}{{else if .FabricState}}<span class="bad">{{.FabricState}}</span>{{else}}<span class="note">—</span>{{end}}</td>
</tr>{{end}}
</table>
</div>
<p class="note">NV is a pair of cards joined by NVLink; a tensor-parallel group kept to NV pairs runs its
all-reduces at NVLink speed, and placement picks it that way. The nearest adapter is where a card's share of
traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS is across the processors.</p>
{{end}}
<h2>What runs here</h2>
{{if .Engines}}
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
+54
View File
@@ -0,0 +1,54 @@
{{define "nvlink"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<meta http-equiv="refresh" content="30">
<title>NVLink domains · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
</head><body><div class="wrap wide">
<header>
<h1>NVLink domains</h1>
<p class="sub">{{len .Domains}} multi-node domain{{if ne (len .Domains) 1}}s{{end}} — racks whose GPUs reach each other over NVLink across nodes</p>
{{if .OnAdminListener}}<nav><a href="/">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header>
{{if not .Domains}}
<p class="empty">No GPU reports a multi-node NVLink domain. An eight-GPU machine's NVSwitch ends at its
chassis and is shown on its node's page; a domain appears here once the fabric manager has joined the
GPUs of an NVL72 rack, or an NVLink Switch system, to one.</p>
{{end}}
<div class="racks">
{{range .Domains}}
<section class="rack{{if .Problems}} troubled{{end}}">
<header class="rack-head">
<div><h2>{{.Short}}</h2><p class="note"><code>{{.ClusterUUID}}</code> · clique {{.CliqueID}}</p></div>
<div class="rack-stats">
<div><b>{{.Usable}}/{{len .Members}}</b><span>trays usable</span></div>
<div><b>{{.Joined}}/{{.GPUs}}</b><span>GPUs joined</span></div>
</div>
</header>
<div class="rack-body">
<div class="nvswitch"><span>NVLink Switch</span><i></i></div>
<div class="rack-trays">
<div class="backplane" aria-hidden="true"></div>
{{range .Members}}
<div class="tray{{if not .Usable}} out{{end}}">
<a class="tray-name" href="{{$.Nodes}}/{{.Node}}">{{.Node}}</a>
<div class="chips">{{range .GPUs}}<span class="chip{{if .Joined}} joined{{else}} lost{{end}}{{if and .Bandwidth (ne .Bandwidth "Full")}} slow{{end}}"
title="GPU {{.Index}} · {{.Name}} · {{or .State "no fabric"}}{{if .Bandwidth}} · {{.Bandwidth}} bandwidth{{end}}">{{.Index}}</span>{{end}}</div>
<span class="imex{{if and (eq .IMEX "active") .IMEXChannels}} up{{end}}" title="nvidia-imex {{or .IMEX "not installed"}}, {{.IMEXChannels}} channel(s)">IMEX</span>
</div>
{{end}}
</div>
</div>
{{if .Problems}}<ul class="fabric-problems rack-problems">{{range .Problems}}
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}</ul>{{end}}
</section>
{{end}}
</div>
<p class="note">A replica that spans nodes is placed inside one of these first: tensor parallelism between its
nodes then runs over NVLink rather than InfiniBand. Only trays with every GPU joined and IMEX up are used —
a tray short of either would leave the replica a card, or its memory, short.</p>
<footer><span>llmgw {{.Version}} · refreshes every 30s</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}
+6 -3
View File
@@ -26,6 +26,7 @@ import (
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -128,9 +129,11 @@ var funcs = template.FuncMap{
// shows. Binary units because that is what `ls -lh` and every disk-full
// message on the host will say, and a page that disagrees with the shell
// about the size of the same file starts an argument about neither.
"bytes": humanBytes,
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
"bytes": humanBytes,
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
"hasInt": func(list []int, v int) bool { return slices.Contains(list, v) },
"shortDomain": nvlink.Short,
// dict lets one chart template be called with named arguments. Go
// templates pass a single value, and four charts differing in three
// strings each is four copies of the same markup otherwise.