Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
@@ -145,3 +146,31 @@ func TestAFabricsPageCarriesItsMapSafely(t *testing.T) {
|
||||
t.Errorf("json: %d %s", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
type nvlSource struct{ busySource }
|
||||
|
||||
func (nvlSource) NVLink() nvlink.Report {
|
||||
return nvlink.Report{
|
||||
Domains: []nvlink.Domain{{Key: "76b1ad76-7c24/7", ClusterUUID: "76b1ad76-7c24", CliqueID: "7", GPUs: 8, Joined: 7, Usable: 1,
|
||||
Members: []nvlink.Member{
|
||||
{Node: "t1", Usable: true, IMEX: "active", IMEXChannels: 1, GPUs: []nvlink.GPU{{Index: 0, Joined: true, State: "Completed"}}},
|
||||
{Node: "t2", IMEX: "inactive", GPUs: []nvlink.GPU{{Index: 3, State: "In Progress"}}},
|
||||
}}},
|
||||
Problems: []nvlink.Problem{{Level: nvlink.Fail, Domain: "76b1ad76-7c24/7", Message: "t2: nvidia-imex is inactive"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
|
||||
_, body := get(t, NVLinkPage(nvlSource{}, false, "v1"), "http://x/admin/nvlink", nil)
|
||||
for _, want := range []string{`class="rack troubled"`, "76b1ad76·7", "1/2</b><span>trays usable", `class="chip joined"`,
|
||||
`class="chip lost"`, `class="tray out"`, "t2: nvidia-imex is inactive", `class="imex up"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
// And the cluster page points at it.
|
||||
_, body = get(t, Dashboard(nvlSource{}, fakeUsage{}, "v1", time.Second, false, NavExtras{}, nil), "http://x/admin", nil)
|
||||
if !strings.Contains(body, "see the racks") {
|
||||
t.Error("the cluster page does not lead to the racks")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user