Files
llmgw/internal/doctor/doctor_test.go
T
Ilker ManapandClaude Opus 5.5 20479a2d5f Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 09:07:59 +02:00

144 lines
5.4 KiB
Go

package doctor
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
// right.
func healthy() Facts {
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
for i := range 8 {
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
for j := range 8 {
if j != i {
g.NVLinkPeers = append(g.NVLinkPeers, j)
}
}
f.GPUs = append(f.GPUs, g)
}
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
return f
}
func levels(fs []Finding, l Level) []string {
var out []string
for _, f := range fs {
if f.Level == l {
out = append(out, f.Message)
}
}
return out
}
func has(t *testing.T, fs []Finding, l Level, fragment string) {
t.Helper()
for _, f := range fs {
if f.Level == l && strings.Contains(f.Message, fragment) {
return
}
}
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
}
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
fs := Evaluate(healthy())
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
t.Errorf("a healthy node was faulted: %v", bad)
}
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
has(t, fs, OK, "NCCL bootstraps over ens3")
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
}
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
cases := map[string]struct {
change func(*Facts)
level Level
says string
}{
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
"no subnet manager": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "INIT"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no subnet manager has configured it"},
"cable out": {func(f *Facts) {
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no link"},
"fabric not advertised": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "DOWN"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "will not be advertised"},
"roce without v2": {func(f *Facts) {
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
}, Fail, "no RoCE v2 GID"},
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
"no nvlink on nvswitch": {func(f *Facts) {
for i := range f.GPUs {
f.GPUs[i].NVLinkPeers = nil
}
}, Warn, "no GPU reports an NVLink peer"},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
f := healthy()
c.change(&f)
fs := Evaluate(f)
has(t, fs, c.level, c.says)
// Most serious first, so the thing to fix is at the top.
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
t.Error("a failure is not first")
}
})
}
}
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
// An adapter no declared fabric names being down is worth a warning, not
// a failure: nothing here was going to use it.
f := healthy()
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
fs := Evaluate(f)
has(t, fs, Warn, "mlx5_4/1 is DOWN")
if len(levels(fs, Fail)) > 0 {
t.Errorf("failures: %v", levels(fs, Fail))
}
}
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
f := healthy()
for i := range f.GPUs {
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
}
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
fs := Evaluate(f)
has(t, fs, OK, "multi-node NVLink domain")
has(t, fs, OK, "nvidia-imex is active")
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
f.IMEX.Channels = 0
fs = Evaluate(f)
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
has(t, fs, Fail, "no IMEX channel")
}