A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
144 lines
5.4 KiB
Go
144 lines
5.4 KiB
Go
package doctor
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
)
|
|
|
|
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
|
|
// right.
|
|
func healthy() Facts {
|
|
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
|
|
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
|
|
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
|
|
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
|
|
for i := range 8 {
|
|
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
|
|
for j := range 8 {
|
|
if j != i {
|
|
g.NVLinkPeers = append(g.NVLinkPeers, j)
|
|
}
|
|
}
|
|
f.GPUs = append(f.GPUs, g)
|
|
}
|
|
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
|
|
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
|
|
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
|
|
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
|
|
return f
|
|
}
|
|
|
|
func levels(fs []Finding, l Level) []string {
|
|
var out []string
|
|
for _, f := range fs {
|
|
if f.Level == l {
|
|
out = append(out, f.Message)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func has(t *testing.T, fs []Finding, l Level, fragment string) {
|
|
t.Helper()
|
|
for _, f := range fs {
|
|
if f.Level == l && strings.Contains(f.Message, fragment) {
|
|
return
|
|
}
|
|
}
|
|
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
|
|
}
|
|
|
|
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
|
|
fs := Evaluate(healthy())
|
|
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
|
|
t.Errorf("a healthy node was faulted: %v", bad)
|
|
}
|
|
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
|
|
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
|
|
has(t, fs, OK, "NCCL bootstraps over ens3")
|
|
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
|
|
}
|
|
|
|
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
|
|
cases := map[string]struct {
|
|
change func(*Facts)
|
|
level Level
|
|
says string
|
|
}{
|
|
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
|
|
"no subnet manager": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State = "INIT"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "no subnet manager has configured it"},
|
|
"cable out": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "no link"},
|
|
"fabric not advertised": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State = "DOWN"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "will not be advertised"},
|
|
"roce without v2": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
|
|
}, Fail, "no RoCE v2 GID"},
|
|
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
|
|
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
|
|
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
|
|
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
|
|
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
|
|
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
|
|
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
|
|
"no nvlink on nvswitch": {func(f *Facts) {
|
|
for i := range f.GPUs {
|
|
f.GPUs[i].NVLinkPeers = nil
|
|
}
|
|
}, Warn, "no GPU reports an NVLink peer"},
|
|
}
|
|
for name, c := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
f := healthy()
|
|
c.change(&f)
|
|
fs := Evaluate(f)
|
|
has(t, fs, c.level, c.says)
|
|
// Most serious first, so the thing to fix is at the top.
|
|
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
|
|
t.Error("a failure is not first")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
|
|
// An adapter no declared fabric names being down is worth a warning, not
|
|
// a failure: nothing here was going to use it.
|
|
f := healthy()
|
|
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
|
|
fs := Evaluate(f)
|
|
has(t, fs, Warn, "mlx5_4/1 is DOWN")
|
|
if len(levels(fs, Fail)) > 0 {
|
|
t.Errorf("failures: %v", levels(fs, Fail))
|
|
}
|
|
}
|
|
|
|
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
|
|
f := healthy()
|
|
for i := range f.GPUs {
|
|
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
|
|
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
|
|
}
|
|
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
|
fs := Evaluate(f)
|
|
has(t, fs, OK, "multi-node NVLink domain")
|
|
has(t, fs, OK, "nvidia-imex is active")
|
|
|
|
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
|
f.IMEX.Channels = 0
|
|
fs = Evaluate(f)
|
|
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
|
|
has(t, fs, Fail, "no IMEX channel")
|
|
}
|