package doctor import ( "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) // healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up // right. func healthy() Facts { f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active", Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true}, ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true, LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}} for i := range 8 { g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"} for j := range 8 { if j != i { g.NVLinkPeers = append(g.NVLinkPeers, j) } } f.GPUs = append(f.GPUs, g) } f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1, SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}} f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": { {GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}} return f } func levels(fs []Finding, l Level) []string { var out []string for _, f := range fs { if f.Level == l { out = append(out, f.Message) } } return out } func has(t *testing.T, fs []Finding, l Level, fragment string) { t.Helper() for _, f := range fs { if f.Level == l && strings.Contains(f.Message, fragment) { return } } t.Errorf("no %s finding containing %q in %+v", l, fragment, fs) } func TestAHealthyNodeHasNothingToFix(t *testing.T) { fs := Evaluate(healthy()) if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 { t.Errorf("a healthy node was faulted: %v", bad) } has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]") has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s") has(t, fs, OK, "NCCL bootstraps over ens3") has(t, fs, OK, "subnet manager 0x98039b03009fcfd6") } func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) { cases := map[string]struct { change func(*Facts) level Level says string }{ "fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"}, "no subnet manager": {func(f *Facts) { f.HCAs[0].Ports[0].State = "INIT" f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) }, Fail, "no subnet manager has configured it"}, "cable out": {func(f *Facts) { f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling" f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) }, Fail, "no link"}, "fabric not advertised": {func(f *Facts) { f.HCAs[0].Ports[0].State = "DOWN" f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) }, Fail, "will not be advertised"}, "roce without v2": {func(f *Facts) { f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet }, Fail, "no RoCE v2 GID"}, "no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"}, "no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"}, "nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"}, "wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"}, "errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"}, "small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"}, "ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"}, "no nvlink on nvswitch": {func(f *Facts) { for i := range f.GPUs { f.GPUs[i].NVLinkPeers = nil } }, Warn, "no GPU reports an NVLink peer"}, } for name, c := range cases { t.Run(name, func(t *testing.T) { f := healthy() c.change(&f) fs := Evaluate(f) has(t, fs, c.level, c.says) // Most serious first, so the thing to fix is at the top. if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 { t.Error("a failure is not first") } }) } } func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) { // An adapter no declared fabric names being down is worth a warning, not // a failure: nothing here was going to use it. f := healthy() f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}}) fs := Evaluate(f) has(t, fs, Warn, "mlx5_4/1 is DOWN") if len(levels(fs, Fail)) > 0 { t.Errorf("failures: %v", levels(fs, Fail)) } } func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) { f := healthy() for i := range f.GPUs { f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"} } f.IMEX = discovery.IMEX{State: "active", Channels: 1} fs := Evaluate(f) has(t, fs, OK, "multi-node NVLink domain") has(t, fs, OK, "nvidia-imex is active") f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"} f.IMEX.Channels = 0 fs = Evaluate(f) has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)") has(t, fs, Fail, "no IMEX channel") } func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) { gpu := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0} cases := []struct { name string f Facts level Level says string }{ {"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"}, // What vLLM cannot do keeps vLLM off the node, not the node out of // the cluster: it still serves llama.cpp. {"a driver too old for either build", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, VLLMVersion: "0.30.0"}, Warn, "driver 525"}, // A driver too old for CUDA 13 runs the same version's CUDA 12.9 build. {"turing, old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5}, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu129) can be installed"}, // A V100 has no build of vLLM 0.20 or later; it gets the newest // that has Volta kernels, and doctor says which. {"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "vLLM 0.19.1 (cu12) can be installed here"}, {"pascal", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.1}, VLLMVersion: "0.30.0"}, Warn, "no vLLM is built for GPUs of compute capability 6.1"}, {"volta, the last vLLM built for it", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, VLLMVersion: "0.19.1", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu12) can be installed"}, {"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Warn, "9.0 GB"}, {"ready to install", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "can be installed"}, {"installed", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", VLLMInstalled: true}, OK, "is installed"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { var found bool for _, fd := range Evaluate(c.f) { if fd.Area == "engines" { found = true if fd.Level != c.level || !strings.Contains(fd.Message, c.says) { t.Errorf("got %s %q, want %s saying %q", fd.Level, fd.Message, c.level, c.says) } } } if !found { t.Error("no finding about engines") } }) } } func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) { area := func(fs []Finding, a string) *Finding { for i := range fs { if fs[i].Area == a { return &fs[i] } } return nil } // A VM with QEMU's default CPU model: nothing looks wrong but the speed. f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu") if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") { t.Errorf("no AVX2: %+v", f) } f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu") if f == nil || f.Level != OK { t.Errorf("with AVX2: %+v", f) } // Not read at all, or not x86: nothing to say. if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil { t.Errorf("unknown CPU: %+v", f) } if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil { t.Errorf("ARM: %+v", f) } } func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) { // The field: a V100 node meant for llama.cpp, refused for a vLLM it was // never going to run. f := Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0}, VLLMVersion: "0.30.0", DisabledEngines: []string{"vllm"}} for _, fd := range Evaluate(f) { if fd.Area == "engines" { t.Errorf("a disabled vLLM was reported on: %s %q", fd.Level, fd.Message) } } } func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) { // The field: a RoCE NIC with no v2 GID on a node added for single-node // inference, no fabric given, and the node refused for it. roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}} f := healthy() f.HCAs = append(f.HCAs, roce) f.Modules["ib_uverbs"] = false f.Fabrics = nil // nothing declared fs := Evaluate(f) if fails := levels(fs, Fail); len(fails) > 0 { t.Errorf("an undeclared adapter failed the node: %v", fails) } has(t, fs, Warn, "no declared fabric uses it") has(t, fs, Warn, "runs single-node replicas without it") // Declared, the same adapter and module are what the fabric needs. f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs) fs = Evaluate(f) has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry") has(t, fs, Fail, "ib_uverbs is not loaded") } func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) { f := Facts{ConfigPath: "/etc/llmgw/agent.yaml", Firewall: "absent"} has(t, Evaluate(f), Warn, "no llmgw firewall") f.ConfigPath = "" // before onboarding has written one for _, x := range Evaluate(f) { if x.Area == "firewall" { t.Errorf("said before the node is set up: %+v", x) } } } func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) { r := ReadinessOf([]Finding{ {Level: OK, Area: "gpu", Message: "2 GPUs"}, {Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"}, }) if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" { t.Errorf("%+v", r) } r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}}) if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" { t.Errorf("%+v", r) } if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" { t.Errorf("%+v", r) } }