Files
llmgw/internal/doctor/doctor_test.go
T
Ilker ManapandClaude Opus 5.5 f2b2e24e1f A node's own readiness and its multi-node readiness are kept apart
doctor sums findings up as two readinesses - the node's, which decides
whether it joins, and multi-node, its fabrics for replicas spanning nodes -
prints fabric findings in a section of their own, and gives both in --json.
node add says the multi-node one. NodeView.multi_node says whether a node's
declared fabrics are up, naming adapters that are down, and ctl nodes shows
it beside the fabric.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 13:11:16 +02:00

279 lines
11 KiB
Go

package doctor
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
// right.
func healthy() Facts {
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
for i := range 8 {
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
for j := range 8 {
if j != i {
g.NVLinkPeers = append(g.NVLinkPeers, j)
}
}
f.GPUs = append(f.GPUs, g)
}
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
return f
}
func levels(fs []Finding, l Level) []string {
var out []string
for _, f := range fs {
if f.Level == l {
out = append(out, f.Message)
}
}
return out
}
func has(t *testing.T, fs []Finding, l Level, fragment string) {
t.Helper()
for _, f := range fs {
if f.Level == l && strings.Contains(f.Message, fragment) {
return
}
}
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
}
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
fs := Evaluate(healthy())
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
t.Errorf("a healthy node was faulted: %v", bad)
}
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
has(t, fs, OK, "NCCL bootstraps over ens3")
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
}
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
cases := map[string]struct {
change func(*Facts)
level Level
says string
}{
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
"no subnet manager": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "INIT"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no subnet manager has configured it"},
"cable out": {func(f *Facts) {
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no link"},
"fabric not advertised": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "DOWN"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "will not be advertised"},
"roce without v2": {func(f *Facts) {
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
}, Fail, "no RoCE v2 GID"},
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
"no nvlink on nvswitch": {func(f *Facts) {
for i := range f.GPUs {
f.GPUs[i].NVLinkPeers = nil
}
}, Warn, "no GPU reports an NVLink peer"},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
f := healthy()
c.change(&f)
fs := Evaluate(f)
has(t, fs, c.level, c.says)
// Most serious first, so the thing to fix is at the top.
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
t.Error("a failure is not first")
}
})
}
}
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
// An adapter no declared fabric names being down is worth a warning, not
// a failure: nothing here was going to use it.
f := healthy()
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
fs := Evaluate(f)
has(t, fs, Warn, "mlx5_4/1 is DOWN")
if len(levels(fs, Fail)) > 0 {
t.Errorf("failures: %v", levels(fs, Fail))
}
}
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
f := healthy()
for i := range f.GPUs {
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
}
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
fs := Evaluate(f)
has(t, fs, OK, "multi-node NVLink domain")
has(t, fs, OK, "nvidia-imex is active")
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
f.IMEX.Channels = 0
fs = Evaluate(f)
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
has(t, fs, Fail, "no IMEX channel")
}
func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) {
gpu := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}
cases := []struct {
name string
f Facts
level Level
says string
}{
{"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"},
// What vLLM cannot do keeps vLLM off the node, not the node out of
// the cluster: it still serves llama.cpp.
{"a driver too old for either build", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, VLLMVersion: "0.30.0"}, Warn, "driver 525"},
// A driver too old for CUDA 13 runs the same version's CUDA 12.9 build.
{"turing, old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5},
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu129) can be installed"},
// A V100 has no build of vLLM 0.20 or later; it gets the newest
// that has Volta kernels, and doctor says which.
{"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "vLLM 0.19.1 (cu12) can be installed here"},
{"pascal", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.1},
VLLMVersion: "0.30.0"}, Warn, "no vLLM is built for GPUs of compute capability 6.1"},
{"volta, the last vLLM built for it", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
VLLMVersion: "0.19.1", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu12) can be installed"},
{"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Warn, "9.0 GB"},
{"ready to install", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "can be installed"},
{"installed", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", VLLMInstalled: true}, OK, "is installed"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var found bool
for _, fd := range Evaluate(c.f) {
if fd.Area == "engines" {
found = true
if fd.Level != c.level || !strings.Contains(fd.Message, c.says) {
t.Errorf("got %s %q, want %s saying %q", fd.Level, fd.Message, c.level, c.says)
}
}
}
if !found {
t.Error("no finding about engines")
}
})
}
}
func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) {
area := func(fs []Finding, a string) *Finding {
for i := range fs {
if fs[i].Area == a {
return &fs[i]
}
}
return nil
}
// A VM with QEMU's default CPU model: nothing looks wrong but the speed.
f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu")
if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") {
t.Errorf("no AVX2: %+v", f)
}
f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu")
if f == nil || f.Level != OK {
t.Errorf("with AVX2: %+v", f)
}
// Not read at all, or not x86: nothing to say.
if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil {
t.Errorf("unknown CPU: %+v", f)
}
if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil {
t.Errorf("ARM: %+v", f)
}
}
func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) {
// The field: a V100 node meant for llama.cpp, refused for a vLLM it was
// never going to run.
f := Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0},
VLLMVersion: "0.30.0", DisabledEngines: []string{"vllm"}}
for _, fd := range Evaluate(f) {
if fd.Area == "engines" {
t.Errorf("a disabled vLLM was reported on: %s %q", fd.Level, fd.Message)
}
}
}
func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) {
// The field: a RoCE NIC with no v2 GID on a node added for single-node
// inference, no fabric given, and the node refused for it.
roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}}
f := healthy()
f.HCAs = append(f.HCAs, roce)
f.Modules["ib_uverbs"] = false
f.Fabrics = nil // nothing declared
fs := Evaluate(f)
if fails := levels(fs, Fail); len(fails) > 0 {
t.Errorf("an undeclared adapter failed the node: %v", fails)
}
has(t, fs, Warn, "no declared fabric uses it")
has(t, fs, Warn, "runs single-node replicas without it")
// Declared, the same adapter and module are what the fabric needs.
f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs)
fs = Evaluate(f)
has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry")
has(t, fs, Fail, "ib_uverbs is not loaded")
}
func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) {
f := Facts{ConfigPath: "/etc/llmgw/agent.yaml", Firewall: "absent"}
has(t, Evaluate(f), Warn, "no llmgw firewall")
f.ConfigPath = "" // before onboarding has written one
for _, x := range Evaluate(f) {
if x.Area == "firewall" {
t.Errorf("said before the node is set up: %+v", x)
}
}
}
func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) {
r := ReadinessOf([]Finding{
{Level: OK, Area: "gpu", Message: "2 GPUs"},
{Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"},
})
if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" {
t.Errorf("%+v", r)
}
r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}})
if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" {
t.Errorf("%+v", r)
}
if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" {
t.Errorf("%+v", r)
}
}