Files
llmgw/internal/doctor/doctor_test.go
T
Ilker ManapandClaude Opus 5.5 dc483d8c68
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m52s
Choose each engine's build from the node's hardware, for every GPU generation
Each engine has a list of builds, best first, from what its project
publishes and how each was compiled: llama.cpp CUDA 13.3, CUDA 12.8
(driver 570 for the GPUs it ships as PTX), then Vulkan, never the CPU
build on an NVIDIA node; vLLM CUDA 13, then CUDA 12.9, and for GPUs
older than every build of the version asked for, the newest built for
them (V100: 0.19.1). The node takes the first its hardware meets and
that installs and computes on the GPU, moving on when one does not, and
reports the version it runs. GPU processes are read from nvidia-smi's
full report, so Vulkan engines count as on their GPU.

Refs #2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 14:35:58 +02:00

251 lines
10 KiB
Go

package doctor
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
// right.
func healthy() Facts {
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
for i := range 8 {
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
for j := range 8 {
if j != i {
g.NVLinkPeers = append(g.NVLinkPeers, j)
}
}
f.GPUs = append(f.GPUs, g)
}
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
return f
}
func levels(fs []Finding, l Level) []string {
var out []string
for _, f := range fs {
if f.Level == l {
out = append(out, f.Message)
}
}
return out
}
func has(t *testing.T, fs []Finding, l Level, fragment string) {
t.Helper()
for _, f := range fs {
if f.Level == l && strings.Contains(f.Message, fragment) {
return
}
}
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
}
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
fs := Evaluate(healthy())
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
t.Errorf("a healthy node was faulted: %v", bad)
}
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
has(t, fs, OK, "NCCL bootstraps over ens3")
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
}
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
cases := map[string]struct {
change func(*Facts)
level Level
says string
}{
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
"no subnet manager": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "INIT"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no subnet manager has configured it"},
"cable out": {func(f *Facts) {
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no link"},
"fabric not advertised": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "DOWN"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "will not be advertised"},
"roce without v2": {func(f *Facts) {
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
}, Fail, "no RoCE v2 GID"},
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
"no nvlink on nvswitch": {func(f *Facts) {
for i := range f.GPUs {
f.GPUs[i].NVLinkPeers = nil
}
}, Warn, "no GPU reports an NVLink peer"},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
f := healthy()
c.change(&f)
fs := Evaluate(f)
has(t, fs, c.level, c.says)
// Most serious first, so the thing to fix is at the top.
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
t.Error("a failure is not first")
}
})
}
}
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
// An adapter no declared fabric names being down is worth a warning, not
// a failure: nothing here was going to use it.
f := healthy()
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
fs := Evaluate(f)
has(t, fs, Warn, "mlx5_4/1 is DOWN")
if len(levels(fs, Fail)) > 0 {
t.Errorf("failures: %v", levels(fs, Fail))
}
}
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
f := healthy()
for i := range f.GPUs {
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
}
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
fs := Evaluate(f)
has(t, fs, OK, "multi-node NVLink domain")
has(t, fs, OK, "nvidia-imex is active")
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
f.IMEX.Channels = 0
fs = Evaluate(f)
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
has(t, fs, Fail, "no IMEX channel")
}
func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) {
gpu := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}
cases := []struct {
name string
f Facts
level Level
says string
}{
{"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"},
// What vLLM cannot do keeps vLLM off the node, not the node out of
// the cluster: it still serves llama.cpp.
{"a driver too old for either build", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, VLLMVersion: "0.30.0"}, Warn, "driver 525"},
// A driver too old for CUDA 13 runs the same version's CUDA 12.9 build.
{"turing, old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5},
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu129) can be installed"},
// A V100 has no build of vLLM 0.20 or later; it gets the newest
// that has Volta kernels, and doctor says which.
{"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "vLLM 0.19.1 (cu12) can be installed here"},
{"pascal", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.1},
VLLMVersion: "0.30.0"}, Warn, "no vLLM is built for GPUs of compute capability 6.1"},
{"volta, the last vLLM built for it", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
VLLMVersion: "0.19.1", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu12) can be installed"},
{"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Warn, "9.0 GB"},
{"ready to install", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "can be installed"},
{"installed", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", VLLMInstalled: true}, OK, "is installed"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var found bool
for _, fd := range Evaluate(c.f) {
if fd.Area == "engines" {
found = true
if fd.Level != c.level || !strings.Contains(fd.Message, c.says) {
t.Errorf("got %s %q, want %s saying %q", fd.Level, fd.Message, c.level, c.says)
}
}
}
if !found {
t.Error("no finding about engines")
}
})
}
}
func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) {
area := func(fs []Finding, a string) *Finding {
for i := range fs {
if fs[i].Area == a {
return &fs[i]
}
}
return nil
}
// A VM with QEMU's default CPU model: nothing looks wrong but the speed.
f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu")
if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") {
t.Errorf("no AVX2: %+v", f)
}
f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu")
if f == nil || f.Level != OK {
t.Errorf("with AVX2: %+v", f)
}
// Not read at all, or not x86: nothing to say.
if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil {
t.Errorf("unknown CPU: %+v", f)
}
if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil {
t.Errorf("ARM: %+v", f)
}
}
func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) {
// The field: a V100 node meant for llama.cpp, refused for a vLLM it was
// never going to run.
f := Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0},
VLLMVersion: "0.30.0", DisabledEngines: []string{"vllm"}}
for _, fd := range Evaluate(f) {
if fd.Area == "engines" {
t.Errorf("a disabled vLLM was reported on: %s %q", fd.Level, fd.Message)
}
}
}
func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) {
// The field: a RoCE NIC with no v2 GID on a node added for single-node
// inference, no fabric given, and the node refused for it.
roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}}
f := healthy()
f.HCAs = append(f.HCAs, roce)
f.Modules["ib_uverbs"] = false
f.Fabrics = nil // nothing declared
fs := Evaluate(f)
if fails := levels(fs, Fail); len(fails) > 0 {
t.Errorf("an undeclared adapter failed the node: %v", fails)
}
has(t, fs, Warn, "no declared fabric uses it")
has(t, fs, Warn, "runs single-node replicas without it")
// Declared, the same adapter and module are what the fabric needs.
f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs)
fs = Evaluate(f)
has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry")
has(t, fs, Fail, "ib_uverbs is not loaded")
}