Files
llmgw/internal/doctor/doctor_test.go
T
Ilker ManapandClaude Opus 5.5 590e5b9f3e Ask a node only for engines it can install; doctor says when AVX2 is hidden
A control plane of this release asks every node for vLLM. An agent of the
last one does not know how to install it, and answered "unsupported", so a
cluster half way through its upgrade showed every node as failing. Agents
now say which engines they can install (Register.installable_engines), and
are asked for no others; an agent that says nothing is asked for llama.cpp
alone, as it always was.

doctor reports the CPU's vector extensions, and warns when AVX2 is missing,
with the fix for a VM: the test node for node add was one, on QEMU's default
CPU model, where llama.cpp runs several times slower and nothing else looks
wrong. node add shows it, since it runs doctor.

The changelog's unreleased section was two releases' worth of sections one
after the other, with two lists of fixes and the upgrade notes in the
middle. It is one entry now: largest changes first, the fixes together, and
every upgrade note at the end, with vLLM's default install and how to keep
your own.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 21:03:03 +02:00

203 lines
7.6 KiB
Go

package doctor
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
// right.
func healthy() Facts {
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
for i := range 8 {
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
for j := range 8 {
if j != i {
g.NVLinkPeers = append(g.NVLinkPeers, j)
}
}
f.GPUs = append(f.GPUs, g)
}
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
return f
}
func levels(fs []Finding, l Level) []string {
var out []string
for _, f := range fs {
if f.Level == l {
out = append(out, f.Message)
}
}
return out
}
func has(t *testing.T, fs []Finding, l Level, fragment string) {
t.Helper()
for _, f := range fs {
if f.Level == l && strings.Contains(f.Message, fragment) {
return
}
}
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
}
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
fs := Evaluate(healthy())
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
t.Errorf("a healthy node was faulted: %v", bad)
}
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
has(t, fs, OK, "NCCL bootstraps over ens3")
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
}
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
cases := map[string]struct {
change func(*Facts)
level Level
says string
}{
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
"no subnet manager": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "INIT"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no subnet manager has configured it"},
"cable out": {func(f *Facts) {
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "no link"},
"fabric not advertised": {func(f *Facts) {
f.HCAs[0].Ports[0].State = "DOWN"
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
}, Fail, "will not be advertised"},
"roce without v2": {func(f *Facts) {
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
}, Fail, "no RoCE v2 GID"},
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
"no nvlink on nvswitch": {func(f *Facts) {
for i := range f.GPUs {
f.GPUs[i].NVLinkPeers = nil
}
}, Warn, "no GPU reports an NVLink peer"},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
f := healthy()
c.change(&f)
fs := Evaluate(f)
has(t, fs, c.level, c.says)
// Most serious first, so the thing to fix is at the top.
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
t.Error("a failure is not first")
}
})
}
}
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
// An adapter no declared fabric names being down is worth a warning, not
// a failure: nothing here was going to use it.
f := healthy()
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
fs := Evaluate(f)
has(t, fs, Warn, "mlx5_4/1 is DOWN")
if len(levels(fs, Fail)) > 0 {
t.Errorf("failures: %v", levels(fs, Fail))
}
}
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
f := healthy()
for i := range f.GPUs {
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
}
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
fs := Evaluate(f)
has(t, fs, OK, "multi-node NVLink domain")
has(t, fs, OK, "nvidia-imex is active")
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
f.IMEX.Channels = 0
fs = Evaluate(f)
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
has(t, fs, Fail, "no IMEX channel")
}
func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) {
gpu := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}
cases := []struct {
name string
f Facts
level Level
says string
}{
{"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"},
{"old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15"}, VLLMVersion: "0.30.0"}, Fail, "driver 580"},
{"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Fail, "9.0 GB"},
{"ready to install", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "can be installed"},
{"installed", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", VLLMInstalled: true}, OK, "is installed"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var found bool
for _, fd := range Evaluate(c.f) {
if fd.Area == "engines" {
found = true
if fd.Level != c.level || !strings.Contains(fd.Message, c.says) {
t.Errorf("got %s %q, want %s saying %q", fd.Level, fd.Message, c.level, c.says)
}
}
}
if !found {
t.Error("no finding about engines")
}
})
}
}
func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) {
area := func(fs []Finding, a string) *Finding {
for i := range fs {
if fs[i].Area == a {
return &fs[i]
}
}
return nil
}
// A VM with QEMU's default CPU model: nothing looks wrong but the speed.
f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu")
if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") {
t.Errorf("no AVX2: %+v", f)
}
f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu")
if f == nil || f.Level != OK {
t.Errorf("with AVX2: %+v", f)
}
// Not read at all, or not x86: nothing to say.
if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil {
t.Errorf("unknown CPU: %+v", f)
}
if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil {
t.Errorf("ARM: %+v", f)
}
}