doctor sums findings up as two readinesses - the node's, which decides whether it joins, and multi-node, its fabrics for replicas spanning nodes - prints fabric findings in a section of their own, and gives both in --json. node add says the multi-node one. NodeView.multi_node says whether a node's declared fabrics are up, naming adapters that are down, and ctl nodes shows it beside the fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
279 lines
11 KiB
Go
279 lines
11 KiB
Go
package doctor
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
)
|
|
|
|
// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up
|
|
// right.
|
|
func healthy() Facts {
|
|
f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active",
|
|
Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true},
|
|
ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true,
|
|
LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}}
|
|
for i := range 8 {
|
|
g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"}
|
|
for j := range 8 {
|
|
if j != i {
|
|
g.NVLinkPeers = append(g.NVLinkPeers, j)
|
|
}
|
|
}
|
|
f.GPUs = append(f.GPUs, g)
|
|
}
|
|
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
|
|
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
|
|
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
|
|
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
|
|
return f
|
|
}
|
|
|
|
func levels(fs []Finding, l Level) []string {
|
|
var out []string
|
|
for _, f := range fs {
|
|
if f.Level == l {
|
|
out = append(out, f.Message)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func has(t *testing.T, fs []Finding, l Level, fragment string) {
|
|
t.Helper()
|
|
for _, f := range fs {
|
|
if f.Level == l && strings.Contains(f.Message, fragment) {
|
|
return
|
|
}
|
|
}
|
|
t.Errorf("no %s finding containing %q in %+v", l, fragment, fs)
|
|
}
|
|
|
|
func TestAHealthyNodeHasNothingToFix(t *testing.T) {
|
|
fs := Evaluate(healthy())
|
|
if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 {
|
|
t.Errorf("a healthy node was faulted: %v", bad)
|
|
}
|
|
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
|
|
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
|
|
has(t, fs, OK, "NCCL bootstraps over ens3")
|
|
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
|
|
}
|
|
|
|
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
|
|
cases := map[string]struct {
|
|
change func(*Facts)
|
|
level Level
|
|
says string
|
|
}{
|
|
"fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"},
|
|
"no subnet manager": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State = "INIT"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "no subnet manager has configured it"},
|
|
"cable out": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "no link"},
|
|
"fabric not advertised": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].State = "DOWN"
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
|
}, Fail, "will not be advertised"},
|
|
"roce without v2": {func(f *Facts) {
|
|
f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet
|
|
}, Fail, "no RoCE v2 GID"},
|
|
"no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"},
|
|
"no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"},
|
|
"nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"},
|
|
"wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"},
|
|
"errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"},
|
|
"small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"},
|
|
"ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"},
|
|
"no nvlink on nvswitch": {func(f *Facts) {
|
|
for i := range f.GPUs {
|
|
f.GPUs[i].NVLinkPeers = nil
|
|
}
|
|
}, Warn, "no GPU reports an NVLink peer"},
|
|
}
|
|
for name, c := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
f := healthy()
|
|
c.change(&f)
|
|
fs := Evaluate(f)
|
|
has(t, fs, c.level, c.says)
|
|
// Most serious first, so the thing to fix is at the top.
|
|
if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 {
|
|
t.Error("a failure is not first")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) {
|
|
// An adapter no declared fabric names being down is worth a warning, not
|
|
// a failure: nothing here was going to use it.
|
|
f := healthy()
|
|
f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}})
|
|
fs := Evaluate(f)
|
|
has(t, fs, Warn, "mlx5_4/1 is DOWN")
|
|
if len(levels(fs, Fail)) > 0 {
|
|
t.Errorf("failures: %v", levels(fs, Fail))
|
|
}
|
|
}
|
|
|
|
func TestAnNVLinkDomainNeedsEveryGPUJoinedAndIMEX(t *testing.T) {
|
|
f := healthy()
|
|
for i := range f.GPUs {
|
|
f.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", Status: "Success",
|
|
ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: "1", Bandwidth: "Full"}
|
|
}
|
|
f.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
|
fs := Evaluate(f)
|
|
has(t, fs, OK, "multi-node NVLink domain")
|
|
has(t, fs, OK, "nvidia-imex is active")
|
|
|
|
f.GPUs[5].Fabric = &discovery.GPUFabric{State: "In Progress"}
|
|
f.IMEX.Channels = 0
|
|
fs = Evaluate(f)
|
|
has(t, fs, Fail, "GPU 5 is not joined to its NVLink domain (In Progress)")
|
|
has(t, fs, Fail, "no IMEX channel")
|
|
}
|
|
|
|
func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) {
|
|
gpu := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}
|
|
cases := []struct {
|
|
name string
|
|
f Facts
|
|
level Level
|
|
says string
|
|
}{
|
|
{"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"},
|
|
// What vLLM cannot do keeps vLLM off the node, not the node out of
|
|
// the cluster: it still serves llama.cpp.
|
|
{"a driver too old for either build", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, VLLMVersion: "0.30.0"}, Warn, "driver 525"},
|
|
// A driver too old for CUDA 13 runs the same version's CUDA 12.9 build.
|
|
{"turing, old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5},
|
|
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu129) can be installed"},
|
|
// A V100 has no build of vLLM 0.20 or later; it gets the newest
|
|
// that has Volta kernels, and doctor says which.
|
|
{"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
|
|
VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "vLLM 0.19.1 (cu12) can be installed here"},
|
|
{"pascal", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.1},
|
|
VLLMVersion: "0.30.0"}, Warn, "no vLLM is built for GPUs of compute capability 6.1"},
|
|
{"volta, the last vLLM built for it", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0},
|
|
VLLMVersion: "0.19.1", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu12) can be installed"},
|
|
{"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Warn, "9.0 GB"},
|
|
{"ready to install", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "can be installed"},
|
|
{"installed", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", VLLMInstalled: true}, OK, "is installed"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
var found bool
|
|
for _, fd := range Evaluate(c.f) {
|
|
if fd.Area == "engines" {
|
|
found = true
|
|
if fd.Level != c.level || !strings.Contains(fd.Message, c.says) {
|
|
t.Errorf("got %s %q, want %s saying %q", fd.Level, fd.Message, c.level, c.says)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("no finding about engines")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) {
|
|
area := func(fs []Finding, a string) *Finding {
|
|
for i := range fs {
|
|
if fs[i].Area == a {
|
|
return &fs[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
// A VM with QEMU's default CPU model: nothing looks wrong but the speed.
|
|
f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu")
|
|
if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") {
|
|
t.Errorf("no AVX2: %+v", f)
|
|
}
|
|
f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu")
|
|
if f == nil || f.Level != OK {
|
|
t.Errorf("with AVX2: %+v", f)
|
|
}
|
|
// Not read at all, or not x86: nothing to say.
|
|
if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil {
|
|
t.Errorf("unknown CPU: %+v", f)
|
|
}
|
|
if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil {
|
|
t.Errorf("ARM: %+v", f)
|
|
}
|
|
}
|
|
|
|
func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) {
|
|
// The field: a V100 node meant for llama.cpp, refused for a vLLM it was
|
|
// never going to run.
|
|
f := Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0},
|
|
VLLMVersion: "0.30.0", DisabledEngines: []string{"vllm"}}
|
|
for _, fd := range Evaluate(f) {
|
|
if fd.Area == "engines" {
|
|
t.Errorf("a disabled vLLM was reported on: %s %q", fd.Level, fd.Message)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) {
|
|
// The field: a RoCE NIC with no v2 GID on a node added for single-node
|
|
// inference, no fabric given, and the node refused for it.
|
|
roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
|
|
PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}}
|
|
f := healthy()
|
|
f.HCAs = append(f.HCAs, roce)
|
|
f.Modules["ib_uverbs"] = false
|
|
f.Fabrics = nil // nothing declared
|
|
fs := Evaluate(f)
|
|
if fails := levels(fs, Fail); len(fails) > 0 {
|
|
t.Errorf("an undeclared adapter failed the node: %v", fails)
|
|
}
|
|
has(t, fs, Warn, "no declared fabric uses it")
|
|
has(t, fs, Warn, "runs single-node replicas without it")
|
|
|
|
// Declared, the same adapter and module are what the fabric needs.
|
|
f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs)
|
|
fs = Evaluate(f)
|
|
has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry")
|
|
has(t, fs, Fail, "ib_uverbs is not loaded")
|
|
}
|
|
|
|
func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) {
|
|
f := Facts{ConfigPath: "/etc/llmgw/agent.yaml", Firewall: "absent"}
|
|
has(t, Evaluate(f), Warn, "no llmgw firewall")
|
|
f.ConfigPath = "" // before onboarding has written one
|
|
for _, x := range Evaluate(f) {
|
|
if x.Area == "firewall" {
|
|
t.Errorf("said before the node is set up: %+v", x)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) {
|
|
r := ReadinessOf([]Finding{
|
|
{Level: OK, Area: "gpu", Message: "2 GPUs"},
|
|
{Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"},
|
|
})
|
|
if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" {
|
|
t.Errorf("%+v", r)
|
|
}
|
|
r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}})
|
|
if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" {
|
|
t.Errorf("%+v", r)
|
|
}
|
|
if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" {
|
|
t.Errorf("%+v", r)
|
|
}
|
|
}
|