diff --git a/CHANGELOG.md b/CHANGELOG.md index 20a37d46..45fcd663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,22 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### An adapter no fabric uses does not keep a node out + +`llmgw doctor` failed a node for a RoCE port with no RoCE v2 GID entry, or for +`ib_uverbs` not being loaded, whether or not any fabric used them; on a node +added for single-node inference, with no fabric given, that refused it for +RDMA nobody had asked for. + +- Those are failures only for ports a declared fabric uses, and only when a + fabric is declared. Otherwise they are warnings saying nothing uses them yet. +- Which fabric an undeclared InfiniBand port is on, and whether it has a + subnet manager LID, are notes rather than warnings; `node add` still warns + once that adapters are up with no fabric given. +- `node add --fabric` hands its fabrics to doctor (`doctor --fabric + name:mlx5_0,…`), so they are checked as declared even though agent.yaml is + written afterwards. + ### A restart with no room beside it `ctl restart` places each replacement beside the replica it replaces. With diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index f61a8f8c..a46250c0 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -23,6 +23,7 @@ func newDoctorCmd() *cobra.Command { opts doctor.Options asJSON bool verbose bool + fabrics []string ) cmd := &cobra.Command{ Use: "doctor", @@ -38,6 +39,16 @@ that file when the report is not enough to go on.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { doctor.SetConfigLoader(config.LoadAgent) + for _, spec := range fabrics { + name, hcas, _ := strings.Cut(spec, ":") + if name == "" || hcas == "" { + return fmt.Errorf("--fabric %q: give it as name:mlx5_0,mlx5_1", spec) + } + if opts.Fabrics == nil { + opts.Fabrics = map[string][]string{} + } + opts.Fabrics[name] = strings.Split(hcas, ",") + } switch opts.VLLMVersion { case "": opts.VLLMVersion = config.DefaultEngines["vllm"] @@ -69,7 +80,8 @@ that file when the report is not enough to go on.`, f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)") f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory") f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi") - f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)") + f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs; none to skip)") + f.StringArrayVar(&fabrics, "fabric", nil, "a fabric about to be declared, name:mlx5_0,mlx5_1, checked as if it were; repeatable") f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back") f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology") return cmd diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 6cb5afcd..ef931d2e 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "maps" "net" "os" "os/exec" @@ -105,6 +106,10 @@ type Options struct { // VLLMVersion is the vLLM this llmgw installs; its prerequisites are // checked when set. VLLMVersion string + // Fabrics are fabrics about to be declared, by name, with their + // adapters: what `node add --fabric` will write into agent.yaml, which + // does not exist yet when doctor runs before it. + Fabrics map[string][]string } // Collect reads the node. @@ -185,6 +190,14 @@ func Collect(ctx context.Context, o Options) Facts { } f.readConfig(o) + if len(o.Fabrics) > 0 { + declared := map[string][]string{} + for _, c := range f.Fabrics { + declared[c.Fabric] = c.Declared + } + maps.Copy(declared, o.Fabrics) + f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) + } f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs) if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" { if f.EnginesDir == "" { @@ -422,8 +435,15 @@ func Evaluate(f Facts) []Finding { } switch { case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0: - add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang", - "give the interface an IPv4 address, or check the adapter's RoCE mode") + // Fatal only for a port a declared fabric uses. An adapter + // nothing is configured to use — a RoCE NIC on a node added + // for single-node inference — kept nodes out for a fabric + // nobody asked for. + msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang" + if declared[h.Name] == "" { + msg += "; no declared fabric uses it, so nothing will until one does" + } + add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode") case p.Active(): on := "" switch { @@ -433,17 +453,24 @@ func Evaluate(f Facts) []Finding { on = ", network " + p.Network } add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "") + // Questions about which fabric a port is on matter to the + // fabrics that use it; for a port none does they are noted, + // not warned of. + quiet := Warn + if declared[h.Name] == "" { + quiet = Info + } if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil { why := f.SMErrors[where] fix := "" if strings.Contains(why, "infiniband-diags") { fix = "install infiniband-diags" } - add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+ + add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+ "; without it, separate fabrics given one name cannot be caught", fix) } if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 { - add(Warn, "fabric", where+" is active but reports no subnet manager LID", "") + add(quiet, "fabric", where+" is active but reports no subnet manager LID", "") } case strings.EqualFold(p.PhysState, "LinkUp"): add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it", @@ -482,7 +509,14 @@ func Evaluate(f Facts) []Finding { } if len(f.HCAs) > 0 { if !f.Modules["ib_uverbs"] { - add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") + // RDMA is for replicas spanning nodes; a node that declares no + // fabric runs none, and is a node without it. + if len(declared) > 0 { + add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") + } else { + add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+ + "later would need; this node runs single-node replicas without it", "modprobe ib_uverbs") + } } if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] { add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory", diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 22c69f75..16097192 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -218,3 +218,26 @@ func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) { } } } + +func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) { + // The field: a RoCE NIC with no v2 GID on a node added for single-node + // inference, no fabric given, and the node refused for it. + roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", + PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}} + f := healthy() + f.HCAs = append(f.HCAs, roce) + f.Modules["ib_uverbs"] = false + f.Fabrics = nil // nothing declared + fs := Evaluate(f) + if fails := levels(fs, Fail); len(fails) > 0 { + t.Errorf("an undeclared adapter failed the node: %v", fails) + } + has(t, fs, Warn, "no declared fabric uses it") + has(t, fs, Warn, "runs single-node replicas without it") + + // Declared, the same adapter and module are what the fabric needs. + f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs) + fs = Evaluate(f) + has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry") + has(t, fs, Fail, "ib_uverbs is not loaded") +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index ff1fa5a5..48b595f6 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -400,6 +400,11 @@ func (r *Runner) check(ctx context.Context, res *Result) error { if r.Opts.IBRoot != "" { args += " --ib-root " + shellQuote(r.Opts.IBRoot) } + // The fabrics given, checked as the ones they will be; any adapter they + // do not name is checked as one nothing uses. + for _, fab := range r.Opts.Fabrics { + args += " --fabric " + shellQuote(fab.Name+":"+strings.Join(fab.HCAs, ",")) + } switch { case slices.Contains(r.Opts.DisableEngines, "vllm"): args += " --vllm none" diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go index 0d69b9b2..81991ae6 100644 --- a/internal/onboard/onboard_test.go +++ b/internal/onboard/onboard_test.go @@ -308,3 +308,16 @@ func TestAnEngineDisabledOnANodeIsWrittenAndNotChecked(t *testing.T) { t.Errorf("doctor was not told the control plane's vLLM: %v", m.scripts) } } + +func TestTheFabricsGivenAreCheckedAsDeclared(t *testing.T) { + // agent.yaml is written after doctor runs, so the fabrics on the command + // line are handed to doctor: checked strictly, and nothing else is. + m := newFake() + r, _ := runner(t, m, Options{Version: "dev", Fabrics: []Fabric{{Name: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}}}) + if _, err := r.Run(t.Context()); err != nil { + t.Fatal(err) + } + if !m.ran("--fabric 'ib-a:mlx5_0,mlx5_1'") { + t.Errorf("doctor was not told the fabric: %v", m.scripts) + } +}