Fail a node only for the fabrics it declares

A RoCE port without a v2 GID and a missing ib_uverbs failed doctor, and
node add, whatever fabric used them. They are now failures only for a
declared fabric and warnings otherwise; node add passes its --fabric
list to doctor so those are checked as declared.

Refs #2 (item 5).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 12:13:18 +02:00
1 parent 6014d1fee8
commit 2ef4220643
6 files changed
+109 -6

No files matched your search

+16
View File
@@ -47,6 +47,22 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
node with two kinds of GPU, a shard could get a different card from the one it
was given.
### An adapter no fabric uses does not keep a node out
`llmgw doctor` failed a node for a RoCE port with no RoCE v2 GID entry, or for
`ib_uverbs` not being loaded, whether or not any fabric used them; on a node
added for single-node inference, with no fabric given, that refused it for
RDMA nobody had asked for.
- Those are failures only for ports a declared fabric uses, and only when a
fabric is declared. Otherwise they are warnings saying nothing uses them yet.
- Which fabric an undeclared InfiniBand port is on, and whether it has a
subnet manager LID, are notes rather than warnings; `node add` still warns
once that adapters are up with no fabric given.
- `node add --fabric` hands its fabrics to doctor (`doctor --fabric
name:mlx5_0,…`), so they are checked as declared even though agent.yaml is
written afterwards.
### A restart with no room beside it
`ctl restart` places each replacement beside the replica it replaces. With
+13 -1
View File
@@ -23,6 +23,7 @@ func newDoctorCmd() *cobra.Command {
opts doctor.Options
asJSON bool
verbose bool
fabrics []string
)
cmd := &cobra.Command{
Use: "doctor",
@@ -38,6 +39,16 @@ that file when the report is not enough to go on.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
doctor.SetConfigLoader(config.LoadAgent)
for _, spec := range fabrics {
name, hcas, _ := strings.Cut(spec, ":")
if name == "" || hcas == "" {
return fmt.Errorf("--fabric %q: give it as name:mlx5_0,mlx5_1", spec)
}
if opts.Fabrics == nil {
opts.Fabrics = map[string][]string{}
}
opts.Fabrics[name] = strings.Split(hcas, ",")
}
switch opts.VLLMVersion {
case "":
opts.VLLMVersion = config.DefaultEngines["vllm"]
@@ -69,7 +80,8 @@ that file when the report is not enough to go on.`,
f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)")
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs; none to skip)")
f.StringArrayVar(&fabrics, "fabric", nil, "a fabric about to be declared, name:mlx5_0,mlx5_1, checked as if it were; repeatable")
f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
return cmd
+39 -5
View File
@@ -10,6 +10,7 @@ import (
"context"
"errors"
"fmt"
"maps"
"net"
"os"
"os/exec"
@@ -105,6 +106,10 @@ type Options struct {
// VLLMVersion is the vLLM this llmgw installs; its prerequisites are
// checked when set.
VLLMVersion string
// Fabrics are fabrics about to be declared, by name, with their
// adapters: what `node add --fabric` will write into agent.yaml, which
// does not exist yet when doctor runs before it.
Fabrics map[string][]string
}
// Collect reads the node.
@@ -185,6 +190,14 @@ func Collect(ctx context.Context, o Options) Facts {
}
f.readConfig(o)
if len(o.Fabrics) > 0 {
declared := map[string][]string{}
for _, c := range f.Fabrics {
declared[c.Fabric] = c.Declared
}
maps.Copy(declared, o.Fabrics)
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
}
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
if f.EnginesDir == "" {
@@ -422,8 +435,15 @@ func Evaluate(f Facts) []Finding {
}
switch {
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang",
"give the interface an IPv4 address, or check the adapter's RoCE mode")
// Fatal only for a port a declared fabric uses. An adapter
// nothing is configured to use — a RoCE NIC on a node added
// for single-node inference — kept nodes out for a fabric
// nobody asked for.
msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang"
if declared[h.Name] == "" {
msg += "; no declared fabric uses it, so nothing will until one does"
}
add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode")
case p.Active():
on := ""
switch {
@@ -433,17 +453,24 @@ func Evaluate(f Facts) []Finding {
on = ", network " + p.Network
}
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
// Questions about which fabric a port is on matter to the
// fabrics that use it; for a port none does they are noted,
// not warned of.
quiet := Warn
if declared[h.Name] == "" {
quiet = Info
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
why := f.SMErrors[where]
fix := ""
if strings.Contains(why, "infiniband-diags") {
fix = "install infiniband-diags"
}
add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
"; without it, separate fabrics given one name cannot be caught", fix)
}
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
add(Warn, "fabric", where+" is active but reports no subnet manager LID", "")
add(quiet, "fabric", where+" is active but reports no subnet manager LID", "")
}
case strings.EqualFold(p.PhysState, "LinkUp"):
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
@@ -482,7 +509,14 @@ func Evaluate(f Facts) []Finding {
}
if len(f.HCAs) > 0 {
if !f.Modules["ib_uverbs"] {
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
// RDMA is for replicas spanning nodes; a node that declares no
// fabric runs none, and is a node without it.
if len(declared) > 0 {
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
} else {
add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+
"later would need; this node runs single-node replicas without it", "modprobe ib_uverbs")
}
}
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
+23
View File
@@ -218,3 +218,26 @@ func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) {
}
}
}
func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) {
// The field: a RoCE NIC with no v2 GID on a node added for single-node
// inference, no fabric given, and the node refused for it.
roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}}
f := healthy()
f.HCAs = append(f.HCAs, roce)
f.Modules["ib_uverbs"] = false
f.Fabrics = nil // nothing declared
fs := Evaluate(f)
if fails := levels(fs, Fail); len(fails) > 0 {
t.Errorf("an undeclared adapter failed the node: %v", fails)
}
has(t, fs, Warn, "no declared fabric uses it")
has(t, fs, Warn, "runs single-node replicas without it")
// Declared, the same adapter and module are what the fabric needs.
f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs)
fs = Evaluate(f)
has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry")
has(t, fs, Fail, "ib_uverbs is not loaded")
}
+5
View File
@@ -400,6 +400,11 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
if r.Opts.IBRoot != "" {
args += " --ib-root " + shellQuote(r.Opts.IBRoot)
}
// The fabrics given, checked as the ones they will be; any adapter they
// do not name is checked as one nothing uses.
for _, fab := range r.Opts.Fabrics {
args += " --fabric " + shellQuote(fab.Name+":"+strings.Join(fab.HCAs, ","))
}
switch {
case slices.Contains(r.Opts.DisableEngines, "vllm"):
args += " --vllm none"
+13
View File
@@ -308,3 +308,16 @@ func TestAnEngineDisabledOnANodeIsWrittenAndNotChecked(t *testing.T) {
t.Errorf("doctor was not told the control plane's vLLM: %v", m.scripts)
}
}
func TestTheFabricsGivenAreCheckedAsDeclared(t *testing.T) {
// agent.yaml is written after doctor runs, so the fabrics on the command
// line are handed to doctor: checked strictly, and nothing else is.
m := newFake()
r, _ := runner(t, m, Options{Version: "dev", Fabrics: []Fabric{{Name: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}}})
if _, err := r.Run(t.Context()); err != nil {
t.Fatal(err)
}
if !m.ran("--fabric 'ib-a:mlx5_0,mlx5_1'") {
t.Errorf("doctor was not told the fabric: %v", m.scripts)
}
}