Fail a node only for the fabrics it declares
A RoCE port without a v2 GID and a missing ib_uverbs failed doctor, and node add, whatever fabric used them. They are now failures only for a declared fabric and warnings otherwise; node add passes its --fabric list to doctor so those are checked as declared. Refs #2 (item 5). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6014d1fee8
commit
2ef4220643
6 files changed
+109
-6
No files matched your search
@@ -47,6 +47,22 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
|
||||
node with two kinds of GPU, a shard could get a different card from the one it
|
||||
was given.
|
||||
|
||||
### An adapter no fabric uses does not keep a node out
|
||||
|
||||
`llmgw doctor` failed a node for a RoCE port with no RoCE v2 GID entry, or for
|
||||
`ib_uverbs` not being loaded, whether or not any fabric used them; on a node
|
||||
added for single-node inference, with no fabric given, that refused it for
|
||||
RDMA nobody had asked for.
|
||||
|
||||
- Those are failures only for ports a declared fabric uses, and only when a
|
||||
fabric is declared. Otherwise they are warnings saying nothing uses them yet.
|
||||
- Which fabric an undeclared InfiniBand port is on, and whether it has a
|
||||
subnet manager LID, are notes rather than warnings; `node add` still warns
|
||||
once that adapters are up with no fabric given.
|
||||
- `node add --fabric` hands its fabrics to doctor (`doctor --fabric
|
||||
name:mlx5_0,…`), so they are checked as declared even though agent.yaml is
|
||||
written afterwards.
|
||||
|
||||
### A restart with no room beside it
|
||||
|
||||
`ctl restart` places each replacement beside the replica it replaces. With
|
||||
|
||||
+13
-1
@@ -23,6 +23,7 @@ func newDoctorCmd() *cobra.Command {
|
||||
opts doctor.Options
|
||||
asJSON bool
|
||||
verbose bool
|
||||
fabrics []string
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "doctor",
|
||||
@@ -38,6 +39,16 @@ that file when the report is not enough to go on.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
doctor.SetConfigLoader(config.LoadAgent)
|
||||
for _, spec := range fabrics {
|
||||
name, hcas, _ := strings.Cut(spec, ":")
|
||||
if name == "" || hcas == "" {
|
||||
return fmt.Errorf("--fabric %q: give it as name:mlx5_0,mlx5_1", spec)
|
||||
}
|
||||
if opts.Fabrics == nil {
|
||||
opts.Fabrics = map[string][]string{}
|
||||
}
|
||||
opts.Fabrics[name] = strings.Split(hcas, ",")
|
||||
}
|
||||
switch opts.VLLMVersion {
|
||||
case "":
|
||||
opts.VLLMVersion = config.DefaultEngines["vllm"]
|
||||
@@ -69,7 +80,8 @@ that file when the report is not enough to go on.`,
|
||||
f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
|
||||
f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
||||
f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
|
||||
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)")
|
||||
f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs; none to skip)")
|
||||
f.StringArrayVar(&fabrics, "fabric", nil, "a fabric about to be declared, name:mlx5_0,mlx5_1, checked as if it were; repeatable")
|
||||
f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
|
||||
f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
|
||||
return cmd
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -105,6 +106,10 @@ type Options struct {
|
||||
// VLLMVersion is the vLLM this llmgw installs; its prerequisites are
|
||||
// checked when set.
|
||||
VLLMVersion string
|
||||
// Fabrics are fabrics about to be declared, by name, with their
|
||||
// adapters: what `node add --fabric` will write into agent.yaml, which
|
||||
// does not exist yet when doctor runs before it.
|
||||
Fabrics map[string][]string
|
||||
}
|
||||
|
||||
// Collect reads the node.
|
||||
@@ -185,6 +190,14 @@ func Collect(ctx context.Context, o Options) Facts {
|
||||
}
|
||||
|
||||
f.readConfig(o)
|
||||
if len(o.Fabrics) > 0 {
|
||||
declared := map[string][]string{}
|
||||
for _, c := range f.Fabrics {
|
||||
declared[c.Fabric] = c.Declared
|
||||
}
|
||||
maps.Copy(declared, o.Fabrics)
|
||||
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
|
||||
}
|
||||
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
|
||||
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
|
||||
if f.EnginesDir == "" {
|
||||
@@ -422,8 +435,15 @@ func Evaluate(f Facts) []Finding {
|
||||
}
|
||||
switch {
|
||||
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
|
||||
add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang",
|
||||
"give the interface an IPv4 address, or check the adapter's RoCE mode")
|
||||
// Fatal only for a port a declared fabric uses. An adapter
|
||||
// nothing is configured to use — a RoCE NIC on a node added
|
||||
// for single-node inference — kept nodes out for a fabric
|
||||
// nobody asked for.
|
||||
msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang"
|
||||
if declared[h.Name] == "" {
|
||||
msg += "; no declared fabric uses it, so nothing will until one does"
|
||||
}
|
||||
add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode")
|
||||
case p.Active():
|
||||
on := ""
|
||||
switch {
|
||||
@@ -433,17 +453,24 @@ func Evaluate(f Facts) []Finding {
|
||||
on = ", network " + p.Network
|
||||
}
|
||||
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
|
||||
// Questions about which fabric a port is on matter to the
|
||||
// fabrics that use it; for a port none does they are noted,
|
||||
// not warned of.
|
||||
quiet := Warn
|
||||
if declared[h.Name] == "" {
|
||||
quiet = Info
|
||||
}
|
||||
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
|
||||
why := f.SMErrors[where]
|
||||
fix := ""
|
||||
if strings.Contains(why, "infiniband-diags") {
|
||||
fix = "install infiniband-diags"
|
||||
}
|
||||
add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
|
||||
add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
|
||||
"; without it, separate fabrics given one name cannot be caught", fix)
|
||||
}
|
||||
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
|
||||
add(Warn, "fabric", where+" is active but reports no subnet manager LID", "")
|
||||
add(quiet, "fabric", where+" is active but reports no subnet manager LID", "")
|
||||
}
|
||||
case strings.EqualFold(p.PhysState, "LinkUp"):
|
||||
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
|
||||
@@ -482,7 +509,14 @@ func Evaluate(f Facts) []Finding {
|
||||
}
|
||||
if len(f.HCAs) > 0 {
|
||||
if !f.Modules["ib_uverbs"] {
|
||||
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
|
||||
// RDMA is for replicas spanning nodes; a node that declares no
|
||||
// fabric runs none, and is a node without it.
|
||||
if len(declared) > 0 {
|
||||
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
|
||||
} else {
|
||||
add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+
|
||||
"later would need; this node runs single-node replicas without it", "modprobe ib_uverbs")
|
||||
}
|
||||
}
|
||||
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
|
||||
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
|
||||
|
||||
@@ -218,3 +218,26 @@ func TestAnEngineDisabledOnTheNodeIsNotChecked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdapterNoFabricUsesDoesNotKeepTheNodeOut(t *testing.T) {
|
||||
// The field: a RoCE NIC with no v2 GID on a node added for single-node
|
||||
// inference, no fabric given, and the node refused for it.
|
||||
roce := discovery.HCA{Name: "mlx5_2", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
|
||||
PhysState: "LinkUp", RateGbps: 100, LinkLayer: discovery.LinkEthernet, RoCEv2GID: -1}}}
|
||||
f := healthy()
|
||||
f.HCAs = append(f.HCAs, roce)
|
||||
f.Modules["ib_uverbs"] = false
|
||||
f.Fabrics = nil // nothing declared
|
||||
fs := Evaluate(f)
|
||||
if fails := levels(fs, Fail); len(fails) > 0 {
|
||||
t.Errorf("an undeclared adapter failed the node: %v", fails)
|
||||
}
|
||||
has(t, fs, Warn, "no declared fabric uses it")
|
||||
has(t, fs, Warn, "runs single-node replicas without it")
|
||||
|
||||
// Declared, the same adapter and module are what the fabric needs.
|
||||
f.Fabrics = discovery.CheckFabrics(map[string][]string{"roce-a": {"mlx5_2"}}, f.HCAs)
|
||||
fs = Evaluate(f)
|
||||
has(t, fs, Fail, "mlx5_2/1 is RoCE with no RoCE v2 GID entry")
|
||||
has(t, fs, Fail, "ib_uverbs is not loaded")
|
||||
}
|
||||
@@ -400,6 +400,11 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
|
||||
if r.Opts.IBRoot != "" {
|
||||
args += " --ib-root " + shellQuote(r.Opts.IBRoot)
|
||||
}
|
||||
// The fabrics given, checked as the ones they will be; any adapter they
|
||||
// do not name is checked as one nothing uses.
|
||||
for _, fab := range r.Opts.Fabrics {
|
||||
args += " --fabric " + shellQuote(fab.Name+":"+strings.Join(fab.HCAs, ","))
|
||||
}
|
||||
switch {
|
||||
case slices.Contains(r.Opts.DisableEngines, "vllm"):
|
||||
args += " --vllm none"
|
||||
|
||||
@@ -308,3 +308,16 @@ func TestAnEngineDisabledOnANodeIsWrittenAndNotChecked(t *testing.T) {
|
||||
t.Errorf("doctor was not told the control plane's vLLM: %v", m.scripts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFabricsGivenAreCheckedAsDeclared(t *testing.T) {
|
||||
// agent.yaml is written after doctor runs, so the fabrics on the command
|
||||
// line are handed to doctor: checked strictly, and nothing else is.
|
||||
m := newFake()
|
||||
r, _ := runner(t, m, Options{Version: "dev", Fabrics: []Fabric{{Name: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}}})
|
||||
if _, err := r.Run(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.ran("--fabric 'ib-a:mlx5_0,mlx5_1'") {
|
||||
t.Errorf("doctor was not told the fabric: %v", m.scripts)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user