From 1b1ca67e350d6859270184404b8c5dead6dc066b Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 03:51:09 +0200 Subject: [PATCH] Add llmgw doctor: what a node's GPUs, NVLink and fabric need, checked For machines nobody here can log into. It reads the GPUs and their nvidia-smi topology, NVSwitch and the fabric manager, every InfiniBand and RoCE port with its counters and RoCE v2 GID, the RDMA and GPUDirect kernel modules, the agent's declared fabrics against the adapters, the management interface NCCL will bootstrap over, the agent unit's memlock limit and the engines installed - and says what is wrong and how to fix it, most serious first, with the NCCL environment a shard on each fabric would be given. --json writes every fact it concluded from, to send back. It exits non-zero when something fails. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/agent/agent.go | 13 + internal/agent/discovery/fabrics.go | 4 +- internal/agent/discovery/topo.go | 3 + internal/agent/engines/engine.go | 4 + internal/cli/agent.go | 8 +- internal/cli/doctor.go | 180 +++++++++++ internal/cli/root.go | 1 + internal/doctor/doctor.go | 447 ++++++++++++++++++++++++++++ internal/doctor/doctor_test.go | 121 ++++++++ 9 files changed, 776 insertions(+), 5 deletions(-) create mode 100644 internal/cli/doctor.go create mode 100644 internal/doctor/doctor.go create mode 100644 internal/doctor/doctor_test.go diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4755291..0d28bb9 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1591,3 +1591,16 @@ func (a *Agent) localizeSpec(spec *engines.ShardSpec) { } } } + +// ResolveManagement is the address and interface this node's agent would use, +// found the way normalize finds them, for a tool that reports them. +func ResolveManagement(c Config) (addr, iface string) { + addr, iface = c.MgmtAddr, c.MgmtInterface + if addr == "" { + addr = addrToward(c.Server) + } + if iface == "" { + iface = interfaceOf(addr) + } + return addr, iface +} diff --git a/internal/agent/discovery/fabrics.go b/internal/agent/discovery/fabrics.go index 27d699d..3892d95 100644 --- a/internal/agent/discovery/fabrics.go +++ b/internal/agent/discovery/fabrics.go @@ -10,6 +10,8 @@ import ( // has. type FabricCheck struct { Fabric string + // Declared are the adapters the configuration names for it. + Declared []string // Usable are the declared adapters that exist and have an active port. Usable []string // RateGbps is the slowest active port among them, because a collective @@ -49,7 +51,7 @@ func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck out := make([]FabricCheck, 0, len(names)) for _, fabric := range names { - c := FabricCheck{Fabric: fabric} + c := FabricCheck{Fabric: fabric, Declared: declared[fabric]} for _, want := range declared[fabric] { hca, found := byName[want] if !found { diff --git a/internal/agent/discovery/topo.go b/internal/agent/discovery/topo.go index d2db624..13230a4 100644 --- a/internal/agent/discovery/topo.go +++ b/internal/agent/discovery/topo.go @@ -64,6 +64,9 @@ func DiscoverTopology(ctx context.Context, binary string) (Topology, error) { return parseTopo(string(out)) } +// ParseTopology reads `nvidia-smi topo -m` output. +func ParseTopology(out string) (Topology, error) { return parseTopo(out) } + var ( ansi = regexp.MustCompile(`\x1b\[[0-9;]*m`) nvLinks = regexp.MustCompile(`^NV(\d+)$`) diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 3d9bbe6..23baf1f 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -196,6 +196,10 @@ type Config struct { // --- shared helpers --- +// NCCLEnv is the environment a shard with this spec is given, for a tool that +// shows an operator what an engine would see. +func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) } + // baseEnv builds the environment every engine shares. // // This is the part of §5.3 the plan marks critical. The rules encoded here: diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 5d34474..102bf3f 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -163,10 +163,10 @@ func printInventory(inv discovery.Inventory) { g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health) } fmt.Fprintf(w, " classes: %v\n", inv.Classes()) - if !discovery.NVMLSupported() { - fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.") - fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside") - fmt.Fprintln(w, " one NVLink domain (plan §6.2).") + for _, g := range inv.GPUs { + if len(g.NVLinkPeers) > 0 { + fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers) + } } } diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go new file mode 100644 index 0000000..36f1372 --- /dev/null +++ b/internal/cli/doctor.go @@ -0,0 +1,180 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/doctor" +) + +func newDoctorCmd() *cobra.Command { + var ( + opts doctor.Options + asJSON bool + verbose bool + ) + cmd := &cobra.Command{ + Use: "doctor", + Short: "Check this node's GPUs, NVLink and InfiniBand for what llmgw needs", + Long: `Reads this node's GPUs, their NVLink and PCIe topology, its InfiniBand and +RoCE ports, the kernel modules RDMA needs and the agent's configuration, and +says what is wrong and how to fix it — most serious first. + +Run it on a node before it joins, and whenever a multi-node replica will not +come up. --json writes everything it read as well as what it concluded; send +that file when the report is not enough to go on.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + doctor.SetConfigLoader(config.LoadAgent) + ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) + defer cancel() + facts := doctor.Collect(ctx, opts) + findings := doctor.Evaluate(facts) + if asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(struct { + Findings []doctor.Finding `json:"findings"` + Facts doctor.Facts `json:"facts"` + }{findings, facts}) + } + printDoctor(os.Stdout, facts, findings, verbose) + for _, f := range findings { + if f.Level == doctor.Fail { + return fmt.Errorf("%d problem(s) need fixing", countLevel(findings, doctor.Fail)) + } + } + return nil + }, + } + f := cmd.Flags() + f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)") + f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory") + f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi") + f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back") + f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology") + return cmd +} + +func countLevel(fs []doctor.Finding, l doctor.Level) int { + n := 0 + for _, f := range fs { + if f.Level == l { + n++ + } + } + return n +} + +func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbose bool) { + fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion, + f.Host.Arch, orDash(f.Host.NVIDIADriver)) + + for _, x := range findings { + fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message) + if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) { + fmt.Fprintf(out, " → %s\n", x.Fix) + } + } + + w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + if len(f.GPUs) > 0 { + fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER") + for _, g := range f.GPUs { + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode, + orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths))) + } + } + if len(f.HCAs) > 0 { + fmt.Fprintln(w, "\nADAPTER\tSTATE\tPHYS\tRATE\tLINK\tNUMA\tROCE GID\tERRORS") + for _, h := range f.HCAs { + for _, p := range h.Ports { + gid := "-" + if p.RoCEv2GID >= 0 { + gid = fmt.Sprint(p.RoCEv2GID) + } + fmt.Fprintf(w, "%s/%d\t%s\t%s\t%d Gb/s\t%s\t%d\t%s\t%s\n", h.Name, p.Num, p.State, + orDash(p.PhysState), p.RateGbps, p.LinkLayer, h.NUMANode, gid, orDash(errorsOf(p.Counters))) + } + } + } + _ = w.Flush() + + for _, fabric := range sortedKeys(f.NCCL) { + fmt.Fprintf(out, "\nA shard on %s is given:\n", fabric) + env := f.NCCL[fabric] + for _, k := range sortedKeys(env) { + fmt.Fprintf(out, " %s=%s\n", k, env[k]) + } + } + if len(f.Engines) > 0 { + fmt.Fprintln(out, "\nEngines:") + for _, k := range sortedKeys(f.Engines) { + fmt.Fprintf(out, " %s: %s\n", k, f.Engines[k]) + } + } + if verbose && f.TopoRaw != "" { + fmt.Fprintf(out, "\nnvidia-smi topo -m:\n%s\n", f.TopoRaw) + } + fmt.Fprintf(out, "\n%d failed, %d warnings. `llmgw doctor --json > report.json` keeps everything read, to send back.\n", + countLevel(findings, doctor.Fail), countLevel(findings, doctor.Warn)) +} + +func nearest(paths map[string]string) string { + best, rel := "", "" + for h, r := range paths { + if best == "" || discovery.PathRank(r) < discovery.PathRank(rel) || + (discovery.PathRank(r) == discovery.PathRank(rel) && h < best) { + best, rel = h, r + } + } + if best == "" { + return "" + } + return best + " (" + rel + ")" +} + +func errorsOf(c map[string]uint64) string { + var out []string + for _, k := range sortedKeys(c) { + if c[k] > 0 && k != "port_xmit_data" && k != "port_rcv_data" { + out = append(out, fmt.Sprintf("%s=%d", k, c[k])) + } + } + return strings.Join(out, " ") +} + +func ints(v []int) string { + s := make([]string, len(v)) + for i, x := range v { + s[i] = fmt.Sprint(x) + } + return strings.Join(s, ",") +} + +func orDash(s string) string { + if s == "" { + return "-" + } + return s +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/cli/root.go b/internal/cli/root.go index a320980..39f895d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -45,6 +45,7 @@ func Root() *cobra.Command { newOrgCmd(), newMirrorCmd(), newSimEngineCmd(), + newDoctorCmd(), ) return root } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go new file mode 100644 index 0000000..5e7ecde --- /dev/null +++ b/internal/doctor/doctor.go @@ -0,0 +1,447 @@ +// Package doctor checks a node for what llmgw needs of its GPUs and fabric, +// and says what is wrong in words an operator can act on. +// +// Written for a machine nobody here can log into. The report is meant to be +// run on the node, read there, and sent back when it is not enough: every +// fact it concluded from is in it, so the conclusion can be checked. +package doctor + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +// Level is how much a finding matters. +type Level string + +// The levels, most serious first. +const ( + Fail Level = "FAIL" + Warn Level = "WARN" + OK Level = "OK" + Info Level = "INFO" +) + +// Finding is one conclusion, and what to do about it. +type Finding struct { + Level Level `json:"level"` + Area string `json:"area"` + Message string `json:"message"` + Fix string `json:"fix,omitempty"` +} + +// Facts are what was read from the node. Every finding is drawn from these +// and nothing else, so a report sent back can be re-read here. +type Facts struct { + At time.Time `json:"at"` + Hostname string `json:"hostname"` + Host discovery.Host `json:"host"` + + SMIFound bool `json:"nvidia_smi_found"` + GPUs []discovery.GPU `json:"gpus"` + GPUError string `json:"gpu_error,omitempty"` + TopoRaw string `json:"topo_raw,omitempty"` + TopoErr string `json:"topo_error,omitempty"` + // NVSwitch is set when the machine has NVSwitch devices, which need the + // fabric manager running before CUDA will start. + NVSwitch bool `json:"nvswitch"` + FabricManager string `json:"fabric_manager,omitempty"` + + HCAs []discovery.HCA `json:"hcas"` + Modules map[string]bool `json:"modules"` + + // ConfigPath is the agent configuration read, empty when there was none. + ConfigPath string `json:"config_path,omitempty"` + ConfigErr string `json:"config_error,omitempty"` + Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"` + MgmtAddr string `json:"mgmt_addr,omitempty"` + MgmtIface string `json:"mgmt_interface,omitempty"` + IfaceExists bool `json:"mgmt_interface_exists"` + LaunchMode string `json:"vllm_launch_mode,omitempty"` + NCCL map[string]map[string]string `json:"nccl_env,omitempty"` + UnitMemlock string `json:"agent_unit_memlock,omitempty"` + Engines map[string]string `json:"engines"` +} + +// Modules checked, and why each matters. +var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"} + +// Options says where to look. +type Options struct { + ConfigPath string + IBRoot string + NvidiaSMI string +} + +// Collect reads the node. +func Collect(ctx context.Context, o Options) Facts { + f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}} + f.Hostname, _ = os.Hostname() + f.Host = discovery.DiscoverHost() + + smi := o.NvidiaSMI + if smi == "" { + if p, err := exec.LookPath("nvidia-smi"); err == nil { + smi = p + } + } + if smi != "" { + f.SMIFound = true + gpus, err := discovery.DiscoverGPUsSMI(ctx, smi) + if err != nil { + f.GPUError = err.Error() + } + out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output() + if err != nil { + f.TopoErr = err.Error() + } else { + f.TopoRaw = string(out) + if t, err := discovery.ParseTopology(f.TopoRaw); err == nil { + t.Apply(gpus) + } else { + f.TopoErr = err.Error() + } + } + f.GPUs = gpus + } + if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 { + f.NVSwitch = true + f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager") + } + + root := o.IBRoot + if root == "" { + root = discovery.SysfsRoot + } + f.HCAs, _ = discovery.DiscoverIB(root) + for _, m := range watchedModules { + _, err := os.Stat(filepath.Join("/sys/module", m)) + f.Modules[m] = err == nil + } + + f.readConfig(o) + f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=") + + for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} { + if name == "sglang" { + continue // no binary of its own to ask + } + if p, err := exec.LookPath(bin); err == nil { + out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput() + f.Engines[name] = firstLine(string(out)) + } + } + return f +} + +func (f *Facts) readConfig(o Options) { + path := o.ConfigPath + if path == "" { + if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil { + return + } + path = "/etc/llmgw/agent.yaml" + } + f.ConfigPath = path + cfg, err := loadAgent(path) + if err != nil { + f.ConfigErr = err.Error() + return + } + declared := map[string][]string{} + for _, fa := range cfg.Fabrics { + declared[fa.Fabric] = fa.HCAs + } + f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) + f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg) + if f.MgmtIface == "" { + f.MgmtIface = "eth0" + } + _, err = net.InterfaceByName(f.MgmtIface) + f.IfaceExists = err == nil + f.LaunchMode = cfg.Engines.VLLM.LaunchMode + if f.LaunchMode == "" { + f.LaunchMode = "ray" + } + + // What a two-node shard on each fabric would be given, the part of an + // engine's environment that decides whether NCCL finds its peers. + f.NCCL = map[string]map[string]string{} + for _, c := range f.Fabrics { + if !c.OK() { + continue + } + spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)} + env := engines.NCCLEnv(spec) + delete(env, "CUDA_VISIBLE_DEVICES") + f.NCCL[c.Fabric] = env + } +} + +// loadAgent is replaced in tests; the real one lives in config, which this +// package cannot import without a cycle through the CLI. +var loadAgent = func(string) (agent.Config, error) { + return agent.Config{}, fmt.Errorf("no agent configuration loader") +} + +// SetConfigLoader gives Collect the configuration loader. +func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn } + +func rocev2(hcas []discovery.HCA, usable []string) *int { + for _, h := range hcas { + if !slices.Contains(usable, h.Name) { + continue + } + for _, p := range h.Ports { + if p.Active() && p.RoCEv2GID >= 0 { + v := p.RoCEv2GID + return &v + } + } + } + return nil +} + +func systemctl(ctx context.Context, args ...string) string { + out, err := exec.CommandContext(ctx, "systemctl", args...).Output() + if err != nil && len(out) == 0 { + return "unknown" + } + return strings.TrimSpace(string(out)) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return s +} + +// Evaluate draws the findings from the facts, most serious first. +func Evaluate(f Facts) []Finding { + var out []Finding + add := func(l Level, area, msg, fix string) { + out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix}) + } + + // --- GPUs --- + switch { + case !f.SMIFound: + add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "") + case f.GPUError != "": + add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand") + case len(f.GPUs) == 0: + add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "") + default: + add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "") + for _, g := range f.GPUs { + if !g.Health.OK() { + add(Warn, "gpu", fmt.Sprintf("GPU %d is kept out of placement: %s", g.Index, strings.Join(g.Health.Problems, "; ")), "") + } + } + } + if f.NVSwitch && f.FabricManager != "active" { + add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+ + ": CUDA will not initialise on its GPUs", + "systemctl enable --now nvidia-fabricmanager (its version must match the driver's)") + } + if len(f.GPUs) > 1 { + pairs, groups := nvlinkGroups(f.GPUs) + switch { + case f.TopoErr != "": + add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+ + "; tensor-parallel groups will be chosen without NVLink", "") + case pairs == 0 && f.NVSwitch: + add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status") + case pairs == 0: + add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "") + default: + add(OK, "nvlink", "NVLink groups: "+groups, "") + } + } + + // --- InfiniBand and RoCE --- + if len(f.HCAs) == 0 { + add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "") + } + declared := map[string]string{} // hca -> fabric + for _, c := range f.Fabrics { + for _, h := range c.Declared { + declared[h] = c.Fabric + } + } + for _, h := range f.HCAs { + for _, p := range h.Ports { + where := fmt.Sprintf("%s/%d", h.Name, p.Num) + level := Warn + if declared[h.Name] != "" { + level = Fail + } + switch { + case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0: + add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang", + "give the interface an IPv4 address, or check the adapter's RoCE mode") + case p.Active(): + add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)", where, p.State, p.RateGbps, p.LinkLayer), "") + if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 { + add(Warn, "fabric", where+" is active but reports no subnet manager LID", "") + } + case strings.EqualFold(p.PhysState, "LinkUp"): + add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it", + "run a subnet manager on the fabric (opensm, or the switch's own)") + default: + add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState), + "check the cable and the switch port") + } + for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} { + if v := p.Counters[name]; v > 0 { + add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v), + "a count that keeps growing is a cable or transceiver going bad") + } + } + } + } + if len(f.HCAs) > 0 { + if !f.Modules["ib_uverbs"] { + add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") + } + if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] { + add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory", + "modprobe nvidia_peermem") + } + } + + // --- the agent's view --- + switch { + case f.ConfigErr != "": + add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "") + case f.ConfigPath == "": + add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config") + default: + for _, c := range f.Fabrics { + if c.OK() { + add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "") + } + for _, p := range c.Problems { + lvl := Warn + if !c.OK() { + lvl = Fail + } + add(lvl, "agent", "fabric "+c.Fabric+": "+p, "") + } + if !c.OK() { + add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "") + } + } + active := activeHCAs(f.HCAs) + if len(f.Fabrics) == 0 && len(active) > 0 { + add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work", + "add fabrics: [{name: , hcas: ["+strings.Join(active, ", ")+"]}]") + } + if !f.IfaceExists { + add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing", + "set mgmt_interface in agent.yaml") + } else { + add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "") + } + if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" { + add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp") + } + } + if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 { + add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers", + "LimitMEMLOCK=infinity in the unit") + } + + sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) }) + return out +} + +func rank(l Level) int { + switch l { + case Fail: + return 0 + case Warn: + return 1 + case OK: + return 2 + } + return 3 +} + +func gpuNames(gpus []discovery.GPU) string { + count := map[string]int{} + var order []string + for _, g := range gpus { + if count[g.Name] == 0 { + order = append(order, g.Name) + } + count[g.Name]++ + } + parts := make([]string, len(order)) + for i, n := range order { + parts[i] = fmt.Sprintf("%d× %s", count[n], n) + } + return strings.Join(parts, ", ") +} + +// nvlinkGroups counts NVLink pairs and names the connected groups. +func nvlinkGroups(gpus []discovery.GPU) (int, string) { + parent := map[int]int{} + var find func(int) int + find = func(x int) int { + if p, ok := parent[x]; ok && p != x { + parent[x] = find(p) + return parent[x] + } + parent[x] = x + return x + } + pairs := 0 + for _, g := range gpus { + find(g.Index) + for _, p := range g.NVLinkPeers { + if p > g.Index { + pairs++ + } + parent[find(g.Index)] = find(p) + } + } + groups := map[int][]int{} + for _, g := range gpus { + r := find(g.Index) + groups[r] = append(groups[r], g.Index) + } + var names []string + for _, members := range groups { + sort.Ints(members) + if len(members) > 1 { + names = append(names, fmt.Sprint(members)) + } + } + sort.Strings(names) + return pairs, strings.Join(names, " ") +} + +func activeHCAs(hcas []discovery.HCA) []string { + var out []string + for _, h := range hcas { + if len(h.ActivePorts()) > 0 { + out = append(out, h.Name) + } + } + return out +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go new file mode 100644 index 0000000..218c572 --- /dev/null +++ b/internal/doctor/doctor_test.go @@ -0,0 +1,121 @@ +package doctor + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up +// right. +func healthy() Facts { + f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active", + Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true}, + ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true, + LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}} + for i := range 8 { + g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"} + for j := range 8 { + if j != i { + g.NVLinkPeers = append(g.NVLinkPeers, j) + } + } + f.GPUs = append(f.GPUs, g) + } + f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", + PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1}}}} + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + return f +} + +func levels(fs []Finding, l Level) []string { + var out []string + for _, f := range fs { + if f.Level == l { + out = append(out, f.Message) + } + } + return out +} + +func has(t *testing.T, fs []Finding, l Level, fragment string) { + t.Helper() + for _, f := range fs { + if f.Level == l && strings.Contains(f.Message, fragment) { + return + } + } + t.Errorf("no %s finding containing %q in %+v", l, fragment, fs) +} + +func TestAHealthyNodeHasNothingToFix(t *testing.T) { + fs := Evaluate(healthy()) + if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 { + t.Errorf("a healthy node was faulted: %v", bad) + } + has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]") + has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s") + has(t, fs, OK, "NCCL bootstraps over ens3") +} + +func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) { + cases := map[string]struct { + change func(*Facts) + level Level + says string + }{ + "fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"}, + "no subnet manager": {func(f *Facts) { + f.HCAs[0].Ports[0].State = "INIT" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "no subnet manager has configured it"}, + "cable out": {func(f *Facts) { + f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "no link"}, + "fabric not advertised": {func(f *Facts) { + f.HCAs[0].Ports[0].State = "DOWN" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "will not be advertised"}, + "roce without v2": {func(f *Facts) { + f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet + }, Fail, "no RoCE v2 GID"}, + "no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"}, + "no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"}, + "nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"}, + "wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"}, + "errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"}, + "small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"}, + "ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"}, + "no nvlink on nvswitch": {func(f *Facts) { + for i := range f.GPUs { + f.GPUs[i].NVLinkPeers = nil + } + }, Warn, "no GPU reports an NVLink peer"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + f := healthy() + c.change(&f) + fs := Evaluate(f) + has(t, fs, c.level, c.says) + // Most serious first, so the thing to fix is at the top. + if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 { + t.Error("a failure is not first") + } + }) + } +} + +func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) { + // An adapter no declared fabric names being down is worth a warning, not + // a failure: nothing here was going to use it. + f := healthy() + f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}}) + fs := Evaluate(f) + has(t, fs, Warn, "mlx5_4/1 is DOWN") + if len(levels(fs, Fail)) > 0 { + t.Errorf("failures: %v", levels(fs, Fail)) + } +}