diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4755291..0d28bb9 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1591,3 +1591,16 @@ func (a *Agent) localizeSpec(spec *engines.ShardSpec) { } } } + +// ResolveManagement is the address and interface this node's agent would use, +// found the way normalize finds them, for a tool that reports them. +func ResolveManagement(c Config) (addr, iface string) { + addr, iface = c.MgmtAddr, c.MgmtInterface + if addr == "" { + addr = addrToward(c.Server) + } + if iface == "" { + iface = interfaceOf(addr) + } + return addr, iface +} diff --git a/internal/agent/discovery/fabrics.go b/internal/agent/discovery/fabrics.go index 27d699d..3892d95 100644 --- a/internal/agent/discovery/fabrics.go +++ b/internal/agent/discovery/fabrics.go @@ -10,6 +10,8 @@ import ( // has. type FabricCheck struct { Fabric string + // Declared are the adapters the configuration names for it. + Declared []string // Usable are the declared adapters that exist and have an active port. Usable []string // RateGbps is the slowest active port among them, because a collective @@ -49,7 +51,7 @@ func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck out := make([]FabricCheck, 0, len(names)) for _, fabric := range names { - c := FabricCheck{Fabric: fabric} + c := FabricCheck{Fabric: fabric, Declared: declared[fabric]} for _, want := range declared[fabric] { hca, found := byName[want] if !found { diff --git a/internal/agent/discovery/topo.go b/internal/agent/discovery/topo.go index d2db624..13230a4 100644 --- a/internal/agent/discovery/topo.go +++ b/internal/agent/discovery/topo.go @@ -64,6 +64,9 @@ func DiscoverTopology(ctx context.Context, binary string) (Topology, error) { return parseTopo(string(out)) } +// ParseTopology reads `nvidia-smi topo -m` output. +func ParseTopology(out string) (Topology, error) { return parseTopo(out) } + var ( ansi = regexp.MustCompile(`\x1b\[[0-9;]*m`) nvLinks = regexp.MustCompile(`^NV(\d+)$`) diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 3d9bbe6..23baf1f 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -196,6 +196,10 @@ type Config struct { // --- shared helpers --- +// NCCLEnv is the environment a shard with this spec is given, for a tool that +// shows an operator what an engine would see. +func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) } + // baseEnv builds the environment every engine shares. // // This is the part of §5.3 the plan marks critical. The rules encoded here: diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 5d34474..102bf3f 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -163,10 +163,10 @@ func printInventory(inv discovery.Inventory) { g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health) } fmt.Fprintf(w, " classes: %v\n", inv.Classes()) - if !discovery.NVMLSupported() { - fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.") - fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside") - fmt.Fprintln(w, " one NVLink domain (plan §6.2).") + for _, g := range inv.GPUs { + if len(g.NVLinkPeers) > 0 { + fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers) + } } } diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go new file mode 100644 index 0000000..36f1372 --- /dev/null +++ b/internal/cli/doctor.go @@ -0,0 +1,180 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/doctor" +) + +func newDoctorCmd() *cobra.Command { + var ( + opts doctor.Options + asJSON bool + verbose bool + ) + cmd := &cobra.Command{ + Use: "doctor", + Short: "Check this node's GPUs, NVLink and InfiniBand for what llmgw needs", + Long: `Reads this node's GPUs, their NVLink and PCIe topology, its InfiniBand and +RoCE ports, the kernel modules RDMA needs and the agent's configuration, and +says what is wrong and how to fix it — most serious first. + +Run it on a node before it joins, and whenever a multi-node replica will not +come up. --json writes everything it read as well as what it concluded; send +that file when the report is not enough to go on.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + doctor.SetConfigLoader(config.LoadAgent) + ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) + defer cancel() + facts := doctor.Collect(ctx, opts) + findings := doctor.Evaluate(facts) + if asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(struct { + Findings []doctor.Finding `json:"findings"` + Facts doctor.Facts `json:"facts"` + }{findings, facts}) + } + printDoctor(os.Stdout, facts, findings, verbose) + for _, f := range findings { + if f.Level == doctor.Fail { + return fmt.Errorf("%d problem(s) need fixing", countLevel(findings, doctor.Fail)) + } + } + return nil + }, + } + f := cmd.Flags() + f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)") + f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory") + f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi") + f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back") + f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology") + return cmd +} + +func countLevel(fs []doctor.Finding, l doctor.Level) int { + n := 0 + for _, f := range fs { + if f.Level == l { + n++ + } + } + return n +} + +func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbose bool) { + fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion, + f.Host.Arch, orDash(f.Host.NVIDIADriver)) + + for _, x := range findings { + fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message) + if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) { + fmt.Fprintf(out, " → %s\n", x.Fix) + } + } + + w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + if len(f.GPUs) > 0 { + fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER") + for _, g := range f.GPUs { + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode, + orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths))) + } + } + if len(f.HCAs) > 0 { + fmt.Fprintln(w, "\nADAPTER\tSTATE\tPHYS\tRATE\tLINK\tNUMA\tROCE GID\tERRORS") + for _, h := range f.HCAs { + for _, p := range h.Ports { + gid := "-" + if p.RoCEv2GID >= 0 { + gid = fmt.Sprint(p.RoCEv2GID) + } + fmt.Fprintf(w, "%s/%d\t%s\t%s\t%d Gb/s\t%s\t%d\t%s\t%s\n", h.Name, p.Num, p.State, + orDash(p.PhysState), p.RateGbps, p.LinkLayer, h.NUMANode, gid, orDash(errorsOf(p.Counters))) + } + } + } + _ = w.Flush() + + for _, fabric := range sortedKeys(f.NCCL) { + fmt.Fprintf(out, "\nA shard on %s is given:\n", fabric) + env := f.NCCL[fabric] + for _, k := range sortedKeys(env) { + fmt.Fprintf(out, " %s=%s\n", k, env[k]) + } + } + if len(f.Engines) > 0 { + fmt.Fprintln(out, "\nEngines:") + for _, k := range sortedKeys(f.Engines) { + fmt.Fprintf(out, " %s: %s\n", k, f.Engines[k]) + } + } + if verbose && f.TopoRaw != "" { + fmt.Fprintf(out, "\nnvidia-smi topo -m:\n%s\n", f.TopoRaw) + } + fmt.Fprintf(out, "\n%d failed, %d warnings. `llmgw doctor --json > report.json` keeps everything read, to send back.\n", + countLevel(findings, doctor.Fail), countLevel(findings, doctor.Warn)) +} + +func nearest(paths map[string]string) string { + best, rel := "", "" + for h, r := range paths { + if best == "" || discovery.PathRank(r) < discovery.PathRank(rel) || + (discovery.PathRank(r) == discovery.PathRank(rel) && h < best) { + best, rel = h, r + } + } + if best == "" { + return "" + } + return best + " (" + rel + ")" +} + +func errorsOf(c map[string]uint64) string { + var out []string + for _, k := range sortedKeys(c) { + if c[k] > 0 && k != "port_xmit_data" && k != "port_rcv_data" { + out = append(out, fmt.Sprintf("%s=%d", k, c[k])) + } + } + return strings.Join(out, " ") +} + +func ints(v []int) string { + s := make([]string, len(v)) + for i, x := range v { + s[i] = fmt.Sprint(x) + } + return strings.Join(s, ",") +} + +func orDash(s string) string { + if s == "" { + return "-" + } + return s +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/cli/root.go b/internal/cli/root.go index a320980..39f895d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -45,6 +45,7 @@ func Root() *cobra.Command { newOrgCmd(), newMirrorCmd(), newSimEngineCmd(), + newDoctorCmd(), ) return root } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go new file mode 100644 index 0000000..5e7ecde --- /dev/null +++ b/internal/doctor/doctor.go @@ -0,0 +1,447 @@ +// Package doctor checks a node for what llmgw needs of its GPUs and fabric, +// and says what is wrong in words an operator can act on. +// +// Written for a machine nobody here can log into. The report is meant to be +// run on the node, read there, and sent back when it is not enough: every +// fact it concluded from is in it, so the conclusion can be checked. +package doctor + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +// Level is how much a finding matters. +type Level string + +// The levels, most serious first. +const ( + Fail Level = "FAIL" + Warn Level = "WARN" + OK Level = "OK" + Info Level = "INFO" +) + +// Finding is one conclusion, and what to do about it. +type Finding struct { + Level Level `json:"level"` + Area string `json:"area"` + Message string `json:"message"` + Fix string `json:"fix,omitempty"` +} + +// Facts are what was read from the node. Every finding is drawn from these +// and nothing else, so a report sent back can be re-read here. +type Facts struct { + At time.Time `json:"at"` + Hostname string `json:"hostname"` + Host discovery.Host `json:"host"` + + SMIFound bool `json:"nvidia_smi_found"` + GPUs []discovery.GPU `json:"gpus"` + GPUError string `json:"gpu_error,omitempty"` + TopoRaw string `json:"topo_raw,omitempty"` + TopoErr string `json:"topo_error,omitempty"` + // NVSwitch is set when the machine has NVSwitch devices, which need the + // fabric manager running before CUDA will start. + NVSwitch bool `json:"nvswitch"` + FabricManager string `json:"fabric_manager,omitempty"` + + HCAs []discovery.HCA `json:"hcas"` + Modules map[string]bool `json:"modules"` + + // ConfigPath is the agent configuration read, empty when there was none. + ConfigPath string `json:"config_path,omitempty"` + ConfigErr string `json:"config_error,omitempty"` + Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"` + MgmtAddr string `json:"mgmt_addr,omitempty"` + MgmtIface string `json:"mgmt_interface,omitempty"` + IfaceExists bool `json:"mgmt_interface_exists"` + LaunchMode string `json:"vllm_launch_mode,omitempty"` + NCCL map[string]map[string]string `json:"nccl_env,omitempty"` + UnitMemlock string `json:"agent_unit_memlock,omitempty"` + Engines map[string]string `json:"engines"` +} + +// Modules checked, and why each matters. +var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"} + +// Options says where to look. +type Options struct { + ConfigPath string + IBRoot string + NvidiaSMI string +} + +// Collect reads the node. +func Collect(ctx context.Context, o Options) Facts { + f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}} + f.Hostname, _ = os.Hostname() + f.Host = discovery.DiscoverHost() + + smi := o.NvidiaSMI + if smi == "" { + if p, err := exec.LookPath("nvidia-smi"); err == nil { + smi = p + } + } + if smi != "" { + f.SMIFound = true + gpus, err := discovery.DiscoverGPUsSMI(ctx, smi) + if err != nil { + f.GPUError = err.Error() + } + out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output() + if err != nil { + f.TopoErr = err.Error() + } else { + f.TopoRaw = string(out) + if t, err := discovery.ParseTopology(f.TopoRaw); err == nil { + t.Apply(gpus) + } else { + f.TopoErr = err.Error() + } + } + f.GPUs = gpus + } + if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 { + f.NVSwitch = true + f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager") + } + + root := o.IBRoot + if root == "" { + root = discovery.SysfsRoot + } + f.HCAs, _ = discovery.DiscoverIB(root) + for _, m := range watchedModules { + _, err := os.Stat(filepath.Join("/sys/module", m)) + f.Modules[m] = err == nil + } + + f.readConfig(o) + f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=") + + for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} { + if name == "sglang" { + continue // no binary of its own to ask + } + if p, err := exec.LookPath(bin); err == nil { + out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput() + f.Engines[name] = firstLine(string(out)) + } + } + return f +} + +func (f *Facts) readConfig(o Options) { + path := o.ConfigPath + if path == "" { + if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil { + return + } + path = "/etc/llmgw/agent.yaml" + } + f.ConfigPath = path + cfg, err := loadAgent(path) + if err != nil { + f.ConfigErr = err.Error() + return + } + declared := map[string][]string{} + for _, fa := range cfg.Fabrics { + declared[fa.Fabric] = fa.HCAs + } + f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) + f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg) + if f.MgmtIface == "" { + f.MgmtIface = "eth0" + } + _, err = net.InterfaceByName(f.MgmtIface) + f.IfaceExists = err == nil + f.LaunchMode = cfg.Engines.VLLM.LaunchMode + if f.LaunchMode == "" { + f.LaunchMode = "ray" + } + + // What a two-node shard on each fabric would be given, the part of an + // engine's environment that decides whether NCCL finds its peers. + f.NCCL = map[string]map[string]string{} + for _, c := range f.Fabrics { + if !c.OK() { + continue + } + spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)} + env := engines.NCCLEnv(spec) + delete(env, "CUDA_VISIBLE_DEVICES") + f.NCCL[c.Fabric] = env + } +} + +// loadAgent is replaced in tests; the real one lives in config, which this +// package cannot import without a cycle through the CLI. +var loadAgent = func(string) (agent.Config, error) { + return agent.Config{}, fmt.Errorf("no agent configuration loader") +} + +// SetConfigLoader gives Collect the configuration loader. +func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn } + +func rocev2(hcas []discovery.HCA, usable []string) *int { + for _, h := range hcas { + if !slices.Contains(usable, h.Name) { + continue + } + for _, p := range h.Ports { + if p.Active() && p.RoCEv2GID >= 0 { + v := p.RoCEv2GID + return &v + } + } + } + return nil +} + +func systemctl(ctx context.Context, args ...string) string { + out, err := exec.CommandContext(ctx, "systemctl", args...).Output() + if err != nil && len(out) == 0 { + return "unknown" + } + return strings.TrimSpace(string(out)) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return s +} + +// Evaluate draws the findings from the facts, most serious first. +func Evaluate(f Facts) []Finding { + var out []Finding + add := func(l Level, area, msg, fix string) { + out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix}) + } + + // --- GPUs --- + switch { + case !f.SMIFound: + add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "") + case f.GPUError != "": + add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand") + case len(f.GPUs) == 0: + add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "") + default: + add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "") + for _, g := range f.GPUs { + if !g.Health.OK() { + add(Warn, "gpu", fmt.Sprintf("GPU %d is kept out of placement: %s", g.Index, strings.Join(g.Health.Problems, "; ")), "") + } + } + } + if f.NVSwitch && f.FabricManager != "active" { + add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+ + ": CUDA will not initialise on its GPUs", + "systemctl enable --now nvidia-fabricmanager (its version must match the driver's)") + } + if len(f.GPUs) > 1 { + pairs, groups := nvlinkGroups(f.GPUs) + switch { + case f.TopoErr != "": + add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+ + "; tensor-parallel groups will be chosen without NVLink", "") + case pairs == 0 && f.NVSwitch: + add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status") + case pairs == 0: + add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "") + default: + add(OK, "nvlink", "NVLink groups: "+groups, "") + } + } + + // --- InfiniBand and RoCE --- + if len(f.HCAs) == 0 { + add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "") + } + declared := map[string]string{} // hca -> fabric + for _, c := range f.Fabrics { + for _, h := range c.Declared { + declared[h] = c.Fabric + } + } + for _, h := range f.HCAs { + for _, p := range h.Ports { + where := fmt.Sprintf("%s/%d", h.Name, p.Num) + level := Warn + if declared[h.Name] != "" { + level = Fail + } + switch { + case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0: + add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang", + "give the interface an IPv4 address, or check the adapter's RoCE mode") + case p.Active(): + add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)", where, p.State, p.RateGbps, p.LinkLayer), "") + if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 { + add(Warn, "fabric", where+" is active but reports no subnet manager LID", "") + } + case strings.EqualFold(p.PhysState, "LinkUp"): + add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it", + "run a subnet manager on the fabric (opensm, or the switch's own)") + default: + add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState), + "check the cable and the switch port") + } + for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} { + if v := p.Counters[name]; v > 0 { + add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v), + "a count that keeps growing is a cable or transceiver going bad") + } + } + } + } + if len(f.HCAs) > 0 { + if !f.Modules["ib_uverbs"] { + add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") + } + if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] { + add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory", + "modprobe nvidia_peermem") + } + } + + // --- the agent's view --- + switch { + case f.ConfigErr != "": + add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "") + case f.ConfigPath == "": + add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config") + default: + for _, c := range f.Fabrics { + if c.OK() { + add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "") + } + for _, p := range c.Problems { + lvl := Warn + if !c.OK() { + lvl = Fail + } + add(lvl, "agent", "fabric "+c.Fabric+": "+p, "") + } + if !c.OK() { + add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "") + } + } + active := activeHCAs(f.HCAs) + if len(f.Fabrics) == 0 && len(active) > 0 { + add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work", + "add fabrics: [{name: , hcas: ["+strings.Join(active, ", ")+"]}]") + } + if !f.IfaceExists { + add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing", + "set mgmt_interface in agent.yaml") + } else { + add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "") + } + if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" { + add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp") + } + } + if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 { + add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers", + "LimitMEMLOCK=infinity in the unit") + } + + sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) }) + return out +} + +func rank(l Level) int { + switch l { + case Fail: + return 0 + case Warn: + return 1 + case OK: + return 2 + } + return 3 +} + +func gpuNames(gpus []discovery.GPU) string { + count := map[string]int{} + var order []string + for _, g := range gpus { + if count[g.Name] == 0 { + order = append(order, g.Name) + } + count[g.Name]++ + } + parts := make([]string, len(order)) + for i, n := range order { + parts[i] = fmt.Sprintf("%d× %s", count[n], n) + } + return strings.Join(parts, ", ") +} + +// nvlinkGroups counts NVLink pairs and names the connected groups. +func nvlinkGroups(gpus []discovery.GPU) (int, string) { + parent := map[int]int{} + var find func(int) int + find = func(x int) int { + if p, ok := parent[x]; ok && p != x { + parent[x] = find(p) + return parent[x] + } + parent[x] = x + return x + } + pairs := 0 + for _, g := range gpus { + find(g.Index) + for _, p := range g.NVLinkPeers { + if p > g.Index { + pairs++ + } + parent[find(g.Index)] = find(p) + } + } + groups := map[int][]int{} + for _, g := range gpus { + r := find(g.Index) + groups[r] = append(groups[r], g.Index) + } + var names []string + for _, members := range groups { + sort.Ints(members) + if len(members) > 1 { + names = append(names, fmt.Sprint(members)) + } + } + sort.Strings(names) + return pairs, strings.Join(names, " ") +} + +func activeHCAs(hcas []discovery.HCA) []string { + var out []string + for _, h := range hcas { + if len(h.ActivePorts()) > 0 { + out = append(out, h.Name) + } + } + return out +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go new file mode 100644 index 0000000..218c572 --- /dev/null +++ b/internal/doctor/doctor_test.go @@ -0,0 +1,121 @@ +package doctor + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// healthy is an eight-GPU NVSwitch node on one InfiniBand fabric, set up +// right. +func healthy() Facts { + f := Facts{SMIFound: true, NVSwitch: true, FabricManager: "active", + Modules: map[string]bool{"ib_uverbs": true, "nvidia_peermem": true}, + ConfigPath: "/etc/llmgw/agent.yaml", MgmtIface: "ens3", MgmtAddr: "10.1.0.5", IfaceExists: true, + LaunchMode: "mp", UnitMemlock: "infinity", Engines: map[string]string{"vllm": "0.11.0"}} + for i := range 8 { + g := discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3"} + for j := range 8 { + if j != i { + g.NVLinkPeers = append(g.NVLinkPeers, j) + } + } + f.GPUs = append(f.GPUs, g) + } + f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", + PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1}}}} + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + return f +} + +func levels(fs []Finding, l Level) []string { + var out []string + for _, f := range fs { + if f.Level == l { + out = append(out, f.Message) + } + } + return out +} + +func has(t *testing.T, fs []Finding, l Level, fragment string) { + t.Helper() + for _, f := range fs { + if f.Level == l && strings.Contains(f.Message, fragment) { + return + } + } + t.Errorf("no %s finding containing %q in %+v", l, fragment, fs) +} + +func TestAHealthyNodeHasNothingToFix(t *testing.T) { + fs := Evaluate(healthy()) + if bad := append(levels(fs, Fail), levels(fs, Warn)...); len(bad) > 0 { + t.Errorf("a healthy node was faulted: %v", bad) + } + has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]") + has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s") + has(t, fs, OK, "NCCL bootstraps over ens3") +} + +func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) { + cases := map[string]struct { + change func(*Facts) + level Level + says string + }{ + "fabric manager down": {func(f *Facts) { f.FabricManager = "inactive" }, Fail, "nvidia-fabricmanager is inactive"}, + "no subnet manager": {func(f *Facts) { + f.HCAs[0].Ports[0].State = "INIT" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "no subnet manager has configured it"}, + "cable out": {func(f *Facts) { + f.HCAs[0].Ports[0].State, f.HCAs[0].Ports[0].PhysState = "DOWN", "Polling" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "no link"}, + "fabric not advertised": {func(f *Facts) { + f.HCAs[0].Ports[0].State = "DOWN" + f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + }, Fail, "will not be advertised"}, + "roce without v2": {func(f *Facts) { + f.HCAs[0].Ports[0].LinkLayer = discovery.LinkEthernet + }, Fail, "no RoCE v2 GID"}, + "no peermem": {func(f *Facts) { f.Modules["nvidia_peermem"] = false }, Warn, "GPUDirect RDMA is off"}, + "no uverbs": {func(f *Facts) { f.Modules["ib_uverbs"] = false }, Fail, "ib_uverbs"}, + "nothing declared": {func(f *Facts) { f.Fabrics = nil }, Warn, "declares no fabric"}, + "wrong interface": {func(f *Facts) { f.IfaceExists = false }, Fail, "does not exist"}, + "errors on a cable": {func(f *Facts) { f.HCAs[0].Ports[0].Counters = map[string]uint64{"symbol_error": 40} }, Warn, "symbol_error = 40"}, + "small memlock": {func(f *Facts) { f.UnitMemlock = "8388608" }, Warn, "limits locked memory"}, + "ray missing": {func(f *Facts) { f.LaunchMode = "ray" }, Fail, "ray is not installed"}, + "no nvlink on nvswitch": {func(f *Facts) { + for i := range f.GPUs { + f.GPUs[i].NVLinkPeers = nil + } + }, Warn, "no GPU reports an NVLink peer"}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + f := healthy() + c.change(&f) + fs := Evaluate(f) + has(t, fs, c.level, c.says) + // Most serious first, so the thing to fix is at the top. + if fs[0].Level != Fail && len(levels(fs, Fail)) > 0 { + t.Error("a failure is not first") + } + }) + } +} + +func TestANodeOutsideTheFabricIsAnAdapterLeftAlone(t *testing.T) { + // An adapter no declared fabric names being down is worth a warning, not + // a failure: nothing here was going to use it. + f := healthy() + f.HCAs = append(f.HCAs, discovery.HCA{Name: "mlx5_4", Ports: []discovery.Port{{Num: 1, State: "DOWN", PhysState: "Disabled", RoCEv2GID: -1}}}) + fs := Evaluate(f) + has(t, fs, Warn, "mlx5_4/1 is DOWN") + if len(levels(fs, Fail)) > 0 { + t.Errorf("failures: %v", levels(fs, Fail)) + } +}