// Package doctor checks a node for what llmgw needs of its GPUs and fabric, // and says what is wrong in words an operator can act on. // // Written for a machine nobody here can log into. The report is meant to be // run on the node, read there, and sent back when it is not enough: every // fact it concluded from is in it, so the conclusion can be checked. package doctor import ( "context" "errors" "fmt" "maps" "net" "os" "os/exec" "path/filepath" "slices" "sort" "strconv" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" ) // Level is how much a finding matters. type Level string // The levels, most serious first. const ( Fail Level = "FAIL" Warn Level = "WARN" OK Level = "OK" Info Level = "INFO" ) // Finding is one conclusion, and what to do about it. type Finding struct { Level Level `json:"level"` Area string `json:"area"` Message string `json:"message"` Fix string `json:"fix,omitempty"` } // Facts are what was read from the node. Every finding is drawn from these // and nothing else, so a report sent back can be re-read here. type Facts struct { At time.Time `json:"at"` Hostname string `json:"hostname"` Host discovery.Host `json:"host"` SMIFound bool `json:"nvidia_smi_found"` GPUs []discovery.GPU `json:"gpus"` GPUError string `json:"gpu_error,omitempty"` TopoRaw string `json:"topo_raw,omitempty"` TopoErr string `json:"topo_error,omitempty"` // MIG is each MIG-capable card's mode and instances; MIGErr why it could // not be read. MIG []discovery.MIGCard `json:"mig,omitempty"` MIGErr string `json:"mig_error,omitempty"` // NVSwitch is set when the machine has NVSwitch devices, which need the // fabric manager running before CUDA will start. NVSwitch bool `json:"nvswitch"` // IMEX is the service multi-node NVLink needs. IMEX discovery.IMEX `json:"imex"` FabricManager string `json:"fabric_manager,omitempty"` HCAs []discovery.HCA `json:"hcas"` // SMErrors say, per port, why the subnet manager could not be asked. SMErrors map[string]string `json:"sm_errors,omitempty"` // SMLists are every subnet manager on each fabric, by master GUID; // SMListErrors why a fabric's could not be listed. SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"` SMListErrors map[string]string `json:"sm_list_errors,omitempty"` Modules map[string]bool `json:"modules"` // ConfigPath is the agent configuration read, empty when there was none. ConfigPath string `json:"config_path,omitempty"` ConfigErr string `json:"config_error,omitempty"` Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"` MgmtAddr string `json:"mgmt_addr,omitempty"` MgmtIface string `json:"mgmt_interface,omitempty"` IfaceExists bool `json:"mgmt_interface_exists"` LaunchMode string `json:"vllm_launch_mode,omitempty"` NCCL map[string]map[string]string `json:"nccl_env,omitempty"` UnitMemlock string `json:"agent_unit_memlock,omitempty"` Engines map[string]string `json:"engines"` // What installing vLLM here would meet: the version this llmgw installs, // where, the space there, and whether it already has. VLLMVersion string `json:"vllm_version,omitempty"` // DisabledEngines are those agent.yaml says this node never runs. DisabledEngines []string `json:"disabled_engines,omitempty"` EnginesDir string `json:"engines_dir,omitempty"` EnginesFreeGB float64 `json:"engines_free_gb,omitempty"` VLLMInstalled bool `json:"vllm_installed"` // Firewall is whether llmgw's nftables table is loaded: "present", // "absent", or "" when it could not be read (not root). Firewall string `json:"firewall,omitempty"` } // Modules checked, and why each matters. var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"} // Options says where to look. type Options struct { ConfigPath string IBRoot string NvidiaSMI string // VLLMVersion is the vLLM this llmgw installs; its prerequisites are // checked when set. VLLMVersion string // Fabrics are fabrics about to be declared, by name, with their // adapters: what `node add --fabric` will write into agent.yaml, which // does not exist yet when doctor runs before it. Fabrics map[string][]string } // Collect reads the node. func Collect(ctx context.Context, o Options) Facts { f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}} f.Hostname, _ = os.Hostname() f.Host = discovery.DiscoverHost() smi := o.NvidiaSMI if smi == "" { if p, err := exec.LookPath("nvidia-smi"); err == nil { smi = p } } if smi != "" { f.SMIFound = true gpus, err := discovery.DiscoverGPUsSMI(ctx, smi) if err != nil { f.GPUError = err.Error() } out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output() if err != nil { f.TopoErr = err.Error() } else { f.TopoRaw = string(out) if t, err := discovery.ParseTopology(f.TopoRaw); err == nil { t.Apply(gpus) } else { f.TopoErr = err.Error() } } if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil { discovery.ApplyFabric(gpus, fab) } f.GPUs = gpus f.IMEX = discovery.DiscoverIMEX(ctx) if slices.ContainsFunc(gpus, func(g discovery.GPU) bool { cc, _ := strconv.ParseFloat(g.ComputeCapability, 64) return g.IsMIG() || cc >= 8 }) { if mig, err := discovery.DiscoverMIG(ctx, smi); err != nil { f.MIGErr = err.Error() } else { f.MIG = mig } } } if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 { f.NVSwitch = true f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager") } root := o.IBRoot if root == "" { root = discovery.SysfsRoot } f.HCAs, _ = discovery.DiscoverIB(root) f.SMErrors = map[string]string{} _ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) { sm, err := discovery.QuerySM(ctx, hca, port) if err != nil { f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error() } return sm, err }) f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{} for _, h := range f.HCAs { for _, p := range h.Ports { if p.SM == nil { continue } if _, done := f.SMLists[p.SM.GUID]; done { continue } if _, done := f.SMListErrors[p.SM.GUID]; done { continue } if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil { f.SMListErrors[p.SM.GUID] = err.Error() } else { f.SMLists[p.SM.GUID] = recs } } } for _, m := range watchedModules { _, err := os.Stat(filepath.Join("/sys/module", m)) f.Modules[m] = err == nil } f.readConfig(o) if len(o.Fabrics) > 0 { declared := map[string][]string{} for _, c := range f.Fabrics { declared[c.Fabric] = c.Declared } maps.Copy(declared, o.Fabrics) f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) } f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs) if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" { if f.EnginesDir == "" { f.EnginesDir = provision.DefaultRoot } // The directory may not exist yet: the space is the filesystem's. for dir := f.EnginesDir; dir != "/" && dir != "."; dir = filepath.Dir(dir) { if gb, err := provision.FreeGB(dir); err == nil { f.EnginesFreeGB = gb break } } matches, _ := filepath.Glob(filepath.Join(f.EnginesDir, "vllm", f.VLLMVersion+"-*", ".llmgw-installed")) f.VLLMInstalled = len(matches) > 0 } f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=") for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} { if name == "sglang" { continue // no binary of its own to ask } if p, err := exec.LookPath(bin); err == nil { out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput() f.Engines[name] = firstLine(string(out)) } } f.Firewall = firewallState(ctx) return f } // firewallState reads whether llmgw's table is loaded. func firewallState(ctx context.Context) string { out, err := exec.CommandContext(ctx, "nft", "list", "table", "inet", "llmgw").CombinedOutput() switch { case err == nil: return "present" case strings.Contains(string(out), "No such file or directory"): return "absent" } return "" // no nft, or not root: unknown } func (f *Facts) readConfig(o Options) { path := o.ConfigPath if path == "" { if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil { return } path = "/etc/llmgw/agent.yaml" } f.ConfigPath = path cfg, err := loadAgent(path) if err != nil { f.ConfigErr = err.Error() return } declared := map[string][]string{} for _, fa := range cfg.Fabrics { declared[fa.Fabric] = fa.HCAs } f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg) if f.MgmtIface == "" { f.MgmtIface = "eth0" } _, err = net.InterfaceByName(f.MgmtIface) f.IfaceExists = err == nil f.EnginesDir = cfg.EnginesDir f.DisabledEngines = cfg.Engines.Disabled f.LaunchMode = cfg.Engines.VLLM.LaunchMode if f.LaunchMode == "" { f.LaunchMode = "mp" } // What a two-node shard on each fabric would be given, the part of an // engine's environment that decides whether NCCL finds its peers. f.NCCL = map[string]map[string]string{} for _, c := range f.Fabrics { if !c.OK() { continue } spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)} env := engines.NCCLEnv(spec) delete(env, "CUDA_VISIBLE_DEVICES") f.NCCL[c.Fabric] = env } } // loadAgent is replaced in tests; the real one lives in config, which this // package cannot import without a cycle through the CLI. var loadAgent = func(string) (agent.Config, error) { return agent.Config{}, fmt.Errorf("no agent configuration loader") } // SetConfigLoader gives Collect the configuration loader. func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn } func rocev2(hcas []discovery.HCA, usable []string) *int { for _, h := range hcas { if !slices.Contains(usable, h.Name) { continue } for _, p := range h.Ports { if p.Active() && p.RoCEv2GID >= 0 { v := p.RoCEv2GID return &v } } } return nil } func systemctl(ctx context.Context, args ...string) string { out, err := exec.CommandContext(ctx, "systemctl", args...).Output() if err != nil && len(out) == 0 { return "unknown" } return strings.TrimSpace(string(out)) } func firstLine(s string) string { s = strings.TrimSpace(s) if i := strings.IndexByte(s, '\n'); i >= 0 { s = s[:i] } return s } // Evaluate draws the findings from the facts, most serious first. func Evaluate(f Facts) []Finding { var out []Finding add := func(l Level, area, msg, fix string) { out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix}) } // --- CPU --- // Read from /proc/cpuinfo, so only on a node whose OS was read too; // elsewhere an empty list means unknown, not missing. if f.Host.Arch == "amd64" && f.Host.OSID != "" { if f.Host.HasCPUFlag("avx2") { add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "") } else { has := "none of AVX, AVX2 or FMA" if len(f.Host.CPUFlags) > 0 { has = "only " + strings.Join(f.Host.CPUFlags, ", ") } // Nothing else looks wrong on such a node: it registers, answers // correctly, and is several times slower than the hardware. add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it", "on a VM, give it the host's CPU type (Proxmox: qm set --cpu host, then restart the VM)") } } // --- GPUs --- switch { case !f.SMIFound: add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "") case f.GPUError != "": add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand") case len(f.GPUs) == 0: add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "") default: add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "") for _, g := range f.GPUs { if !g.Health.OK() { add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s", discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "") } } } evaluateMIG(f, add) if f.NVSwitch && f.FabricManager != "active" { add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+ ": CUDA will not initialise on its GPUs", "systemctl enable --now nvidia-fabricmanager (its version must match the driver's)") } // NVLink is between whole cards: a MIG instance has none to report, and // counting instances would call a node of MIG cards one without NVLink. cards := slices.DeleteFunc(slices.Clone(f.GPUs), discovery.GPU.IsMIG) if len(cards) > 1 { pairs, groups := nvlinkGroups(cards) switch { case f.TopoErr != "": add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+ "; tensor-parallel groups will be chosen without NVLink", "") case pairs == 0 && f.NVSwitch: add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status") case pairs == 0: add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "") default: add(OK, "nvlink", "NVLink groups: "+groups, "") } } // --- multi-node NVLink --- domain := "" for _, g := range cards { fb := g.Fabric if fb == nil || strings.EqualFold(fb.State, "Not Supported") { continue } if d := fb.Domain(); d != "" { domain = d if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") { add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)), "check its NVLink cables and trays: nvidia-smi -q -d FABRIC") } continue } if !strings.EqualFold(fb.State, "Completed") { why := fb.State if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") { why += ": " + fb.Status } add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why), "the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays") } } if domain != "" { add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "") switch { case f.IMEX.State != "active": state := f.IMEX.State if state == "" { state = "not installed" } add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes", "systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg") case f.IMEX.Channels == 0: add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open", "load the driver with options nvidia NVreg_CreateImexChannel0=1") default: add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "") } } // --- InfiniBand and RoCE --- if len(f.HCAs) == 0 { add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "") } declared := map[string]string{} // hca -> fabric for _, c := range f.Fabrics { for _, h := range c.Declared { declared[h] = c.Fabric } } for _, h := range f.HCAs { for _, p := range h.Ports { where := fmt.Sprintf("%s/%d", h.Name, p.Num) level := Warn if declared[h.Name] != "" { level = Fail } switch { case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0: // Fatal only for a port a declared fabric uses. An adapter // nothing is configured to use — a RoCE NIC on a node added // for single-node inference — kept nodes out for a fabric // nobody asked for. msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang" if declared[h.Name] == "" { msg += "; no declared fabric uses it, so nothing will until one does" } add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode") case p.Active(): on := "" switch { case p.SM != nil: on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State) case p.Network != "": on = ", network " + p.Network } add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "") // Questions about which fabric a port is on matter to the // fabrics that use it; for a port none does they are noted, // not warned of. quiet := Warn if declared[h.Name] == "" { quiet = Info } if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil { why := f.SMErrors[where] fix := "" if strings.Contains(why, "infiniband-diags") { fix = "install infiniband-diags" } add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+ "; without it, separate fabrics given one name cannot be caught", fix) } if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 { add(quiet, "fabric", where+" is active but reports no subnet manager LID", "") } case strings.EqualFold(p.PhysState, "LinkUp"): add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it", "run a subnet manager on the fabric (opensm, or the switch's own)") default: add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState), "check the cable and the switch port") } for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} { if v := p.Counters[name]; v > 0 { add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v), "a count that keeps growing is a cable or transceiver going bad") } } } } for _, master := range sortedKeys(f.SMLists) { recs := f.SMLists[master] var parts []string standby := 0 for _, r := range recs { parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority)) if r.State == "standby" { standby++ } } if standby == 0 { add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "), "run a second subnet manager, on a switch or a host, at a lower priority") } else { add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "") } } for _, master := range sortedKeys(f.SMListErrors) { add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "") } if len(f.HCAs) > 0 { if !f.Modules["ib_uverbs"] { // RDMA is for replicas spanning nodes; a node that declares no // fabric runs none, and is a node without it. if len(declared) > 0 { add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") } else { add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+ "later would need; this node runs single-node replicas without it", "modprobe ib_uverbs") } } if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] { add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory", "modprobe nvidia_peermem") } } // --- the agent's view --- switch { case f.ConfigErr != "": add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "") case f.ConfigPath == "": add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config") default: for _, c := range f.Fabrics { if c.OK() { add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "") } for _, p := range c.Problems { lvl := Warn if !c.OK() { lvl = Fail } add(lvl, "agent", "fabric "+c.Fabric+": "+p, "") } if !c.OK() { add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "") } } active := activeHCAs(f.HCAs) if len(f.Fabrics) == 0 && len(active) > 0 { add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work", "add fabrics: [{name: , hcas: ["+strings.Join(active, ", ")+"]}]") } if !f.IfaceExists { add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing", "set mgmt_interface in agent.yaml") } else { add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "") } if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" { add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp") } } // --- engines --- // An engine disabled on this node: whether it could be installed is not // a question anybody is asking. if f.VLLMVersion != "" && !slices.Contains(f.DisabledEngines, "vllm") { vllmFindings(f, add) } if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 { add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers", "LimitMEMLOCK=infinity in the unit") } // --- firewall --- // Only on a node already set up: before that, onboarding writes it. if f.ConfigPath != "" { switch f.Firewall { case "present": add(OK, "firewall", "llmgw's ports answer the cluster's networks only", "") case "absent": add(Warn, "firewall", "no llmgw firewall: the engines' ports answer every network this machine is on", "run node add for it again, or list extra networks in firewall_networks first") } } sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) }) return out } // vllmFindings says whether the agent can install vLLM here, and if not why: // the same decision the agent makes, so the report and the node agree. func vllmFindings(f Facts, add func(Level, string, string, string)) { v := f.VLLMVersion builds, err := provision.Recipes["vllm"].Candidates(v, "", f.Host) var b provision.Build if err == nil { // The first the agent would try; it moves to the next by itself if // that one does not work here. b = builds[0] if b.Version != "" { v = b.Version } } var nh provision.NotHere switch { case errors.As(err, &nh): add(Info, "engines", "vLLM will not be installed: "+nh.Reason, "") return case err != nil: // A warning, not a failure: the node is still a node, and serves // llama.cpp. Refusing it here kept older GPU generations out of a // cluster for an engine nobody meant to run on them. add(Warn, "engines", "vLLM "+v+" will not run here, and the node joins without it: "+err.Error(), "add it with --disable-engine vllm if it is meant for llama.cpp alone") return case f.VLLMInstalled: add(OK, "engines", "vLLM "+v+" is installed under "+f.EnginesDir, "") return } if f.EnginesFreeGB > 0 && f.EnginesFreeGB < float64(b.FreeGB) { add(Warn, "engines", fmt.Sprintf("vLLM %s needs about %d GB free under %s and there are %.1f GB; the node joins without it until there is", v, b.FreeGB, f.EnginesDir, f.EnginesFreeGB), "free space there, or point engines_dir in agent.yaml at a larger disk") return } note := "" if b.Note != "" { note = "; " + b.Note } add(OK, "engines", fmt.Sprintf("vLLM %s (%s) can be installed here: driver %s, %.1f GB free under %s%s", v, b.Variant, f.Host.NVIDIADriver, f.EnginesFreeGB, f.EnginesDir, note), "") } func rank(l Level) int { switch l { case Fail: return 0 case Warn: return 1 case OK: return 2 } return 3 } // evaluateMIG reports each MIG-capable card's mode and instances. Quiet for a // card with MIG off, which is every card on most machines; what it warns of // are the states in which a card is in MIG mode and nothing can run on it. func evaluateMIG(f Facts, add func(Level, string, string, string)) { if f.MIGErr != "" { add(Warn, "mig", "could not read the cards' MIG state: "+f.MIGErr, "run nvidia-smi -q by hand") return } for _, c := range f.MIG { card := fmt.Sprintf("GPU %d (%s)", c.Index, c.Name) if c.Enabled != c.PendingEnabled { want := "off" if c.PendingEnabled { want = "on" } add(Warn, "mig", fmt.Sprintf("%s has MIG turning %s at its next reset, and until then runs as before", card, want), fmt.Sprintf("reset the card when nothing runs on it: nvidia-smi -i %d -r (or reboot)", c.Index)) } if !c.Enabled { continue } // A GPU instance with no compute instance is carved but not usable: // it is not a device until one is made inside it. var bare []string for _, gi := range c.GPUInstances { has := slices.ContainsFunc(c.ComputeInstances, func(ci discovery.MIGInstance) bool { return ci.GPUInstance == gi.GPUInstance }) if !has { bare = append(bare, fmt.Sprintf("%d (%s)", gi.GPUInstance, gi.Profile)) } } switch { case len(c.Instances) == 0 && len(c.GPUInstances) == 0: add(Warn, "mig", card+" is in MIG mode with no instances created: nothing can run on it", fmt.Sprintf("create instances, for example nvidia-smi mig -i %d -cgi 3g.40gb,3g.40gb -C "+ "(nvidia-smi mig -i %d -lgip lists the profiles), or turn MIG off: nvidia-smi -i %d -mig 0", c.Index, c.Index, c.Index)) case len(c.Instances) == 0: add(Warn, "mig", fmt.Sprintf("%s is in MIG mode with GPU instances %s but no compute instances: nothing can run on it", card, strings.Join(bare, ", ")), fmt.Sprintf("create them: nvidia-smi mig -i %d -cci", c.Index)) default: var names []string for _, in := range c.Instances { names = append(names, fmt.Sprintf("%d: %s %s", in.Device, in.Profile, in.UUID)) } add(OK, "mig", fmt.Sprintf("%s is in MIG mode with %d instance(s), each a device of its own: %s", card, len(c.Instances), strings.Join(names, "; ")), "") if len(bare) > 0 { add(Warn, "mig", fmt.Sprintf("%s has GPU instances %s with no compute instance, which are not devices yet", card, strings.Join(bare, ", ")), fmt.Sprintf("create them: nvidia-smi mig -i %d -cci, or destroy the GPU instances", c.Index)) } } } } func gpuNames(gpus []discovery.GPU) string { count := map[string]int{} var order []string for _, g := range gpus { if count[g.Name] == 0 { order = append(order, g.Name) } count[g.Name]++ } parts := make([]string, len(order)) for i, n := range order { parts[i] = fmt.Sprintf("%d× %s", count[n], n) } return strings.Join(parts, ", ") } // nvlinkGroups counts NVLink pairs and names the connected groups. func nvlinkGroups(gpus []discovery.GPU) (int, string) { parent := map[int]int{} var find func(int) int find = func(x int) int { if p, ok := parent[x]; ok && p != x { parent[x] = find(p) return parent[x] } parent[x] = x return x } pairs := 0 for _, g := range gpus { find(g.Index) for _, p := range g.NVLinkPeers { if p > g.Index { pairs++ } parent[find(g.Index)] = find(p) } } groups := map[int][]int{} for _, g := range gpus { r := find(g.Index) groups[r] = append(groups[r], g.Index) } var names []string for _, members := range groups { sort.Ints(members) if len(members) > 1 { names = append(names, fmt.Sprint(members)) } } sort.Strings(names) return pairs, strings.Join(names, " ") } func activeHCAs(hcas []discovery.HCA) []string { var out []string for _, h := range hcas { if len(h.ActivePorts()) > 0 { out = append(out, h.Name) } } return out } func orUnknown(s string) string { if s == "" { return "no answer from the subnet manager" } return s } func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // MultiNode is whether a finding is about the fabrics replicas spanning // nodes use, rather than about the node itself. func (f Finding) MultiNode() bool { return f.Area == "fabric" } // Readiness is a node's two readinesses, kept apart: whether it can join and // serve replicas of its own, and whether it can also take part in replicas // spanning nodes over a fabric. The second never keeps a node out unless a // fabric was declared for it, whose problems are then the node's. type Readiness struct { // Node is "ready", or how many problems keep it out. Node string `json:"node"` // MultiNode is "ready", "not used" (no fabric adapter), or "not ready:" // and the first reason. MultiNode string `json:"multi_node"` } // ReadinessOf sums findings up as the two readinesses. func ReadinessOf(fs []Finding) Readiness { r := Readiness{Node: "ready", MultiNode: "not used"} fails := 0 for _, f := range fs { if f.Level == Fail { fails++ } } if fails > 0 { r.Node = fmt.Sprintf("%d problem(s) keep it out", fails) } for _, f := range fs { if !f.MultiNode() { continue } if r.MultiNode == "not used" { r.MultiNode = "ready" } if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") { r.MultiNode = "not ready: " + f.Message } } return r }