doctor sums findings up as two readinesses - the node's, which decides whether it joins, and multi-node, its fabrics for replicas spanning nodes - prints fabric findings in a section of their own, and gives both in --json. node add says the multi-node one. NodeView.multi_node says whether a node's declared fabrics are up, naming adapters that are down, and ctl nodes shows it beside the fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
864 lines
29 KiB
Go
864 lines
29 KiB
Go
// Package doctor checks a node for what llmgw needs of its GPUs and fabric,
|
||
// and says what is wrong in words an operator can act on.
|
||
//
|
||
// Written for a machine nobody here can log into. The report is meant to be
|
||
// run on the node, read there, and sent back when it is not enough: every
|
||
// fact it concluded from is in it, so the conclusion can be checked.
|
||
package doctor
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"maps"
|
||
"net"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"slices"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||
)
|
||
|
||
// Level is how much a finding matters.
|
||
type Level string
|
||
|
||
// The levels, most serious first.
|
||
const (
|
||
Fail Level = "FAIL"
|
||
Warn Level = "WARN"
|
||
OK Level = "OK"
|
||
Info Level = "INFO"
|
||
)
|
||
|
||
// Finding is one conclusion, and what to do about it.
|
||
type Finding struct {
|
||
Level Level `json:"level"`
|
||
Area string `json:"area"`
|
||
Message string `json:"message"`
|
||
Fix string `json:"fix,omitempty"`
|
||
}
|
||
|
||
// Facts are what was read from the node. Every finding is drawn from these
|
||
// and nothing else, so a report sent back can be re-read here.
|
||
type Facts struct {
|
||
At time.Time `json:"at"`
|
||
Hostname string `json:"hostname"`
|
||
Host discovery.Host `json:"host"`
|
||
|
||
SMIFound bool `json:"nvidia_smi_found"`
|
||
GPUs []discovery.GPU `json:"gpus"`
|
||
GPUError string `json:"gpu_error,omitempty"`
|
||
TopoRaw string `json:"topo_raw,omitempty"`
|
||
TopoErr string `json:"topo_error,omitempty"`
|
||
// MIG is each MIG-capable card's mode and instances; MIGErr why it could
|
||
// not be read.
|
||
MIG []discovery.MIGCard `json:"mig,omitempty"`
|
||
MIGErr string `json:"mig_error,omitempty"`
|
||
// NVSwitch is set when the machine has NVSwitch devices, which need the
|
||
// fabric manager running before CUDA will start.
|
||
NVSwitch bool `json:"nvswitch"`
|
||
// IMEX is the service multi-node NVLink needs.
|
||
IMEX discovery.IMEX `json:"imex"`
|
||
FabricManager string `json:"fabric_manager,omitempty"`
|
||
|
||
HCAs []discovery.HCA `json:"hcas"`
|
||
// SMErrors say, per port, why the subnet manager could not be asked.
|
||
SMErrors map[string]string `json:"sm_errors,omitempty"`
|
||
// SMLists are every subnet manager on each fabric, by master GUID;
|
||
// SMListErrors why a fabric's could not be listed.
|
||
SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"`
|
||
SMListErrors map[string]string `json:"sm_list_errors,omitempty"`
|
||
Modules map[string]bool `json:"modules"`
|
||
|
||
// ConfigPath is the agent configuration read, empty when there was none.
|
||
ConfigPath string `json:"config_path,omitempty"`
|
||
ConfigErr string `json:"config_error,omitempty"`
|
||
Fabrics []discovery.FabricCheck `json:"fabrics,omitempty"`
|
||
MgmtAddr string `json:"mgmt_addr,omitempty"`
|
||
MgmtIface string `json:"mgmt_interface,omitempty"`
|
||
IfaceExists bool `json:"mgmt_interface_exists"`
|
||
LaunchMode string `json:"vllm_launch_mode,omitempty"`
|
||
NCCL map[string]map[string]string `json:"nccl_env,omitempty"`
|
||
UnitMemlock string `json:"agent_unit_memlock,omitempty"`
|
||
Engines map[string]string `json:"engines"`
|
||
|
||
// What installing vLLM here would meet: the version this llmgw installs,
|
||
// where, the space there, and whether it already has.
|
||
VLLMVersion string `json:"vllm_version,omitempty"`
|
||
// DisabledEngines are those agent.yaml says this node never runs.
|
||
DisabledEngines []string `json:"disabled_engines,omitempty"`
|
||
EnginesDir string `json:"engines_dir,omitempty"`
|
||
EnginesFreeGB float64 `json:"engines_free_gb,omitempty"`
|
||
VLLMInstalled bool `json:"vllm_installed"`
|
||
|
||
// Firewall is whether llmgw's nftables table is loaded: "present",
|
||
// "absent", or "" when it could not be read (not root).
|
||
Firewall string `json:"firewall,omitempty"`
|
||
}
|
||
|
||
// Modules checked, and why each matters.
|
||
var watchedModules = []string{"ib_core", "ib_uverbs", "mlx5_ib", "rdma_ucm", "nvidia_peermem", "nv_peer_mem"}
|
||
|
||
// Options says where to look.
|
||
type Options struct {
|
||
ConfigPath string
|
||
IBRoot string
|
||
NvidiaSMI string
|
||
// VLLMVersion is the vLLM this llmgw installs; its prerequisites are
|
||
// checked when set.
|
||
VLLMVersion string
|
||
// Fabrics are fabrics about to be declared, by name, with their
|
||
// adapters: what `node add --fabric` will write into agent.yaml, which
|
||
// does not exist yet when doctor runs before it.
|
||
Fabrics map[string][]string
|
||
}
|
||
|
||
// Collect reads the node.
|
||
func Collect(ctx context.Context, o Options) Facts {
|
||
f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}}
|
||
f.Hostname, _ = os.Hostname()
|
||
f.Host = discovery.DiscoverHost()
|
||
|
||
smi := o.NvidiaSMI
|
||
if smi == "" {
|
||
if p, err := exec.LookPath("nvidia-smi"); err == nil {
|
||
smi = p
|
||
}
|
||
}
|
||
if smi != "" {
|
||
f.SMIFound = true
|
||
gpus, err := discovery.DiscoverGPUsSMI(ctx, smi)
|
||
if err != nil {
|
||
f.GPUError = err.Error()
|
||
}
|
||
out, err := exec.CommandContext(ctx, smi, "topo", "-m").Output()
|
||
if err != nil {
|
||
f.TopoErr = err.Error()
|
||
} else {
|
||
f.TopoRaw = string(out)
|
||
if t, err := discovery.ParseTopology(f.TopoRaw); err == nil {
|
||
t.Apply(gpus)
|
||
} else {
|
||
f.TopoErr = err.Error()
|
||
}
|
||
}
|
||
if fab, err := discovery.DiscoverGPUFabric(ctx, smi); err == nil {
|
||
discovery.ApplyFabric(gpus, fab)
|
||
}
|
||
f.GPUs = gpus
|
||
f.IMEX = discovery.DiscoverIMEX(ctx)
|
||
if slices.ContainsFunc(gpus, func(g discovery.GPU) bool {
|
||
cc, _ := strconv.ParseFloat(g.ComputeCapability, 64)
|
||
return g.IsMIG() || cc >= 8
|
||
}) {
|
||
if mig, err := discovery.DiscoverMIG(ctx, smi); err != nil {
|
||
f.MIGErr = err.Error()
|
||
} else {
|
||
f.MIG = mig
|
||
}
|
||
}
|
||
}
|
||
if m, _ := filepath.Glob("/dev/nvidia-nvswitch*"); len(m) > 0 {
|
||
f.NVSwitch = true
|
||
f.FabricManager = systemctl(ctx, "is-active", "nvidia-fabricmanager")
|
||
}
|
||
|
||
root := o.IBRoot
|
||
if root == "" {
|
||
root = discovery.SysfsRoot
|
||
}
|
||
f.HCAs, _ = discovery.DiscoverIB(root)
|
||
f.SMErrors = map[string]string{}
|
||
_ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) {
|
||
sm, err := discovery.QuerySM(ctx, hca, port)
|
||
if err != nil {
|
||
f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error()
|
||
}
|
||
return sm, err
|
||
})
|
||
f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{}
|
||
for _, h := range f.HCAs {
|
||
for _, p := range h.Ports {
|
||
if p.SM == nil {
|
||
continue
|
||
}
|
||
if _, done := f.SMLists[p.SM.GUID]; done {
|
||
continue
|
||
}
|
||
if _, done := f.SMListErrors[p.SM.GUID]; done {
|
||
continue
|
||
}
|
||
if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil {
|
||
f.SMListErrors[p.SM.GUID] = err.Error()
|
||
} else {
|
||
f.SMLists[p.SM.GUID] = recs
|
||
}
|
||
}
|
||
}
|
||
for _, m := range watchedModules {
|
||
_, err := os.Stat(filepath.Join("/sys/module", m))
|
||
f.Modules[m] = err == nil
|
||
}
|
||
|
||
f.readConfig(o)
|
||
if len(o.Fabrics) > 0 {
|
||
declared := map[string][]string{}
|
||
for _, c := range f.Fabrics {
|
||
declared[c.Fabric] = c.Declared
|
||
}
|
||
maps.Copy(declared, o.Fabrics)
|
||
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
|
||
}
|
||
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
|
||
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
|
||
if f.EnginesDir == "" {
|
||
f.EnginesDir = provision.DefaultRoot
|
||
}
|
||
// The directory may not exist yet: the space is the filesystem's.
|
||
for dir := f.EnginesDir; dir != "/" && dir != "."; dir = filepath.Dir(dir) {
|
||
if gb, err := provision.FreeGB(dir); err == nil {
|
||
f.EnginesFreeGB = gb
|
||
break
|
||
}
|
||
}
|
||
matches, _ := filepath.Glob(filepath.Join(f.EnginesDir, "vllm", f.VLLMVersion+"-*", ".llmgw-installed"))
|
||
f.VLLMInstalled = len(matches) > 0
|
||
}
|
||
f.UnitMemlock = strings.TrimPrefix(systemctl(ctx, "show", "llmgw-agent", "-p", "LimitMEMLOCK"), "LimitMEMLOCK=")
|
||
|
||
for name, bin := range map[string]string{"vllm": "vllm", "ray": "ray", "sglang": "python3"} {
|
||
if name == "sglang" {
|
||
continue // no binary of its own to ask
|
||
}
|
||
if p, err := exec.LookPath(bin); err == nil {
|
||
out, _ := exec.CommandContext(ctx, p, "--version").CombinedOutput()
|
||
f.Engines[name] = firstLine(string(out))
|
||
}
|
||
}
|
||
f.Firewall = firewallState(ctx)
|
||
return f
|
||
}
|
||
|
||
// firewallState reads whether llmgw's table is loaded.
|
||
func firewallState(ctx context.Context) string {
|
||
out, err := exec.CommandContext(ctx, "nft", "list", "table", "inet", "llmgw").CombinedOutput()
|
||
switch {
|
||
case err == nil:
|
||
return "present"
|
||
case strings.Contains(string(out), "No such file or directory"):
|
||
return "absent"
|
||
}
|
||
return "" // no nft, or not root: unknown
|
||
}
|
||
|
||
func (f *Facts) readConfig(o Options) {
|
||
path := o.ConfigPath
|
||
if path == "" {
|
||
if _, err := os.Stat("/etc/llmgw/agent.yaml"); err != nil {
|
||
return
|
||
}
|
||
path = "/etc/llmgw/agent.yaml"
|
||
}
|
||
f.ConfigPath = path
|
||
cfg, err := loadAgent(path)
|
||
if err != nil {
|
||
f.ConfigErr = err.Error()
|
||
return
|
||
}
|
||
declared := map[string][]string{}
|
||
for _, fa := range cfg.Fabrics {
|
||
declared[fa.Fabric] = fa.HCAs
|
||
}
|
||
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
|
||
f.MgmtAddr, f.MgmtIface = agent.ResolveManagement(cfg)
|
||
if f.MgmtIface == "" {
|
||
f.MgmtIface = "eth0"
|
||
}
|
||
_, err = net.InterfaceByName(f.MgmtIface)
|
||
f.IfaceExists = err == nil
|
||
f.EnginesDir = cfg.EnginesDir
|
||
f.DisabledEngines = cfg.Engines.Disabled
|
||
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
|
||
if f.LaunchMode == "" {
|
||
f.LaunchMode = "mp"
|
||
}
|
||
|
||
// What a two-node shard on each fabric would be given, the part of an
|
||
// engine's environment that decides whether NCCL finds its peers.
|
||
f.NCCL = map[string]map[string]string{}
|
||
for _, c := range f.Fabrics {
|
||
if !c.OK() {
|
||
continue
|
||
}
|
||
spec := engines.ShardSpec{NNodes: 2, HCAs: c.Usable, MgmtInterface: f.MgmtIface, GIDIndex: rocev2(f.HCAs, c.Usable)}
|
||
env := engines.NCCLEnv(spec)
|
||
delete(env, "CUDA_VISIBLE_DEVICES")
|
||
f.NCCL[c.Fabric] = env
|
||
}
|
||
}
|
||
|
||
// loadAgent is replaced in tests; the real one lives in config, which this
|
||
// package cannot import without a cycle through the CLI.
|
||
var loadAgent = func(string) (agent.Config, error) {
|
||
return agent.Config{}, fmt.Errorf("no agent configuration loader")
|
||
}
|
||
|
||
// SetConfigLoader gives Collect the configuration loader.
|
||
func SetConfigLoader(fn func(string) (agent.Config, error)) { loadAgent = fn }
|
||
|
||
func rocev2(hcas []discovery.HCA, usable []string) *int {
|
||
for _, h := range hcas {
|
||
if !slices.Contains(usable, h.Name) {
|
||
continue
|
||
}
|
||
for _, p := range h.Ports {
|
||
if p.Active() && p.RoCEv2GID >= 0 {
|
||
v := p.RoCEv2GID
|
||
return &v
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func systemctl(ctx context.Context, args ...string) string {
|
||
out, err := exec.CommandContext(ctx, "systemctl", args...).Output()
|
||
if err != nil && len(out) == 0 {
|
||
return "unknown"
|
||
}
|
||
return strings.TrimSpace(string(out))
|
||
}
|
||
|
||
func firstLine(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||
s = s[:i]
|
||
}
|
||
return s
|
||
}
|
||
|
||
// Evaluate draws the findings from the facts, most serious first.
|
||
func Evaluate(f Facts) []Finding {
|
||
var out []Finding
|
||
add := func(l Level, area, msg, fix string) {
|
||
out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix})
|
||
}
|
||
|
||
// --- CPU ---
|
||
// Read from /proc/cpuinfo, so only on a node whose OS was read too;
|
||
// elsewhere an empty list means unknown, not missing.
|
||
if f.Host.Arch == "amd64" && f.Host.OSID != "" {
|
||
if f.Host.HasCPUFlag("avx2") {
|
||
add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "")
|
||
} else {
|
||
has := "none of AVX, AVX2 or FMA"
|
||
if len(f.Host.CPUFlags) > 0 {
|
||
has = "only " + strings.Join(f.Host.CPUFlags, ", ")
|
||
}
|
||
// Nothing else looks wrong on such a node: it registers, answers
|
||
// correctly, and is several times slower than the hardware.
|
||
add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it",
|
||
"on a VM, give it the host's CPU type (Proxmox: qm set <vmid> --cpu host, then restart the VM)")
|
||
}
|
||
}
|
||
|
||
// --- GPUs ---
|
||
switch {
|
||
case !f.SMIFound:
|
||
add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "")
|
||
case f.GPUError != "":
|
||
add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand")
|
||
case len(f.GPUs) == 0:
|
||
add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "")
|
||
default:
|
||
add(OK, "gpu", fmt.Sprintf("%d GPUs: %s", len(f.GPUs), gpuNames(f.GPUs)), "")
|
||
for _, g := range f.GPUs {
|
||
if !g.Health.OK() {
|
||
add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s",
|
||
discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "")
|
||
}
|
||
}
|
||
}
|
||
evaluateMIG(f, add)
|
||
if f.NVSwitch && f.FabricManager != "active" {
|
||
add(Fail, "nvlink", "this machine has NVSwitch but nvidia-fabricmanager is "+f.FabricManager+
|
||
": CUDA will not initialise on its GPUs",
|
||
"systemctl enable --now nvidia-fabricmanager (its version must match the driver's)")
|
||
}
|
||
// NVLink is between whole cards: a MIG instance has none to report, and
|
||
// counting instances would call a node of MIG cards one without NVLink.
|
||
cards := slices.DeleteFunc(slices.Clone(f.GPUs), discovery.GPU.IsMIG)
|
||
if len(cards) > 1 {
|
||
pairs, groups := nvlinkGroups(cards)
|
||
switch {
|
||
case f.TopoErr != "":
|
||
add(Warn, "nvlink", "could not read nvidia-smi topo -m: "+f.TopoErr+
|
||
"; tensor-parallel groups will be chosen without NVLink", "")
|
||
case pairs == 0 && f.NVSwitch:
|
||
add(Warn, "nvlink", "NVSwitch is present but no GPU reports an NVLink peer", "check nvidia-smi nvlink --status")
|
||
case pairs == 0:
|
||
add(Info, "nvlink", "no NVLink: GPUs talk over PCIe, and tensor parallelism across them is slow", "")
|
||
default:
|
||
add(OK, "nvlink", "NVLink groups: "+groups, "")
|
||
}
|
||
}
|
||
|
||
// --- multi-node NVLink ---
|
||
domain := ""
|
||
for _, g := range cards {
|
||
fb := g.Fabric
|
||
if fb == nil || strings.EqualFold(fb.State, "Not Supported") {
|
||
continue
|
||
}
|
||
if d := fb.Domain(); d != "" {
|
||
domain = d
|
||
if fb.Bandwidth != "" && !strings.EqualFold(fb.Bandwidth, "Full") {
|
||
add(Warn, "nvlink", fmt.Sprintf("GPU %d's NVLink bandwidth is %s", g.Index, strings.ToLower(fb.Bandwidth)),
|
||
"check its NVLink cables and trays: nvidia-smi -q -d FABRIC")
|
||
}
|
||
continue
|
||
}
|
||
if !strings.EqualFold(fb.State, "Completed") {
|
||
why := fb.State
|
||
if fb.Status != "" && !strings.EqualFold(fb.Status, "Success") {
|
||
why += ": " + fb.Status
|
||
}
|
||
add(Fail, "nvlink", fmt.Sprintf("GPU %d is not joined to its NVLink domain (%s)", g.Index, why),
|
||
"the fabric manager joins it: check nvidia-fabricmanager and the NVLink switch trays")
|
||
}
|
||
}
|
||
if domain != "" {
|
||
add(OK, "nvlink", "the GPUs are in multi-node NVLink domain "+domain, "")
|
||
switch {
|
||
case f.IMEX.State != "active":
|
||
state := f.IMEX.State
|
||
if state == "" {
|
||
state = "not installed"
|
||
}
|
||
add(Fail, "nvlink", "nvidia-imex is "+state+": without it the domain's GPUs cannot share memory across nodes",
|
||
"systemctl enable --now nvidia-imex, with every node of the domain in /etc/nvidia-imex/nodes_config.cfg")
|
||
case f.IMEX.Channels == 0:
|
||
add(Fail, "nvlink", "nvidia-imex runs but no IMEX channel exists for an engine to open",
|
||
"load the driver with options nvidia NVreg_CreateImexChannel0=1")
|
||
default:
|
||
add(OK, "nvlink", fmt.Sprintf("nvidia-imex is active with %d channel(s)", f.IMEX.Channels), "")
|
||
}
|
||
}
|
||
|
||
// --- InfiniBand and RoCE ---
|
||
if len(f.HCAs) == 0 {
|
||
add(Info, "fabric", "no InfiniBand or RoCE adapters: this node can only run single-node replicas", "")
|
||
}
|
||
declared := map[string]string{} // hca -> fabric
|
||
for _, c := range f.Fabrics {
|
||
for _, h := range c.Declared {
|
||
declared[h] = c.Fabric
|
||
}
|
||
}
|
||
for _, h := range f.HCAs {
|
||
for _, p := range h.Ports {
|
||
where := fmt.Sprintf("%s/%d", h.Name, p.Num)
|
||
level := Warn
|
||
if declared[h.Name] != "" {
|
||
level = Fail
|
||
}
|
||
switch {
|
||
case p.Active() && p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID < 0:
|
||
// Fatal only for a port a declared fabric uses. An adapter
|
||
// nothing is configured to use — a RoCE NIC on a node added
|
||
// for single-node inference — kept nodes out for a fabric
|
||
// nobody asked for.
|
||
msg := where + " is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang"
|
||
if declared[h.Name] == "" {
|
||
msg += "; no declared fabric uses it, so nothing will until one does"
|
||
}
|
||
add(level, "fabric", msg, "give the interface an IPv4 address, or check the adapter's RoCE mode")
|
||
case p.Active():
|
||
on := ""
|
||
switch {
|
||
case p.SM != nil:
|
||
on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State)
|
||
case p.Network != "":
|
||
on = ", network " + p.Network
|
||
}
|
||
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
|
||
// Questions about which fabric a port is on matter to the
|
||
// fabrics that use it; for a port none does they are noted,
|
||
// not warned of.
|
||
quiet := Warn
|
||
if declared[h.Name] == "" {
|
||
quiet = Info
|
||
}
|
||
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
|
||
why := f.SMErrors[where]
|
||
fix := ""
|
||
if strings.Contains(why, "infiniband-diags") {
|
||
fix = "install infiniband-diags"
|
||
}
|
||
add(quiet, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
|
||
"; without it, separate fabrics given one name cannot be caught", fix)
|
||
}
|
||
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
|
||
add(quiet, "fabric", where+" is active but reports no subnet manager LID", "")
|
||
}
|
||
case strings.EqualFold(p.PhysState, "LinkUp"):
|
||
add(level, "fabric", where+" has a link but is "+p.State+": no subnet manager has configured it",
|
||
"run a subnet manager on the fabric (opensm, or the switch's own)")
|
||
default:
|
||
add(level, "fabric", fmt.Sprintf("%s is %s (%s): no link", where, p.State, p.PhysState),
|
||
"check the cable and the switch port")
|
||
}
|
||
for _, name := range []string{"symbol_error", "link_downed", "link_error_recovery", "port_rcv_errors", "local_link_integrity_errors"} {
|
||
if v := p.Counters[name]; v > 0 {
|
||
add(Warn, "fabric", fmt.Sprintf("%s: %s = %d since the adapter came up", where, name, v),
|
||
"a count that keeps growing is a cable or transceiver going bad")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, master := range sortedKeys(f.SMLists) {
|
||
recs := f.SMLists[master]
|
||
var parts []string
|
||
standby := 0
|
||
for _, r := range recs {
|
||
parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority))
|
||
if r.State == "standby" {
|
||
standby++
|
||
}
|
||
}
|
||
if standby == 0 {
|
||
add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "),
|
||
"run a second subnet manager, on a switch or a host, at a lower priority")
|
||
} else {
|
||
add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "")
|
||
}
|
||
}
|
||
for _, master := range sortedKeys(f.SMListErrors) {
|
||
add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "")
|
||
}
|
||
if len(f.HCAs) > 0 {
|
||
if !f.Modules["ib_uverbs"] {
|
||
// RDMA is for replicas spanning nodes; a node that declares no
|
||
// fabric runs none, and is a node without it.
|
||
if len(declared) > 0 {
|
||
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
|
||
} else {
|
||
add(Warn, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA, which a fabric declared "+
|
||
"later would need; this node runs single-node replicas without it", "modprobe ib_uverbs")
|
||
}
|
||
}
|
||
if len(f.GPUs) > 0 && !f.Modules["nvidia_peermem"] && !f.Modules["nv_peer_mem"] {
|
||
add(Warn, "fabric", "nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory",
|
||
"modprobe nvidia_peermem")
|
||
}
|
||
}
|
||
|
||
// --- the agent's view ---
|
||
switch {
|
||
case f.ConfigErr != "":
|
||
add(Fail, "agent", "the agent configuration "+f.ConfigPath+" does not load: "+f.ConfigErr, "")
|
||
case f.ConfigPath == "":
|
||
add(Info, "agent", "no agent configuration found; declared fabrics were not checked", "pass --config")
|
||
default:
|
||
for _, c := range f.Fabrics {
|
||
if c.OK() {
|
||
add(OK, "agent", fmt.Sprintf("fabric %s: %s usable at %d Gb/s", c.Fabric, strings.Join(c.Usable, ","), c.RateGbps), "")
|
||
}
|
||
for _, p := range c.Problems {
|
||
lvl := Warn
|
||
if !c.OK() {
|
||
lvl = Fail
|
||
}
|
||
add(lvl, "agent", "fabric "+c.Fabric+": "+p, "")
|
||
}
|
||
if !c.OK() {
|
||
add(Fail, "agent", "fabric "+c.Fabric+" will not be advertised: no multi-node replica will be placed here", "")
|
||
}
|
||
}
|
||
active := activeHCAs(f.HCAs)
|
||
if len(f.Fabrics) == 0 && len(active) > 0 {
|
||
add(Warn, "agent", "adapters are up ("+strings.Join(active, ", ")+") but agent.yaml declares no fabric: this node will never get multi-node work",
|
||
"add fabrics: [{name: <fabric>, hcas: ["+strings.Join(active, ", ")+"]}]")
|
||
}
|
||
if !f.IfaceExists {
|
||
add(Fail, "agent", "the management interface "+f.MgmtIface+" does not exist: NCCL and Gloo would bootstrap over nothing",
|
||
"set mgmt_interface in agent.yaml")
|
||
} else {
|
||
add(OK, "agent", "NCCL bootstraps over "+f.MgmtIface+" ("+f.MgmtAddr+")", "")
|
||
}
|
||
if len(f.Fabrics) > 0 && f.LaunchMode == "ray" && f.Engines["ray"] == "" && f.Engines["vllm"] != "" {
|
||
add(Fail, "agent", "vLLM's launch_mode is ray but ray is not installed", "install ray, or set engines.vllm.launch_mode: mp")
|
||
}
|
||
}
|
||
// --- engines ---
|
||
// An engine disabled on this node: whether it could be installed is not
|
||
// a question anybody is asking.
|
||
if f.VLLMVersion != "" && !slices.Contains(f.DisabledEngines, "vllm") {
|
||
vllmFindings(f, add)
|
||
}
|
||
|
||
if f.UnitMemlock != "" && f.UnitMemlock != "unknown" && f.UnitMemlock != "infinity" && len(f.HCAs) > 0 {
|
||
add(Warn, "agent", "the llmgw-agent unit limits locked memory to "+f.UnitMemlock+"; RDMA pins what it registers",
|
||
"LimitMEMLOCK=infinity in the unit")
|
||
}
|
||
|
||
// --- firewall ---
|
||
// Only on a node already set up: before that, onboarding writes it.
|
||
if f.ConfigPath != "" {
|
||
switch f.Firewall {
|
||
case "present":
|
||
add(OK, "firewall", "llmgw's ports answer the cluster's networks only", "")
|
||
case "absent":
|
||
add(Warn, "firewall", "no llmgw firewall: the engines' ports answer every network this machine is on",
|
||
"run node add for it again, or list extra networks in firewall_networks first")
|
||
}
|
||
}
|
||
|
||
sort.SliceStable(out, func(i, j int) bool { return rank(out[i].Level) < rank(out[j].Level) })
|
||
return out
|
||
}
|
||
|
||
// vllmFindings says whether the agent can install vLLM here, and if not why:
|
||
// the same decision the agent makes, so the report and the node agree.
|
||
func vllmFindings(f Facts, add func(Level, string, string, string)) {
|
||
v := f.VLLMVersion
|
||
builds, err := provision.Recipes["vllm"].Candidates(v, "", f.Host)
|
||
var b provision.Build
|
||
if err == nil {
|
||
// The first the agent would try; it moves to the next by itself if
|
||
// that one does not work here.
|
||
b = builds[0]
|
||
if b.Version != "" {
|
||
v = b.Version
|
||
}
|
||
}
|
||
var nh provision.NotHere
|
||
switch {
|
||
case errors.As(err, &nh):
|
||
add(Info, "engines", "vLLM will not be installed: "+nh.Reason, "")
|
||
return
|
||
case err != nil:
|
||
// A warning, not a failure: the node is still a node, and serves
|
||
// llama.cpp. Refusing it here kept older GPU generations out of a
|
||
// cluster for an engine nobody meant to run on them.
|
||
add(Warn, "engines", "vLLM "+v+" will not run here, and the node joins without it: "+err.Error(),
|
||
"add it with --disable-engine vllm if it is meant for llama.cpp alone")
|
||
return
|
||
case f.VLLMInstalled:
|
||
add(OK, "engines", "vLLM "+v+" is installed under "+f.EnginesDir, "")
|
||
return
|
||
}
|
||
if f.EnginesFreeGB > 0 && f.EnginesFreeGB < float64(b.FreeGB) {
|
||
add(Warn, "engines", fmt.Sprintf("vLLM %s needs about %d GB free under %s and there are %.1f GB; the node joins without it until there is",
|
||
v, b.FreeGB, f.EnginesDir, f.EnginesFreeGB),
|
||
"free space there, or point engines_dir in agent.yaml at a larger disk")
|
||
return
|
||
}
|
||
note := ""
|
||
if b.Note != "" {
|
||
note = "; " + b.Note
|
||
}
|
||
add(OK, "engines", fmt.Sprintf("vLLM %s (%s) can be installed here: driver %s, %.1f GB free under %s%s",
|
||
v, b.Variant, f.Host.NVIDIADriver, f.EnginesFreeGB, f.EnginesDir, note), "")
|
||
}
|
||
|
||
func rank(l Level) int {
|
||
switch l {
|
||
case Fail:
|
||
return 0
|
||
case Warn:
|
||
return 1
|
||
case OK:
|
||
return 2
|
||
}
|
||
return 3
|
||
}
|
||
|
||
// evaluateMIG reports each MIG-capable card's mode and instances. Quiet for a
|
||
// card with MIG off, which is every card on most machines; what it warns of
|
||
// are the states in which a card is in MIG mode and nothing can run on it.
|
||
func evaluateMIG(f Facts, add func(Level, string, string, string)) {
|
||
if f.MIGErr != "" {
|
||
add(Warn, "mig", "could not read the cards' MIG state: "+f.MIGErr, "run nvidia-smi -q by hand")
|
||
return
|
||
}
|
||
for _, c := range f.MIG {
|
||
card := fmt.Sprintf("GPU %d (%s)", c.Index, c.Name)
|
||
if c.Enabled != c.PendingEnabled {
|
||
want := "off"
|
||
if c.PendingEnabled {
|
||
want = "on"
|
||
}
|
||
add(Warn, "mig", fmt.Sprintf("%s has MIG turning %s at its next reset, and until then runs as before", card, want),
|
||
fmt.Sprintf("reset the card when nothing runs on it: nvidia-smi -i %d -r (or reboot)", c.Index))
|
||
}
|
||
if !c.Enabled {
|
||
continue
|
||
}
|
||
// A GPU instance with no compute instance is carved but not usable:
|
||
// it is not a device until one is made inside it.
|
||
var bare []string
|
||
for _, gi := range c.GPUInstances {
|
||
has := slices.ContainsFunc(c.ComputeInstances, func(ci discovery.MIGInstance) bool {
|
||
return ci.GPUInstance == gi.GPUInstance
|
||
})
|
||
if !has {
|
||
bare = append(bare, fmt.Sprintf("%d (%s)", gi.GPUInstance, gi.Profile))
|
||
}
|
||
}
|
||
switch {
|
||
case len(c.Instances) == 0 && len(c.GPUInstances) == 0:
|
||
add(Warn, "mig", card+" is in MIG mode with no instances created: nothing can run on it",
|
||
fmt.Sprintf("create instances, for example nvidia-smi mig -i %d -cgi 3g.40gb,3g.40gb -C "+
|
||
"(nvidia-smi mig -i %d -lgip lists the profiles), or turn MIG off: nvidia-smi -i %d -mig 0",
|
||
c.Index, c.Index, c.Index))
|
||
case len(c.Instances) == 0:
|
||
add(Warn, "mig", fmt.Sprintf("%s is in MIG mode with GPU instances %s but no compute instances: nothing can run on it",
|
||
card, strings.Join(bare, ", ")),
|
||
fmt.Sprintf("create them: nvidia-smi mig -i %d -cci", c.Index))
|
||
default:
|
||
var names []string
|
||
for _, in := range c.Instances {
|
||
names = append(names, fmt.Sprintf("%d: %s %s", in.Device, in.Profile, in.UUID))
|
||
}
|
||
add(OK, "mig", fmt.Sprintf("%s is in MIG mode with %d instance(s), each a device of its own: %s",
|
||
card, len(c.Instances), strings.Join(names, "; ")), "")
|
||
if len(bare) > 0 {
|
||
add(Warn, "mig", fmt.Sprintf("%s has GPU instances %s with no compute instance, which are not devices yet",
|
||
card, strings.Join(bare, ", ")),
|
||
fmt.Sprintf("create them: nvidia-smi mig -i %d -cci, or destroy the GPU instances", c.Index))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func gpuNames(gpus []discovery.GPU) string {
|
||
count := map[string]int{}
|
||
var order []string
|
||
for _, g := range gpus {
|
||
if count[g.Name] == 0 {
|
||
order = append(order, g.Name)
|
||
}
|
||
count[g.Name]++
|
||
}
|
||
parts := make([]string, len(order))
|
||
for i, n := range order {
|
||
parts[i] = fmt.Sprintf("%d× %s", count[n], n)
|
||
}
|
||
return strings.Join(parts, ", ")
|
||
}
|
||
|
||
// nvlinkGroups counts NVLink pairs and names the connected groups.
|
||
func nvlinkGroups(gpus []discovery.GPU) (int, string) {
|
||
parent := map[int]int{}
|
||
var find func(int) int
|
||
find = func(x int) int {
|
||
if p, ok := parent[x]; ok && p != x {
|
||
parent[x] = find(p)
|
||
return parent[x]
|
||
}
|
||
parent[x] = x
|
||
return x
|
||
}
|
||
pairs := 0
|
||
for _, g := range gpus {
|
||
find(g.Index)
|
||
for _, p := range g.NVLinkPeers {
|
||
if p > g.Index {
|
||
pairs++
|
||
}
|
||
parent[find(g.Index)] = find(p)
|
||
}
|
||
}
|
||
groups := map[int][]int{}
|
||
for _, g := range gpus {
|
||
r := find(g.Index)
|
||
groups[r] = append(groups[r], g.Index)
|
||
}
|
||
var names []string
|
||
for _, members := range groups {
|
||
sort.Ints(members)
|
||
if len(members) > 1 {
|
||
names = append(names, fmt.Sprint(members))
|
||
}
|
||
}
|
||
sort.Strings(names)
|
||
return pairs, strings.Join(names, " ")
|
||
}
|
||
|
||
func activeHCAs(hcas []discovery.HCA) []string {
|
||
var out []string
|
||
for _, h := range hcas {
|
||
if len(h.ActivePorts()) > 0 {
|
||
out = append(out, h.Name)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func orUnknown(s string) string {
|
||
if s == "" {
|
||
return "no answer from the subnet manager"
|
||
}
|
||
return s
|
||
}
|
||
|
||
func sortedKeys[V any](m map[string]V) []string {
|
||
out := make([]string, 0, len(m))
|
||
for k := range m {
|
||
out = append(out, k)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// MultiNode is whether a finding is about the fabrics replicas spanning
|
||
// nodes use, rather than about the node itself.
|
||
func (f Finding) MultiNode() bool { return f.Area == "fabric" }
|
||
|
||
// Readiness is a node's two readinesses, kept apart: whether it can join and
|
||
// serve replicas of its own, and whether it can also take part in replicas
|
||
// spanning nodes over a fabric. The second never keeps a node out unless a
|
||
// fabric was declared for it, whose problems are then the node's.
|
||
type Readiness struct {
|
||
// Node is "ready", or how many problems keep it out.
|
||
Node string `json:"node"`
|
||
// MultiNode is "ready", "not used" (no fabric adapter), or "not ready:"
|
||
// and the first reason.
|
||
MultiNode string `json:"multi_node"`
|
||
}
|
||
|
||
// ReadinessOf sums findings up as the two readinesses.
|
||
func ReadinessOf(fs []Finding) Readiness {
|
||
r := Readiness{Node: "ready", MultiNode: "not used"}
|
||
fails := 0
|
||
for _, f := range fs {
|
||
if f.Level == Fail {
|
||
fails++
|
||
}
|
||
}
|
||
if fails > 0 {
|
||
r.Node = fmt.Sprintf("%d problem(s) keep it out", fails)
|
||
}
|
||
for _, f := range fs {
|
||
if !f.MultiNode() {
|
||
continue
|
||
}
|
||
if r.MultiNode == "not used" {
|
||
r.MultiNode = "ready"
|
||
}
|
||
if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") {
|
||
r.MultiNode = "not ready: " + f.Message
|
||
}
|
||
}
|
||
return r
|
||
}
|