package discovery import ( "context" "errors" "fmt" "log/slog" ) // ErrNoNVML means this binary was built without the NVML backend. // // It is declared here rather than in the build-tagged file so callers — and // tests — can refer to it in either build. var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology") // Options configure a full inventory scan. type Options struct { // NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH. // Worth configuring on nodes where the driver lives outside the default // paths, which containerized driver installs often do. NvidiaSMI string `koanf:"nvidia_smi"` // IBRoot is the sysfs InfiniBand directory. IBRoot string `koanf:"ib_root"` // PreferNVML uses the NVML backend when this binary has one. It is the only // source of NVLink topology. PreferNVML bool `koanf:"prefer_nvml"` // SkipGPUs omits GPU discovery, for nodes that have none. SkipGPUs bool `koanf:"skip_gpus"` } // Discover builds the node's inventory. // // When NVML is preferred but fails, it falls back to nvidia-smi and says so in // Inventory.Source. The fallback is logged rather than silent: on a node that // is supposed to have working NVML, a quiet downgrade hides a broken driver and // costs the scheduler its NVLink topology. func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) { if log == nil { log = slog.Default() } inv := Inventory{Source: "none"} // First, because it has nothing to do with GPUs and every path below can // return early. Read after them, a node with no nvidia-smi reported no // memory at all: GPU discovery fails, Discover returns, and the fact that // has nothing to do with cards goes with it. That is exactly the node whose // memory matters most, since a CPU profile is what it will be asked to run. inv.MemoryTotalMB = MemoryTotalMB() if !opts.SkipGPUs { switch { case opts.PreferNVML && nvmlAvailable: gpus, err := DiscoverGPUsNVML(ctx) if err == nil { inv.GPUs, inv.Source = gpus, "nvml" break } log.Warn("NVML discovery failed, falling back to nvidia-smi; "+ "NVLink topology will be unavailable", "err", err) fallthrough default: gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI) if err != nil { return inv, fmt.Errorf("gpu discovery: %w", err) } inv.GPUs, inv.Source = gpus, "nvidia-smi" } } hcas, err := DiscoverIB(opts.IBRoot) if err != nil { return inv, fmt.Errorf("infiniband discovery: %w", err) } inv.HCAs = hcas return inv, nil } // NVMLSupported reports whether this binary includes the NVML backend. func NVMLSupported() bool { return nvmlAvailable }