Deploying the memory-aware placement showed node01 to node04 reporting 7G and node05 reporting nothing, on a machine whose /proc/meminfo plainly says 32 GB. node05 has no nvidia-smi, so GPU discovery returns an error, Discover returns with it, and the memory read — which has nothing to do with cards — never happened. That is the node whose memory matters most, because a CPU profile is what a GPU-less node will be asked to run, and min_memory_gb would have excluded it from everything. Read first now, before any path that can return early. The test fails against the old order. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
79 lines
2.6 KiB
Go
79 lines
2.6 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
)
|
|
|
|
// ErrNoNVML means this binary was built without the NVML backend.
|
|
//
|
|
// It is declared here rather than in the build-tagged file so callers — and
|
|
// tests — can refer to it in either build.
|
|
var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
|
|
|
|
// Options configure a full inventory scan.
|
|
type Options struct {
|
|
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
|
|
// Worth configuring on nodes where the driver lives outside the default
|
|
// paths, which containerized driver installs often do.
|
|
NvidiaSMI string `koanf:"nvidia_smi"`
|
|
// IBRoot is the sysfs InfiniBand directory.
|
|
IBRoot string `koanf:"ib_root"`
|
|
// PreferNVML uses the NVML backend when this binary has one. It is the only
|
|
// source of NVLink topology.
|
|
PreferNVML bool `koanf:"prefer_nvml"`
|
|
// SkipGPUs omits GPU discovery, for nodes that have none.
|
|
SkipGPUs bool `koanf:"skip_gpus"`
|
|
}
|
|
|
|
// Discover builds the node's inventory.
|
|
//
|
|
// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
|
|
// Inventory.Source. The fallback is logged rather than silent: on a node that
|
|
// is supposed to have working NVML, a quiet downgrade hides a broken driver and
|
|
// costs the scheduler its NVLink topology.
|
|
func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
inv := Inventory{Source: "none"}
|
|
// First, because it has nothing to do with GPUs and every path below can
|
|
// return early. Read after them, a node with no nvidia-smi reported no
|
|
// memory at all: GPU discovery fails, Discover returns, and the fact that
|
|
// has nothing to do with cards goes with it. That is exactly the node whose
|
|
// memory matters most, since a CPU profile is what it will be asked to run.
|
|
inv.MemoryTotalMB = MemoryTotalMB()
|
|
|
|
if !opts.SkipGPUs {
|
|
switch {
|
|
case opts.PreferNVML && nvmlAvailable:
|
|
gpus, err := DiscoverGPUsNVML(ctx)
|
|
if err == nil {
|
|
inv.GPUs, inv.Source = gpus, "nvml"
|
|
break
|
|
}
|
|
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
|
|
"NVLink topology will be unavailable", "err", err)
|
|
fallthrough
|
|
default:
|
|
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
|
|
if err != nil {
|
|
return inv, fmt.Errorf("gpu discovery: %w", err)
|
|
}
|
|
inv.GPUs, inv.Source = gpus, "nvidia-smi"
|
|
}
|
|
}
|
|
|
|
hcas, err := DiscoverIB(opts.IBRoot)
|
|
if err != nil {
|
|
return inv, fmt.Errorf("infiniband discovery: %w", err)
|
|
}
|
|
inv.HCAs = hcas
|
|
return inv, nil
|
|
}
|
|
|
|
// NVMLSupported reports whether this binary includes the NVML backend.
|
|
func NVMLSupported() bool { return nvmlAvailable }
|