Files
llmgw/internal/agent/discovery/discover.go
T
Ilker ManapandClaude Opus 5 d71d785598 Report node memory even when GPU discovery fails
Deploying the memory-aware placement showed node01 to node04 reporting 7G
and node05 reporting nothing, on a machine whose /proc/meminfo plainly says
32 GB. node05 has no nvidia-smi, so GPU discovery returns an error, Discover
returns with it, and the memory read — which has nothing to do with cards —
never happened.

That is the node whose memory matters most, because a CPU profile is what a
GPU-less node will be asked to run, and min_memory_gb would have excluded it
from everything.

Read first now, before any path that can return early. The test fails
against the old order.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 23:50:53 +02:00

79 lines
2.6 KiB
Go

package discovery
import (
"context"
"errors"
"fmt"
"log/slog"
)
// ErrNoNVML means this binary was built without the NVML backend.
//
// It is declared here rather than in the build-tagged file so callers — and
// tests — can refer to it in either build.
var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
// Options configure a full inventory scan.
type Options struct {
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
// Worth configuring on nodes where the driver lives outside the default
// paths, which containerized driver installs often do.
NvidiaSMI string `koanf:"nvidia_smi"`
// IBRoot is the sysfs InfiniBand directory.
IBRoot string `koanf:"ib_root"`
// PreferNVML uses the NVML backend when this binary has one. It is the only
// source of NVLink topology.
PreferNVML bool `koanf:"prefer_nvml"`
// SkipGPUs omits GPU discovery, for nodes that have none.
SkipGPUs bool `koanf:"skip_gpus"`
}
// Discover builds the node's inventory.
//
// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
// Inventory.Source. The fallback is logged rather than silent: on a node that
// is supposed to have working NVML, a quiet downgrade hides a broken driver and
// costs the scheduler its NVLink topology.
func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
if log == nil {
log = slog.Default()
}
inv := Inventory{Source: "none"}
// First, because it has nothing to do with GPUs and every path below can
// return early. Read after them, a node with no nvidia-smi reported no
// memory at all: GPU discovery fails, Discover returns, and the fact that
// has nothing to do with cards goes with it. That is exactly the node whose
// memory matters most, since a CPU profile is what it will be asked to run.
inv.MemoryTotalMB = MemoryTotalMB()
if !opts.SkipGPUs {
switch {
case opts.PreferNVML && nvmlAvailable:
gpus, err := DiscoverGPUsNVML(ctx)
if err == nil {
inv.GPUs, inv.Source = gpus, "nvml"
break
}
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
"NVLink topology will be unavailable", "err", err)
fallthrough
default:
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
if err != nil {
return inv, fmt.Errorf("gpu discovery: %w", err)
}
inv.GPUs, inv.Source = gpus, "nvidia-smi"
}
}
hcas, err := DiscoverIB(opts.IBRoot)
if err != nil {
return inv, fmt.Errorf("infiniband discovery: %w", err)
}
inv.HCAs = hcas
return inv, nil
}
// NVMLSupported reports whether this binary includes the NVML backend.
func NVMLSupported() bool { return nvmlAvailable }