Files
llmgw/internal/agent/discovery/discover.go
T
ilkerandClaude Opus 5 55417546fc
CI / test (push) Successful in 1m37s
Phase 2 (2/2): NVML backend behind a build tag
NVML is the only source of NVLink topology, and that topology is what keeps a
tensor-parallel group off the slow path inside a node. It needs cgo, so it sits
behind the nvml tag and the default build keeps using nvidia-smi.

Discover() picks the backend and falls back to nvidia-smi when NVML fails,
logging the fallback rather than performing it silently: on a node that is meant
to have working NVML, a quiet downgrade hides a broken driver installation and
costs the scheduler its topology.

CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML
was declared only in the untagged file, so the tests did not compile under
-tags nvml. It is now declared where both builds can see it.

Not verified: the NVML path has never run against a driver, because there is no
GPU machine available. The NVLink domains it reports should be treated as
unverified until someone runs `llmgw agent inventory` on a real node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 02:05:25 +02:00

71 lines
2.0 KiB
Go

package discovery
import (
"context"
"errors"
"fmt"
"log/slog"
)
// ErrNoNVML means this binary was built without the NVML backend.
//
// It is declared here rather than in the build-tagged file so callers — and
// tests — can refer to it in either build.
var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
// Options configure a full inventory scan.
type Options struct {
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
NvidiaSMI string
// IBRoot is the sysfs InfiniBand directory.
IBRoot string
// PreferNVML uses the NVML backend when this binary has one. It is the only
// source of NVLink topology.
PreferNVML bool
// SkipGPUs omits GPU discovery, for nodes that have none.
SkipGPUs bool
}
// Discover builds the node's inventory.
//
// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
// Inventory.Source. The fallback is logged rather than silent: on a node that
// is supposed to have working NVML, a quiet downgrade hides a broken driver and
// costs the scheduler its NVLink topology.
func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
if log == nil {
log = slog.Default()
}
inv := Inventory{Source: "none"}
if !opts.SkipGPUs {
switch {
case opts.PreferNVML && nvmlAvailable:
gpus, err := DiscoverGPUsNVML(ctx)
if err == nil {
inv.GPUs, inv.Source = gpus, "nvml"
break
}
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
"NVLink topology will be unavailable", "err", err)
fallthrough
default:
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
if err != nil {
return inv, fmt.Errorf("gpu discovery: %w", err)
}
inv.GPUs, inv.Source = gpus, "nvidia-smi"
}
}
hcas, err := DiscoverIB(opts.IBRoot)
if err != nil {
return inv, fmt.Errorf("infiniband discovery: %w", err)
}
inv.HCAs = hcas
return inv, nil
}
// NVMLSupported reports whether this binary includes the NVML backend.
func NVMLSupported() bool { return nvmlAvailable }