Phase 2 (2/2): NVML backend behind a build tag
CI / test (push) Successful in 1m37s

NVML is the only source of NVLink topology, and that topology is what keeps a
tensor-parallel group off the slow path inside a node. It needs cgo, so it sits
behind the nvml tag and the default build keeps using nvidia-smi.

Discover() picks the backend and falls back to nvidia-smi when NVML fails,
logging the fallback rather than performing it silently: on a node that is meant
to have working NVML, a quiet downgrade hides a broken driver installation and
costs the scheduler its topology.

CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML
was declared only in the untagged file, so the tests did not compile under
-tags nvml. It is now declared where both builds can see it.

Not verified: the NVML path has never run against a driver, because there is no
GPU machine available. The NVLink domains it reports should be treated as
unverified until someone runs `llmgw agent inventory` on a real node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 02:05:25 +02:00
1 parent 6a13e3dddf
commit 55417546fc
10 files changed
+374 -23

No files matched your search

+18 -17
View File
@@ -36,6 +36,7 @@ func newInventoryCmd() *cobra.Command {
smiBin string
ibRoot string
skipGPU bool
useNVML bool
)
cmd := &cobra.Command{
@@ -46,24 +47,18 @@ func newInventoryCmd() *cobra.Command {
"Run it before adding a node to the cluster — it is the fastest way to find\n" +
"a card another process is already using, or a port with no subnet manager.",
RunE: func(cmd *cobra.Command, _ []string) error {
inv := discovery.Inventory{Source: "nvidia-smi"}
if !skipGPU {
gpus, err := discovery.DiscoverGPUsSMI(cmd.Context(), smiBin)
if err != nil {
// Not fatal: a node with no GPUs still has IB worth showing,
// and the error itself is the useful output.
fmt.Fprintf(os.Stderr, "GPU discovery failed: %v\n", err)
inv.Source = "unavailable"
}
inv.GPUs = gpus
}
hcas, err := discovery.DiscoverIB(ibRoot)
inv, err := discovery.Discover(cmd.Context(), discovery.Options{
NvidiaSMI: smiBin,
IBRoot: ibRoot,
PreferNVML: useNVML,
SkipGPUs: skipGPU,
}, logger())
if err != nil {
return err
// Not fatal: a node whose GPUs cannot be read still has IB
// worth showing, and the error is itself the useful output.
fmt.Fprintf(os.Stderr, "%v\n", err)
inv.Source = "unavailable"
}
inv.HCAs = hcas
if asJSON {
enc := json.NewEncoder(os.Stdout)
@@ -80,6 +75,7 @@ func newInventoryCmd() *cobra.Command {
f.StringVar(&smiBin, "nvidia-smi", "", "path to nvidia-smi")
f.StringVar(&ibRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
f.BoolVar(&skipGPU, "no-gpu", false, "skip GPU discovery")
f.BoolVar(&useNVML, "nvml", true, "prefer the NVML backend when this build has one (it is the only source of NVLink topology)")
return cmd
}
@@ -87,7 +83,7 @@ func newInventoryCmd() *cobra.Command {
func printInventory(inv discovery.Inventory) {
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintf(w, "GPUs (%d)\n", len(inv.GPUs))
fmt.Fprintf(w, "GPUs (%d, via %s)\n", len(inv.GPUs), inv.Source)
if len(inv.GPUs) == 0 {
fmt.Fprintln(w, " none")
} else {
@@ -101,6 +97,11 @@ func printInventory(inv discovery.Inventory) {
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
}
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
if !discovery.NVMLSupported() {
fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.")
fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside")
fmt.Fprintln(w, " one NVLink domain (plan §6.2).")
}
}
fmt.Fprintf(w, "\nInfiniBand adapters (%d)\n", len(inv.HCAs))