Read NVLink and GPU-to-adapter distance from nvidia-smi topo

The release binary is built without cgo, so without NVML, and NVML was
the only source of NVLink peers: on every NVLink machine the portable
agent reported none, and tensor-parallel groups were chosen by GPU index.
Discovery now reads `nvidia-smi topo -m` after either path, for the
NVLink peers (kept from NVML where it gave them), each card's NUMA node,
and how far each network adapter is from it - PIX, PXB, PHB, NODE or
SYS - which travels to the control plane as nic_paths for placement to
use. Both the NICn-with-legend layout of newer drivers and the
adapter-named columns of older ones are read.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 00:09:26 +02:00
1 parent 22bb098247
commit bf2d171e89
8 files changed
+363 -30

No files matched your search

+14 -2
View File
@@ -64,8 +64,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
inv.GPUs, inv.Source = gpus, "nvml"
break
}
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
"NVLink topology will be unavailable", "err", err)
log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err)
fallthrough
default:
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
@@ -74,6 +73,19 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
}
inv.GPUs, inv.Source = gpus, "nvidia-smi"
}
// NVLink and which adapter sits nearest each card, for either path:
// NVML knows NVLink but not the adapters, nvidia-smi's query knows
// neither. A failure costs the topology and nothing else — the cards
// are still cards.
if len(inv.GPUs) > 0 {
topo, err := DiscoverTopology(ctx, opts.NvidiaSMI)
if err != nil {
log.Warn("could not read the GPU topology; tensor-parallel groups "+
"will be chosen without NVLink", "err", err)
} else {
topo.Apply(inv.GPUs)
}
}
}
hcas, err := DiscoverIB(opts.IBRoot)