Read NVLink and GPU-to-adapter distance from nvidia-smi topo
The release binary is built without cgo, so without NVML, and NVML was the only source of NVLink peers: on every NVLink machine the portable agent reported none, and tensor-parallel groups were chosen by GPU index. Discovery now reads `nvidia-smi topo -m` after either path, for the NVLink peers (kept from NVML where it gave them), each card's NUMA node, and how far each network adapter is from it - PIX, PXB, PHB, NODE or SYS - which travels to the control plane as nic_paths for placement to use. Both the NICn-with-legend layout of newer drivers and the adapter-named columns of older ones are read. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
22bb098247
commit
bf2d171e89
8 files changed
+363
-30
No files matched your search
@@ -64,8 +64,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
inv.GPUs, inv.Source = gpus, "nvml"
|
||||
break
|
||||
}
|
||||
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
|
||||
"NVLink topology will be unavailable", "err", err)
|
||||
log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err)
|
||||
fallthrough
|
||||
default:
|
||||
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
|
||||
@@ -74,6 +73,19 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
}
|
||||
inv.GPUs, inv.Source = gpus, "nvidia-smi"
|
||||
}
|
||||
// NVLink and which adapter sits nearest each card, for either path:
|
||||
// NVML knows NVLink but not the adapters, nvidia-smi's query knows
|
||||
// neither. A failure costs the topology and nothing else — the cards
|
||||
// are still cards.
|
||||
if len(inv.GPUs) > 0 {
|
||||
topo, err := DiscoverTopology(ctx, opts.NvidiaSMI)
|
||||
if err != nil {
|
||||
log.Warn("could not read the GPU topology; tensor-parallel groups "+
|
||||
"will be chosen without NVLink", "err", err)
|
||||
} else {
|
||||
topo.Apply(inv.GPUs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hcas, err := DiscoverIB(opts.IBRoot)
|
||||
|
||||
Reference in new issue
Block a user