Read NVLink and GPU-to-adapter distance from nvidia-smi topo
The release binary is built without cgo, so without NVML, and NVML was the only source of NVLink peers: on every NVLink machine the portable agent reported none, and tensor-parallel groups were chosen by GPU index. Discovery now reads `nvidia-smi topo -m` after either path, for the NVLink peers (kept from NVML where it gave them), each card's NUMA node, and how far each network adapter is from it - PIX, PXB, PHB, NODE or SYS - which travels to the control plane as nic_paths for placement to use. Both the NICn-with-legend layout of newer drivers and the adapter-named columns of older ones are read. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
22bb098247
commit
bf2d171e89
8 files changed
+363
-30
No files matched your search
@@ -0,0 +1,113 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// row joins cells the way nvidia-smi prints them: tab-separated, each padded.
|
||||
func row(cells ...string) string { return strings.Join(cells, "\t") }
|
||||
|
||||
// dgxH100 is `nvidia-smi topo -m` on an eight-GPU NVSwitch machine with two
|
||||
// adapters, as newer drivers print it: NICn columns and a legend naming them.
|
||||
var dgxH100 = strings.Join([]string{
|
||||
row("\x1b[4m", "GPU0", "GPU1", "GPU2", "GPU3", "NIC0", "NIC1", "CPU Affinity", "NUMA Affinity", "GPU NUMA ID\x1b[0m"),
|
||||
row("GPU0", " X ", "NV18", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
|
||||
row("GPU1", "NV18", " X ", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
|
||||
row("GPU2", "NV18", "NV18", " X ", "NV18", "SYS", "PXB", "56-111,168-223", "1", "N/A"),
|
||||
row("GPU3", "NV18", "NV18", "NV18", " X ", "SYS", "PIX", "56-111,168-223", "1", "N/A"),
|
||||
row("NIC0", "PXB", "PXB", "SYS", "SYS", " X ", "SYS"),
|
||||
row("NIC1", "SYS", "SYS", "PXB", "PIX", "SYS", " X "),
|
||||
"",
|
||||
"Legend:",
|
||||
"",
|
||||
" X = Self",
|
||||
" SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)",
|
||||
" NV# = Connection traversing a bonded set of # NVLinks",
|
||||
"",
|
||||
"NIC Legend:",
|
||||
"",
|
||||
" NIC0: mlx5_0",
|
||||
" NIC1: mlx5_1",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
func TestNVLinkAndAdapterDistanceAreReadFromTopo(t *testing.T) {
|
||||
topo, err := parseTopo(dgxH100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := topo.NVLink[0]; !reflect.DeepEqual(got, map[int]int{1: 18, 2: 18, 3: 18}) {
|
||||
t.Errorf("GPU0's NVLink peers = %v", got)
|
||||
}
|
||||
// The legend turns NIC1 into the adapter's own name.
|
||||
if got := topo.NICPath[3]; !reflect.DeepEqual(got, map[string]string{"mlx5_0": "SYS", "mlx5_1": "PIX"}) {
|
||||
t.Errorf("GPU3's adapters = %v", got)
|
||||
}
|
||||
if topo.NUMA[2] != 1 || topo.NUMA[0] != 0 {
|
||||
t.Errorf("NUMA = %v", topo.NUMA)
|
||||
}
|
||||
|
||||
gpus := []GPU{{Index: 0}, {Index: 3}}
|
||||
topo.Apply(gpus)
|
||||
if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1, 2, 3}) || gpus[1].NUMANode != 1 ||
|
||||
gpus[1].NICPaths["mlx5_1"] != "PIX" {
|
||||
t.Errorf("applied = %+v", gpus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOlderDriversNameAdaptersInTheHeader(t *testing.T) {
|
||||
// No legend: the columns are the adapters' own names, and NVLink between
|
||||
// pairs only, as on an A100 PCIe box with bridges.
|
||||
out := strings.Join([]string{
|
||||
row("", "GPU0", "GPU1", "GPU2", "GPU3", "mlx5_0", "CPU Affinity", "NUMA Affinity"),
|
||||
row("GPU0", " X ", "NV12", "SYS", "SYS", "NODE", "0-31", "0"),
|
||||
row("GPU1", "NV12", " X ", "SYS", "SYS", "NODE", "0-31", "0"),
|
||||
row("GPU2", "SYS", "SYS", " X ", "NV12", "SYS", "32-63", "1"),
|
||||
row("GPU3", "SYS", "SYS", "NV12", " X ", "SYS", "32-63", "1"),
|
||||
row("mlx5_0", "NODE", "NODE", "SYS", "SYS", " X "),
|
||||
"",
|
||||
}, "\n")
|
||||
topo, err := parseTopo(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(topo.NVLink[2], map[int]int{3: 12}) || len(topo.NVLink[0]) != 1 {
|
||||
t.Errorf("NVLink = %v", topo.NVLink)
|
||||
}
|
||||
if topo.NICPath[0]["mlx5_0"] != "NODE" {
|
||||
t.Errorf("adapters = %v", topo.NICPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPCIeMachineHasNoNVLink(t *testing.T) {
|
||||
out := strings.Join([]string{
|
||||
row("", "GPU0", "GPU1", "CPU Affinity", "NUMA Affinity"),
|
||||
row("GPU0", " X ", "PHB", "0-15", "0"),
|
||||
row("GPU1", "PHB", " X ", "0-15", "0"),
|
||||
"",
|
||||
}, "\n")
|
||||
topo, err := parseTopo(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(topo.NVLink) != 0 {
|
||||
t.Errorf("NVLink on a PCIe machine: %v", topo.NVLink)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNVMLPeersAreKeptOverTopo(t *testing.T) {
|
||||
topo, _ := parseTopo(dgxH100)
|
||||
gpus := []GPU{{Index: 0, NVLinkPeers: []int{1}}}
|
||||
topo.Apply(gpus)
|
||||
if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1}) {
|
||||
t.Errorf("NVML's answer was replaced: %v", gpus[0].NVLinkPeers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSomethingThatIsNotAMatrixIsAnError(t *testing.T) {
|
||||
if _, err := parseTopo("NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver."); err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user