Read NVLink and GPU-to-adapter distance from nvidia-smi topo

The release binary is built without cgo, so without NVML, and NVML was
the only source of NVLink peers: on every NVLink machine the portable
agent reported none, and tensor-parallel groups were chosen by GPU index.
Discovery now reads `nvidia-smi topo -m` after either path, for the
NVLink peers (kept from NVML where it gave them), each card's NUMA node,
and how far each network adapter is from it - PIX, PXB, PHB, NODE or
SYS - which travels to the control plane as nic_paths for placement to
use. Both the NICn-with-legend layout of newer drivers and the
adapter-named columns of older ones are read.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

api/gen/llmgw/v1/agent.pb.go

@@ -933,8 +933,11 @@ type GPU struct {
 	// Zero when the driver is too old to report it, which is not the same as
 	// idle: the metric is simply absent rather than wrong.
 	UtilizationPercent int32 `protobuf:"varint,15,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"`
-	unknownFields      protoimpl.UnknownFields
-	sizeCache          protoimpl.SizeCache
+	// nic_paths is how far each network adapter is from this card, by HCA
+	// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
+	NicPaths      map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *GPU) Reset() {
@@ -1072,6 +1075,13 @@ func (x *GPU) GetUtilizationPercent() int32 {
 	return 0
 }
 
+func (x *GPU) GetNicPaths() map[string]string {
+	if x != nil {
+		return x.NicPaths
+	}
+	return nil
+}
+
 type FabricAttachment struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
 	// fabric is the operator-assigned name. It is not derived from the GID
@@ -2604,7 +2614,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\vleader_port\x18\x0e \x01(\x05R\n" +
 	"leaderPort\x12\x1d\n" +
 	"\n" +
-	"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf9\x03\n" +
+	"profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf0\x04\n" +
 	"\x03GPU\x12\x14\n" +
 	"\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" +
 	"\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" +
@@ -2623,7 +2633,11 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\rtemperature_c\x18\r \x01(\x05R\ftemperatureC\x12\x1d\n" +
 	"\n" +
 	"ecc_errors\x18\x0e \x01(\x05R\teccErrors\x12/\n" +
-	"\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\"[\n" +
+	"\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\x128\n" +
+	"\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x1a;\n" +
+	"\rNicPathsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"[\n" +
 	"\x10FabricAttachment\x12\x16\n" +
 	"\x06fabric\x18\x01 \x01(\tR\x06fabric\x12\x12\n" +
 	"\x04hcas\x18\x02 \x03(\tR\x04hcas\x12\x1b\n" +
@@ -2783,7 +2797,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
 }
 
 var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
+var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 30)
 var file_llmgw_v1_agent_proto_goTypes = []any{
 	(ShardStatus_State)(0),   // 0: llmgw.v1.ShardStatus.State
 	(*AgentMessage)(nil),     // 1: llmgw.v1.AgentMessage
@@ -2813,8 +2827,9 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
 	(*LogsReply)(nil),        // 25: llmgw.v1.LogsReply
 	nil,                      // 26: llmgw.v1.Register.LabelsEntry
 	nil,                      // 27: llmgw.v1.Register.EngineVersionsEntry
-	nil,                      // 28: llmgw.v1.Heartbeat.EngineVersionsEntry
-	nil,                      // 29: llmgw.v1.EngineProfile.EnvEntry
+	nil,                      // 28: llmgw.v1.GPU.NicPathsEntry
+	nil,                      // 29: llmgw.v1.Heartbeat.EngineVersionsEntry
+	nil,                      // 30: llmgw.v1.EngineProfile.EnvEntry
 }
 var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	3,  // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -2837,24 +2852,25 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	5,  // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
 	4,  // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
 	0,  // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
-	11, // 20: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
-	5,  // 21: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
-	9,  // 22: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
-	28, // 23: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
-	15, // 24: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
-	10, // 25: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
-	14, // 26: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
-	0,  // 27: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
-	17, // 28: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
-	19, // 29: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
-	29, // 30: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
-	1,  // 31: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
-	2,  // 32: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
-	32, // [32:33] is the sub-list for method output_type
-	31, // [31:32] is the sub-list for method input_type
-	31, // [31:31] is the sub-list for extension type_name
-	31, // [31:31] is the sub-list for extension extendee
-	0,  // [0:31] is the sub-list for field type_name
+	28, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
+	11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
+	5,  // 22: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
+	9,  // 23: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
+	29, // 24: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
+	15, // 25: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
+	10, // 26: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
+	14, // 27: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
+	0,  // 28: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
+	17, // 29: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
+	19, // 30: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
+	30, // 31: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
+	1,  // 32: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
+	2,  // 33: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
+	33, // [33:34] is the sub-list for method output_type
+	32, // [32:33] is the sub-list for method input_type
+	32, // [32:32] is the sub-list for extension type_name
+	32, // [32:32] is the sub-list for extension extendee
+	0,  // [0:32] is the sub-list for field type_name
 }
 
 func init() { file_llmgw_v1_agent_proto_init() }
@@ -2883,7 +2899,7 @@ func file_llmgw_v1_agent_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
 			NumEnums:      1,
-			NumMessages:   29,
+			NumMessages:   30,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

api/proto/llmgw/v1/agent.proto

@@ -207,6 +207,9 @@ message GPU {
   // Zero when the driver is too old to report it, which is not the same as
   // idle: the metric is simply absent rather than wrong.
   int32 utilization_percent = 15;
+  // nic_paths is how far each network adapter is from this card, by HCA
+  // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
+  map<string, string> nic_paths = 16;
 }
 
 message FabricAttachment {

internal/agent/discovery/discover.go

@@ -64,8 +64,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
 				inv.GPUs, inv.Source = gpus, "nvml"
 				break
 			}
-			log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
-				"NVLink topology will be unavailable", "err", err)
+			log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err)
 			fallthrough
 		default:
 			gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
@@ -74,6 +73,19 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
 			}
 			inv.GPUs, inv.Source = gpus, "nvidia-smi"
 		}
+		// NVLink and which adapter sits nearest each card, for either path:
+		// NVML knows NVLink but not the adapters, nvidia-smi's query knows
+		// neither. A failure costs the topology and nothing else — the cards
+		// are still cards.
+		if len(inv.GPUs) > 0 {
+			topo, err := DiscoverTopology(ctx, opts.NvidiaSMI)
+			if err != nil {
+				log.Warn("could not read the GPU topology; tensor-parallel groups "+
+					"will be chosen without NVLink", "err", err)
+			} else {
+				topo.Apply(inv.GPUs)
+			}
+		}
 	}
 
 	hcas, err := DiscoverIB(opts.IBRoot)

internal/agent/discovery/gpu.go

@@ -38,6 +38,10 @@ type GPU struct {
 	// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
 	// a TP group should stay inside one NVLink domain.
 	NVLinkPeers []int `json:"nvlink_peers,omitempty"`
+	// NICPaths is how far each network adapter is from this GPU, by HCA name:
+	// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
+	// which adapter a GPU's share of a multi-node job should go through.
+	NICPaths map[string]string `json:"nic_paths,omitempty"`
 	// Health carries anything that should keep the GPU out of placement.
 	Health GPUHealth `json:"health"`
 }

internal/agent/discovery/gpu_smi.go

@@ -40,8 +40,8 @@ const mandatorySMIFields = 10
 //
 // This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
 // would force the whole binary — server included — to be built with a C
-// toolchain. nvidia-smi gives everything except NVLink topology, which the
-// NVML build adds.
+// toolchain. The query gives everything except topology, which Discover
+// reads from `nvidia-smi topo -m` afterwards.
 func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
 	if binary == "" {
 		// Not configured and not installed is a node without an NVIDIA card,

internal/agent/discovery/topo.go

@@ -0,0 +1,182 @@
+package discovery
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Topology is what `nvidia-smi topo -m` says about how a node's GPUs and
+// network adapters are wired together.
+//
+// Read from nvidia-smi rather than NVML so the portable, cgo-free binary knows
+// it too. Without it that binary saw no NVLink at all: every GPU reported no
+// peers, and a tensor-parallel group was chosen by index on machines where
+// NVLink decides whether it runs at full speed.
+type Topology struct {
+	// NVLink is, per GPU, the GPUs it reaches over NVLink and how many links
+	// are bonded to each.
+	NVLink map[int]map[int]int
+	// NICPath is, per GPU, how far each network adapter is, by its HCA name:
+	// PIX, PXB, PHB, NODE or SYS, nearest first. A GPU should talk to the
+	// fabric through the adapter nearest it; one across the socket costs a
+	// trip over the processor interconnect on every transfer.
+	NICPath map[int]map[string]string
+	// NUMA is the NUMA node each GPU is attached to, where reported.
+	NUMA map[int]int
+}
+
+// PathRank orders the relations nvidia-smi reports, nearest first. Lower is
+// closer; an unknown relation ranks last.
+func PathRank(rel string) int {
+	switch strings.ToUpper(rel) {
+	case "PIX":
+		return 0
+	case "PXB":
+		return 1
+	case "PHB":
+		return 2
+	case "NODE":
+		return 3
+	case "SYS":
+		return 4
+	}
+	return 5
+}
+
+// DiscoverTopology runs `nvidia-smi topo -m`. A node without nvidia-smi has
+// no topology to report, which is not an error.
+func DiscoverTopology(ctx context.Context, binary string) (Topology, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return Topology{}, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "topo", "-m").Output()
+	if err != nil {
+		return Topology{}, fmt.Errorf("running %s topo -m: %w", binary, err)
+	}
+	return parseTopo(string(out))
+}
+
+var (
+	ansi       = regexp.MustCompile(`\x1b\[[0-9;]*m`)
+	nvLinks    = regexp.MustCompile(`^NV(\d+)$`)
+	nicLegend  = regexp.MustCompile(`^\s*(NIC\d+)\s*:\s*(\S+)\s*$`)
+	gpuColumn  = regexp.MustCompile(`^GPU(\d+)$`)
+	numaColumn = "NUMA Affinity"
+)
+
+// parseTopo reads the matrix. Columns are tab-separated; the header names
+// each one — GPUn, then the adapters, as NICn with a legend below on newer
+// drivers or by HCA name on older ones — then CPU and NUMA affinity.
+func parseTopo(out string) (Topology, error) {
+	out = ansi.ReplaceAllString(out, "")
+	lines := strings.Split(out, "\n")
+
+	legend := map[string]string{}
+	for _, l := range lines {
+		if m := nicLegend.FindStringSubmatch(l); m != nil {
+			legend[m[1]] = m[2]
+		}
+	}
+
+	var header []string
+	start := -1
+	for i, l := range lines {
+		cells := splitTabs(l)
+		if len(cells) > 1 && cells[0] == "" && gpuColumn.MatchString(cells[1]) {
+			header, start = cells, i
+			break
+		}
+	}
+	if start < 0 {
+		return Topology{}, fmt.Errorf("no GPU matrix in nvidia-smi topo output")
+	}
+
+	t := Topology{NVLink: map[int]map[int]int{}, NICPath: map[int]map[string]string{}, NUMA: map[int]int{}}
+	for _, l := range lines[start+1:] {
+		cells := splitTabs(l)
+		if len(cells) < 2 {
+			// The matrix ends at the first blank line, before the legend.
+			if strings.TrimSpace(l) == "" {
+				break
+			}
+			continue
+		}
+		m := gpuColumn.FindStringSubmatch(cells[0])
+		if m == nil {
+			continue // an adapter's row: its view of the GPUs is the same matrix
+		}
+		gpu, _ := strconv.Atoi(m[1])
+		for c := 1; c < len(cells) && c < len(header); c++ {
+			col, v := header[c], strings.TrimSpace(cells[c])
+			switch {
+			case gpuColumn.MatchString(col):
+				peer, _ := strconv.Atoi(gpuColumn.FindStringSubmatch(col)[1])
+				if n := nvLinks.FindStringSubmatch(v); n != nil && peer != gpu {
+					links, _ := strconv.Atoi(n[1])
+					if t.NVLink[gpu] == nil {
+						t.NVLink[gpu] = map[int]int{}
+					}
+					t.NVLink[gpu][peer] = links
+				}
+			case col == numaColumn:
+				if n, err := strconv.Atoi(v); err == nil {
+					t.NUMA[gpu] = n
+				}
+			case col == "CPU Affinity" || col == "GPU NUMA ID" || col == "":
+			default:
+				// An adapter: NICn named by the legend, or an HCA by name.
+				hca := col
+				if name, ok := legend[col]; ok {
+					hca = name
+				}
+				if PathRank(v) < 5 {
+					if t.NICPath[gpu] == nil {
+						t.NICPath[gpu] = map[string]string{}
+					}
+					t.NICPath[gpu][hca] = strings.ToUpper(v)
+				}
+			}
+		}
+	}
+	if len(t.NVLink) == 0 && len(t.NICPath) == 0 && len(t.NUMA) == 0 {
+		return t, fmt.Errorf("the nvidia-smi topo matrix had no rows")
+	}
+	return t, nil
+}
+
+func splitTabs(l string) []string {
+	l = strings.TrimRight(l, "\r")
+	cells := strings.Split(l, "\t")
+	for i := range cells {
+		cells[i] = strings.TrimSpace(cells[i])
+	}
+	return cells
+}
+
+// Apply puts the topology on the GPUs it describes. What a GPU already knows
+// — NVLink peers from NVML, which is the more exact source — is kept.
+func (t Topology) Apply(gpus []GPU) {
+	for i := range gpus {
+		g := &gpus[i]
+		if len(g.NVLinkPeers) == 0 {
+			for peer := range t.NVLink[g.Index] {
+				g.NVLinkPeers = append(g.NVLinkPeers, peer)
+			}
+			sort.Ints(g.NVLinkPeers)
+		}
+		if n, ok := t.NUMA[g.Index]; ok {
+			g.NUMANode = n
+		}
+		if paths := t.NICPath[g.Index]; len(paths) > 0 {
+			g.NICPaths = paths
+		}
+	}
+}

internal/agent/discovery/topo_test.go

@@ -0,0 +1,113 @@
+package discovery
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// row joins cells the way nvidia-smi prints them: tab-separated, each padded.
+func row(cells ...string) string { return strings.Join(cells, "\t") }
+
+// dgxH100 is `nvidia-smi topo -m` on an eight-GPU NVSwitch machine with two
+// adapters, as newer drivers print it: NICn columns and a legend naming them.
+var dgxH100 = strings.Join([]string{
+	row("\x1b[4m", "GPU0", "GPU1", "GPU2", "GPU3", "NIC0", "NIC1", "CPU Affinity", "NUMA Affinity", "GPU NUMA ID\x1b[0m"),
+	row("GPU0", " X ", "NV18", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
+	row("GPU1", "NV18", " X ", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
+	row("GPU2", "NV18", "NV18", " X ", "NV18", "SYS", "PXB", "56-111,168-223", "1", "N/A"),
+	row("GPU3", "NV18", "NV18", "NV18", " X ", "SYS", "PIX", "56-111,168-223", "1", "N/A"),
+	row("NIC0", "PXB", "PXB", "SYS", "SYS", " X ", "SYS"),
+	row("NIC1", "SYS", "SYS", "PXB", "PIX", "SYS", " X "),
+	"",
+	"Legend:",
+	"",
+	"  X    = Self",
+	"  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)",
+	"  NV#  = Connection traversing a bonded set of # NVLinks",
+	"",
+	"NIC Legend:",
+	"",
+	"  NIC0: mlx5_0",
+	"  NIC1: mlx5_1",
+	"",
+}, "\n")
+
+func TestNVLinkAndAdapterDistanceAreReadFromTopo(t *testing.T) {
+	topo, err := parseTopo(dgxH100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := topo.NVLink[0]; !reflect.DeepEqual(got, map[int]int{1: 18, 2: 18, 3: 18}) {
+		t.Errorf("GPU0's NVLink peers = %v", got)
+	}
+	// The legend turns NIC1 into the adapter's own name.
+	if got := topo.NICPath[3]; !reflect.DeepEqual(got, map[string]string{"mlx5_0": "SYS", "mlx5_1": "PIX"}) {
+		t.Errorf("GPU3's adapters = %v", got)
+	}
+	if topo.NUMA[2] != 1 || topo.NUMA[0] != 0 {
+		t.Errorf("NUMA = %v", topo.NUMA)
+	}
+
+	gpus := []GPU{{Index: 0}, {Index: 3}}
+	topo.Apply(gpus)
+	if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1, 2, 3}) || gpus[1].NUMANode != 1 ||
+		gpus[1].NICPaths["mlx5_1"] != "PIX" {
+		t.Errorf("applied = %+v", gpus)
+	}
+}
+
+func TestOlderDriversNameAdaptersInTheHeader(t *testing.T) {
+	// No legend: the columns are the adapters' own names, and NVLink between
+	// pairs only, as on an A100 PCIe box with bridges.
+	out := strings.Join([]string{
+		row("", "GPU0", "GPU1", "GPU2", "GPU3", "mlx5_0", "CPU Affinity", "NUMA Affinity"),
+		row("GPU0", " X ", "NV12", "SYS", "SYS", "NODE", "0-31", "0"),
+		row("GPU1", "NV12", " X ", "SYS", "SYS", "NODE", "0-31", "0"),
+		row("GPU2", "SYS", "SYS", " X ", "NV12", "SYS", "32-63", "1"),
+		row("GPU3", "SYS", "SYS", "NV12", " X ", "SYS", "32-63", "1"),
+		row("mlx5_0", "NODE", "NODE", "SYS", "SYS", " X "),
+		"",
+	}, "\n")
+	topo, err := parseTopo(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(topo.NVLink[2], map[int]int{3: 12}) || len(topo.NVLink[0]) != 1 {
+		t.Errorf("NVLink = %v", topo.NVLink)
+	}
+	if topo.NICPath[0]["mlx5_0"] != "NODE" {
+		t.Errorf("adapters = %v", topo.NICPath)
+	}
+}
+
+func TestAPCIeMachineHasNoNVLink(t *testing.T) {
+	out := strings.Join([]string{
+		row("", "GPU0", "GPU1", "CPU Affinity", "NUMA Affinity"),
+		row("GPU0", " X ", "PHB", "0-15", "0"),
+		row("GPU1", "PHB", " X ", "0-15", "0"),
+		"",
+	}, "\n")
+	topo, err := parseTopo(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(topo.NVLink) != 0 {
+		t.Errorf("NVLink on a PCIe machine: %v", topo.NVLink)
+	}
+}
+
+func TestNVMLPeersAreKeptOverTopo(t *testing.T) {
+	topo, _ := parseTopo(dgxH100)
+	gpus := []GPU{{Index: 0, NVLinkPeers: []int{1}}}
+	topo.Apply(gpus)
+	if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1}) {
+		t.Errorf("NVML's answer was replaced: %v", gpus[0].NVLinkPeers)
+	}
+}
+
+func TestSomethingThatIsNotAMatrixIsAnError(t *testing.T) {
+	if _, err := parseTopo("NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver."); err == nil {
+		t.Error("expected an error")
+	}
+}

internal/control/agentsvc/convert.go

@@ -3,6 +3,7 @@ package agentsvc
 
 import (
 	"fmt"
+	"maps"
 	"time"
 
 	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
@@ -59,6 +60,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
 			UtilizationPercent: int(g.GetUtilizationPercent()),
 			CUDAVersion:        g.GetCudaVersion(),
 			NVLinkPeers:        intSlice(g.GetNvlinkPeers()),
+			NICPaths:           maps.Clone(g.GetNicPaths()),
 			Health: discovery.GPUHealth{
 				ECCErrors:    int(g.GetEccErrors()),
 				TemperatureC: int(g.GetTemperatureC()),
@@ -112,6 +114,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
 			UtilizationPercent: int32(g.UtilizationPercent),
 			CudaVersion:        g.CUDAVersion,
 			NvlinkPeers:        int32Slice(g.NVLinkPeers),
+			NicPaths:           maps.Clone(g.NICPaths),
 			Problems:           g.Health.Problems,
 			TemperatureC:       int32(g.Health.TemperatureC),
 			EccErrors:          int32(g.Health.ECCErrors),