From bf2d171e8941bbff7cbbfba078a5ceb18c388f9e Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 00:09:26 +0200 Subject: [PATCH] Read NVLink and GPU-to-adapter distance from nvidia-smi topo The release binary is built without cgo, so without NVML, and NVML was the only source of NVLink peers: on every NVLink machine the portable agent reported none, and tensor-parallel groups were chosen by GPU index. Discovery now reads `nvidia-smi topo -m` after either path, for the NVLink peers (kept from NVML where it gave them), each card's NUMA node, and how far each network adapter is from it - PIX, PXB, PHB, NODE or SYS - which travels to the control plane as nic_paths for placement to use. Both the NICn-with-legend layout of newer drivers and the adapter-named columns of older ones are read. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 68 ++++++---- api/proto/llmgw/v1/agent.proto | 3 + internal/agent/discovery/discover.go | 16 ++- internal/agent/discovery/gpu.go | 4 + internal/agent/discovery/gpu_smi.go | 4 +- internal/agent/discovery/topo.go | 182 ++++++++++++++++++++++++++ internal/agent/discovery/topo_test.go | 113 ++++++++++++++++ internal/control/agentsvc/convert.go | 3 + 8 files changed, 363 insertions(+), 30 deletions(-) create mode 100644 internal/agent/discovery/topo.go create mode 100644 internal/agent/discovery/topo_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 5265e854..67dc12c0 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -933,8 +933,11 @@ type GPU struct { // Zero when the driver is too old to report it, which is not the same as // idle: the metric is simply absent rather than wrong. UtilizationPercent int32 `protobuf:"varint,15,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // nic_paths is how far each network adapter is from this card, by HCA + // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo. + NicPaths map[string]string `protobuf:"bytes,16,rep,name=nic_paths,json=nicPaths,proto3" json:"nic_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPU) Reset() { @@ -1072,6 +1075,13 @@ func (x *GPU) GetUtilizationPercent() int32 { return 0 } +func (x *GPU) GetNicPaths() map[string]string { + if x != nil { + return x.NicPaths + } + return nil +} + type FabricAttachment struct { state protoimpl.MessageState `protogen:"open.v1"` // fabric is the operator-assigned name. It is not derived from the GID @@ -2604,7 +2614,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vleader_port\x18\x0e \x01(\x05R\n" + "leaderPort\x12\x1d\n" + "\n" + - "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf9\x03\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf0\x04\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -2623,7 +2633,11 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\rtemperature_c\x18\r \x01(\x05R\ftemperatureC\x12\x1d\n" + "\n" + "ecc_errors\x18\x0e \x01(\x05R\teccErrors\x12/\n" + - "\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\"[\n" + + "\x13utilization_percent\x18\x0f \x01(\x05R\x12utilizationPercent\x128\n" + + "\tnic_paths\x18\x10 \x03(\v2\x1b.llmgw.v1.GPU.NicPathsEntryR\bnicPaths\x1a;\n" + + "\rNicPathsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"[\n" + "\x10FabricAttachment\x12\x16\n" + "\x06fabric\x18\x01 \x01(\tR\x06fabric\x12\x12\n" + "\x04hcas\x18\x02 \x03(\tR\x04hcas\x12\x1b\n" + @@ -2783,7 +2797,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 30) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -2813,8 +2827,9 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*LogsReply)(nil), // 25: llmgw.v1.LogsReply nil, // 26: llmgw.v1.Register.LabelsEntry nil, // 27: llmgw.v1.Register.EngineVersionsEntry - nil, // 28: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 29: llmgw.v1.EngineProfile.EnvEntry + nil, // 28: llmgw.v1.GPU.NicPathsEntry + nil, // 29: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 30: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -2837,24 +2852,25 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 11, // 20: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 5, // 21: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 22: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 28, // 23: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 15, // 24: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 25: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 14, // 26: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 27: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 17, // 28: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 19, // 29: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 29, // 30: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 31: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 32: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 32, // [32:33] is the sub-list for method output_type - 31, // [31:32] is the sub-list for method input_type - 31, // [31:31] is the sub-list for extension type_name - 31, // [31:31] is the sub-list for extension extendee - 0, // [0:31] is the sub-list for field type_name + 28, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 5, // 22: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 23: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 29, // 24: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 15, // 25: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 26: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 14, // 27: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 28: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 17, // 29: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 19, // 30: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 30, // 31: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 32: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 33: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 33, // [33:34] is the sub-list for method output_type + 32, // [32:33] is the sub-list for method input_type + 32, // [32:32] is the sub-list for extension type_name + 32, // [32:32] is the sub-list for extension extendee + 0, // [0:32] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -2883,7 +2899,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 29, + NumMessages: 30, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 908a3fdd..00bbaf29 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -207,6 +207,9 @@ message GPU { // Zero when the driver is too old to report it, which is not the same as // idle: the metric is simply absent rather than wrong. int32 utilization_percent = 15; + // nic_paths is how far each network adapter is from this card, by HCA + // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo. + map nic_paths = 16; } message FabricAttachment { diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 6e72ad26..b459e88f 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -64,8 +64,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e inv.GPUs, inv.Source = gpus, "nvml" break } - log.Warn("NVML discovery failed, falling back to nvidia-smi; "+ - "NVLink topology will be unavailable", "err", err) + log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err) fallthrough default: gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI) @@ -74,6 +73,19 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e } inv.GPUs, inv.Source = gpus, "nvidia-smi" } + // NVLink and which adapter sits nearest each card, for either path: + // NVML knows NVLink but not the adapters, nvidia-smi's query knows + // neither. A failure costs the topology and nothing else — the cards + // are still cards. + if len(inv.GPUs) > 0 { + topo, err := DiscoverTopology(ctx, opts.NvidiaSMI) + if err != nil { + log.Warn("could not read the GPU topology; tensor-parallel groups "+ + "will be chosen without NVLink", "err", err) + } else { + topo.Apply(inv.GPUs) + } + } } hcas, err := DiscoverIB(opts.IBRoot) diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index 4d475bf1..cd335159 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -38,6 +38,10 @@ type GPU struct { // NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node // a TP group should stay inside one NVLink domain. NVLinkPeers []int `json:"nvlink_peers,omitempty"` + // NICPaths is how far each network adapter is from this GPU, by HCA name: + // PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides + // which adapter a GPU's share of a multi-node job should go through. + NICPaths map[string]string `json:"nic_paths,omitempty"` // Health carries anything that should keep the GPU out of placement. Health GPUHealth `json:"health"` } diff --git a/internal/agent/discovery/gpu_smi.go b/internal/agent/discovery/gpu_smi.go index ed011726..f04c7774 100644 --- a/internal/agent/discovery/gpu_smi.go +++ b/internal/agent/discovery/gpu_smi.go @@ -40,8 +40,8 @@ const mandatorySMIFields = 10 // // This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which // would force the whole binary — server included — to be built with a C -// toolchain. nvidia-smi gives everything except NVLink topology, which the -// NVML build adds. +// toolchain. The query gives everything except topology, which Discover +// reads from `nvidia-smi topo -m` afterwards. func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { if binary == "" { // Not configured and not installed is a node without an NVIDIA card, diff --git a/internal/agent/discovery/topo.go b/internal/agent/discovery/topo.go new file mode 100644 index 00000000..d2db624f --- /dev/null +++ b/internal/agent/discovery/topo.go @@ -0,0 +1,182 @@ +package discovery + +import ( + "context" + "fmt" + "os/exec" + "regexp" + "sort" + "strconv" + "strings" +) + +// Topology is what `nvidia-smi topo -m` says about how a node's GPUs and +// network adapters are wired together. +// +// Read from nvidia-smi rather than NVML so the portable, cgo-free binary knows +// it too. Without it that binary saw no NVLink at all: every GPU reported no +// peers, and a tensor-parallel group was chosen by index on machines where +// NVLink decides whether it runs at full speed. +type Topology struct { + // NVLink is, per GPU, the GPUs it reaches over NVLink and how many links + // are bonded to each. + NVLink map[int]map[int]int + // NICPath is, per GPU, how far each network adapter is, by its HCA name: + // PIX, PXB, PHB, NODE or SYS, nearest first. A GPU should talk to the + // fabric through the adapter nearest it; one across the socket costs a + // trip over the processor interconnect on every transfer. + NICPath map[int]map[string]string + // NUMA is the NUMA node each GPU is attached to, where reported. + NUMA map[int]int +} + +// PathRank orders the relations nvidia-smi reports, nearest first. Lower is +// closer; an unknown relation ranks last. +func PathRank(rel string) int { + switch strings.ToUpper(rel) { + case "PIX": + return 0 + case "PXB": + return 1 + case "PHB": + return 2 + case "NODE": + return 3 + case "SYS": + return 4 + } + return 5 +} + +// DiscoverTopology runs `nvidia-smi topo -m`. A node without nvidia-smi has +// no topology to report, which is not an error. +func DiscoverTopology(ctx context.Context, binary string) (Topology, error) { + if binary == "" { + if _, err := exec.LookPath("nvidia-smi"); err != nil { + return Topology{}, nil + } + binary = "nvidia-smi" + } + out, err := exec.CommandContext(ctx, binary, "topo", "-m").Output() + if err != nil { + return Topology{}, fmt.Errorf("running %s topo -m: %w", binary, err) + } + return parseTopo(string(out)) +} + +var ( + ansi = regexp.MustCompile(`\x1b\[[0-9;]*m`) + nvLinks = regexp.MustCompile(`^NV(\d+)$`) + nicLegend = regexp.MustCompile(`^\s*(NIC\d+)\s*:\s*(\S+)\s*$`) + gpuColumn = regexp.MustCompile(`^GPU(\d+)$`) + numaColumn = "NUMA Affinity" +) + +// parseTopo reads the matrix. Columns are tab-separated; the header names +// each one — GPUn, then the adapters, as NICn with a legend below on newer +// drivers or by HCA name on older ones — then CPU and NUMA affinity. +func parseTopo(out string) (Topology, error) { + out = ansi.ReplaceAllString(out, "") + lines := strings.Split(out, "\n") + + legend := map[string]string{} + for _, l := range lines { + if m := nicLegend.FindStringSubmatch(l); m != nil { + legend[m[1]] = m[2] + } + } + + var header []string + start := -1 + for i, l := range lines { + cells := splitTabs(l) + if len(cells) > 1 && cells[0] == "" && gpuColumn.MatchString(cells[1]) { + header, start = cells, i + break + } + } + if start < 0 { + return Topology{}, fmt.Errorf("no GPU matrix in nvidia-smi topo output") + } + + t := Topology{NVLink: map[int]map[int]int{}, NICPath: map[int]map[string]string{}, NUMA: map[int]int{}} + for _, l := range lines[start+1:] { + cells := splitTabs(l) + if len(cells) < 2 { + // The matrix ends at the first blank line, before the legend. + if strings.TrimSpace(l) == "" { + break + } + continue + } + m := gpuColumn.FindStringSubmatch(cells[0]) + if m == nil { + continue // an adapter's row: its view of the GPUs is the same matrix + } + gpu, _ := strconv.Atoi(m[1]) + for c := 1; c < len(cells) && c < len(header); c++ { + col, v := header[c], strings.TrimSpace(cells[c]) + switch { + case gpuColumn.MatchString(col): + peer, _ := strconv.Atoi(gpuColumn.FindStringSubmatch(col)[1]) + if n := nvLinks.FindStringSubmatch(v); n != nil && peer != gpu { + links, _ := strconv.Atoi(n[1]) + if t.NVLink[gpu] == nil { + t.NVLink[gpu] = map[int]int{} + } + t.NVLink[gpu][peer] = links + } + case col == numaColumn: + if n, err := strconv.Atoi(v); err == nil { + t.NUMA[gpu] = n + } + case col == "CPU Affinity" || col == "GPU NUMA ID" || col == "": + default: + // An adapter: NICn named by the legend, or an HCA by name. + hca := col + if name, ok := legend[col]; ok { + hca = name + } + if PathRank(v) < 5 { + if t.NICPath[gpu] == nil { + t.NICPath[gpu] = map[string]string{} + } + t.NICPath[gpu][hca] = strings.ToUpper(v) + } + } + } + } + if len(t.NVLink) == 0 && len(t.NICPath) == 0 && len(t.NUMA) == 0 { + return t, fmt.Errorf("the nvidia-smi topo matrix had no rows") + } + return t, nil +} + +func splitTabs(l string) []string { + l = strings.TrimRight(l, "\r") + cells := strings.Split(l, "\t") + for i := range cells { + cells[i] = strings.TrimSpace(cells[i]) + } + return cells +} + +// Apply puts the topology on the GPUs it describes. What a GPU already knows +// — NVLink peers from NVML, which is the more exact source — is kept. +func (t Topology) Apply(gpus []GPU) { + for i := range gpus { + g := &gpus[i] + if len(g.NVLinkPeers) == 0 { + for peer := range t.NVLink[g.Index] { + g.NVLinkPeers = append(g.NVLinkPeers, peer) + } + sort.Ints(g.NVLinkPeers) + } + if n, ok := t.NUMA[g.Index]; ok { + g.NUMANode = n + } + if paths := t.NICPath[g.Index]; len(paths) > 0 { + g.NICPaths = paths + } + } +} diff --git a/internal/agent/discovery/topo_test.go b/internal/agent/discovery/topo_test.go new file mode 100644 index 00000000..fe2420f1 --- /dev/null +++ b/internal/agent/discovery/topo_test.go @@ -0,0 +1,113 @@ +package discovery + +import ( + "reflect" + "strings" + "testing" +) + +// row joins cells the way nvidia-smi prints them: tab-separated, each padded. +func row(cells ...string) string { return strings.Join(cells, "\t") } + +// dgxH100 is `nvidia-smi topo -m` on an eight-GPU NVSwitch machine with two +// adapters, as newer drivers print it: NICn columns and a legend naming them. +var dgxH100 = strings.Join([]string{ + row("\x1b[4m", "GPU0", "GPU1", "GPU2", "GPU3", "NIC0", "NIC1", "CPU Affinity", "NUMA Affinity", "GPU NUMA ID\x1b[0m"), + row("GPU0", " X ", "NV18", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"), + row("GPU1", "NV18", " X ", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"), + row("GPU2", "NV18", "NV18", " X ", "NV18", "SYS", "PXB", "56-111,168-223", "1", "N/A"), + row("GPU3", "NV18", "NV18", "NV18", " X ", "SYS", "PIX", "56-111,168-223", "1", "N/A"), + row("NIC0", "PXB", "PXB", "SYS", "SYS", " X ", "SYS"), + row("NIC1", "SYS", "SYS", "PXB", "PIX", "SYS", " X "), + "", + "Legend:", + "", + " X = Self", + " SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)", + " NV# = Connection traversing a bonded set of # NVLinks", + "", + "NIC Legend:", + "", + " NIC0: mlx5_0", + " NIC1: mlx5_1", + "", +}, "\n") + +func TestNVLinkAndAdapterDistanceAreReadFromTopo(t *testing.T) { + topo, err := parseTopo(dgxH100) + if err != nil { + t.Fatal(err) + } + if got := topo.NVLink[0]; !reflect.DeepEqual(got, map[int]int{1: 18, 2: 18, 3: 18}) { + t.Errorf("GPU0's NVLink peers = %v", got) + } + // The legend turns NIC1 into the adapter's own name. + if got := topo.NICPath[3]; !reflect.DeepEqual(got, map[string]string{"mlx5_0": "SYS", "mlx5_1": "PIX"}) { + t.Errorf("GPU3's adapters = %v", got) + } + if topo.NUMA[2] != 1 || topo.NUMA[0] != 0 { + t.Errorf("NUMA = %v", topo.NUMA) + } + + gpus := []GPU{{Index: 0}, {Index: 3}} + topo.Apply(gpus) + if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1, 2, 3}) || gpus[1].NUMANode != 1 || + gpus[1].NICPaths["mlx5_1"] != "PIX" { + t.Errorf("applied = %+v", gpus) + } +} + +func TestOlderDriversNameAdaptersInTheHeader(t *testing.T) { + // No legend: the columns are the adapters' own names, and NVLink between + // pairs only, as on an A100 PCIe box with bridges. + out := strings.Join([]string{ + row("", "GPU0", "GPU1", "GPU2", "GPU3", "mlx5_0", "CPU Affinity", "NUMA Affinity"), + row("GPU0", " X ", "NV12", "SYS", "SYS", "NODE", "0-31", "0"), + row("GPU1", "NV12", " X ", "SYS", "SYS", "NODE", "0-31", "0"), + row("GPU2", "SYS", "SYS", " X ", "NV12", "SYS", "32-63", "1"), + row("GPU3", "SYS", "SYS", "NV12", " X ", "SYS", "32-63", "1"), + row("mlx5_0", "NODE", "NODE", "SYS", "SYS", " X "), + "", + }, "\n") + topo, err := parseTopo(out) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(topo.NVLink[2], map[int]int{3: 12}) || len(topo.NVLink[0]) != 1 { + t.Errorf("NVLink = %v", topo.NVLink) + } + if topo.NICPath[0]["mlx5_0"] != "NODE" { + t.Errorf("adapters = %v", topo.NICPath) + } +} + +func TestAPCIeMachineHasNoNVLink(t *testing.T) { + out := strings.Join([]string{ + row("", "GPU0", "GPU1", "CPU Affinity", "NUMA Affinity"), + row("GPU0", " X ", "PHB", "0-15", "0"), + row("GPU1", "PHB", " X ", "0-15", "0"), + "", + }, "\n") + topo, err := parseTopo(out) + if err != nil { + t.Fatal(err) + } + if len(topo.NVLink) != 0 { + t.Errorf("NVLink on a PCIe machine: %v", topo.NVLink) + } +} + +func TestNVMLPeersAreKeptOverTopo(t *testing.T) { + topo, _ := parseTopo(dgxH100) + gpus := []GPU{{Index: 0, NVLinkPeers: []int{1}}} + topo.Apply(gpus) + if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1}) { + t.Errorf("NVML's answer was replaced: %v", gpus[0].NVLinkPeers) + } +} + +func TestSomethingThatIsNotAMatrixIsAnError(t *testing.T) { + if _, err := parseTopo("NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver."); err == nil { + t.Error("expected an error") + } +} diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 0fe0e1c4..0d1c0f04 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -3,6 +3,7 @@ package agentsvc import ( "fmt" + "maps" "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" @@ -59,6 +60,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { UtilizationPercent: int(g.GetUtilizationPercent()), CUDAVersion: g.GetCudaVersion(), NVLinkPeers: intSlice(g.GetNvlinkPeers()), + NICPaths: maps.Clone(g.GetNicPaths()), Health: discovery.GPUHealth{ ECCErrors: int(g.GetEccErrors()), TemperatureC: int(g.GetTemperatureC()), @@ -112,6 +114,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, UtilizationPercent: int32(g.UtilizationPercent), CudaVersion: g.CUDAVersion, NvlinkPeers: int32Slice(g.NVLinkPeers), + NicPaths: maps.Clone(g.NICPaths), Problems: g.Health.Problems, TemperatureC: int32(g.Health.TemperatureC), EccErrors: int32(g.Health.ECCErrors),