Read NVLink and GPU-to-adapter distance from nvidia-smi topo
The release binary is built without cgo, so without NVML, and NVML was the only source of NVLink peers: on every NVLink machine the portable agent reported none, and tensor-parallel groups were chosen by GPU index. Discovery now reads `nvidia-smi topo -m` after either path, for the NVLink peers (kept from NVML where it gave them), each card's NUMA node, and how far each network adapter is from it - PIX, PXB, PHB, NODE or SYS - which travels to the control plane as nic_paths for placement to use. Both the NICn-with-legend layout of newer drivers and the adapter-named columns of older ones are read. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
22bb098247
commit
bf2d171e89
8 files changed
+363
-30
No files matched your search
@@ -64,8 +64,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
inv.GPUs, inv.Source = gpus, "nvml"
|
||||
break
|
||||
}
|
||||
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
|
||||
"NVLink topology will be unavailable", "err", err)
|
||||
log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err)
|
||||
fallthrough
|
||||
default:
|
||||
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
|
||||
@@ -74,6 +73,19 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
}
|
||||
inv.GPUs, inv.Source = gpus, "nvidia-smi"
|
||||
}
|
||||
// NVLink and which adapter sits nearest each card, for either path:
|
||||
// NVML knows NVLink but not the adapters, nvidia-smi's query knows
|
||||
// neither. A failure costs the topology and nothing else — the cards
|
||||
// are still cards.
|
||||
if len(inv.GPUs) > 0 {
|
||||
topo, err := DiscoverTopology(ctx, opts.NvidiaSMI)
|
||||
if err != nil {
|
||||
log.Warn("could not read the GPU topology; tensor-parallel groups "+
|
||||
"will be chosen without NVLink", "err", err)
|
||||
} else {
|
||||
topo.Apply(inv.GPUs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hcas, err := DiscoverIB(opts.IBRoot)
|
||||
|
||||
@@ -38,6 +38,10 @@ type GPU struct {
|
||||
// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
|
||||
// a TP group should stay inside one NVLink domain.
|
||||
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
|
||||
// NICPaths is how far each network adapter is from this GPU, by HCA name:
|
||||
// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
|
||||
// which adapter a GPU's share of a multi-node job should go through.
|
||||
NICPaths map[string]string `json:"nic_paths,omitempty"`
|
||||
// Health carries anything that should keep the GPU out of placement.
|
||||
Health GPUHealth `json:"health"`
|
||||
}
|
||||
|
||||
@@ -40,8 +40,8 @@ const mandatorySMIFields = 10
|
||||
//
|
||||
// This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
|
||||
// would force the whole binary — server included — to be built with a C
|
||||
// toolchain. nvidia-smi gives everything except NVLink topology, which the
|
||||
// NVML build adds.
|
||||
// toolchain. The query gives everything except topology, which Discover
|
||||
// reads from `nvidia-smi topo -m` afterwards.
|
||||
func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
|
||||
if binary == "" {
|
||||
// Not configured and not installed is a node without an NVIDIA card,
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Topology is what `nvidia-smi topo -m` says about how a node's GPUs and
|
||||
// network adapters are wired together.
|
||||
//
|
||||
// Read from nvidia-smi rather than NVML so the portable, cgo-free binary knows
|
||||
// it too. Without it that binary saw no NVLink at all: every GPU reported no
|
||||
// peers, and a tensor-parallel group was chosen by index on machines where
|
||||
// NVLink decides whether it runs at full speed.
|
||||
type Topology struct {
|
||||
// NVLink is, per GPU, the GPUs it reaches over NVLink and how many links
|
||||
// are bonded to each.
|
||||
NVLink map[int]map[int]int
|
||||
// NICPath is, per GPU, how far each network adapter is, by its HCA name:
|
||||
// PIX, PXB, PHB, NODE or SYS, nearest first. A GPU should talk to the
|
||||
// fabric through the adapter nearest it; one across the socket costs a
|
||||
// trip over the processor interconnect on every transfer.
|
||||
NICPath map[int]map[string]string
|
||||
// NUMA is the NUMA node each GPU is attached to, where reported.
|
||||
NUMA map[int]int
|
||||
}
|
||||
|
||||
// PathRank orders the relations nvidia-smi reports, nearest first. Lower is
|
||||
// closer; an unknown relation ranks last.
|
||||
func PathRank(rel string) int {
|
||||
switch strings.ToUpper(rel) {
|
||||
case "PIX":
|
||||
return 0
|
||||
case "PXB":
|
||||
return 1
|
||||
case "PHB":
|
||||
return 2
|
||||
case "NODE":
|
||||
return 3
|
||||
case "SYS":
|
||||
return 4
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// DiscoverTopology runs `nvidia-smi topo -m`. A node without nvidia-smi has
|
||||
// no topology to report, which is not an error.
|
||||
func DiscoverTopology(ctx context.Context, binary string) (Topology, error) {
|
||||
if binary == "" {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return Topology{}, nil
|
||||
}
|
||||
binary = "nvidia-smi"
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, binary, "topo", "-m").Output()
|
||||
if err != nil {
|
||||
return Topology{}, fmt.Errorf("running %s topo -m: %w", binary, err)
|
||||
}
|
||||
return parseTopo(string(out))
|
||||
}
|
||||
|
||||
var (
|
||||
ansi = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||||
nvLinks = regexp.MustCompile(`^NV(\d+)$`)
|
||||
nicLegend = regexp.MustCompile(`^\s*(NIC\d+)\s*:\s*(\S+)\s*$`)
|
||||
gpuColumn = regexp.MustCompile(`^GPU(\d+)$`)
|
||||
numaColumn = "NUMA Affinity"
|
||||
)
|
||||
|
||||
// parseTopo reads the matrix. Columns are tab-separated; the header names
|
||||
// each one — GPUn, then the adapters, as NICn with a legend below on newer
|
||||
// drivers or by HCA name on older ones — then CPU and NUMA affinity.
|
||||
func parseTopo(out string) (Topology, error) {
|
||||
out = ansi.ReplaceAllString(out, "")
|
||||
lines := strings.Split(out, "\n")
|
||||
|
||||
legend := map[string]string{}
|
||||
for _, l := range lines {
|
||||
if m := nicLegend.FindStringSubmatch(l); m != nil {
|
||||
legend[m[1]] = m[2]
|
||||
}
|
||||
}
|
||||
|
||||
var header []string
|
||||
start := -1
|
||||
for i, l := range lines {
|
||||
cells := splitTabs(l)
|
||||
if len(cells) > 1 && cells[0] == "" && gpuColumn.MatchString(cells[1]) {
|
||||
header, start = cells, i
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 {
|
||||
return Topology{}, fmt.Errorf("no GPU matrix in nvidia-smi topo output")
|
||||
}
|
||||
|
||||
t := Topology{NVLink: map[int]map[int]int{}, NICPath: map[int]map[string]string{}, NUMA: map[int]int{}}
|
||||
for _, l := range lines[start+1:] {
|
||||
cells := splitTabs(l)
|
||||
if len(cells) < 2 {
|
||||
// The matrix ends at the first blank line, before the legend.
|
||||
if strings.TrimSpace(l) == "" {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
m := gpuColumn.FindStringSubmatch(cells[0])
|
||||
if m == nil {
|
||||
continue // an adapter's row: its view of the GPUs is the same matrix
|
||||
}
|
||||
gpu, _ := strconv.Atoi(m[1])
|
||||
for c := 1; c < len(cells) && c < len(header); c++ {
|
||||
col, v := header[c], strings.TrimSpace(cells[c])
|
||||
switch {
|
||||
case gpuColumn.MatchString(col):
|
||||
peer, _ := strconv.Atoi(gpuColumn.FindStringSubmatch(col)[1])
|
||||
if n := nvLinks.FindStringSubmatch(v); n != nil && peer != gpu {
|
||||
links, _ := strconv.Atoi(n[1])
|
||||
if t.NVLink[gpu] == nil {
|
||||
t.NVLink[gpu] = map[int]int{}
|
||||
}
|
||||
t.NVLink[gpu][peer] = links
|
||||
}
|
||||
case col == numaColumn:
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
t.NUMA[gpu] = n
|
||||
}
|
||||
case col == "CPU Affinity" || col == "GPU NUMA ID" || col == "":
|
||||
default:
|
||||
// An adapter: NICn named by the legend, or an HCA by name.
|
||||
hca := col
|
||||
if name, ok := legend[col]; ok {
|
||||
hca = name
|
||||
}
|
||||
if PathRank(v) < 5 {
|
||||
if t.NICPath[gpu] == nil {
|
||||
t.NICPath[gpu] = map[string]string{}
|
||||
}
|
||||
t.NICPath[gpu][hca] = strings.ToUpper(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(t.NVLink) == 0 && len(t.NICPath) == 0 && len(t.NUMA) == 0 {
|
||||
return t, fmt.Errorf("the nvidia-smi topo matrix had no rows")
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func splitTabs(l string) []string {
|
||||
l = strings.TrimRight(l, "\r")
|
||||
cells := strings.Split(l, "\t")
|
||||
for i := range cells {
|
||||
cells[i] = strings.TrimSpace(cells[i])
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
// Apply puts the topology on the GPUs it describes. What a GPU already knows
|
||||
// — NVLink peers from NVML, which is the more exact source — is kept.
|
||||
func (t Topology) Apply(gpus []GPU) {
|
||||
for i := range gpus {
|
||||
g := &gpus[i]
|
||||
if len(g.NVLinkPeers) == 0 {
|
||||
for peer := range t.NVLink[g.Index] {
|
||||
g.NVLinkPeers = append(g.NVLinkPeers, peer)
|
||||
}
|
||||
sort.Ints(g.NVLinkPeers)
|
||||
}
|
||||
if n, ok := t.NUMA[g.Index]; ok {
|
||||
g.NUMANode = n
|
||||
}
|
||||
if paths := t.NICPath[g.Index]; len(paths) > 0 {
|
||||
g.NICPaths = paths
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// row joins cells the way nvidia-smi prints them: tab-separated, each padded.
|
||||
func row(cells ...string) string { return strings.Join(cells, "\t") }
|
||||
|
||||
// dgxH100 is `nvidia-smi topo -m` on an eight-GPU NVSwitch machine with two
|
||||
// adapters, as newer drivers print it: NICn columns and a legend naming them.
|
||||
var dgxH100 = strings.Join([]string{
|
||||
row("\x1b[4m", "GPU0", "GPU1", "GPU2", "GPU3", "NIC0", "NIC1", "CPU Affinity", "NUMA Affinity", "GPU NUMA ID\x1b[0m"),
|
||||
row("GPU0", " X ", "NV18", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
|
||||
row("GPU1", "NV18", " X ", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
|
||||
row("GPU2", "NV18", "NV18", " X ", "NV18", "SYS", "PXB", "56-111,168-223", "1", "N/A"),
|
||||
row("GPU3", "NV18", "NV18", "NV18", " X ", "SYS", "PIX", "56-111,168-223", "1", "N/A"),
|
||||
row("NIC0", "PXB", "PXB", "SYS", "SYS", " X ", "SYS"),
|
||||
row("NIC1", "SYS", "SYS", "PXB", "PIX", "SYS", " X "),
|
||||
"",
|
||||
"Legend:",
|
||||
"",
|
||||
" X = Self",
|
||||
" SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)",
|
||||
" NV# = Connection traversing a bonded set of # NVLinks",
|
||||
"",
|
||||
"NIC Legend:",
|
||||
"",
|
||||
" NIC0: mlx5_0",
|
||||
" NIC1: mlx5_1",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
func TestNVLinkAndAdapterDistanceAreReadFromTopo(t *testing.T) {
|
||||
topo, err := parseTopo(dgxH100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := topo.NVLink[0]; !reflect.DeepEqual(got, map[int]int{1: 18, 2: 18, 3: 18}) {
|
||||
t.Errorf("GPU0's NVLink peers = %v", got)
|
||||
}
|
||||
// The legend turns NIC1 into the adapter's own name.
|
||||
if got := topo.NICPath[3]; !reflect.DeepEqual(got, map[string]string{"mlx5_0": "SYS", "mlx5_1": "PIX"}) {
|
||||
t.Errorf("GPU3's adapters = %v", got)
|
||||
}
|
||||
if topo.NUMA[2] != 1 || topo.NUMA[0] != 0 {
|
||||
t.Errorf("NUMA = %v", topo.NUMA)
|
||||
}
|
||||
|
||||
gpus := []GPU{{Index: 0}, {Index: 3}}
|
||||
topo.Apply(gpus)
|
||||
if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1, 2, 3}) || gpus[1].NUMANode != 1 ||
|
||||
gpus[1].NICPaths["mlx5_1"] != "PIX" {
|
||||
t.Errorf("applied = %+v", gpus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOlderDriversNameAdaptersInTheHeader(t *testing.T) {
|
||||
// No legend: the columns are the adapters' own names, and NVLink between
|
||||
// pairs only, as on an A100 PCIe box with bridges.
|
||||
out := strings.Join([]string{
|
||||
row("", "GPU0", "GPU1", "GPU2", "GPU3", "mlx5_0", "CPU Affinity", "NUMA Affinity"),
|
||||
row("GPU0", " X ", "NV12", "SYS", "SYS", "NODE", "0-31", "0"),
|
||||
row("GPU1", "NV12", " X ", "SYS", "SYS", "NODE", "0-31", "0"),
|
||||
row("GPU2", "SYS", "SYS", " X ", "NV12", "SYS", "32-63", "1"),
|
||||
row("GPU3", "SYS", "SYS", "NV12", " X ", "SYS", "32-63", "1"),
|
||||
row("mlx5_0", "NODE", "NODE", "SYS", "SYS", " X "),
|
||||
"",
|
||||
}, "\n")
|
||||
topo, err := parseTopo(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(topo.NVLink[2], map[int]int{3: 12}) || len(topo.NVLink[0]) != 1 {
|
||||
t.Errorf("NVLink = %v", topo.NVLink)
|
||||
}
|
||||
if topo.NICPath[0]["mlx5_0"] != "NODE" {
|
||||
t.Errorf("adapters = %v", topo.NICPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPCIeMachineHasNoNVLink(t *testing.T) {
|
||||
out := strings.Join([]string{
|
||||
row("", "GPU0", "GPU1", "CPU Affinity", "NUMA Affinity"),
|
||||
row("GPU0", " X ", "PHB", "0-15", "0"),
|
||||
row("GPU1", "PHB", " X ", "0-15", "0"),
|
||||
"",
|
||||
}, "\n")
|
||||
topo, err := parseTopo(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(topo.NVLink) != 0 {
|
||||
t.Errorf("NVLink on a PCIe machine: %v", topo.NVLink)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNVMLPeersAreKeptOverTopo(t *testing.T) {
|
||||
topo, _ := parseTopo(dgxH100)
|
||||
gpus := []GPU{{Index: 0, NVLinkPeers: []int{1}}}
|
||||
topo.Apply(gpus)
|
||||
if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1}) {
|
||||
t.Errorf("NVML's answer was replaced: %v", gpus[0].NVLinkPeers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSomethingThatIsNotAMatrixIsAnError(t *testing.T) {
|
||||
if _, err := parseTopo("NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver."); err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package agentsvc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -59,6 +60,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
UtilizationPercent: int(g.GetUtilizationPercent()),
|
||||
CUDAVersion: g.GetCudaVersion(),
|
||||
NVLinkPeers: intSlice(g.GetNvlinkPeers()),
|
||||
NICPaths: maps.Clone(g.GetNicPaths()),
|
||||
Health: discovery.GPUHealth{
|
||||
ECCErrors: int(g.GetEccErrors()),
|
||||
TemperatureC: int(g.GetTemperatureC()),
|
||||
@@ -112,6 +114,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
UtilizationPercent: int32(g.UtilizationPercent),
|
||||
CudaVersion: g.CUDAVersion,
|
||||
NvlinkPeers: int32Slice(g.NVLinkPeers),
|
||||
NicPaths: maps.Clone(g.NICPaths),
|
||||
Problems: g.Health.Problems,
|
||||
TemperatureC: int32(g.Health.TemperatureC),
|
||||
EccErrors: int32(g.Health.ECCErrors),
|
||||
|
||||
Reference in new issue
Block a user