Find multi-node NVLink domains, place across them, and show them as racks

A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:07:59 +02:00
1 parent 5bbc619a31
commit 20479a2d5f
31 files changed
+1535 -131

No files matched your search

+45
View File
@@ -365,6 +365,8 @@ type Agent struct {
sm map[string]*discovery.SMInfo
smErr map[string]string
smAsked time.Time
// nvlAsked is when the GPUs' NVLink fabric state was last read.
nvlAsked time.Time
// sms is every subnet manager on a fabric, asked through one port per
// master, keyed by that port; smsErr says why a port could not ask.
sms map[string][]discovery.SMRecord
@@ -999,6 +1001,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
}
hb.Load = a.sampleLoad()
a.sampleIB(hb)
a.sampleNVLink(hb)
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
}
@@ -1752,3 +1755,45 @@ func (a *Agent) ensureDiags(ctx context.Context) {
"err", err)
}
}
// discoverGPUFabric and discoverIMEX are replaced in tests.
var (
discoverGPUFabric = discovery.DiscoverGPUFabric
discoverIMEX = discovery.DiscoverIMEX
)
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
// for every heartbeat, and a fabric that was joined an hour ago rarely
// changes in five seconds — but when a fabric manager restarts it does, for
// every card at once.
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
a.mu.Lock()
gpus := slices.Clone(a.inv.GPUs)
due := time.Since(a.nvlAsked) > time.Minute
if due {
a.nvlAsked = time.Now()
}
a.mu.Unlock()
if len(gpus) == 0 || !due {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
if err != nil || len(fab) == 0 {
return
}
discovery.ApplyFabric(gpus, fab)
for _, g := range gpus {
if g.Fabric == nil {
continue
}
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
})
}
m := discoverIMEX(ctx)
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
}
+7
View File
@@ -85,6 +85,13 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
} else {
topo.Apply(inv.GPUs)
}
// Multi-node NVLink: which domain each card has joined.
if fab, err := DiscoverGPUFabric(ctx, opts.NvidiaSMI); err != nil {
log.Warn("could not read the GPUs' NVLink fabric state", "err", err)
} else {
ApplyFabric(inv.GPUs, fab)
}
inv.IMEX = DiscoverIMEX(ctx)
}
}
+6
View File
@@ -42,6 +42,9 @@ type GPU struct {
// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
// which adapter a GPU's share of a multi-node job should go through.
NICPaths map[string]string `json:"nic_paths,omitempty"`
// Fabric is the GPU's multi-node NVLink membership, nil where the
// driver reports none.
Fabric *GPUFabric `json:"fabric,omitempty"`
// Health carries anything that should keep the GPU out of placement.
Health GPUHealth `json:"health"`
}
@@ -121,6 +124,9 @@ type Inventory struct {
// Storage is the disk behind those weights, one entry per filesystem.
// Empty means the same thing it means for Weights: not asked.
Storage []Storage `json:"storage,omitempty"`
// IMEX is the service a multi-node NVLink domain needs, read where the
// node has GPUs.
IMEX IMEX `json:"imex"`
}
// Classes returns the distinct GPU classes present, sorted.
+160
View File
@@ -0,0 +1,160 @@
package discovery
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// GPUFabric is a GPU's place in a multi-node NVLink domain, as the driver
// reports it: the NVL72 rack, or the NVLink Switch system, it belongs to.
type GPUFabric struct {
// State is Completed once the fabric manager has joined the GPU to its
// domain; In Progress, Not Started or Not Supported otherwise.
State string `json:"state"`
// Status is Success, or the reason the join failed.
Status string `json:"status,omitempty"`
// ClusterUUID and CliqueID name the domain: GPUs sharing both can reach
// each other over NVLink whatever node they are in.
ClusterUUID string `json:"cluster_uuid,omitempty"`
CliqueID string `json:"clique_id,omitempty"`
// Bandwidth is Full, or how the domain's NVLink is degraded.
Bandwidth string `json:"bandwidth,omitempty"`
}
// Domain is the NVLink domain the GPU is joined to, empty when it is in none
// that crosses nodes: a card with no fabric, one not joined, or one whose
// cluster is all zeros, which is what a single machine's NVSwitch reports.
func (f *GPUFabric) Domain() string {
if f == nil || !strings.EqualFold(f.State, "Completed") {
return ""
}
u := strings.ToLower(strings.TrimSpace(f.ClusterUUID))
if u == "" || strings.Trim(u, "0-") == "" {
return ""
}
return u + "/" + f.CliqueID
}
// ParseGPUFabric reads the Fabric sections of `nvidia-smi -q`, by the PCI
// address each GPU's section starts with.
func ParseGPUFabric(out string) map[string]GPUFabric {
res := map[string]GPUFabric{}
var bus string
fabricIndent := -1
var cur *GPUFabric
flush := func() {
if bus != "" && cur != nil && cur.State != "" {
res[bus] = *cur
}
cur, fabricIndent = nil, -1
}
for _, line := range strings.Split(out, "\n") {
trim := strings.TrimSpace(line)
indent := len(line) - len(strings.TrimLeft(line, " \t"))
if strings.HasPrefix(line, "GPU ") && indent == 0 {
flush()
bus = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "GPU ")))
continue
}
if trim == "Fabric" {
flush()
cur, fabricIndent = &GPUFabric{}, indent
continue
}
if cur == nil {
continue
}
if trim != "" && indent <= fabricIndent {
flush()
continue
}
k, v, ok := strings.Cut(trim, ":")
if !ok {
continue
}
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
if strings.EqualFold(v, "N/A") {
v = ""
}
switch k {
case "State":
cur.State = v
case "Status":
cur.Status = v
case "ClusterUUID", "Cluster UUID":
cur.ClusterUUID = v
case "CliqueId", "Clique ID":
cur.CliqueID = v
case "Bandwidth":
cur.Bandwidth = v
}
}
flush()
return res
}
// DiscoverGPUFabric runs `nvidia-smi -q` for the Fabric sections. Nothing
// is reported on a driver or a card without them.
func DiscoverGPUFabric(ctx context.Context, binary string) (map[string]GPUFabric, error) {
if binary == "" {
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, binary, "-q").Output()
if err != nil {
return nil, err
}
return ParseGPUFabric(string(out)), nil
}
// ApplyFabric puts each GPU's fabric on it, matched by PCI address.
func ApplyFabric(gpus []GPU, fab map[string]GPUFabric) {
for i := range gpus {
if f, ok := fab[strings.ToLower(gpus[i].PCIBusID)]; ok {
f := f
gpus[i].Fabric = &f
}
}
}
// IMEX is the service that lets GPUs in different nodes of one NVLink
// domain share memory: without it, multi-node NVLink carries nothing.
type IMEX struct {
// State is systemd's word for nvidia-imex: active, inactive, or empty
// where it is not installed.
State string `json:"state,omitempty"`
// Channels is how many IMEX channels exist; a process needs one.
Channels int `json:"channels"`
}
// DiscoverIMEX reads the IMEX service and its channels.
func DiscoverIMEX(ctx context.Context) IMEX {
var m IMEX
if entries, err := os.ReadDir("/dev/nvidia-caps-imex-channels"); err == nil {
m.Channels = len(entries)
}
if _, err := exec.LookPath("systemctl"); err == nil {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "nvidia-imex").Output()
m.State = strings.TrimSpace(string(out))
if m.State == "inactive" {
// "inactive" is also what systemd says of a unit that does not
// exist; the binary says which.
if _, err := exec.LookPath("nvidia-imex"); err != nil {
if _, err := os.Stat(filepath.Join("/usr/bin", "nvidia-imex")); err != nil {
m.State = ""
}
}
}
}
return m
}
@@ -0,0 +1,77 @@
package discovery
import "testing"
// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to
// one NVL72 domain, then a PCIe card that has no fabric at all.
const smiQ = `
==============NVSMI LOG==============
Timestamp : Mon Sep 29 10:00:00 2026
Driver Version : 570.86.15
Attached GPUs : 3
GPU 00000008:01:00.0
Product Name : NVIDIA GB200
Minor Number : 0
Fabric
State : Completed
Status : Success
CliqueId : 32766
ClusterUUID : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa
Health
Bandwidth : Full
Processes : None
GPU 00000009:01:00.0
Product Name : NVIDIA GB200
Fabric
State : In Progress
Status : N/A
CliqueId : N/A
ClusterUUID : N/A
Processes : None
GPU 0000000A:01:00.0
Product Name : NVIDIA L40S
Fabric
State : N/A
Status : N/A
`
func TestAGPUsNVLinkDomainIsRead(t *testing.T) {
fab := ParseGPUFabric(smiQ)
a, ok := fab["00000008:01:00.0"]
if !ok {
t.Fatalf("fabric = %+v", fab)
}
if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" {
t.Errorf("GB200 = %+v", a)
}
if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" {
t.Errorf("domain = %q", got)
}
// Still joining: no domain yet, and said as such.
b := fab["00000009:01:00.0"]
if b.State != "In Progress" || b.Domain() != "" {
t.Errorf("joining GPU = %+v", b)
}
// A card with no fabric reports nothing rather than an empty one.
if _, ok := fab["0000000a:01:00.0"]; ok {
t.Error("a card without a fabric was given one")
}
gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}}
ApplyFabric(gpus, fab)
if gpus[0].Fabric == nil || gpus[1].Fabric != nil {
t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric)
}
}
func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) {
// An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its
// NVLink ends at the chassis.
f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"}
if f.Domain() != "" {
t.Errorf("a single machine read as a domain: %q", f.Domain())
}
}
+11 -2
View File
@@ -71,8 +71,9 @@ type ShardSpec struct {
Profile Profile
// Pool and Fabric are control plane concepts the agent cannot derive. They
// are carried so the agent can report them back after a restart, and are
// not used when building a command.
// are carried so the agent can report them back after a restart; Fabric
// also says, with an "nvl:" prefix, that the replica is inside a
// multi-node NVLink domain, which changes NCCL's environment.
Pool string
Fabric string
// ProfileFP identifies the version of the profile this shard was started
@@ -232,6 +233,14 @@ func baseEnv(s ShardSpec) map[string]string {
env["NCCL_IB_DISABLE"] = "1"
}
// Placed inside a multi-node NVLink domain: NCCL is to carry the
// replica's collectives over NVLink between its nodes, which it only
// does when told to, and needs IMEX for.
if s.NNodes > 1 && strings.HasPrefix(s.Fabric, "nvl:") {
env["NCCL_MNNVL_ENABLE"] = "1"
env["NCCL_CUMEM_ENABLE"] = "1"
}
if s.MgmtInterface != "" {
env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
+12
View File
@@ -451,3 +451,15 @@ func TestQuantizationOf(t *testing.T) {
t.Errorf("a dangling flag should yield nothing, got %q", got)
}
}
func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
s := multiNodeSpec(0)
s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7"
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
if env["NCCL_MNNVL_ENABLE"] != "1" {
t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"])
}
if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" {
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
}
}