Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -365,6 +365,8 @@ type Agent struct {
|
||||
sm map[string]*discovery.SMInfo
|
||||
smErr map[string]string
|
||||
smAsked time.Time
|
||||
// nvlAsked is when the GPUs' NVLink fabric state was last read.
|
||||
nvlAsked time.Time
|
||||
// sms is every subnet manager on a fabric, asked through one port per
|
||||
// master, keyed by that port; smsErr says why a port could not ask.
|
||||
sms map[string][]discovery.SMRecord
|
||||
@@ -999,6 +1001,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
}
|
||||
hb.Load = a.sampleLoad()
|
||||
a.sampleIB(hb)
|
||||
a.sampleNVLink(hb)
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||
}
|
||||
|
||||
@@ -1752,3 +1755,45 @@ func (a *Agent) ensureDiags(ctx context.Context) {
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
||||
var (
|
||||
discoverGPUFabric = discovery.DiscoverGPUFabric
|
||||
discoverIMEX = discovery.DiscoverIMEX
|
||||
)
|
||||
|
||||
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
|
||||
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
|
||||
// for every heartbeat, and a fabric that was joined an hour ago rarely
|
||||
// changes in five seconds — but when a fabric manager restarts it does, for
|
||||
// every card at once.
|
||||
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
a.mu.Lock()
|
||||
gpus := slices.Clone(a.inv.GPUs)
|
||||
due := time.Since(a.nvlAsked) > time.Minute
|
||||
if due {
|
||||
a.nvlAsked = time.Now()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if len(gpus) == 0 || !due {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || len(fab) == 0 {
|
||||
return
|
||||
}
|
||||
discovery.ApplyFabric(gpus, fab)
|
||||
for _, g := range gpus {
|
||||
if g.Fabric == nil {
|
||||
continue
|
||||
}
|
||||
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
|
||||
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
|
||||
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
|
||||
})
|
||||
}
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
@@ -85,6 +85,13 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
} else {
|
||||
topo.Apply(inv.GPUs)
|
||||
}
|
||||
// Multi-node NVLink: which domain each card has joined.
|
||||
if fab, err := DiscoverGPUFabric(ctx, opts.NvidiaSMI); err != nil {
|
||||
log.Warn("could not read the GPUs' NVLink fabric state", "err", err)
|
||||
} else {
|
||||
ApplyFabric(inv.GPUs, fab)
|
||||
}
|
||||
inv.IMEX = DiscoverIMEX(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,9 @@ type GPU struct {
|
||||
// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
|
||||
// which adapter a GPU's share of a multi-node job should go through.
|
||||
NICPaths map[string]string `json:"nic_paths,omitempty"`
|
||||
// Fabric is the GPU's multi-node NVLink membership, nil where the
|
||||
// driver reports none.
|
||||
Fabric *GPUFabric `json:"fabric,omitempty"`
|
||||
// Health carries anything that should keep the GPU out of placement.
|
||||
Health GPUHealth `json:"health"`
|
||||
}
|
||||
@@ -121,6 +124,9 @@ type Inventory struct {
|
||||
// Storage is the disk behind those weights, one entry per filesystem.
|
||||
// Empty means the same thing it means for Weights: not asked.
|
||||
Storage []Storage `json:"storage,omitempty"`
|
||||
// IMEX is the service a multi-node NVLink domain needs, read where the
|
||||
// node has GPUs.
|
||||
IMEX IMEX `json:"imex"`
|
||||
}
|
||||
|
||||
// Classes returns the distinct GPU classes present, sorted.
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GPUFabric is a GPU's place in a multi-node NVLink domain, as the driver
|
||||
// reports it: the NVL72 rack, or the NVLink Switch system, it belongs to.
|
||||
type GPUFabric struct {
|
||||
// State is Completed once the fabric manager has joined the GPU to its
|
||||
// domain; In Progress, Not Started or Not Supported otherwise.
|
||||
State string `json:"state"`
|
||||
// Status is Success, or the reason the join failed.
|
||||
Status string `json:"status,omitempty"`
|
||||
// ClusterUUID and CliqueID name the domain: GPUs sharing both can reach
|
||||
// each other over NVLink whatever node they are in.
|
||||
ClusterUUID string `json:"cluster_uuid,omitempty"`
|
||||
CliqueID string `json:"clique_id,omitempty"`
|
||||
// Bandwidth is Full, or how the domain's NVLink is degraded.
|
||||
Bandwidth string `json:"bandwidth,omitempty"`
|
||||
}
|
||||
|
||||
// Domain is the NVLink domain the GPU is joined to, empty when it is in none
|
||||
// that crosses nodes: a card with no fabric, one not joined, or one whose
|
||||
// cluster is all zeros, which is what a single machine's NVSwitch reports.
|
||||
func (f *GPUFabric) Domain() string {
|
||||
if f == nil || !strings.EqualFold(f.State, "Completed") {
|
||||
return ""
|
||||
}
|
||||
u := strings.ToLower(strings.TrimSpace(f.ClusterUUID))
|
||||
if u == "" || strings.Trim(u, "0-") == "" {
|
||||
return ""
|
||||
}
|
||||
return u + "/" + f.CliqueID
|
||||
}
|
||||
|
||||
// ParseGPUFabric reads the Fabric sections of `nvidia-smi -q`, by the PCI
|
||||
// address each GPU's section starts with.
|
||||
func ParseGPUFabric(out string) map[string]GPUFabric {
|
||||
res := map[string]GPUFabric{}
|
||||
var bus string
|
||||
fabricIndent := -1
|
||||
var cur *GPUFabric
|
||||
flush := func() {
|
||||
if bus != "" && cur != nil && cur.State != "" {
|
||||
res[bus] = *cur
|
||||
}
|
||||
cur, fabricIndent = nil, -1
|
||||
}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
trim := strings.TrimSpace(line)
|
||||
indent := len(line) - len(strings.TrimLeft(line, " \t"))
|
||||
if strings.HasPrefix(line, "GPU ") && indent == 0 {
|
||||
flush()
|
||||
bus = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "GPU ")))
|
||||
continue
|
||||
}
|
||||
if trim == "Fabric" {
|
||||
flush()
|
||||
cur, fabricIndent = &GPUFabric{}, indent
|
||||
continue
|
||||
}
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
if trim != "" && indent <= fabricIndent {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
k, v, ok := strings.Cut(trim, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||||
if strings.EqualFold(v, "N/A") {
|
||||
v = ""
|
||||
}
|
||||
switch k {
|
||||
case "State":
|
||||
cur.State = v
|
||||
case "Status":
|
||||
cur.Status = v
|
||||
case "ClusterUUID", "Cluster UUID":
|
||||
cur.ClusterUUID = v
|
||||
case "CliqueId", "Clique ID":
|
||||
cur.CliqueID = v
|
||||
case "Bandwidth":
|
||||
cur.Bandwidth = v
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return res
|
||||
}
|
||||
|
||||
// DiscoverGPUFabric runs `nvidia-smi -q` for the Fabric sections. Nothing
|
||||
// is reported on a driver or a card without them.
|
||||
func DiscoverGPUFabric(ctx context.Context, binary string) (map[string]GPUFabric, error) {
|
||||
if binary == "" {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
binary = "nvidia-smi"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, binary, "-q").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseGPUFabric(string(out)), nil
|
||||
}
|
||||
|
||||
// ApplyFabric puts each GPU's fabric on it, matched by PCI address.
|
||||
func ApplyFabric(gpus []GPU, fab map[string]GPUFabric) {
|
||||
for i := range gpus {
|
||||
if f, ok := fab[strings.ToLower(gpus[i].PCIBusID)]; ok {
|
||||
f := f
|
||||
gpus[i].Fabric = &f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IMEX is the service that lets GPUs in different nodes of one NVLink
|
||||
// domain share memory: without it, multi-node NVLink carries nothing.
|
||||
type IMEX struct {
|
||||
// State is systemd's word for nvidia-imex: active, inactive, or empty
|
||||
// where it is not installed.
|
||||
State string `json:"state,omitempty"`
|
||||
// Channels is how many IMEX channels exist; a process needs one.
|
||||
Channels int `json:"channels"`
|
||||
}
|
||||
|
||||
// DiscoverIMEX reads the IMEX service and its channels.
|
||||
func DiscoverIMEX(ctx context.Context) IMEX {
|
||||
var m IMEX
|
||||
if entries, err := os.ReadDir("/dev/nvidia-caps-imex-channels"); err == nil {
|
||||
m.Channels = len(entries)
|
||||
}
|
||||
if _, err := exec.LookPath("systemctl"); err == nil {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "nvidia-imex").Output()
|
||||
m.State = strings.TrimSpace(string(out))
|
||||
if m.State == "inactive" {
|
||||
// "inactive" is also what systemd says of a unit that does not
|
||||
// exist; the binary says which.
|
||||
if _, err := exec.LookPath("nvidia-imex"); err != nil {
|
||||
if _, err := os.Stat(filepath.Join("/usr/bin", "nvidia-imex")); err != nil {
|
||||
m.State = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package discovery
|
||||
|
||||
import "testing"
|
||||
|
||||
// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to
|
||||
// one NVL72 domain, then a PCIe card that has no fabric at all.
|
||||
const smiQ = `
|
||||
==============NVSMI LOG==============
|
||||
|
||||
Timestamp : Mon Sep 29 10:00:00 2026
|
||||
Driver Version : 570.86.15
|
||||
Attached GPUs : 3
|
||||
GPU 00000008:01:00.0
|
||||
Product Name : NVIDIA GB200
|
||||
Minor Number : 0
|
||||
Fabric
|
||||
State : Completed
|
||||
Status : Success
|
||||
CliqueId : 32766
|
||||
ClusterUUID : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa
|
||||
Health
|
||||
Bandwidth : Full
|
||||
Processes : None
|
||||
|
||||
GPU 00000009:01:00.0
|
||||
Product Name : NVIDIA GB200
|
||||
Fabric
|
||||
State : In Progress
|
||||
Status : N/A
|
||||
CliqueId : N/A
|
||||
ClusterUUID : N/A
|
||||
Processes : None
|
||||
|
||||
GPU 0000000A:01:00.0
|
||||
Product Name : NVIDIA L40S
|
||||
Fabric
|
||||
State : N/A
|
||||
Status : N/A
|
||||
`
|
||||
|
||||
func TestAGPUsNVLinkDomainIsRead(t *testing.T) {
|
||||
fab := ParseGPUFabric(smiQ)
|
||||
a, ok := fab["00000008:01:00.0"]
|
||||
if !ok {
|
||||
t.Fatalf("fabric = %+v", fab)
|
||||
}
|
||||
if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" {
|
||||
t.Errorf("GB200 = %+v", a)
|
||||
}
|
||||
if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" {
|
||||
t.Errorf("domain = %q", got)
|
||||
}
|
||||
// Still joining: no domain yet, and said as such.
|
||||
b := fab["00000009:01:00.0"]
|
||||
if b.State != "In Progress" || b.Domain() != "" {
|
||||
t.Errorf("joining GPU = %+v", b)
|
||||
}
|
||||
// A card with no fabric reports nothing rather than an empty one.
|
||||
if _, ok := fab["0000000a:01:00.0"]; ok {
|
||||
t.Error("a card without a fabric was given one")
|
||||
}
|
||||
|
||||
gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}}
|
||||
ApplyFabric(gpus, fab)
|
||||
if gpus[0].Fabric == nil || gpus[1].Fabric != nil {
|
||||
t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) {
|
||||
// An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its
|
||||
// NVLink ends at the chassis.
|
||||
f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"}
|
||||
if f.Domain() != "" {
|
||||
t.Errorf("a single machine read as a domain: %q", f.Domain())
|
||||
}
|
||||
}
|
||||
@@ -71,8 +71,9 @@ type ShardSpec struct {
|
||||
Profile Profile
|
||||
|
||||
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||
// are carried so the agent can report them back after a restart, and are
|
||||
// not used when building a command.
|
||||
// are carried so the agent can report them back after a restart; Fabric
|
||||
// also says, with an "nvl:" prefix, that the replica is inside a
|
||||
// multi-node NVLink domain, which changes NCCL's environment.
|
||||
Pool string
|
||||
Fabric string
|
||||
// ProfileFP identifies the version of the profile this shard was started
|
||||
@@ -232,6 +233,14 @@ func baseEnv(s ShardSpec) map[string]string {
|
||||
env["NCCL_IB_DISABLE"] = "1"
|
||||
}
|
||||
|
||||
// Placed inside a multi-node NVLink domain: NCCL is to carry the
|
||||
// replica's collectives over NVLink between its nodes, which it only
|
||||
// does when told to, and needs IMEX for.
|
||||
if s.NNodes > 1 && strings.HasPrefix(s.Fabric, "nvl:") {
|
||||
env["NCCL_MNNVL_ENABLE"] = "1"
|
||||
env["NCCL_CUMEM_ENABLE"] = "1"
|
||||
}
|
||||
|
||||
if s.MgmtInterface != "" {
|
||||
env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
|
||||
env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
|
||||
|
||||
@@ -451,3 +451,15 @@ func TestQuantizationOf(t *testing.T) {
|
||||
t.Errorf("a dangling flag should yield nothing, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
|
||||
s := multiNodeSpec(0)
|
||||
s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7"
|
||||
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
|
||||
if env["NCCL_MNNVL_ENABLE"] != "1" {
|
||||
t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"])
|
||||
}
|
||||
if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" {
|
||||
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user