Ilker ManapandClaude Opus 5.5
20479a2d5f
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
78 lines
2.7 KiB
Go
78 lines
2.7 KiB
Go
package discovery
|
|
|
|
import "testing"
|
|
|
|
// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to
|
|
// one NVL72 domain, then a PCIe card that has no fabric at all.
|
|
const smiQ = `
|
|
==============NVSMI LOG==============
|
|
|
|
Timestamp : Mon Sep 29 10:00:00 2026
|
|
Driver Version : 570.86.15
|
|
Attached GPUs : 3
|
|
GPU 00000008:01:00.0
|
|
Product Name : NVIDIA GB200
|
|
Minor Number : 0
|
|
Fabric
|
|
State : Completed
|
|
Status : Success
|
|
CliqueId : 32766
|
|
ClusterUUID : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa
|
|
Health
|
|
Bandwidth : Full
|
|
Processes : None
|
|
|
|
GPU 00000009:01:00.0
|
|
Product Name : NVIDIA GB200
|
|
Fabric
|
|
State : In Progress
|
|
Status : N/A
|
|
CliqueId : N/A
|
|
ClusterUUID : N/A
|
|
Processes : None
|
|
|
|
GPU 0000000A:01:00.0
|
|
Product Name : NVIDIA L40S
|
|
Fabric
|
|
State : N/A
|
|
Status : N/A
|
|
`
|
|
|
|
func TestAGPUsNVLinkDomainIsRead(t *testing.T) {
|
|
fab := ParseGPUFabric(smiQ)
|
|
a, ok := fab["00000008:01:00.0"]
|
|
if !ok {
|
|
t.Fatalf("fabric = %+v", fab)
|
|
}
|
|
if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" {
|
|
t.Errorf("GB200 = %+v", a)
|
|
}
|
|
if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" {
|
|
t.Errorf("domain = %q", got)
|
|
}
|
|
// Still joining: no domain yet, and said as such.
|
|
b := fab["00000009:01:00.0"]
|
|
if b.State != "In Progress" || b.Domain() != "" {
|
|
t.Errorf("joining GPU = %+v", b)
|
|
}
|
|
// A card with no fabric reports nothing rather than an empty one.
|
|
if _, ok := fab["0000000a:01:00.0"]; ok {
|
|
t.Error("a card without a fabric was given one")
|
|
}
|
|
|
|
gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}}
|
|
ApplyFabric(gpus, fab)
|
|
if gpus[0].Fabric == nil || gpus[1].Fabric != nil {
|
|
t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric)
|
|
}
|
|
}
|
|
|
|
func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) {
|
|
// An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its
|
|
// NVLink ends at the chassis.
|
|
f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"}
|
|
if f.Domain() != "" {
|
|
t.Errorf("a single machine read as a domain: %q", f.Domain())
|
|
}
|
|
}
|