Find multi-node NVLink domains, place across them, and show them as racks

A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:07:59 +02:00
1 parent 5bbc619a31
commit 20479a2d5f
31 files changed
+1535 -131

No files matched your search

+68 -1
View File
@@ -239,13 +239,20 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
var rejections []Rejection
fabrics := s.inv.Fabrics()
if len(fabrics) == 0 {
if len(fabrics) == 0 && !s.anyNVLinkDomain() {
return nil, []Rejection{{
Profile: p.Name,
Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes),
}}
}
// A multi-node NVLink domain first: a replica inside one runs tensor
// parallelism across its nodes at NVLink speed, which no InfiniBand
// fabric matches.
if pl := s.placeOnNVLink(p, avoid, &rejections); pl != nil {
return pl, rejections
}
canon := inventory.CanonicalSMs(s.inv.Nodes())
for _, fabric := range fabrics {
all := s.eligibleNodes(p, fabric, avoid, &rejections)
@@ -262,6 +269,66 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
return nil, rejections
}
// NVLinkFabric is how a placement names a multi-node NVLink domain in its
// Fabric, so the shards on it are told to use it.
const NVLinkFabric = "nvl:"
func (s *Scheduler) anyNVLinkDomain() bool {
for _, n := range s.inv.Nodes() {
if n.NVLinkDomain() != "" {
return true
}
}
return false
}
// placeOnNVLink forms a replica inside one multi-node NVLink domain, from
// nodes whose every card is joined to it and whose IMEX is up: a node
// missing either would leave the replica a card, or its memory, short.
func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejection) *Placement {
var none []Rejection
cands := s.eligibleNodes(p, "", avoid, &none)
by := map[string][]candidate{}
for _, c := range cands {
d := c.node.NVLinkDomain()
if d == "" {
continue
}
if c.node.IMEX.State != "active" || c.node.IMEX.Channels == 0 {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: NVLinkFabric + shortDomain(d), Node: c.node.ID,
Reason: "in an NVLink domain, but nvidia-imex is not up"})
continue
}
by[d] = append(by[d], c)
}
keys := make([]string, 0, len(by))
for k := range by {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if len(by[keys[i]]) != len(by[keys[j]]) {
return len(by[keys[i]]) > len(by[keys[j]])
}
return keys[i] < keys[j]
})
for _, d := range keys {
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
return pl
}
}
return nil
}
// shortDomain is an NVLink domain as a pool name carries it.
func shortDomain(d string) string {
uuid, clique, _ := strings.Cut(d, "/")
if i := strings.IndexByte(uuid, '-'); i > 0 {
uuid = uuid[:i]
}
return uuid + "." + clique
}
// fabricGroup is the candidates on one physical fabric.
type fabricGroup struct {
key string
@@ -1018,3 +1018,56 @@ func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
}
}
// inDomain joins every card of a node to one multi-node NVLink domain, with
// IMEX up.
func inDomain(n inventory.Node, clique string) inventory.Node {
for i := range n.GPUs {
n.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: clique}
}
n.IMEX = discovery.IMEX{State: "active", Channels: 1}
return n
}
func TestAReplicaSpansNodesOverNVLinkBeforeInfiniBand(t *testing.T) {
// Four trays on one InfiniBand fabric, two of them in an NVLink domain:
// the pair is placed across the domain, where tensor parallelism between
// nodes runs at NVLink speed, and given no adapters to use.
st := store(
onSM(node("ib1", "h100-80g", 4, "ib-a"), "0x01"), onSM(node("ib2", "h100-80g", 4, "ib-a"), "0x01"),
inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7"),
inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7"),
)
res := New(st).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if !strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) || !strings.HasPrefix(res.Placement.Pool, "nvl:76b1ad76.7/") {
t.Errorf("fabric %q pool %q", res.Placement.Fabric, res.Placement.Pool)
}
for _, a := range res.Placement.Nodes {
if !strings.HasPrefix(a.NodeID, "nv") || len(a.HCAs) != 0 {
t.Errorf("assignment %+v", a)
}
}
}
func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
a := inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7")
b := inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7")
b.IMEX.State = "inactive"
res := New(store(a, b)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
// Not across the domain; InfiniBand still carries it.
if res.Placement == nil || strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) {
t.Fatalf("placement %+v\n%s", res.Placement, res.Explain())
}
// With no InfiniBand to fall back on, the refusal says why.
a2 := inDomain(node("nv1", "h100-80g", 4), "7")
b2 := inDomain(node("nv2", "h100-80g", 4), "7")
b2.IMEX.State = "inactive"
res = New(store(a2, b2)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
if res.Placement != nil || !strings.Contains(res.Explain(), "nvidia-imex is not up") {
t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
}
}