Find multi-node NVLink domains, place across them, and show them as racks

A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its
GPUs reach each other over NVLink whatever node they are in. Each GPU
now reports its fabric from nvidia-smi -q - state, status, ClusterUUID,
CliqueId, bandwidth - at registration and once a minute after, since a
fabric manager restarting drops every card out at once; and each node
reports nvidia-imex and its channels, without which the domain's GPUs
cannot share memory across nodes. A single machine's NVSwitch, which
reports a cluster of zeros, is no domain.

A replica spanning nodes is placed inside one domain first, on nodes
whose every card is joined and whose IMEX is up, and its shards are
told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback.

/admin/nvlink draws each domain as a rack - the NVLink Switch band on
top, light running down a backplane, trays of GPU chips that glow when
joined, pulse red when not, turn amber on degraded bandwidth - and
lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink
matrix, each card's NUMA node and nearest adapter, and its domain and
IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics
llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:07:59 +02:00
1 parent 5bbc619a31
commit 20479a2d5f
31 files changed
+1535 -131

No files matched your search

+18
View File
@@ -61,6 +61,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
CUDAVersion: g.GetCudaVersion(),
NVLinkPeers: intSlice(g.GetNvlinkPeers()),
NICPaths: maps.Clone(g.GetNicPaths()),
Fabric: fabricFrom(g.GetFabricState(), g.GetFabricStatus(), g.GetClusterUuid(),
g.GetCliqueId(), g.GetFabricBandwidth()),
Health: discovery.GPUHealth{
ECCErrors: int(g.GetEccErrors()),
TemperatureC: int(g.GetTemperatureC()),
@@ -68,6 +70,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
},
})
}
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
for _, f := range r.GetFabrics() {
n.Fabrics = append(n.Fabrics, inventory.FabricAttachment{
Fabric: f.GetFabric(),
@@ -119,7 +122,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
TemperatureC: int32(g.Health.TemperatureC),
EccErrors: int32(g.Health.ECCErrors),
})
if f := g.Fabric; f != nil {
last := r.Gpus[len(r.Gpus)-1]
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
}
}
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
for _, f := range fabrics {
r.Fabrics = append(r.Fabrics, &llmgwv1.FabricAttachment{
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
@@ -266,3 +275,12 @@ func int32Slice(v []int) []int32 {
}
return out
}
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
// driver reported none.
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
if state == "" {
return nil
}
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
}
@@ -242,3 +242,25 @@ func TestAPortGoingDownTakesItsFabricAway(t *testing.T) {
t.Error("a heartbeat without a fabric change emptied the fabrics")
}
}
func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
// A fabric manager restarting drops cards out of the domain; the node has
// to stop being offered work that spans it until they are back.
s := newTestService()
fab := &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24", CliqueID: "1"}
nodeWith(s, inventory.Node{ID: "t1", State: inventory.Ready, GPUs: []discovery.GPU{
{Index: 0, Fabric: fab}, {Index: 1, Fabric: fab}}})
if n, _ := s.inv.Get("t1"); n.NVLinkDomain() == "" {
t.Fatal("the node should start in its domain")
}
s.applyHeartbeat("t1", &llmgwv1.Heartbeat{NodeId: "t1",
GpuFabric: []*llmgwv1.GPUFabricState{{Index: 1, State: "In Progress"}},
Imex: &llmgwv1.IMEX{State: "active", Channels: 1}})
n, _ := s.inv.Get("t1")
if n.NVLinkDomain() != "" {
t.Errorf("a node with a card out of the domain is still in it")
}
if n.IMEX.State != "active" || n.IMEX.Channels != 1 {
t.Errorf("imex = %+v", n.IMEX)
}
}
+19 -1
View File
@@ -322,7 +322,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
!hb.GetFabricsChanged() {
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
return
}
@@ -457,6 +457,24 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
if len(hb.GetIbPorts()) > 0 {
n.IBAt = time.Now()
}
// NVLink domain membership, which a fabric manager restarting takes
// away from every card at once.
if fab := hb.GetGpuFabric(); len(fab) > 0 {
before := n.NVLinkDomain()
for _, f := range fab {
for i := range n.GPUs {
if n.GPUs[i].Index == int(f.GetIndex()) {
n.GPUs[i].Fabric = fabricFrom(f.GetState(), f.GetStatus(), f.GetClusterUuid(), f.GetCliqueId(), f.GetBandwidth())
}
}
}
if after := n.NVLinkDomain(); after != before {
s.log.Warn("a node's NVLink domain changed", "node", nodeID, "from", before, "to", after)
}
}
if m := hb.GetImex(); m != nil {
n.IMEX = discovery.IMEX{State: m.GetState(), Channels: int(m.GetChannels())}
}
// A port went down, or came back. Placement reads the fabrics, so this is
// what stops a node being offered multi-node work it can no longer carry.
if hb.GetFabricsChanged() {