Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -61,6 +61,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
CUDAVersion: g.GetCudaVersion(),
|
||||
NVLinkPeers: intSlice(g.GetNvlinkPeers()),
|
||||
NICPaths: maps.Clone(g.GetNicPaths()),
|
||||
Fabric: fabricFrom(g.GetFabricState(), g.GetFabricStatus(), g.GetClusterUuid(),
|
||||
g.GetCliqueId(), g.GetFabricBandwidth()),
|
||||
Health: discovery.GPUHealth{
|
||||
ECCErrors: int(g.GetEccErrors()),
|
||||
TemperatureC: int(g.GetTemperatureC()),
|
||||
@@ -68,6 +70,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
},
|
||||
})
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
|
||||
for _, f := range r.GetFabrics() {
|
||||
n.Fabrics = append(n.Fabrics, inventory.FabricAttachment{
|
||||
Fabric: f.GetFabric(),
|
||||
@@ -119,7 +122,13 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
TemperatureC: int32(g.Health.TemperatureC),
|
||||
EccErrors: int32(g.Health.ECCErrors),
|
||||
})
|
||||
if f := g.Fabric; f != nil {
|
||||
last := r.Gpus[len(r.Gpus)-1]
|
||||
last.FabricState, last.FabricStatus, last.ClusterUuid = f.State, f.Status, f.ClusterUUID
|
||||
last.CliqueId, last.FabricBandwidth = f.CliqueID, f.Bandwidth
|
||||
}
|
||||
}
|
||||
r.Imex = &llmgwv1.IMEX{State: inv.IMEX.State, Channels: int32(inv.IMEX.Channels)}
|
||||
for _, f := range fabrics {
|
||||
r.Fabrics = append(r.Fabrics, &llmgwv1.FabricAttachment{
|
||||
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
|
||||
@@ -266,3 +275,12 @@ func int32Slice(v []int) []int32 {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fabricFrom is a GPU's NVLink fabric as the wire carries it, nil when the
|
||||
// driver reported none.
|
||||
func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPUFabric {
|
||||
if state == "" {
|
||||
return nil
|
||||
}
|
||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||
}
|
||||
@@ -242,3 +242,25 @@ func TestAPortGoingDownTakesItsFabricAway(t *testing.T) {
|
||||
t.Error("a heartbeat without a fabric change emptied the fabrics")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
// A fabric manager restarting drops cards out of the domain; the node has
|
||||
// to stop being offered work that spans it until they are back.
|
||||
s := newTestService()
|
||||
fab := &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24", CliqueID: "1"}
|
||||
nodeWith(s, inventory.Node{ID: "t1", State: inventory.Ready, GPUs: []discovery.GPU{
|
||||
{Index: 0, Fabric: fab}, {Index: 1, Fabric: fab}}})
|
||||
if n, _ := s.inv.Get("t1"); n.NVLinkDomain() == "" {
|
||||
t.Fatal("the node should start in its domain")
|
||||
}
|
||||
s.applyHeartbeat("t1", &llmgwv1.Heartbeat{NodeId: "t1",
|
||||
GpuFabric: []*llmgwv1.GPUFabricState{{Index: 1, State: "In Progress"}},
|
||||
Imex: &llmgwv1.IMEX{State: "active", Channels: 1}})
|
||||
n, _ := s.inv.Get("t1")
|
||||
if n.NVLinkDomain() != "" {
|
||||
t.Errorf("a node with a card out of the domain is still in it")
|
||||
}
|
||||
if n.IMEX.State != "active" || n.IMEX.Channels != 1 {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
@@ -322,7 +322,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||
!hb.GetFabricsChanged() {
|
||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -457,6 +457,24 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
if len(hb.GetIbPorts()) > 0 {
|
||||
n.IBAt = time.Now()
|
||||
}
|
||||
// NVLink domain membership, which a fabric manager restarting takes
|
||||
// away from every card at once.
|
||||
if fab := hb.GetGpuFabric(); len(fab) > 0 {
|
||||
before := n.NVLinkDomain()
|
||||
for _, f := range fab {
|
||||
for i := range n.GPUs {
|
||||
if n.GPUs[i].Index == int(f.GetIndex()) {
|
||||
n.GPUs[i].Fabric = fabricFrom(f.GetState(), f.GetStatus(), f.GetClusterUuid(), f.GetCliqueId(), f.GetBandwidth())
|
||||
}
|
||||
}
|
||||
}
|
||||
if after := n.NVLinkDomain(); after != before {
|
||||
s.log.Warn("a node's NVLink domain changed", "node", nodeID, "from", before, "to", after)
|
||||
}
|
||||
}
|
||||
if m := hb.GetImex(); m != nil {
|
||||
n.IMEX = discovery.IMEX{State: m.GetState(), Channels: int(m.GetChannels())}
|
||||
}
|
||||
// A port went down, or came back. Placement reads the fabrics, so this is
|
||||
// what stops a node being offered multi-node work it can no longer carry.
|
||||
if hb.GetFabricsChanged() {
|
||||
|
||||
Reference in new issue
Block a user