Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -365,6 +365,8 @@ type Agent struct {
|
||||
sm map[string]*discovery.SMInfo
|
||||
smErr map[string]string
|
||||
smAsked time.Time
|
||||
// nvlAsked is when the GPUs' NVLink fabric state was last read.
|
||||
nvlAsked time.Time
|
||||
// sms is every subnet manager on a fabric, asked through one port per
|
||||
// master, keyed by that port; smsErr says why a port could not ask.
|
||||
sms map[string][]discovery.SMRecord
|
||||
@@ -999,6 +1001,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
}
|
||||
hb.Load = a.sampleLoad()
|
||||
a.sampleIB(hb)
|
||||
a.sampleNVLink(hb)
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||
}
|
||||
|
||||
@@ -1752,3 +1755,45 @@ func (a *Agent) ensureDiags(ctx context.Context) {
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
||||
var (
|
||||
discoverGPUFabric = discovery.DiscoverGPUFabric
|
||||
discoverIMEX = discovery.DiscoverIMEX
|
||||
)
|
||||
|
||||
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
|
||||
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
|
||||
// for every heartbeat, and a fabric that was joined an hour ago rarely
|
||||
// changes in five seconds — but when a fabric manager restarts it does, for
|
||||
// every card at once.
|
||||
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
a.mu.Lock()
|
||||
gpus := slices.Clone(a.inv.GPUs)
|
||||
due := time.Since(a.nvlAsked) > time.Minute
|
||||
if due {
|
||||
a.nvlAsked = time.Now()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if len(gpus) == 0 || !due {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || len(fab) == 0 {
|
||||
return
|
||||
}
|
||||
discovery.ApplyFabric(gpus, fab)
|
||||
for _, g := range gpus {
|
||||
if g.Fabric == nil {
|
||||
continue
|
||||
}
|
||||
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
|
||||
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
|
||||
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
|
||||
})
|
||||
}
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
Reference in new issue
Block a user