Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -116,6 +116,8 @@ message Register {
|
||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||
// built node with no engine installed yet was a candidate for every model.
|
||||
bool engines_probed = 14;
|
||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||
IMEX imex = 15;
|
||||
}
|
||||
|
||||
// Host describes the operating system and processor of a node.
|
||||
@@ -212,6 +214,14 @@ message GPU {
|
||||
// nic_paths is how far each network adapter is from this card, by HCA
|
||||
// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
|
||||
map<string, string> nic_paths = 16;
|
||||
// The card's multi-node NVLink membership, from nvidia-smi -q: state is
|
||||
// Completed once the fabric manager has joined it; cluster_uuid and
|
||||
// clique_id name the domain it joined.
|
||||
string fabric_state = 17;
|
||||
string fabric_status = 18;
|
||||
string cluster_uuid = 19;
|
||||
string clique_id = 20;
|
||||
string fabric_bandwidth = 21;
|
||||
}
|
||||
|
||||
message FabricAttachment {
|
||||
@@ -315,6 +325,23 @@ message SubnetManager {
|
||||
string state = 4;
|
||||
}
|
||||
|
||||
// IMEX is the nvidia-imex service and its channels.
|
||||
message IMEX {
|
||||
// state is systemd's word for it, empty where it is not installed.
|
||||
string state = 1;
|
||||
int32 channels = 2;
|
||||
}
|
||||
|
||||
// GPUFabricState is one GPU's NVLink domain membership.
|
||||
message GPUFabricState {
|
||||
int32 index = 1;
|
||||
string state = 2;
|
||||
string status = 3;
|
||||
string cluster_uuid = 4;
|
||||
string clique_id = 5;
|
||||
string bandwidth = 6;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
string node_id = 1;
|
||||
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
||||
@@ -370,6 +397,12 @@ message Heartbeat {
|
||||
// port going down takes its fabric away, and one coming back returns it.
|
||||
bool fabrics_changed = 14;
|
||||
repeated FabricAttachment fabrics = 15;
|
||||
|
||||
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
||||
// once a minute: a fabric manager restarting drops every card out of its
|
||||
// domain, and the node has to stop being offered work that spans it.
|
||||
repeated GPUFabricState gpu_fabric = 16;
|
||||
IMEX imex = 17;
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
|
||||
Reference in new issue
Block a user