Find multi-node NVLink domains, place across them, and show them as racks
A GB200 NVL72 rack or an NVLink Switch system is a scale-up island: its GPUs reach each other over NVLink whatever node they are in. Each GPU now reports its fabric from nvidia-smi -q - state, status, ClusterUUID, CliqueId, bandwidth - at registration and once a minute after, since a fabric manager restarting drops every card out at once; and each node reports nvidia-imex and its channels, without which the domain's GPUs cannot share memory across nodes. A single machine's NVSwitch, which reports a cluster of zeros, is no domain. A replica spanning nodes is placed inside one domain first, on nodes whose every card is joined and whose IMEX is up, and its shards are told NCCL_MNNVL_ENABLE=1; InfiniBand is the fallback. /admin/nvlink draws each domain as a rack - the NVLink Switch band on top, light running down a backplane, trays of GPU chips that glow when joined, pulse red when not, turn amber on degraded bandwidth - and lists what keeps a tray out. A node's page shows its GPU-by-GPU NVLink matrix, each card's NUMA node and nearest adapter, and its domain and IMEX. doctor checks joining, bandwidth, IMEX and its channel; metrics llmgw_gpu_nvlink_joined and llmgw_imex_up; /api/v1/nvlink. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5bbc619a31
commit
20479a2d5f
31 files changed
+1535
-131
No files matched your search
@@ -239,13 +239,20 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
var rejections []Rejection
|
||||
|
||||
fabrics := s.inv.Fabrics()
|
||||
if len(fabrics) == 0 {
|
||||
if len(fabrics) == 0 && !s.anyNVLinkDomain() {
|
||||
return nil, []Rejection{{
|
||||
Profile: p.Name,
|
||||
Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes),
|
||||
}}
|
||||
}
|
||||
|
||||
// A multi-node NVLink domain first: a replica inside one runs tensor
|
||||
// parallelism across its nodes at NVLink speed, which no InfiniBand
|
||||
// fabric matches.
|
||||
if pl := s.placeOnNVLink(p, avoid, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
}
|
||||
|
||||
canon := inventory.CanonicalSMs(s.inv.Nodes())
|
||||
for _, fabric := range fabrics {
|
||||
all := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
@@ -262,6 +269,66 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
return nil, rejections
|
||||
}
|
||||
|
||||
// NVLinkFabric is how a placement names a multi-node NVLink domain in its
|
||||
// Fabric, so the shards on it are told to use it.
|
||||
const NVLinkFabric = "nvl:"
|
||||
|
||||
func (s *Scheduler) anyNVLinkDomain() bool {
|
||||
for _, n := range s.inv.Nodes() {
|
||||
if n.NVLinkDomain() != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// placeOnNVLink forms a replica inside one multi-node NVLink domain, from
|
||||
// nodes whose every card is joined to it and whose IMEX is up: a node
|
||||
// missing either would leave the replica a card, or its memory, short.
|
||||
func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejection) *Placement {
|
||||
var none []Rejection
|
||||
cands := s.eligibleNodes(p, "", avoid, &none)
|
||||
by := map[string][]candidate{}
|
||||
for _, c := range cands {
|
||||
d := c.node.NVLinkDomain()
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if c.node.IMEX.State != "active" || c.node.IMEX.Channels == 0 {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: NVLinkFabric + shortDomain(d), Node: c.node.ID,
|
||||
Reason: "in an NVLink domain, but nvidia-imex is not up"})
|
||||
continue
|
||||
}
|
||||
by[d] = append(by[d], c)
|
||||
}
|
||||
keys := make([]string, 0, len(by))
|
||||
for k := range by {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(by[keys[i]]) != len(by[keys[j]]) {
|
||||
return len(by[keys[i]]) > len(by[keys[j]])
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
for _, d := range keys {
|
||||
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
|
||||
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
|
||||
return pl
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shortDomain is an NVLink domain as a pool name carries it.
|
||||
func shortDomain(d string) string {
|
||||
uuid, clique, _ := strings.Cut(d, "/")
|
||||
if i := strings.IndexByte(uuid, '-'); i > 0 {
|
||||
uuid = uuid[:i]
|
||||
}
|
||||
return uuid + "." + clique
|
||||
}
|
||||
|
||||
// fabricGroup is the candidates on one physical fabric.
|
||||
type fabricGroup struct {
|
||||
key string
|
||||
|
||||
@@ -1018,3 +1018,56 @@ func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
|
||||
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
|
||||
}
|
||||
}
|
||||
|
||||
// inDomain joins every card of a node to one multi-node NVLink domain, with
|
||||
// IMEX up.
|
||||
func inDomain(n inventory.Node, clique string) inventory.Node {
|
||||
for i := range n.GPUs {
|
||||
n.GPUs[i].Fabric = &discovery.GPUFabric{State: "Completed", ClusterUUID: "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa", CliqueID: clique}
|
||||
}
|
||||
n.IMEX = discovery.IMEX{State: "active", Channels: 1}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestAReplicaSpansNodesOverNVLinkBeforeInfiniBand(t *testing.T) {
|
||||
// Four trays on one InfiniBand fabric, two of them in an NVLink domain:
|
||||
// the pair is placed across the domain, where tensor parallelism between
|
||||
// nodes runs at NVLink speed, and given no adapters to use.
|
||||
st := store(
|
||||
onSM(node("ib1", "h100-80g", 4, "ib-a"), "0x01"), onSM(node("ib2", "h100-80g", 4, "ib-a"), "0x01"),
|
||||
inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7"),
|
||||
)
|
||||
res := New(st).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
if !strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) || !strings.HasPrefix(res.Placement.Pool, "nvl:76b1ad76.7/") {
|
||||
t.Errorf("fabric %q pool %q", res.Placement.Fabric, res.Placement.Pool)
|
||||
}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
if !strings.HasPrefix(a.NodeID, "nv") || len(a.HCAs) != 0 {
|
||||
t.Errorf("assignment %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
|
||||
a := inDomain(onSM(node("nv1", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b := inDomain(onSM(node("nv2", "h100-80g", 4, "ib-a"), "0x01"), "7")
|
||||
b.IMEX.State = "inactive"
|
||||
res := New(store(a, b)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
// Not across the domain; InfiniBand still carries it.
|
||||
if res.Placement == nil || strings.HasPrefix(res.Placement.Fabric, NVLinkFabric) {
|
||||
t.Fatalf("placement %+v\n%s", res.Placement, res.Explain())
|
||||
}
|
||||
|
||||
// With no InfiniBand to fall back on, the refusal says why.
|
||||
a2 := inDomain(node("nv1", "h100-80g", 4), "7")
|
||||
b2 := inDomain(node("nv2", "h100-80g", 4), "7")
|
||||
b2.IMEX.State = "inactive"
|
||||
res = New(store(a2, b2)).Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil)
|
||||
if res.Placement != nil || !strings.Contains(res.Explain(), "nvidia-imex is not up") {
|
||||
t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user