From 3ba36e23a65a03f58b4285ed4f1ed988d5162170 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 00:19:52 +0200 Subject: [PATCH] Choose a shard's GPUs by NVLink, nearness to the fabric and NUMA GPUs were chosen by growing one card's NVLink peers, which took cards each joined to that one but not to each other, and knew nothing of the fabric. Every set of the right size is now scored - a node's handful of cards makes that cheap - by NVLink pairs within it, then for a multi-node shard by how near each card is to one of the fabric's adapters (PIX before SYS), then by NUMA nodes spanned, with the lowest indices breaking ties. A node without topology still gets the lowest indices. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/control/placement/placement.go | 115 ++++++++++++++----- internal/control/placement/placement_test.go | 61 ++++++++++ 2 files changed, 148 insertions(+), 28 deletions(-) diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index b10d776..8bb557b 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -213,7 +213,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re preferNodesHoldingWeights(p.Source, candidates) best := candidates[0] - gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode) + gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil) if gpus == nil { rejections = append(rejections, Rejection{ Profile: p.Name, Node: best.node.ID, @@ -267,7 +267,7 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej assignments := make([]NodeAssignment, 0, p.Nodes) for rank, c := range candidates[:p.Nodes] { - gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode) + gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric)) if gpus == nil { assignments = nil rejections = append(rejections, Rejection{ @@ -475,48 +475,107 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // pickGPUs chooses which of a node's free GPUs to use. // -// It prefers a set that shares an NVLink domain: within a node, a -// tensor-parallel group that has to fall back to PCIe for its all-reduces runs -// several times slower. NVLink topology is only available from the NVML build, -// so without it this degrades to taking the lowest indices — which is usually -// right, but only by accident of how vendors number slots. -func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) []int { +// Every set of the right size is scored, which a node's handful of cards +// makes cheap, in this order: +// +// - NVLink: the most pairs in the set joined by it. A tensor-parallel group +// that falls back to PCIe for its all-reduces runs several times slower, +// and "seed and its peers", the rule before, took cards that were each +// joined to the seed but not to each other. +// - For a multi-node shard, the fabric: cards near one of its adapters (PIX +// or PXB rather than across the socket), since every transfer between +// nodes goes through one. +// - One NUMA node rather than two. +// - The lowest indices, so the answer is the same every time. +// +// hcas are the fabric's adapters for a multi-node shard, nil otherwise. +// Without topology — an agent that reported none — every set scores the same +// and it is the lowest indices, as it always was. +func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int, hcas []string) []int { if len(free) < want { return nil } - if want == 1 { + if want == 1 && len(hcas) == 0 { return []int{free[0].Index} } + sorted := slices.Clone(free) + slices.SortFunc(sorted, func(a, b discovery.GPU) int { return a.Index - b.Index }) - freeSet := map[int]struct{}{} - for _, g := range free { - freeSet[g.Index] = struct{}{} + // Too many to score every set: take the lowest, as before topology. + if len(sorted) > 16 { + out := make([]int, 0, want) + for _, g := range sorted[:want] { + out = append(out, g.Index) + } + return out } - // Try each free GPU as a seed and grow its NVLink-connected set. - for _, seed := range free { - group := []int{seed.Index} - for _, peer := range seed.NVLinkPeers { - if _, ok := freeSet[peer]; !ok || peer == seed.Index { - continue - } - group = append(group, peer) - if len(group) == want { - slices.Sort(group) - return group + var best []int + bestScore := [3]int{-1, 0, 0} + pick := make([]int, 0, want) + var walk func(from int) + walk = func(from int) { + if len(pick) == want { + sc := gpuSetScore(sorted, pick, hcas) + if best == nil || better(sc, bestScore) { + bestScore = sc + best = slices.Clone(pick) } + return + } + for i := from; i <= len(sorted)-(want-len(pick)); i++ { + pick = append(pick, i) + walk(i + 1) + pick = pick[:len(pick)-1] } } + walk(0) - // No NVLink domain large enough, or no topology at all. - out := make([]int, 0, want) - for _, g := range free[:want] { - out = append(out, g.Index) + out := make([]int, len(best)) + for i, k := range best { + out[i] = sorted[k].Index } - slices.Sort(out) return out } +// gpuSetScore rates a set, as positions in gpus: NVLink pairs, nearness to +// the fabric (negated, so higher is better) and NUMA nodes spanned (negated). +func gpuSetScore(gpus []discovery.GPU, set []int, hcas []string) [3]int { + var sc [3]int + numa := map[int]bool{} + for a, i := range set { + g := gpus[i] + numa[g.NUMANode] = true + for _, j := range set[a+1:] { + if slices.Contains(g.NVLinkPeers, gpus[j].Index) { + sc[0]++ + } + } + if len(hcas) > 0 && len(g.NICPaths) > 0 { + nearest := 5 + for _, h := range hcas { + if rel, ok := g.NICPaths[h]; ok { + nearest = min(nearest, discovery.PathRank(rel)) + } + } + sc[1] -= nearest + } + } + sc[2] = -len(numa) + return sc +} + +// better reports whether a scores above b, most important first. Equal is +// not better, so the first set found — the lowest indices — keeps a tie. +func better(a, b [3]int) bool { + for i := range a { + if a[i] != b[i] { + return a[i] > b[i] + } + } + return false +} + // claimedMemory is what the shards already on a node asked for. // // Declared, not measured. A profile that says it needs 15 GB is treated as diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 9a241d6..8189d4d 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -827,3 +827,64 @@ func TestANodeStillInstallingItsEngineIsNotACandidate(t *testing.T) { t.Errorf("sim was refused:\n%s", res.Explain()) } } + +func TestAGroupIsJoinedToItselfNotOnlyToOneCard(t *testing.T) { + // Cards bridged in pairs, as on PCIe boxes with NVLink bridges. For two + // cards, a pair joined by NVLink wins over lower indices that are not + // joined at all. + n := node("n1", "h100-80g", 8) + link := func(a, b int) { + n.GPUs[a].NVLinkPeers = append(n.GPUs[a].NVLinkPeers, b) + n.GPUs[b].NVLinkPeers = append(n.GPUs[b].NVLinkPeers, a) + } + link(2, 3) + link(4, 5) + st := store(n) + // 0 and 1 are taken; 2-3 is the lowest pair that shares NVLink. + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) + res := New(st).Place([]Profile{{Name: "tp2", GPUClass: "h100-80g", GPUsPerNode: 2, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].GPUIndices; !slices.Equal(got, []int{2, 3}) { + t.Errorf("chose %v, want the NVLink pair 2-3", got) + } +} + +func TestAMultiNodeShardTakesTheCardsNearestTheFabric(t *testing.T) { + // Every transfer between nodes goes through an adapter. Cards 4-7 sit + // under the same PCIe switch as the fabric's adapter; 0-3 are across the + // socket from it, and would cross the processor interconnect every time. + mk := func(id string) inventory.Node { + n := node(id, "h100-80g", 8, "ib-a") + n.Fabrics[0].HCAs = []string{"mlx5_1"} + for i := range n.GPUs { + rel := "SYS" + if i >= 4 { + rel = "PIX" + } + n.GPUs[i].NICPaths = map[string]string{"mlx5_1": rel} + n.GPUs[i].NUMANode = i / 4 + } + return n + } + res := New(store(mk("a"), mk("b"))).Place( + []Profile{{Name: "tp8pp", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + for _, a := range res.Placement.Nodes { + if !slices.Equal(a.GPUIndices, []int{4, 5, 6, 7}) { + t.Errorf("%s: chose %v, want the cards beside the adapter", a.NodeID, a.GPUIndices) + } + } +} + +func TestWithoutTopologyItIsTheLowestIndices(t *testing.T) { + st := store(node("n1", "h100-80g", 8)) + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{1}}) + res := New(st).Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if got := res.Placement.Nodes[0].GPUIndices; !slices.Equal(got, []int{0, 2, 3, 4}) { + t.Errorf("chose %v", got) + } +}