diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index b10d776..8bb557b 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -213,7 +213,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re preferNodesHoldingWeights(p.Source, candidates) best := candidates[0] - gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode) + gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil) if gpus == nil { rejections = append(rejections, Rejection{ Profile: p.Name, Node: best.node.ID, @@ -267,7 +267,7 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej assignments := make([]NodeAssignment, 0, p.Nodes) for rank, c := range candidates[:p.Nodes] { - gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode) + gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric)) if gpus == nil { assignments = nil rejections = append(rejections, Rejection{ @@ -475,48 +475,107 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // pickGPUs chooses which of a node's free GPUs to use. // -// It prefers a set that shares an NVLink domain: within a node, a -// tensor-parallel group that has to fall back to PCIe for its all-reduces runs -// several times slower. NVLink topology is only available from the NVML build, -// so without it this degrades to taking the lowest indices — which is usually -// right, but only by accident of how vendors number slots. -func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) []int { +// Every set of the right size is scored, which a node's handful of cards +// makes cheap, in this order: +// +// - NVLink: the most pairs in the set joined by it. A tensor-parallel group +// that falls back to PCIe for its all-reduces runs several times slower, +// and "seed and its peers", the rule before, took cards that were each +// joined to the seed but not to each other. +// - For a multi-node shard, the fabric: cards near one of its adapters (PIX +// or PXB rather than across the socket), since every transfer between +// nodes goes through one. +// - One NUMA node rather than two. +// - The lowest indices, so the answer is the same every time. +// +// hcas are the fabric's adapters for a multi-node shard, nil otherwise. +// Without topology — an agent that reported none — every set scores the same +// and it is the lowest indices, as it always was. +func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int, hcas []string) []int { if len(free) < want { return nil } - if want == 1 { + if want == 1 && len(hcas) == 0 { return []int{free[0].Index} } + sorted := slices.Clone(free) + slices.SortFunc(sorted, func(a, b discovery.GPU) int { return a.Index - b.Index }) - freeSet := map[int]struct{}{} - for _, g := range free { - freeSet[g.Index] = struct{}{} + // Too many to score every set: take the lowest, as before topology. + if len(sorted) > 16 { + out := make([]int, 0, want) + for _, g := range sorted[:want] { + out = append(out, g.Index) + } + return out } - // Try each free GPU as a seed and grow its NVLink-connected set. - for _, seed := range free { - group := []int{seed.Index} - for _, peer := range seed.NVLinkPeers { - if _, ok := freeSet[peer]; !ok || peer == seed.Index { - continue - } - group = append(group, peer) - if len(group) == want { - slices.Sort(group) - return group + var best []int + bestScore := [3]int{-1, 0, 0} + pick := make([]int, 0, want) + var walk func(from int) + walk = func(from int) { + if len(pick) == want { + sc := gpuSetScore(sorted, pick, hcas) + if best == nil || better(sc, bestScore) { + bestScore = sc + best = slices.Clone(pick) } + return + } + for i := from; i <= len(sorted)-(want-len(pick)); i++ { + pick = append(pick, i) + walk(i + 1) + pick = pick[:len(pick)-1] } } + walk(0) - // No NVLink domain large enough, or no topology at all. - out := make([]int, 0, want) - for _, g := range free[:want] { - out = append(out, g.Index) + out := make([]int, len(best)) + for i, k := range best { + out[i] = sorted[k].Index } - slices.Sort(out) return out } +// gpuSetScore rates a set, as positions in gpus: NVLink pairs, nearness to +// the fabric (negated, so higher is better) and NUMA nodes spanned (negated). +func gpuSetScore(gpus []discovery.GPU, set []int, hcas []string) [3]int { + var sc [3]int + numa := map[int]bool{} + for a, i := range set { + g := gpus[i] + numa[g.NUMANode] = true + for _, j := range set[a+1:] { + if slices.Contains(g.NVLinkPeers, gpus[j].Index) { + sc[0]++ + } + } + if len(hcas) > 0 && len(g.NICPaths) > 0 { + nearest := 5 + for _, h := range hcas { + if rel, ok := g.NICPaths[h]; ok { + nearest = min(nearest, discovery.PathRank(rel)) + } + } + sc[1] -= nearest + } + } + sc[2] = -len(numa) + return sc +} + +// better reports whether a scores above b, most important first. Equal is +// not better, so the first set found — the lowest indices — keeps a tie. +func better(a, b [3]int) bool { + for i := range a { + if a[i] != b[i] { + return a[i] > b[i] + } + } + return false +} + // claimedMemory is what the shards already on a node asked for. // // Declared, not measured. A profile that says it needs 15 GB is treated as diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 9a241d6..8189d4d 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -827,3 +827,64 @@ func TestANodeStillInstallingItsEngineIsNotACandidate(t *testing.T) { t.Errorf("sim was refused:\n%s", res.Explain()) } } + +func TestAGroupIsJoinedToItselfNotOnlyToOneCard(t *testing.T) { + // Cards bridged in pairs, as on PCIe boxes with NVLink bridges. For two + // cards, a pair joined by NVLink wins over lower indices that are not + // joined at all. + n := node("n1", "h100-80g", 8) + link := func(a, b int) { + n.GPUs[a].NVLinkPeers = append(n.GPUs[a].NVLinkPeers, b) + n.GPUs[b].NVLinkPeers = append(n.GPUs[b].NVLinkPeers, a) + } + link(2, 3) + link(4, 5) + st := store(n) + // 0 and 1 are taken; 2-3 is the lowest pair that shares NVLink. + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) + res := New(st).Place([]Profile{{Name: "tp2", GPUClass: "h100-80g", GPUsPerNode: 2, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].GPUIndices; !slices.Equal(got, []int{2, 3}) { + t.Errorf("chose %v, want the NVLink pair 2-3", got) + } +} + +func TestAMultiNodeShardTakesTheCardsNearestTheFabric(t *testing.T) { + // Every transfer between nodes goes through an adapter. Cards 4-7 sit + // under the same PCIe switch as the fabric's adapter; 0-3 are across the + // socket from it, and would cross the processor interconnect every time. + mk := func(id string) inventory.Node { + n := node(id, "h100-80g", 8, "ib-a") + n.Fabrics[0].HCAs = []string{"mlx5_1"} + for i := range n.GPUs { + rel := "SYS" + if i >= 4 { + rel = "PIX" + } + n.GPUs[i].NICPaths = map[string]string{"mlx5_1": rel} + n.GPUs[i].NUMANode = i / 4 + } + return n + } + res := New(store(mk("a"), mk("b"))).Place( + []Profile{{Name: "tp8pp", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + for _, a := range res.Placement.Nodes { + if !slices.Equal(a.GPUIndices, []int{4, 5, 6, 7}) { + t.Errorf("%s: chose %v, want the cards beside the adapter", a.NodeID, a.GPUIndices) + } + } +} + +func TestWithoutTopologyItIsTheLowestIndices(t *testing.T) { + st := store(node("n1", "h100-80g", 8)) + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{1}}) + res := New(st).Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if got := res.Placement.Nodes[0].GPUIndices; !slices.Equal(got, []int{0, 2, 3, 4}) { + t.Errorf("chose %v", got) + } +}