package placement import ( "fmt" "math/rand/v2" "slices" "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) // node builds a node with n GPUs of one class. func node(id string, class string, n int, fabrics ...string) inventory.Node { name, mem := classToHardware(class) nd := inventory.Node{ID: id, State: inventory.Ready} for i := range n { nd.GPUs = append(nd.GPUs, discovery.GPU{ Index: i, Name: name, MemoryTotalMB: mem, ComputeCapability: "9.0", UUID: fmt.Sprintf("GPU-%s-%d", id, i), }) } for _, f := range fabrics { nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{ Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200, }) } return nd } // classToHardware maps a class back to a name and size that derive it. func classToHardware(class string) (string, int) { switch class { case "h100-80g": return "NVIDIA H100 80GB HBM3", 81559 case "a100-40g": return "Tesla A100-SXM4-40GB", 40960 default: return "NVIDIA L40S", 46068 } } func store(nodes ...inventory.Node) *inventory.Store { s := inventory.NewStore() for _, n := range nodes { s.Upsert(n) } return s } func TestSingleNodePlacement(t *testing.T) { s := New(store(node("n1", "h100-80g", 8))) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } if len(res.Placement.Nodes) != 1 { t.Fatalf("expected 1 node, got %d", len(res.Placement.Nodes)) } if got := len(res.Placement.Nodes[0].GPUIndices); got != 4 { t.Errorf("got %d GPUs, want 4", got) } if res.Placement.Fabric != "" { t.Errorf("a single-node replica was given fabric %q", res.Placement.Fabric) } } func TestNodeWithoutFabricTakesSingleNodeWork(t *testing.T) { // A single-node replica talks to nobody, so it has nothing to be reachable // over. Excluding fabric-less nodes here would waste them entirely. s := New(store(node("no-ib", "h100-80g", 8))) res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("a node without IB was refused single-node work:\n%s", res.Explain()) } } func TestNodeWithoutFabricRefusedMultiNode(t *testing.T) { s := New(store( node("no-ib-1", "h100-80g", 8), node("no-ib-2", "h100-80g", 8), )) res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) if res.Placement != nil { t.Fatal("a multi-node replica was placed on nodes with no fabric") } if !strings.Contains(res.Explain(), "no node reports a fabric") { t.Errorf("the reason is unclear:\n%s", res.Explain()) } } // TestNeverSpansTwoFabrics is the first hard invariant from plan §1.1. func TestNeverSpansTwoFabrics(t *testing.T) { // Two fabrics, one node each: together they would satisfy the count, and a // scheduler that only counted nodes would happily combine them. s := New(store( node("a1", "h100-80g", 8, "ib-a"), node("b1", "h100-80g", 8, "ib-b"), )) res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) if res.Placement != nil { var fabrics []string for _, n := range res.Placement.Nodes { fabrics = append(fabrics, n.NodeID) } t.Fatalf("a replica was spread across fabrics: %v", fabrics) } if !strings.Contains(res.Explain(), "only 1 qualify") { t.Errorf("explanation should say each fabric was short of nodes:\n%s", res.Explain()) } } func TestMultiNodeWithinOneFabric(t *testing.T) { s := New(store( node("a1", "h100-80g", 8, "ib-a"), node("a2", "h100-80g", 8, "ib-a"), node("b1", "h100-80g", 8, "ib-b"), )) res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } if res.Placement.Fabric != "ib-a" { t.Errorf("fabric=%q, only ib-a has two nodes", res.Placement.Fabric) } for _, n := range res.Placement.Nodes { if n.NodeID == "b1" { t.Error("a node from the other fabric was included") } if len(n.HCAs) == 0 { t.Errorf("node %s got no HCAs; NCCL would pick its own", n.NodeID) } } // Ranks must be assigned, since rank 0 serves HTTP. ranks := []int{res.Placement.Nodes[0].Rank, res.Placement.Nodes[1].Rank} slices.Sort(ranks) if ranks[0] != 0 || ranks[1] != 1 { t.Errorf("ranks=%v", ranks) } } // TestNeverMixesGPUClasses is the second hard invariant from plan §1.1. func TestNeverMixesGPUClasses(t *testing.T) { // Enough GPUs in total, but split across classes. n := node("mixed", "h100-80g", 4) a100Name, a100Mem := classToHardware("a100-40g") for i := 4; i < 8; i++ { n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: a100Name, MemoryTotalMB: a100Mem}) } s := New(store(n)) res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) if res.Placement != nil { t.Fatal("a placement used GPUs of more than one class") } if !strings.Contains(res.Explain(), "has 4 free h100-80g GPUs, needs 8") { t.Errorf("explanation should count only the matching class:\n%s", res.Explain()) } // The 4-GPU profile does fit, using only H100s. res = s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("the matching-class profile was refused:\n%s", res.Explain()) } for _, idx := range res.Placement.Nodes[0].GPUIndices { if idx >= 4 { t.Errorf("GPU %d is an A100 and was included in an h100 placement", idx) } } } func TestProfilePriorityOrder(t *testing.T) { s := New(store( node("h", "h100-80g", 8, "ib-a"), node("a", "a100-40g", 8, "ib-a"), )) res := s.Place([]Profile{ {Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50}, {Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100}, }, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } if res.Placement.Profile.Name != "h100" { t.Errorf("chose %q; the higher-priority profile fits", res.Placement.Profile.Name) } } func TestFallsBackToLowerPriorityProfile(t *testing.T) { s := New(store(node("a", "a100-40g", 8))) res := s.Place([]Profile{ {Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100}, {Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50}, }, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } if res.Placement.Profile.Name != "a100" { t.Errorf("chose %q", res.Placement.Profile.Name) } // The rejected first choice must still be explained. if !strings.Contains(res.Explain(), "h100") { t.Errorf("the higher-priority profile's rejection is missing:\n%s", res.Explain()) } } func TestCordonedNodeExcluded(t *testing.T) { st := store(node("n1", "h100-80g", 8)) st.SetState("n1", inventory.Cordoned) s := New(st) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement != nil { t.Fatal("a cordoned node was used") } if !strings.Contains(res.Explain(), "cordoned") { t.Errorf("explanation:\n%s", res.Explain()) } } func TestUnhealthyGPUsExcluded(t *testing.T) { n := node("n1", "h100-80g", 8) n.GPUs[0].Health.Problems = []string{"12 uncorrected ECC errors"} n.GPUs[1].Health.Problems = []string{"40000 MiB already allocated by another process"} s := New(store(n)) res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) if res.Placement != nil { t.Fatal("unhealthy GPUs were counted as available") } res = s.Place([]Profile{{Name: "tp6", GPUClass: "h100-80g", GPUsPerNode: 6, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("the six healthy GPUs should have been usable:\n%s", res.Explain()) } for _, idx := range res.Placement.Nodes[0].GPUIndices { if idx == 0 || idx == 1 { t.Errorf("unhealthy GPU %d was placed on", idx) } } } func TestAllocatedGPUsExcluded(t *testing.T) { st := store(node("n1", "h100-80g", 8)) st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) s := New(st) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("the remaining four GPUs should fit:\n%s", res.Explain()) } for _, idx := range res.Placement.Nodes[0].GPUIndices { if idx < 4 { t.Errorf("GPU %d is already allocated", idx) } } // Another four will not fit. st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{4, 5, 6, 7}}) if res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil); res.Placement != nil { t.Fatal("a full node was placed on") } } func TestAvoidSpreadsAcrossNodes(t *testing.T) { s := New(store( node("n1", "h100-80g", 8), node("n2", "h100-80g", 8), )) p := []Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}} first := s.Place(p, nil) if first.Placement == nil { t.Fatal("no first placement") } placed := first.Placement.Nodes[0].NodeID second := s.Place(p, []string{placed}) if second.Placement == nil { t.Fatalf("the second replica had nowhere to go:\n%s", second.Explain()) } if second.Placement.Nodes[0].NodeID == placed { t.Error("both replicas landed on the same node despite avoid") } } func TestBinPackingPrefersPartlyUsedNodes(t *testing.T) { // Keeping whole nodes free matters: a large model that needs all 8 GPUs // cannot be split, so scattering small replicas would strand it. st := store( node("empty", "h100-80g", 8), node("partial", "h100-80g", 8), ) st.Allocate("partial", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) s := New(st) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } if got := res.Placement.Nodes[0].NodeID; got != "partial" { t.Errorf("placed on %q; the partly used node should be filled first", got) } } func TestNVLinkGroupPreferred(t *testing.T) { // GPUs 0-3 and 4-7 form separate NVLink domains. A TP group crossing them // falls back to PCIe for its all-reduces and runs several times slower. n := node("n1", "h100-80g", 8) for i := range n.GPUs { var peers []int base := (i / 4) * 4 for j := base; j < base+4; j++ { if j != i { peers = append(peers, j) } } n.GPUs[i].NVLinkPeers = peers } st := store(n) // Take GPU 0, so a naive "lowest indices" pick would span the two domains. st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0}}) s := New(st) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement == nil { t.Fatalf("no placement:\n%s", res.Explain()) } got := res.Placement.Nodes[0].GPUIndices if !slices.Equal(got, []int{4, 5, 6, 7}) { t.Errorf("chose %v; 4-7 is the only intact NVLink domain left", got) } } func TestExplainIsUsefulWhenNothingFits(t *testing.T) { s := New(store(node("n1", "a100-40g", 2))) res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) if res.Placement != nil { t.Fatal("unexpected placement") } exp := res.Explain() // It must name the node, the profile, and what was actually wrong. for _, want := range []string{"n1", "tp8", "free h100-80g"} { if !strings.Contains(exp, want) { t.Errorf("explanation is missing %q:\n%s", want, exp) } } } func TestEmptyInventory(t *testing.T) { s := New(inventory.NewStore()) res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) if res.Placement != nil { t.Fatal("something was placed on an empty cluster") } } // --- property-based invariants (plan §13) --- // TestInvariantsHold generates random clusters and profiles and checks the two // hard constraints on every placement produced. // // The table-driven tests above cover the cases someone thought of; this covers // the ones nobody did. func TestInvariantsHold(t *testing.T) { rng := rand.New(rand.NewPCG(1, 2)) classes := []string{"h100-80g", "a100-40g", "l40s-45g"} fabrics := []string{"ib-a", "ib-b", "ib-c"} for iter := range 400 { st := inventory.NewStore() nodeCount := 1 + rng.IntN(6) for i := range nodeCount { class := classes[rng.IntN(len(classes))] gpuCount := 1 + rng.IntN(8) var nf []string switch rng.IntN(4) { case 0: // no fabric at all case 1: nf = []string{fabrics[rng.IntN(len(fabrics))]} default: nf = []string{fabrics[rng.IntN(len(fabrics))]} if rng.IntN(3) == 0 { // A node attached to two fabrics: allowed, and the reason // HCAs must be selected per fabric rather than per node. nf = append(nf, fabrics[rng.IntN(len(fabrics))]) } } n := node(fmt.Sprintf("n%d", i), class, gpuCount, nf...) if rng.IntN(8) == 0 { n.State = inventory.Cordoned } st.Upsert(n) } s := New(st) p := Profile{ Name: "gen", GPUClass: classes[rng.IntN(len(classes))], GPUsPerNode: 1 + rng.IntN(8), Nodes: 1 + rng.IntN(3), } res := s.Place([]Profile{p}, nil) if res.Placement == nil { continue } checkInvariants(t, st, p, res.Placement, iter) } } func checkInvariants(t *testing.T, st *inventory.Store, p Profile, pl *Placement, iter int) { t.Helper() if len(pl.Nodes) != max(p.Nodes, 1) { t.Fatalf("iter %d: placed on %d nodes, profile wants %d", iter, len(pl.Nodes), p.Nodes) } seenNodes := map[string]struct{}{} for _, a := range pl.Nodes { if _, dup := seenNodes[a.NodeID]; dup { t.Fatalf("iter %d: node %s used twice in one replica", iter, a.NodeID) } seenNodes[a.NodeID] = struct{}{} n, ok := st.Get(a.NodeID) if !ok { t.Fatalf("iter %d: placed on unknown node %s", iter, a.NodeID) } if !n.State.Schedulable() { t.Fatalf("iter %d: placed on a %s node", iter, n.State) } if len(a.GPUIndices) != p.GPUsPerNode { t.Fatalf("iter %d: node %s got %d GPUs, wants %d", iter, a.NodeID, len(a.GPUIndices), p.GPUsPerNode) } // Invariant: every GPU is of the profile's class, healthy, and real. byIndex := map[int]discovery.GPU{} for _, g := range n.GPUs { byIndex[g.Index] = g } for _, idx := range a.GPUIndices { g, ok := byIndex[idx] if !ok { t.Fatalf("iter %d: node %s has no GPU %d", iter, a.NodeID, idx) } if g.Class() != p.GPUClass { t.Fatalf("iter %d: GPU %d on %s is %s, profile wants %s", iter, idx, a.NodeID, g.Class(), p.GPUClass) } if !g.Health.OK() { t.Fatalf("iter %d: unhealthy GPU %d was placed on", iter, idx) } } // Invariant: a multi-node replica lives on one fabric, and each shard // gets only that fabric's adapters. if p.Nodes > 1 { if pl.Fabric == "" { t.Fatalf("iter %d: multi-node placement with no fabric", iter) } if !n.HasFabric(pl.Fabric) { t.Fatalf("iter %d: node %s is not attached to %s", iter, a.NodeID, pl.Fabric) } want := n.HCAsFor(pl.Fabric) if !slices.Equal(a.HCAs, want) { t.Fatalf("iter %d: node %s got HCAs %v, want exactly %v for %s", iter, a.NodeID, a.HCAs, want, pl.Fabric) } } } } func TestNodeSelectorKeepsAProfileOffTheWrongNodes(t *testing.T) { // A CPU profile fits everywhere, because there is no card to be free. That // is right for a 0.5B model and wrong for a 14B one: the small nodes have // 8 GB of RAM and the model needs 24, and nothing in the inventory says so. small := node("small", "", 0) small.Labels = map[string]string{"ram": "8g"} big := node("big", "", 0) big.Labels = map[string]string{"ram": "32g"} s := New(store(small, big)) p := Profile{Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"}} res := s.Place([]Profile{p}, nil) if res.Placement == nil { t.Fatalf("nothing was placed:\n%s", res.Explain()) } if got := res.Placement.Nodes[0].NodeID; got != "big" { t.Fatalf("placed on %q; only the labelled node can hold this model", got) } } func TestAnUnsatisfiableSelectorSaysWhichLabelIsMissing(t *testing.T) { // "no candidates were considered" is not something an operator can act on. small := node("small", "", 0) small.Labels = map[string]string{"ram": "8g"} s := New(store(small)) res := s.Place([]Profile{{ Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"}, }}, nil) if res.Placement != nil { t.Fatal("placed on a node that does not carry the label") } if !strings.Contains(res.Explain(), "ram=32g") { t.Errorf("the rejection does not name the missing label:\n%s", res.Explain()) } } func TestAnEmptySelectorConstrainsNothing(t *testing.T) { // Every existing profile has no selector, and must keep fitting everywhere. s := New(store(node("plain", "", 0))) if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil { t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain()) } } func TestAProfileIsNotPlacedOnANodeTooSmallForIt(t *testing.T) { // The failure this prevents: a CPU profile has no card to be free, so every // ready node is a candidate. Qwen2.5-Coder 14B has a working set around // 24 GB and the small nodes here have 8, and the node reports itself // perfectly healthy right up until the OOM killer arrives. small := node("small", "", 0) small.MemoryTotalMB = 8 * 1024 big := node("big", "", 0) big.MemoryTotalMB = 32 * 1024 s := New(store(small, big)) res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil) if res.Placement == nil { t.Fatalf("nothing was placed:\n%s", res.Explain()) } if got := res.Placement.Nodes[0].NodeID; got != "big" { t.Fatalf("placed on %q, which cannot hold the model", got) } } func TestTheRejectionSaysHowMuchMemoryIsMissing(t *testing.T) { small := node("small", "", 0) small.MemoryTotalMB = 8 * 1024 s := New(store(small)) res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil) if res.Placement != nil { t.Fatal("placed on a node with too little memory") } if e := res.Explain(); !strings.Contains(e, "8 GB") || !strings.Contains(e, "24") { t.Errorf("the rejection does not say what was short:\n%s", e) } } func TestANodeThatDoesNotReportMemoryIsStillUsable(t *testing.T) { // An agent older than the field reports nothing. Excluding those would let // a control plane upgrade quietly empty half a fleet, which is worse than // the failure the check prevents. old := node("old", "", 0) // MemoryTotalMB left at zero s := New(store(old)) if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil); res.Placement == nil { t.Fatalf("a node that reports no memory was excluded:\n%s", res.Explain()) } } func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) { // Every profile that exists today. small := node("small", "", 0) small.MemoryTotalMB = 1024 s := New(store(small)) if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil { t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain()) } }