package reconciler import ( "context" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) func inRack(n inventory.Node, rack string) inventory.Node { n.Labels = map[string]string{"llmgw.io/rack": rack} return n } func runPasses(h *harness, n int) { for range n { h.r.Reconcile(context.Background()) h.markReady() h.advance(time.Second) } } func racksOf(h *harness) map[string]int { out := map[string]int{} for _, rep := range h.r.Replicas() { for _, s := range rep.Shards { n, _ := h.inv.Get(s.NodeID) out[n.Location().Rack]++ } } return out } func TestAModelsReplicasLandInDifferentRacks(t *testing.T) { // Two nodes in r1 and one in r2, each with room for one replica of // four cards twice over: without racks the second replica's node is // a coin toss between r1's other node and r2's. m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4}) h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"), inRack(gpuNode("2", 8), "r1"), inRack(gpuNode("3", 8), "r2")) runPasses(h, 6) if got := racksOf(h); got["r1"] != 1 || got["r2"] != 1 { t.Errorf("replicas by rack %v, want one in each", got) } } func TestNothingIsPlacedInAHeldRackAndExplainSaysSo(t *testing.T) { m := testModel(1, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4}) h := newHarness(t, m, inRack(gpuNode("1", 8), "r1")) h.inv.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{ "rack:r1": {Domain: "rack:r1", Reason: "cooling fault"}, }}) runPasses(h, 3) if n := len(h.r.Replicas()); n != 0 { t.Fatalf("%d replicas placed in a held rack", n) } res, _ := h.r.Explain("llama") if !strings.Contains(res.Explain(), "rack r1 is held: cooling fault") { t.Errorf("explain-placement does not say the rack is held:\n%s", res.Explain()) } h.inv.SetFacility(facility.Policy{}) runPasses(h, 3) if n := len(h.r.Replicas()); n != 1 { t.Errorf("released, %d replicas; want 1", n) } } func TestADeclaredDrawIsRecordedSoTheNextReplicaSeesTheRackFull(t *testing.T) { // 6 kW cap; each replica declares 4 kW. The first fits, the second would // take the rack to 8 and is refused, though the node has the cards. m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4, PowerWatts: 4000}) h := newHarness(t, m, inRack(gpuNode("1", 8), "r1")) h.inv.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}}) runPasses(h, 6) if n := len(h.r.Replicas()); n != 1 { t.Fatalf("%d replicas under a cap with room for one", n) } if u := h.inv.PowerUse(time.Now())["rack:r1"]; u.Reserved != 4000 { t.Errorf("rack r1 has %v W reserved, want the declared 4000", u.Reserved) } }