package reconciler import ( "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) func owned(m catalog.Model, owner string) catalog.Model { m.Owner = owner; return m } func gpus(n int) store.Resources { return store.Resources{GPUs: &n} } // Two teams in one org; each 8-GPU node holds one tp8 replica. func withQuotas(h *harness, qs ...store.CapacityQuota) { pol := quota.New(qs, map[string][]string{"team:ml": {"org:acme"}, "team:web": {"org:acme"}}) h.r.SetQuotas(func() *quota.Policy { return pol }) } func TestATenantWithinItsGuaranteeTakesBackWhatAnotherBorrowed(t *testing.T) { // web is guaranteed one node and borrowed the second; ml is guaranteed // one node and wants it. Same class, and web's replica is at its minimum // — by class alone nothing could be taken. web := owned(gpuModel("web-chat", "", 2), "team:web") ml := owned(gpuModel("ml-train", "", 1), "team:ml") h := newFleetHarness(t, []catalog.Model{web, withMin(ml, 0)}, gpuNode("1", 8), gpuNode("2", 8)) withQuotas(h, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)}, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: gpus(8)}) h.settle(t, 4) if replicasOf(h, "web-chat") != 2 { t.Fatalf("setup: web did not borrow the second node: %+v", h.r.Replicas()) } h.advance(h.r.cfg.MinLifetime + time.Minute) cat, _ := catalog.New([]catalog.Model{web, ml}) h.r.cat.Store(cat) h.settle(t, 5) if replicasOf(h, "ml-train") != 1 || replicasOf(h, "web-chat") != 1 { t.Errorf("the borrowed node was not taken back: web %d, ml %d", replicasOf(h, "web-chat"), replicasOf(h, "ml-train")) } } func TestAReplicaWithinItsGuaranteeIsNotTakenByAHigherClassElsewhere(t *testing.T) { web := owned(gpuModel("web-chat", "", 1), "team:web") vital := owned(gpuModel("vital", catalog.ClassCritical, 1), "team:ml") h := fillThenWant(t, vital, web, gpuNode("1", 8)) withQuotas(h, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)}) h.settle(t, 3) if replicasOf(h, "web-chat") != 1 || replicasOf(h, "vital") != 0 { t.Errorf("a guaranteed replica was preempted by another tenant's class: %+v", h.r.Replicas()) } // Without the guarantee, critical takes it: the guarantee was the reason. h2 := fillThenWant(t, vital, web, gpuNode("1", 8)) h2.settle(t, 3) if replicasOf(h2, "vital") != 1 { t.Error("setup: without a guarantee the critical model should have taken the node") } } func TestALimitStopsAModelBeforeItStarts(t *testing.T) { web := owned(gpuModel("web-chat", "", 2), "team:web") h := newFleetHarness(t, []catalog.Model{web}, gpuNode("1", 8), gpuNode("2", 8)) withQuotas(h, store.CapacityQuota{Scope: store.ScopeOrg, ScopeID: "acme", Limit: gpus(8)}) h.settle(t, 4) if replicasOf(h, "web-chat") != 1 { t.Errorf("a second replica went past the org's 8-GPU limit: %+v", h.r.Replicas()) } hs := health(h, "web-chat") if !hs.Unplaceable || !strings.Contains(hs.Placement, "org:acme is at its capacity limit (16 GPUs of 8)") { t.Errorf("the refusal should name the limit: %+v", hs) } room, _ := h.r.Headroom("web-chat", 5) if room.More != 0 || len(room.Blocked) != 1 || !strings.Contains(room.Blocked[0], "capacity limit") { t.Errorf("headroom should stop at the limit, not the empty node: %+v", room) } }