Files
llmgw/internal/control/reconciler/quotas_test.go
T
Ilker ManapandClaude Opus 5.5 9dc1b62fdc
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s
Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope
GPUs, cores and memory and may limit it; what a scope holds is summed from
its models' replicas and those beneath it. A replica within its owner's
guarantee is not taken for another scope; what is held beyond a guarantee
is borrowed and taken back for a scope within its own. A limit stops a
model before it starts, and headroom stops there too. ctl quota and ctl
model owner set them; metrics and a CapacityAtLimit alert report them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:15:38 +02:00

82 lines
3.4 KiB
Go

package reconciler
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func owned(m catalog.Model, owner string) catalog.Model { m.Owner = owner; return m }
func gpus(n int) store.Resources { return store.Resources{GPUs: &n} }
// Two teams in one org; each 8-GPU node holds one tp8 replica.
func withQuotas(h *harness, qs ...store.CapacityQuota) {
pol := quota.New(qs, map[string][]string{"team:ml": {"org:acme"}, "team:web": {"org:acme"}})
h.r.SetQuotas(func() *quota.Policy { return pol })
}
func TestATenantWithinItsGuaranteeTakesBackWhatAnotherBorrowed(t *testing.T) {
// web is guaranteed one node and borrowed the second; ml is guaranteed
// one node and wants it. Same class, and web's replica is at its minimum
// — by class alone nothing could be taken.
web := owned(gpuModel("web-chat", "", 2), "team:web")
ml := owned(gpuModel("ml-train", "", 1), "team:ml")
h := newFleetHarness(t, []catalog.Model{web, withMin(ml, 0)}, gpuNode("1", 8), gpuNode("2", 8))
withQuotas(h,
store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)},
store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: gpus(8)})
h.settle(t, 4)
if replicasOf(h, "web-chat") != 2 {
t.Fatalf("setup: web did not borrow the second node: %+v", h.r.Replicas())
}
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{web, ml})
h.r.cat.Store(cat)
h.settle(t, 5)
if replicasOf(h, "ml-train") != 1 || replicasOf(h, "web-chat") != 1 {
t.Errorf("the borrowed node was not taken back: web %d, ml %d",
replicasOf(h, "web-chat"), replicasOf(h, "ml-train"))
}
}
func TestAReplicaWithinItsGuaranteeIsNotTakenByAHigherClassElsewhere(t *testing.T) {
web := owned(gpuModel("web-chat", "", 1), "team:web")
vital := owned(gpuModel("vital", catalog.ClassCritical, 1), "team:ml")
h := fillThenWant(t, vital, web, gpuNode("1", 8))
withQuotas(h, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)})
h.settle(t, 3)
if replicasOf(h, "web-chat") != 1 || replicasOf(h, "vital") != 0 {
t.Errorf("a guaranteed replica was preempted by another tenant's class: %+v", h.r.Replicas())
}
// Without the guarantee, critical takes it: the guarantee was the reason.
h2 := fillThenWant(t, vital, web, gpuNode("1", 8))
h2.settle(t, 3)
if replicasOf(h2, "vital") != 1 {
t.Error("setup: without a guarantee the critical model should have taken the node")
}
}
func TestALimitStopsAModelBeforeItStarts(t *testing.T) {
web := owned(gpuModel("web-chat", "", 2), "team:web")
h := newFleetHarness(t, []catalog.Model{web}, gpuNode("1", 8), gpuNode("2", 8))
withQuotas(h, store.CapacityQuota{Scope: store.ScopeOrg, ScopeID: "acme", Limit: gpus(8)})
h.settle(t, 4)
if replicasOf(h, "web-chat") != 1 {
t.Errorf("a second replica went past the org's 8-GPU limit: %+v", h.r.Replicas())
}
hs := health(h, "web-chat")
if !hs.Unplaceable || !strings.Contains(hs.Placement, "org:acme is at its capacity limit (16 GPUs of 8)") {
t.Errorf("the refusal should name the limit: %+v", hs)
}
room, _ := h.r.Headroom("web-chat", 5)
if room.More != 0 || len(room.Blocked) != 1 || !strings.Contains(room.Blocked[0], "capacity limit") {
t.Errorf("headroom should stop at the limit, not the empty node: %+v", room)
}
}