A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
82 lines
3.4 KiB
Go
82 lines
3.4 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func owned(m catalog.Model, owner string) catalog.Model { m.Owner = owner; return m }
|
|
|
|
func gpus(n int) store.Resources { return store.Resources{GPUs: &n} }
|
|
|
|
// Two teams in one org; each 8-GPU node holds one tp8 replica.
|
|
func withQuotas(h *harness, qs ...store.CapacityQuota) {
|
|
pol := quota.New(qs, map[string][]string{"team:ml": {"org:acme"}, "team:web": {"org:acme"}})
|
|
h.r.SetQuotas(func() *quota.Policy { return pol })
|
|
}
|
|
|
|
func TestATenantWithinItsGuaranteeTakesBackWhatAnotherBorrowed(t *testing.T) {
|
|
// web is guaranteed one node and borrowed the second; ml is guaranteed
|
|
// one node and wants it. Same class, and web's replica is at its minimum
|
|
// — by class alone nothing could be taken.
|
|
web := owned(gpuModel("web-chat", "", 2), "team:web")
|
|
ml := owned(gpuModel("ml-train", "", 1), "team:ml")
|
|
h := newFleetHarness(t, []catalog.Model{web, withMin(ml, 0)}, gpuNode("1", 8), gpuNode("2", 8))
|
|
withQuotas(h,
|
|
store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)},
|
|
store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: gpus(8)})
|
|
h.settle(t, 4)
|
|
if replicasOf(h, "web-chat") != 2 {
|
|
t.Fatalf("setup: web did not borrow the second node: %+v", h.r.Replicas())
|
|
}
|
|
h.advance(h.r.cfg.MinLifetime + time.Minute)
|
|
cat, _ := catalog.New([]catalog.Model{web, ml})
|
|
h.r.cat.Store(cat)
|
|
|
|
h.settle(t, 5)
|
|
if replicasOf(h, "ml-train") != 1 || replicasOf(h, "web-chat") != 1 {
|
|
t.Errorf("the borrowed node was not taken back: web %d, ml %d",
|
|
replicasOf(h, "web-chat"), replicasOf(h, "ml-train"))
|
|
}
|
|
}
|
|
|
|
func TestAReplicaWithinItsGuaranteeIsNotTakenByAHigherClassElsewhere(t *testing.T) {
|
|
web := owned(gpuModel("web-chat", "", 1), "team:web")
|
|
vital := owned(gpuModel("vital", catalog.ClassCritical, 1), "team:ml")
|
|
h := fillThenWant(t, vital, web, gpuNode("1", 8))
|
|
withQuotas(h, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)})
|
|
h.settle(t, 3)
|
|
if replicasOf(h, "web-chat") != 1 || replicasOf(h, "vital") != 0 {
|
|
t.Errorf("a guaranteed replica was preempted by another tenant's class: %+v", h.r.Replicas())
|
|
}
|
|
// Without the guarantee, critical takes it: the guarantee was the reason.
|
|
h2 := fillThenWant(t, vital, web, gpuNode("1", 8))
|
|
h2.settle(t, 3)
|
|
if replicasOf(h2, "vital") != 1 {
|
|
t.Error("setup: without a guarantee the critical model should have taken the node")
|
|
}
|
|
}
|
|
|
|
func TestALimitStopsAModelBeforeItStarts(t *testing.T) {
|
|
web := owned(gpuModel("web-chat", "", 2), "team:web")
|
|
h := newFleetHarness(t, []catalog.Model{web}, gpuNode("1", 8), gpuNode("2", 8))
|
|
withQuotas(h, store.CapacityQuota{Scope: store.ScopeOrg, ScopeID: "acme", Limit: gpus(8)})
|
|
h.settle(t, 4)
|
|
if replicasOf(h, "web-chat") != 1 {
|
|
t.Errorf("a second replica went past the org's 8-GPU limit: %+v", h.r.Replicas())
|
|
}
|
|
hs := health(h, "web-chat")
|
|
if !hs.Unplaceable || !strings.Contains(hs.Placement, "org:acme is at its capacity limit (16 GPUs of 8)") {
|
|
t.Errorf("the refusal should name the limit: %+v", hs)
|
|
}
|
|
room, _ := h.r.Headroom("web-chat", 5)
|
|
if room.More != 0 || len(room.Blocked) != 1 || !strings.Contains(room.Blocked[0], "capacity limit") {
|
|
t.Errorf("headroom should stop at the limit, not the empty node: %+v", room)
|
|
}
|
|
}
|