A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
84 lines
2.8 KiB
Go
84 lines
2.8 KiB
Go
package quota
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func n(v int) *int { return &v }
|
|
|
|
// acme has two teams; ml is guaranteed 8 GPUs, the org as a whole 12 and
|
|
// no more than 16.
|
|
func policy() *Policy {
|
|
return New([]store.CapacityQuota{
|
|
{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: store.Resources{GPUs: n(8)}},
|
|
{Scope: store.ScopeOrg, ScopeID: "acme", Guarantee: store.Resources{GPUs: n(12)}, Limit: store.Resources{GPUs: n(16)}},
|
|
}, map[string][]string{
|
|
"team:ml": {"org:acme"}, "team:web": {"org:acme"},
|
|
"project:train": {"team:ml", "org:acme"},
|
|
})
|
|
}
|
|
|
|
func at(min int) time.Time { return time.Date(2026, 1, 1, 0, min, 0, 0, time.UTC) }
|
|
|
|
func TestAScopeHoldsWhatEverythingBeneathItHolds(t *testing.T) {
|
|
p := policy()
|
|
u := p.Usage([]Holding{
|
|
{Replica: "a", Owner: "project:train", Amount: Amount{GPUs: 4}},
|
|
{Replica: "b", Owner: "team:web", Amount: Amount{GPUs: 2, CPUCores: 3}},
|
|
{Replica: "c", Owner: "", Amount: Amount{GPUs: 8}}, // shared: nobody's
|
|
})
|
|
if u["team:ml"].GPUs != 4 || u["org:acme"].GPUs != 6 || u["org:acme"].CPUCores != 3 || u["project:train"].GPUs != 4 {
|
|
t.Errorf("usage %+v", u)
|
|
}
|
|
if _, ok := u[""]; ok {
|
|
t.Error("a shared model was counted against somebody")
|
|
}
|
|
}
|
|
|
|
func TestWhatGoesPastAGuaranteeIsBorrowedNewestFirst(t *testing.T) {
|
|
p := policy()
|
|
b := p.Borrowed([]Holding{
|
|
{Replica: "old", Owner: "team:ml", Amount: Amount{GPUs: 8}, Since: at(1)},
|
|
{Replica: "new", Owner: "team:ml", Amount: Amount{GPUs: 4}, Since: at(2)},
|
|
{Replica: "web", Owner: "team:web", Amount: Amount{GPUs: 4}, Since: at(3)},
|
|
})
|
|
// ml: 8 within its 8, then 4 past it. acme: 8 + 4 = 12 within, web's 4
|
|
// takes the org to 16, past its 12.
|
|
if b["old"] || !b["new"] || !b["web"] {
|
|
t.Errorf("borrowed %v", b)
|
|
}
|
|
}
|
|
|
|
func TestAGuaranteeIsOnlyForWhoHasOne(t *testing.T) {
|
|
p := policy()
|
|
u := map[string]Amount{"team:ml": {GPUs: 4}, "org:acme": {GPUs: 4}}
|
|
if !p.Guaranteed("project:train", u, Amount{GPUs: 4}) {
|
|
t.Error("8 of ml's 8 and 8 of acme's 12 is within both")
|
|
}
|
|
if p.Guaranteed("project:train", u, Amount{GPUs: 5}) {
|
|
t.Error("9 GPUs is past ml's 8")
|
|
}
|
|
if p.Guaranteed("", u, Amount{GPUs: 1}) || New(nil, nil).Guaranteed("team:ml", u, Amount{}) {
|
|
t.Error("an owner with no guarantee is not guaranteed anything")
|
|
}
|
|
}
|
|
|
|
func TestALimitStopsGrowthWhereverItIsSet(t *testing.T) {
|
|
p := policy()
|
|
u := map[string]Amount{"org:acme": {GPUs: 14}}
|
|
if why := p.AtLimit("team:web", u, Amount{GPUs: 2}); why != "" {
|
|
t.Errorf("16 of 16 is at the limit, not past it: %s", why)
|
|
}
|
|
why := p.AtLimit("team:web", u, Amount{GPUs: 4})
|
|
if !strings.Contains(why, "org:acme is at its capacity limit (18 GPUs of 16)") {
|
|
t.Errorf("the refusal should name the scope and the figures: %q", why)
|
|
}
|
|
if p.AtLimit("", u, Amount{GPUs: 99}) != "" {
|
|
t.Error("a shared model was limited")
|
|
}
|
|
}
|