Ilker ManapandClaude Opus 5.5
9dc1b62fdc
A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
64 lines
2.2 KiB
Go
64 lines
2.2 KiB
Go
package store
|
|
|
|
import "testing"
|
|
|
|
func ip(v int) *int { return &v }
|
|
|
|
func TestAQuotaKeepsWhatItGovernsAndNothingElse(t *testing.T) {
|
|
s := tenancyStore(t)
|
|
org, _ := s.CreateOrg(t.Context(), "Acme")
|
|
q := CapacityQuota{Scope: ScopeOrg, ScopeID: org, UpdatedBy: "ops",
|
|
Guarantee: Resources{GPUs: ip(8)}, Limit: Resources{GPUs: ip(16), CPUCores: ip(0)}}
|
|
if err := s.PutCapacityQuota(t.Context(), q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
all, err := s.CapacityQuotas(t.Context())
|
|
if err != nil || len(all) != 1 {
|
|
t.Fatalf("%v %+v", err, all)
|
|
}
|
|
got := all[0]
|
|
if *got.Guarantee.GPUs != 8 || got.Guarantee.CPUCores != nil || got.Guarantee.MemoryGB != nil {
|
|
t.Errorf("guarantee %+v: only GPUs were set", got.Guarantee)
|
|
}
|
|
// Zero is a limit of nothing, not an absent one.
|
|
if *got.Limit.GPUs != 16 || got.Limit.CPUCores == nil || *got.Limit.CPUCores != 0 {
|
|
t.Errorf("limit %+v", got.Limit)
|
|
}
|
|
q.Guarantee = Resources{MemoryGB: ip(64)}
|
|
if err := s.PutCapacityQuota(t.Context(), q); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
all, _ = s.CapacityQuotas(t.Context())
|
|
if len(all) != 1 || all[0].Guarantee.GPUs != nil || *all[0].Guarantee.MemoryGB != 64 {
|
|
t.Errorf("a second put should replace the first: %+v", all)
|
|
}
|
|
if err := s.DeleteCapacityQuota(t.Context(), ScopeOrg, org); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if all, _ = s.CapacityQuotas(t.Context()); len(all) != 0 {
|
|
t.Error("the quota survived its deletion")
|
|
}
|
|
if err := s.PutCapacityQuota(t.Context(), CapacityQuota{Scope: ScopeOrg, ScopeID: org,
|
|
Guarantee: Resources{GPUs: ip(-1)}}); err == nil {
|
|
t.Error("a negative quota was stored")
|
|
}
|
|
}
|
|
|
|
func TestEveryScopeKnowsWhatIsAboveIt(t *testing.T) {
|
|
s := tenancyStore(t)
|
|
org, _ := s.CreateOrg(t.Context(), "Acme")
|
|
team, _ := s.CreateTeam(t.Context(), org, "ml")
|
|
proj, _ := s.CreateProject(t.Context(), team, "training")
|
|
p, err := s.ScopeParents(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"team:" + team, "org:" + org}
|
|
if got := p["project:"+proj]; len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
|
t.Errorf("project's parents %v, expected %v", got, want)
|
|
}
|
|
if got := p["team:"+team]; len(got) != 1 || got[0] != "org:"+org {
|
|
t.Errorf("team's parents %v", got)
|
|
}
|
|
}
|