The tenancy tree handed limits down and never pooled what was spent against them. A limit written on a team was inherited by every key beneath it, and then enforced with the key as the counter — so a project given a budget of 1000 and ten keys permitted ten thousand, and a team given 60 RPM got 60 per member. That is "each of you may spend this much", which is a useful thing to be able to say. It is not what somebody writing a budget on a team means, and nothing in the system said which one you were getting. Resolution now records which scope set each value that won, and the counter is keyed by that scope. The value and the counter come from the same place, so a project tightening its team's limit is enforced against the project's own pool rather than eating into a quota another project never agreed to. Each limit is owned separately, because they resolve separately: a team budget with a per-key rate limit — the group's money, the individual's throughput — is the ordinary arrangement and has to stay expressible. Cluster defaults are owned by the key. Read as a shared pool, one default would become a single cluster-wide rate limit that every key in every organization competes for: not a default, an outage. Restoring after a restart had to change with it. Per-key totals rebuilt a counter named after the key while the check read one named after the team, so the control plane came back believing the team had spent nothing — a budget that reset on every restart, for exactly the scopes where it matters most. SpendSinceByScope returns each key's place in the tree and the accountant folds it into every scope that might own a budget. Counters are namespaced by level throughout. A bare key id in one path and a "key:"-prefixed one in another are two counters for the same key, which is how the restore was silently doing nothing. Still missing, and worth naming: no usage report rolls up by project, team or org, so a shared quota can now be enforced but not yet shown. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
148 lines
4.3 KiB
Go
148 lines
4.3 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func treeStore(t *testing.T) *SQLite {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "t.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = s.Close() })
|
|
|
|
for _, err := range []error{
|
|
s.PutOrg(ctx, Org{ID: "o", Name: "o"}),
|
|
s.PutTeam(ctx, Team{ID: "t", OrgID: "o", Name: "t"}),
|
|
s.PutProject(ctx, Project{ID: "p", TeamID: "t", Name: "p"}),
|
|
} {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_, k, _ := NewKey("k1", "n", "p")
|
|
k.CreatedAt = time.Now()
|
|
if err := s.CreateKey(ctx, k); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestAResolvedLimitRemembersWhichScopeSetIt(t *testing.T) {
|
|
s := treeStore(t)
|
|
rpm := 60
|
|
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &rpm}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
e, err := s.Resolve(t.Context(), "k1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Without this the tree hands down values and never pools spending: the
|
|
// team's 60 becomes 60 for each key under it.
|
|
if e.RPMOwner.Scope != ScopeTeam || e.RPMOwner.ID != "t" {
|
|
t.Errorf("expected the team to own the rate limit, got %+v", e.RPMOwner)
|
|
}
|
|
if e.RPMOwner.CounterKey() != "team:t" {
|
|
t.Errorf("counter key should name the level too; got %q", e.RPMOwner.CounterKey())
|
|
}
|
|
}
|
|
|
|
func TestTheTighterScopeTakesOwnershipWithTheValue(t *testing.T) {
|
|
s := treeStore(t)
|
|
loose, tight := 100, 10
|
|
must := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
must(s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &loose}))
|
|
must(s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &tight}))
|
|
|
|
e, err := s.Resolve(t.Context(), "k1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The value and the counter have to come from the same place. Enforcing
|
|
// the project's 10 against the team's pool would let one project's traffic
|
|
// exhaust a limit another project never agreed to.
|
|
if *e.RPM != 10 || e.RPMOwner.ID != "p" {
|
|
t.Errorf("expected the project's value and ownership, got %d from %+v", *e.RPM, e.RPMOwner)
|
|
}
|
|
}
|
|
|
|
func TestALooserInnerScopeDoesNotTakeOwnership(t *testing.T) {
|
|
s := treeStore(t)
|
|
tight, loose := 10, 100
|
|
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &tight}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &loose}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
e, err := s.Resolve(t.Context(), "k1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A project cannot raise what its team set, so the team still owns both the
|
|
// number and the counter it is enforced against.
|
|
if *e.RPM != 10 || e.RPMOwner.ID != "t" {
|
|
t.Errorf("expected the team to keep ownership, got %d from %+v", *e.RPM, e.RPMOwner)
|
|
}
|
|
}
|
|
|
|
func TestAClusterDefaultIsOwnedByTheKey(t *testing.T) {
|
|
s := treeStore(t)
|
|
rpm := 30
|
|
s.SetDefaultLimits(&rpm, nil)
|
|
|
|
e, err := s.Resolve(t.Context(), "k1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A default is a fallback for each caller given no limit of its own. Owned
|
|
// by a shared scope it would become one cluster-wide rate limit that every
|
|
// key in every organization competes for — not a default, an outage.
|
|
if e.RPMOwner.Scope != ScopeKey || e.RPMOwner.ID != "k1" {
|
|
t.Errorf("a default should be counted per key, got %+v", e.RPMOwner)
|
|
}
|
|
}
|
|
|
|
func TestSpendComesBackWithThePlaceInTheTree(t *testing.T) {
|
|
s := treeStore(t)
|
|
if err := s.WriteUsage(t.Context(), []Usage{
|
|
{At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m",
|
|
ServedModel: "m", CostMicros: 700, Status: 200},
|
|
{At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m",
|
|
ServedModel: "m", CostMicros: 300, Status: 200},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
rows, err := s.SpendSinceByScope(t.Context(), time.Now().Add(-time.Hour))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rows) != 1 {
|
|
t.Fatalf("expected one key's total, got %+v", rows)
|
|
}
|
|
// The tree, not just the key: a budget owned by a team is rebuilt by
|
|
// summing every key under it, and per-key totals alone cannot say which
|
|
// keys those are.
|
|
got := rows[0]
|
|
if got.Micros != 1000 {
|
|
t.Errorf("expected the two rows summed, got %d", got.Micros)
|
|
}
|
|
if got.ProjectID != "p" || got.TeamID != "t" || got.OrgID != "o" {
|
|
t.Errorf("the whole chain should come back, got %+v", got)
|
|
}
|
|
}
|