Files
llmgw/internal/accounting/accounting_test.go
T
Ilker ManapandClaude Opus 5 3febb6ef63 Count a group's quota against the group, not against each of its keys
The tenancy tree handed limits down and never pooled what was spent against
them. A limit written on a team was inherited by every key beneath it, and then
enforced with the key as the counter — so a project given a budget of 1000 and
ten keys permitted ten thousand, and a team given 60 RPM got 60 per member.

That is "each of you may spend this much", which is a useful thing to be able
to say. It is not what somebody writing a budget on a team means, and nothing
in the system said which one you were getting.

Resolution now records which scope set each value that won, and the counter is
keyed by that scope. The value and the counter come from the same place, so a
project tightening its team's limit is enforced against the project's own pool
rather than eating into a quota another project never agreed to.

Each limit is owned separately, because they resolve separately: a team budget
with a per-key rate limit — the group's money, the individual's throughput — is
the ordinary arrangement and has to stay expressible.

Cluster defaults are owned by the key. Read as a shared pool, one default would
become a single cluster-wide rate limit that every key in every organization
competes for: not a default, an outage.

Restoring after a restart had to change with it. Per-key totals rebuilt a
counter named after the key while the check read one named after the team, so
the control plane came back believing the team had spent nothing — a budget
that reset on every restart, for exactly the scopes where it matters most.
SpendSinceByScope returns each key's place in the tree and the accountant folds
it into every scope that might own a budget.

Counters are namespaced by level throughout. A bare key id in one path and a
"key:"-prefixed one in another are two counters for the same key, which is how
the restore was silently doing nothing.

Still missing, and worth naming: no usage report rolls up by project, team or
org, so a shared quota can now be enforced but not yet shown.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 12:10:46 +02:00

462 lines
15 KiB
Go

package accounting
import (
"context"
"errors"
"log/slog"
"path/filepath"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func intp(v int) *int { return &v }
func i64p(v int64) *int64 { return &v }
func principal(id string, tune func(*store.Effective)) auth.Principal {
e := store.Effective{KeyID: id, ProjectID: "proj"}
if tune != nil {
tune(&e)
}
return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e}
}
func pricing(t *testing.T) *Pricing {
t.Helper()
return NewPricing(map[string]Price{
"paid": {PromptPer1k: 1000, CompletionPer1k: 3000},
"free": {},
}, Price{})
}
func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
// Rounding down would make a stream of tiny requests cost nothing, which is
// exactly the traffic shape a quota exists to bound.
p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
if got := p.Cost(1, 0); got != 1 {
t.Errorf("one prompt token cost %d", got)
}
if got := p.Cost(1000, 1000); got != 4000 {
t.Errorf("a thousand of each cost %d, want 4000", got)
}
if got := p.Cost(0, 0); got != 0 {
t.Errorf("an empty request cost %d", got)
}
}
func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) {
// This is what makes budget fallback work: a key with nothing left can
// still be served something that costs nothing.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", func(e *store.Effective) {
e.Budget = i64p(100)
e.BudgetPeriod = store.PeriodMonth
})
res, err := a.Check(p, "paid", 10)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0})
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Errorf("a spent budget still served a paid model: %v", err)
}
if _, err := a.Check(p, "free", 10); err != nil {
t.Errorf("a spent budget refused a free model: %v", err)
}
}
func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) {
// A budget does not refill until the period rolls over, so a Retry-After of
// a second would be a lie that turns one client into a retry loop.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", func(e *store.Effective) {
e.Budget = i64p(1)
e.BudgetPeriod = store.PeriodDay
})
res, _ := a.Check(p, "paid", 10)
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000})
_, err := a.Check(p, "paid", 10)
var le *LimitError
if !errors.As(err, &le) {
t.Fatalf("expected a limit error, got %v", err)
}
if le.RetryAfter != 0 {
t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter)
}
}
func TestRateLimitsBind(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) })
for i := range 3 {
if _, err := a.Check(p, "free", 1); err != nil {
t.Fatalf("request %d was refused: %v", i, err)
}
}
if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) {
t.Errorf("the fourth request was allowed: %v", err)
}
}
func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) {
// The real token count only exists once the request has run, so the check
// has to work on an estimate. A caller that consistently underestimates
// must still end up bounded.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) })
// Claims 10 tokens, actually uses 900.
res, err := a.Check(p, "free", 10)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500})
// The settled figure is what the next check sees, so a second request of
// the same shape cannot fit.
if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) {
t.Errorf("the underestimate was never settled: %v", err)
}
}
func TestNoLimitIsNotALimitOfZero(t *testing.T) {
// The distinction the store keeps with a nil pointer. Getting it wrong here
// would silently forbid every request for every key with no limits set.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
p := principal("k1", nil)
for range 50 {
if _, err := a.Check(p, "paid", 10_000); err != nil {
t.Fatalf("a key with no limits was refused: %v", err)
}
}
}
func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) {
// With auth off there is no key to charge, and Check must not refuse.
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
res, err := a.Check(auth.Principal{}, "paid", 1000)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{PromptTokens: 1000})
if spent, _, _ := a.Status(auth.Principal{}); spent != 0 {
t.Errorf("an anonymous request was charged %d", spent)
}
}
func TestBudgetSurvivesARestart(t *testing.T) {
// The counters live in memory; the store is what makes them survive.
ctx := context.Background()
path := filepath.Join(t.TempDir(), "acct.db")
s, err := store.OpenSQLite(ctx, path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"}))
must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"}))
_, k, _ := store.NewKey("k1", "n", "proj")
k.CreatedAt = time.Now()
must(t, s.CreateKey(ctx, k))
must(t, s.WriteUsage(ctx, []store.Usage{
{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
ServedModel: "paid", CostMicros: 5000, Status: 200},
}))
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
p := principal("k1", func(e *store.Effective) {
e.Budget = i64p(5000)
e.BudgetPeriod = store.PeriodMonth
})
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Errorf("the budget was reset by the restart: %v", err)
}
}
func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
b := NewBudgets()
now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
b.now = func() time.Time { return now }
b.Add("k1", store.PeriodMonth, 900)
if got := b.Spent("k1", store.PeriodMonth); got != 900 {
t.Fatalf("spent %d", got)
}
now = now.Add(2 * time.Minute) // into February
if got := b.Spent("k1", store.PeriodMonth); got != 0 {
t.Errorf("the counter carried %d across the month boundary", got)
}
}
func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) {
b := NewBudgets()
now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
b.now = func() time.Time { return now }
b.Add("daily", store.PeriodDay, 100)
b.Add("monthly", store.PeriodMonth, 100)
now = now.Add(24 * time.Hour)
if got := b.Spent("daily", store.PeriodDay); got != 0 {
t.Errorf("the daily counter carried %d across a day", got)
}
if got := b.Spent("monthly", store.PeriodMonth); got != 100 {
t.Errorf("the monthly counter lost %d within a month", 100-got)
}
}
func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) {
// The reason for a sliding window. With a fixed one, a caller sends its
// whole allowance at the end of a window and again at the start of the
// next, so the observed rate is double the limit exactly when the system is
// least able to take it.
l := NewLimiter()
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
now := base.Add(59 * time.Second)
l.now = func() time.Time { return now }
const limit = 60
sent := 0
for range limit {
if l.Allow("k", limit, 1) {
sent++
}
}
if sent != limit {
t.Fatalf("the first burst only managed %d", sent)
}
// One second later, in the next fixed window.
now = base.Add(61 * time.Second)
second := 0
for range limit {
if l.Allow("k", limit, 1) {
second++
}
}
// Some allowance returns as the old window slides out of view, but nowhere
// near the whole of it.
if second > limit/2 {
t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding",
second, sent)
}
}
func TestLimiterIsSafeUnderConcurrency(t *testing.T) {
// Checking and charging have to be one operation, or two concurrent
// requests both see room and both take it.
l := NewLimiter()
const limit = 100
var wg sync.WaitGroup
var mu sync.Mutex
allowed := 0
for range 500 {
wg.Add(1)
go func() {
defer wg.Done()
if l.Allow("k", limit, 1) {
mu.Lock()
allowed++
mu.Unlock()
}
}()
}
wg.Wait()
if allowed != limit {
t.Errorf("%d requests passed a limit of %d", allowed, limit)
}
}
func must(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
// teamMember is a key whose budget and rate limit were set on its team, which
// is the arrangement an enterprise actually asks for: a group gets a quota and
// the people in it share it.
func teamMember(keyID, teamID string, tune func(*store.Effective)) auth.Principal {
e := store.Effective{KeyID: keyID, ProjectID: "proj", TeamID: teamID}
owner := store.LimitOwner{Scope: store.ScopeTeam, ID: teamID}
e.BudgetOwner, e.RPMOwner, e.TPMOwner = owner, owner, owner
if tune != nil {
tune(&e)
}
return auth.Principal{KeyID: keyID, ProjectID: "proj", Limits: e}
}
func TestATeamBudgetIsOnePoolRatherThanOnePerMember(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
budget := func(e *store.Effective) {
e.Budget = i64p(3000)
e.BudgetPeriod = store.PeriodMonth
}
first := teamMember("k1", "t1", budget)
second := teamMember("k2", "t1", budget)
// One member spends the lot.
res, err := a.Check(first, "paid", 1000)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
// The second must now be refused. Counted per key it would have its own
// untouched 3000 — a team budget of 3000 with ten keys would permit thirty
// thousand, which is the bug this change exists for.
if _, err := a.Check(second, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Errorf("the team's budget should be shared; the second member got %v", err)
}
}
func TestATeamRateLimitIsSharedToo(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
rpm := func(e *store.Effective) { e.RPM = intp(1) }
if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
t.Fatal(err)
}
if _, err := a.Check(teamMember("k2", "t1", rpm), "free", 1); !errors.Is(err, ErrRateLimited) {
t.Error("a rate limit set on the team should be shared by its keys")
}
}
func TestTwoTeamsDoNotShareACounter(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
rpm := func(e *store.Effective) { e.RPM = intp(1) }
if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
t.Fatal(err)
}
// Pooling is the point; pooling the wrong things is the danger. One team
// exhausting another's quota would be worse than the bug being fixed.
if _, err := a.Check(teamMember("k2", "t2", rpm), "free", 1); err != nil {
t.Errorf("a second team should have its own counter, got %v", err)
}
}
func TestAKeyLimitStaysPerKey(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
perKey := func(id string) auth.Principal {
return principal(id, func(e *store.Effective) {
e.RPM = intp(1)
e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
})
}
if _, err := a.Check(perKey("k1"), "free", 1); err != nil {
t.Fatal(err)
}
// The tree still has to be able to say "each of you may do this much".
// Sharing a group pool must not become the only thing expressible.
if _, err := a.Check(perKey("k2"), "free", 1); err != nil {
t.Errorf("a limit owned by the key should not be shared, got %v", err)
}
}
func TestTheDifferentLimitsCanBeOwnedByDifferentScopes(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
// A team budget with a per-key rate limit is the ordinary arrangement: the
// group's money, the individual's throughput.
mk := func(id string) auth.Principal {
e := store.Effective{KeyID: id, TeamID: "t1"}
e.Budget, e.BudgetPeriod = i64p(3000), store.PeriodMonth
e.BudgetOwner = store.LimitOwner{Scope: store.ScopeTeam, ID: "t1"}
e.RPM = intp(1)
e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
return auth.Principal{KeyID: id, Limits: e}
}
res, err := a.Check(mk("k1"), "paid", 1000)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
// k2's rate limit is its own and untouched; the budget is the team's and
// gone. The budget is checked first, so that is what comes back.
if _, err := a.Check(mk("k2"), "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Errorf("expected the shared budget to bite, got %v", err)
}
}
func TestStatusReportsTheCounterThatWillRefuseYou(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
budget := func(e *store.Effective) {
e.Budget = i64p(9000)
e.BudgetPeriod = store.PeriodMonth
}
res, err := a.Check(teamMember("k1", "t1", budget), "paid", 1000)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
// A member who has spent nothing themselves still sees the team's spend,
// because that is the number that will stop them. A header disagreeing with
// the refusal it precedes is worse than no header.
spent, _, _ := a.Status(teamMember("k2", "t1", budget))
if spent == 0 {
t.Error("a member should see the pool's spend, not their own")
}
}
func TestATeamBudgetSurvivesARestart(t *testing.T) {
// The per-key restore could not do this: it rebuilt a counter named after
// the key, while the check reads one named after the team. The control
// plane came back believing the team had spent nothing — a budget that
// resets on every restart, for exactly the scope where it matters most.
ctx := context.Background()
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "team.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
must(t, s.PutTeam(ctx, store.Team{ID: "t1", OrgID: "o", Name: "t"}))
must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t1", Name: "p"}))
for _, id := range []string{"k1", "k2"} {
_, k, _ := store.NewKey(id, "n", "proj")
k.CreatedAt = time.Now()
must(t, s.CreateKey(ctx, k))
}
// Two members of one team, 2500 each: neither alone exceeds a 4000 budget,
// together they do.
must(t, s.WriteUsage(ctx, []store.Usage{
{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
ServedModel: "paid", CostMicros: 2500, Status: 200},
{At: time.Now(), KeyID: "k2", ProjectID: "proj", Model: "paid",
ServedModel: "paid", CostMicros: 2500, Status: 200},
}))
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
p := teamMember("k1", "t1", func(e *store.Effective) {
e.Budget = i64p(4000)
e.BudgetPeriod = store.PeriodMonth
})
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Errorf("the team's spend should have been restored and pooled: %v", err)
}
}