diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index 8746279..5ce8dbe 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -100,14 +100,24 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s // Reservation is what Check returns: the estimate it charged, so Record can // settle the difference against the real figure. +// pool is one level's counter and the period its budget runs over. +type pool struct { + Counter string + Period store.Period +} + type Reservation struct { KeyID string - // BudgetCounter and TokenCounter are what the check counted against, carried - // so the settlement lands on the same counter. Deriving them again here - // would mean re-resolving the limits, and a budget checked against the team - // and settled against the key is a budget that never fills up. - BudgetCounter string - TokenCounter string + // Budgets and Tokens are every counter the check went through, carried so + // the settlement lands on the same ones. Every, not the tightest: an + // organization's pool only fills up if each team's spend is added to it + // as well as to the team's own. + // + // Derived again at settlement would mean re-resolving the limits, and a + // budget checked against the team and settled against the key is a budget + // that never fills up. + Budgets []pool + Tokens []string EstimatedTokens int64 Price Price // Period is the key's budget period, carried from the check so the record @@ -125,17 +135,24 @@ type Reservation struct { func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) { price := a.pricing.For(model) lim := p.Limits - // Counted against whichever scope set the limit. A budget written on a team - // is one pool the team shares; counting it per key would hand the team's - // number to every key under it and permit the total times the headcount. - budgetCounter := counterFor(lim.BudgetOwner, p.KeyID) - tokenCounter := counterFor(lim.TPMOwner, p.KeyID) + // Every level that set a limit, each against its own pool. A quota written + // on a team is one pool the team shares, and the organization above it is a + // second pool that the team's spending also fills — both have to hold, or + // an organization given 1000 with four teams given 500 spends 2000. + chain := levels(lim, p.KeyID) res := Reservation{ KeyID: p.KeyID, EstimatedTokens: estimatedTokens, - BudgetCounter: budgetCounter, TokenCounter: tokenCounter, Price: price, Period: lim.BudgetPeriod, } + for _, l := range chain { + if l.Budget != nil { + res.Budgets = append(res.Budgets, pool{l.Counter, l.Period}) + } + if l.TPM != nil { + res.Tokens = append(res.Tokens, l.Counter) + } + } if p.KeyID == "" { return res, nil } @@ -146,15 +163,29 @@ func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64 // // A model that costs nothing is exempt, which is what lets a key with no // budget left still be served a cheap fallback (plan §7.7). - if lim.Budget != nil && !price.Zero() && - a.budgets.Exhausted(budgetCounter, lim.BudgetPeriod, *lim.Budget) { - return res, &LimitError{Err: ErrBudgetSpent} + // Budgets first, and all of them. Exhausted only reads, so the order costs + // nothing and a refusal here charges no counter. + if !price.Zero() { + for _, l := range chain { + if l.Budget != nil && a.budgets.Exhausted(l.Counter, l.Period, *l.Budget) { + return res, &LimitError{Err: ErrBudgetSpent} + } + } } - if lim.RPM != nil && !a.requests.Allow(counterFor(lim.RPMOwner, p.KeyID), *lim.RPM, 1) { + // Rate limits charge as they check — deliberately, so two concurrent + // requests cannot both see room. With several levels that means a refusal + // at one has already taken from the levels before it, so those are handed + // back. Without the refund a team at its ceiling would quietly eat its + // organization's rate limit with requests that were never served. + if err := a.allowAll(chain, func(l level) (string, int, int64, bool) { + return l.Counter, valInt(l.RPM), 1, l.RPM != nil + }, a.requests); err != nil { return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second} } - if lim.TPM != nil && !a.tokens.Allow(tokenCounter, *lim.TPM, estimatedTokens) { + if err := a.allowAll(chain, func(l level) (string, int, int64, bool) { + return l.Counter, valInt(l.TPM), estimatedTokens, l.TPM != nil + }, a.tokens); err != nil { // The request was not charged against the request-rate limit in vain: // it was, and that is correct — it did arrive. return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second} @@ -162,6 +193,88 @@ func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64 return res, nil } +// level is one scope's limits with the counter they are enforced against. +type level struct { + Counter string + Period store.Period + RPM *int + TPM *int + Budget *int64 +} + +// levels turns a resolved chain into the pools a request must pass. +// +// Falls back to the collapsed values when the chain is empty, which is what a +// principal assembled by hand looks like — in tests, and anywhere that builds +// limits without walking the tree. +func levels(e store.Effective, keyID string) []level { + if len(e.Chain) == 0 { + var out []level + if e.RPM != nil || e.TPM != nil || e.Budget != nil { + out = append(out, level{ + Counter: counterFor(e.BudgetOwner, keyID), + Period: e.BudgetPeriod, Budget: e.Budget, + }) + out[0].RPM, out[0].TPM = e.RPM, e.TPM + // One entry, so a hand-built principal with different owners per + // limit still counts each where it said to. + if r, b := counterFor(e.RPMOwner, keyID), out[0].Counter; r != b && e.RPM != nil { + out[0].RPM = nil + out = append(out, level{Counter: r, RPM: e.RPM}) + } + if tp, b := counterFor(e.TPMOwner, keyID), out[0].Counter; tp != b && e.TPM != nil { + out[0].TPM = nil + out = append(out, level{Counter: tp, TPM: e.TPM}) + } + } + return out + } + out := make([]level, 0, len(e.Chain)) + for _, l := range e.Chain { + out = append(out, level{ + Counter: store.LimitOwner{Scope: l.Scope, ID: l.ScopeID}.CounterKey(), + Period: l.BudgetPeriod, RPM: l.RPM, TPM: l.TPM, Budget: l.Budget, + }) + } + return out +} + +// allowAll charges every level that has this limit, refunding on a refusal. +func (a *Accountant) allowAll(chain []level, + pick func(level) (key string, limit int, n int64, has bool), lim *Limiter, +) error { + var charged []struct { + key string + n int64 + } + for _, l := range chain { + key, limit, n, has := pick(l) + if !has { + continue + } + if !lim.Allow(key, limit, n) { + for _, c := range charged { + lim.Charge(c.key, -c.n) + } + return errRefused + } + charged = append(charged, struct { + key string + n int64 + }{key, n}) + } + return nil +} + +var errRefused = errors.New("refused") + +func valInt(v *int) int { + if v == nil { + return 0 + } + return *v +} + // Record settles a finished request: it reconciles the token estimate, adds to // the budget, and queues the usage row. func (a *Accountant) Record(res Reservation, u store.Usage) { @@ -173,10 +286,16 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { // The estimate was charged up front; charge the difference now. A caller // whose requests are consistently larger than estimated is still bounded, // one window later than with a perfect estimate. - a.tokens.Charge(or(res.TokenCounter, keyCounter(res.KeyID)), actual-res.EstimatedTokens) + // Every level the check went through, so an organization's pool fills up + // from its teams' spending rather than only from requests that named it. + for _, c := range res.Tokens { + a.tokens.Charge(c, actual-res.EstimatedTokens) + } u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) - a.budgets.Add(or(res.BudgetCounter, keyCounter(res.KeyID)), res.Period, u.CostMicros) + for _, b := range res.Budgets { + a.budgets.Add(b.Counter, b.Period, u.CostMicros) + } if a.recorder != nil { a.recorder.Record(u) @@ -188,13 +307,29 @@ func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { return 0, 0, 0 } - // Reported against the same counters the check reads, so a header saying - // how much is left agrees with the refusal the next request gets. A member - // of a team sharing a budget sees the team's spend, which is the number - // that will stop them. - return a.budgets.Spent(counterFor(p.Limits.BudgetOwner, p.KeyID), p.Limits.BudgetPeriod), - a.requests.Rate(counterFor(p.Limits.RPMOwner, p.KeyID)), - a.tokens.Rate(counterFor(p.Limits.TPMOwner, p.KeyID)) + // The pool that will refuse them, which is not always the one with the + // smallest number on it. A team given 2000 inside an organization given the + // same is bounded by whichever is nearer its ceiling, and that is the + // organization as soon as a second team spends anything. A header that + // reported the team's would disagree with the refusal that follows it. + for _, l := range levels(p.Limits, p.KeyID) { + if l.Budget != nil { + if v := a.budgets.Spent(l.Counter, l.Period); v > spent { + spent = v + } + } + if l.RPM != nil { + if v := a.requests.Rate(l.Counter); v > rpm { + rpm = v + } + } + if l.TPM != nil { + if v := a.tokens.Rate(l.Counter); v > tpm { + tpm = v + } + } + } + return spent, rpm, tpm } // counterFor is the counter a limit owned by this scope should use. diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go index 492c304..bc6bc01 100644 --- a/internal/accounting/accounting_test.go +++ b/internal/accounting/accounting_test.go @@ -459,3 +459,153 @@ func TestATeamBudgetSurvivesARestart(t *testing.T) { t.Errorf("the team's spend should have been restored and pooled: %v", err) } } + +// inTree is a key under org/team/project, with whatever each level sets. +func inTree(keyID string, chain ...store.Limits) auth.Principal { + e := store.Effective{KeyID: keyID, Chain: chain} + // The collapsed values still exist for headers and the profile page; they + // are not what enforcement reads any more. + for _, l := range chain { + if l.Budget != nil && (e.Budget == nil || *l.Budget < *e.Budget) { + e.Budget, e.BudgetPeriod = l.Budget, l.BudgetPeriod + e.BudgetOwner = store.LimitOwner{Scope: l.Scope, ID: l.ScopeID} + } + } + return auth.Principal{KeyID: keyID, Limits: e} +} + +func orgBudget(id string, v int64) store.Limits { + return store.Limits{Scope: store.ScopeOrg, ScopeID: id, + Budget: i64p(v), BudgetPeriod: store.PeriodMonth} +} + +func teamBudget(id string, v int64) store.Limits { + return store.Limits{Scope: store.ScopeTeam, ScopeID: id, + Budget: i64p(v), BudgetPeriod: store.PeriodMonth} +} + +func TestTheSumOfTeamsCannotExceedTheOrganization(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + + // Each request costs 800. The organization allows 2000 and each of its four + // teams allows 2000, so no team comes close to its own ceiling — but the + // organization's fills after three. Taking only the most restrictive value, + // the organization's number was never looked at and all four went through. + // + // A budget stops spending once it is reached rather than refusing the + // request that would cross it, so the third is served and the fourth is not. + spend := func(team string) error { + p := inTree(team+"-k", orgBudget("o", 2000), teamBudget(team, 2000)) + res, err := a.Check(p, "paid", 1000) + if err != nil { + return err + } + a.Record(res, store.Usage{KeyID: team + "-k", PromptTokens: 200, CompletionTokens: 200}) + return nil + } + + total := 0 + for _, team := range []string{"t1", "t2", "t3", "t4"} { + if err := spend(team); err == nil { + total++ + } + } + if total != 3 { + t.Errorf("expected the organization to stop the fourth team, %d got through", total) + } +} + +func TestATeamStillStopsBeforeItsOrganizationDoes(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := func() auth.Principal { + return inTree("k1", orgBudget("o", 100_000), teamBudget("t1", 800)) + } + + res, err := a.Check(p(), "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 200, CompletionTokens: 200}) + + // The narrower pool binds first, which is what it is for. Enforcing every + // level must not stop the tighter one working. + if _, err := a.Check(p(), "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("the team's own budget should still bite, got %v", err) + } +} + +func TestSpendingFillsEveryPoolAboveIt(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := inTree("k1", orgBudget("o", 5000), teamBudget("t1", 5000)) + + res, err := a.Check(p, "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 200, CompletionTokens: 200}) + + // Settled onto both, or the organization's pool would only ever fill from + // requests that named it directly — which is none of them. + if got := a.budgets.Spent("org:o", store.PeriodMonth); got == 0 { + t.Error("the organization's pool did not fill") + } + if got := a.budgets.Spent("team:t1", store.PeriodMonth); got == 0 { + t.Error("the team's pool did not fill") + } +} + +func TestARefusalHandsBackWhatTheNarrowerLevelsTook(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + // The team allows plenty; the organization allows one request a minute. + chain := []store.Limits{ + {Scope: store.ScopeOrg, ScopeID: "o", RPM: intp(1)}, + {Scope: store.ScopeTeam, ScopeID: "t1", RPM: intp(100)}, + } + if _, err := a.Check(inTree("k1", chain...), "free", 1); err != nil { + t.Fatal(err) + } + before := a.requests.Rate("team:t1") + + if _, err := a.Check(inTree("k2", chain...), "free", 1); !errors.Is(err, ErrRateLimited) { + t.Fatalf("the organization should have refused the second, got %v", err) + } + + // A rate limiter charges as it checks, so a refusal further along has + // already taken from the levels before it. Left there, a team at its + // organization's ceiling would quietly eat its own rate limit with requests + // that were never served. + if after := a.requests.Rate("team:t1"); after != before { + t.Errorf("the refused request stayed charged to the team: %v -> %v", before, after) + } +} + +func TestAChainWithNoLimitsAnywhereAllowsEverything(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + for i := 0; i < 5; i++ { + if _, err := a.Check(inTree("k1"), "paid", 1000); err != nil { + t.Fatalf("nothing is limited, but request %d was refused: %v", i, err) + } + } +} + +func TestStatusReportsTheFullestPoolNotTheSmallestNumber(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + chain := func(team string) auth.Principal { + return inTree(team+"-k", orgBudget("o", 9000), teamBudget(team, 9000)) + } + + // Another team spends; this one has spent nothing of its own. + res, err := a.Check(chain("t1"), "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "t1-k", PromptTokens: 200, CompletionTokens: 200}) + + spent, _, _ := a.Status(chain("t2")) + // The organization's pool is the one nearer its ceiling, so it is the one + // that will refuse them. Reporting the team's nil would disagree with the + // refusal that eventually follows it. + if spent == 0 { + t.Error("a member should see the fullest pool that bounds them") + } +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 96c8057..f265414 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -323,6 +323,9 @@ func (s *DB) Resolve(ctx context.Context, keyID string) (Effective, error) { return Effective{}, err } e.tighten(l) + // Kept whole as well as folded in. Enforcement walks this; the folded + // values are for showing somebody their ceiling. + e.Chain = append(e.Chain, l) } s.applyDefaults(&e) return e, nil @@ -355,13 +358,21 @@ func (s *DB) applyDefaults(e *Effective) { s.mu.RLock() defer s.mu.RUnlock() own := LimitOwner{Scope: ScopeKey, ID: e.KeyID} + fallback := Limits{Scope: ScopeKey, ScopeID: e.KeyID} if e.RPM == nil && s.defaultRPM != nil { v := *s.defaultRPM e.RPM, e.RPMOwner = &v, own + fallback.RPM = &v } if e.TPM == nil && s.defaultTPM != nil { v := *s.defaultTPM e.TPM, e.TPMOwner = &v, own + fallback.TPM = &v + } + // Appended so enforcement sees it like any other level. Left out, a cluster + // default would be reported as the ceiling and then never checked. + if fallback.RPM != nil || fallback.TPM != nil { + e.Chain = append(e.Chain, fallback) } } diff --git a/internal/store/store.go b/internal/store/store.go index 57ef138..2cf517c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -144,6 +144,16 @@ type Effective struct { RPMOwner LimitOwner TPMOwner LimitOwner BudgetOwner LimitOwner + + // Chain is every level that set a limit, widest first. + // + // The collapsed values above are the ceiling a single caller faces, and + // they are what a header or a profile page should show. They are not enough + // to enforce with: once a quota is a pool, two levels are two pools and both + // have to hold. An organization given 1000 and four teams given 500 each + // would otherwise spend 2000 — the team's value wins as the more + // restrictive, and the organization's is never looked at again. + Chain []Limits } // LimitOwner is the scope a resolved limit came from.