Files
llmgw/internal/accounting/accounting.go
T
Ilker ManapandClaude Opus 5 3febb6ef63 Count a group's quota against the group, not against each of its keys
The tenancy tree handed limits down and never pooled what was spent against
them. A limit written on a team was inherited by every key beneath it, and then
enforced with the key as the counter — so a project given a budget of 1000 and
ten keys permitted ten thousand, and a team given 60 RPM got 60 per member.

That is "each of you may spend this much", which is a useful thing to be able
to say. It is not what somebody writing a budget on a team means, and nothing
in the system said which one you were getting.

Resolution now records which scope set each value that won, and the counter is
keyed by that scope. The value and the counter come from the same place, so a
project tightening its team's limit is enforced against the project's own pool
rather than eating into a quota another project never agreed to.

Each limit is owned separately, because they resolve separately: a team budget
with a per-key rate limit — the group's money, the individual's throughput — is
the ordinary arrangement and has to stay expressible.

Cluster defaults are owned by the key. Read as a shared pool, one default would
become a single cluster-wide rate limit that every key in every organization
competes for: not a default, an outage.

Restoring after a restart had to change with it. Per-key totals rebuilt a
counter named after the key while the check read one named after the team, so
the control plane came back believing the team had spent nothing — a budget
that reset on every restart, for exactly the scopes where it matters most.
SpendSinceByScope returns each key's place in the tree and the accountant folds
it into every scope that might own a budget.

Counters are namespaced by level throughout. A bare key id in one path and a
"key:"-prefixed one in another are two counters for the same key, which is how
the restore was silently doing nothing.

Still missing, and worth naming: no usage report rolls up by project, team or
org, so a shared quota can now be enforced but not yet shown.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 12:10:46 +02:00

230 lines
8.0 KiB
Go

package accounting
import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Refusals a key can hit. They are distinguished so the client is told which
// limit it hit, and so retrying is only suggested where retrying helps.
var (
ErrRateLimited = errors.New("request rate limit exceeded")
ErrTokenLimited = errors.New("token rate limit exceeded")
ErrBudgetSpent = errors.New("budget exhausted")
)
// LimitError carries which limit was hit and, where it makes sense, when to
// come back.
type LimitError struct {
Err error
// RetryAfter is zero when waiting does not help, which is the case for a
// spent budget: it will not refill until the period rolls over, and telling
// a client to retry in a second would be a lie.
RetryAfter time.Duration
}
func (e *LimitError) Error() string { return e.Err.Error() }
func (e *LimitError) Unwrap() error { return e.Err }
// Accountant enforces limits and records what was used.
//
// One object because the three halves are not independent: a budget check needs
// the price, the price needs the model, and the record closes the loop by
// feeding the counters that the next check reads.
type Accountant struct {
pricing *Pricing
budgets *Budgets
requests *Limiter
tokens *Limiter
recorder *Recorder
log *slog.Logger
}
// New builds an accountant.
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
if log == nil {
log = slog.Default()
}
return &Accountant{
pricing: pricing,
budgets: NewBudgets(),
requests: NewLimiter(),
tokens: NewLimiter(),
recorder: recorder,
log: log,
}
}
// RestoreBudgets seeds the in-memory counters from the store.
//
// Called at startup. A restart must not reset a budget, which it would if the
// counters only ever lived in memory.
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
start := PeriodStart(period, time.Now())
if start.IsZero() {
return nil
}
rows, err := s.SpendSinceByScope(ctx, start)
if err != nil {
return fmt.Errorf("restoring budgets: %w", err)
}
// Every scope a key sits under, not just the key. A budget owned by a team
// is one counter fed by every key beneath it, and restoring only per-key
// totals would bring the control plane back up believing the team had spent
// nothing — which is a budget that resets on every restart, for exactly the
// scopes where it matters most.
spend := make(map[string]int64, len(rows)*4)
for _, r := range rows {
for _, o := range []store.LimitOwner{
{Scope: store.ScopeKey, ID: r.KeyID},
{Scope: store.ScopeProject, ID: r.ProjectID},
{Scope: store.ScopeTeam, ID: r.TeamID},
{Scope: store.ScopeOrg, ID: r.OrgID},
} {
if k := o.CounterKey(); k != "" {
spend[k] += r.Micros
}
}
}
a.budgets.Restore(period, spend)
a.log.Info("restored budgets from the store",
"keys", len(rows), "counters", len(spend), "since", start)
return nil
}
// Reservation is what Check returns: the estimate it charged, so Record can
// settle the difference against the real figure.
type Reservation struct {
KeyID string
// BudgetCounter and TokenCounter are what the check counted against, carried
// so the settlement lands on the same counter. Deriving them again here
// would mean re-resolving the limits, and a budget checked against the team
// and settled against the key is a budget that never fills up.
BudgetCounter string
TokenCounter string
EstimatedTokens int64
Price Price
// Period is the key's budget period, carried from the check so the record
// counts against the same window the check read. Deriving it later would
// mean guessing, and a monthly budget credited to a daily counter is a
// budget that never binds.
Period store.Period
}
// Check decides whether a request may proceed.
//
// It runs before scheduling and touches no database: every counter it reads is
// in memory. The token limit is checked against an estimate, because the real
// figure only exists once the request has run; Record settles the difference.
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
price := a.pricing.For(model)
lim := p.Limits
// Counted against whichever scope set the limit. A budget written on a team
// is one pool the team shares; counting it per key would hand the team's
// number to every key under it and permit the total times the headcount.
budgetCounter := counterFor(lim.BudgetOwner, p.KeyID)
tokenCounter := counterFor(lim.TPMOwner, p.KeyID)
res := Reservation{
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
BudgetCounter: budgetCounter, TokenCounter: tokenCounter,
Price: price, Period: lim.BudgetPeriod,
}
if p.KeyID == "" {
return res, nil
}
// Budget first. It is the limit whose refusal cannot be retried away, so
// telling the client about it first is more useful than reporting a rate
// limit it would hit again in a second.
//
// A model that costs nothing is exempt, which is what lets a key with no
// budget left still be served a cheap fallback (plan §7.7).
if lim.Budget != nil && !price.Zero() &&
a.budgets.Exhausted(budgetCounter, lim.BudgetPeriod, *lim.Budget) {
return res, &LimitError{Err: ErrBudgetSpent}
}
if lim.RPM != nil && !a.requests.Allow(counterFor(lim.RPMOwner, p.KeyID), *lim.RPM, 1) {
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
}
if lim.TPM != nil && !a.tokens.Allow(tokenCounter, *lim.TPM, estimatedTokens) {
// The request was not charged against the request-rate limit in vain:
// it was, and that is correct — it did arrive.
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
}
return res, nil
}
// Record settles a finished request: it reconciles the token estimate, adds to
// the budget, and queues the usage row.
func (a *Accountant) Record(res Reservation, u store.Usage) {
if res.KeyID == "" {
return
}
actual := u.PromptTokens + u.CompletionTokens
// The estimate was charged up front; charge the difference now. A caller
// whose requests are consistently larger than estimated is still bounded,
// one window later than with a perfect estimate.
a.tokens.Charge(or(res.TokenCounter, keyCounter(res.KeyID)), actual-res.EstimatedTokens)
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
a.budgets.Add(or(res.BudgetCounter, keyCounter(res.KeyID)), res.Period, u.CostMicros)
if a.recorder != nil {
a.recorder.Record(u)
}
}
// Status returns what a key has used, for headers and `ctl`.
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
if p.KeyID == "" {
return 0, 0, 0
}
// Reported against the same counters the check reads, so a header saying
// how much is left agrees with the refusal the next request gets. A member
// of a team sharing a budget sees the team's spend, which is the number
// that will stop them.
return a.budgets.Spent(counterFor(p.Limits.BudgetOwner, p.KeyID), p.Limits.BudgetPeriod),
a.requests.Rate(counterFor(p.Limits.RPMOwner, p.KeyID)),
a.tokens.Rate(counterFor(p.Limits.TPMOwner, p.KeyID))
}
// counterFor is the counter a limit owned by this scope should use.
//
// Falls back to the key's own counter when no scope owns the limit, which
// happens when nothing in the chain set one.
//
// Namespaced even then. A bare key id here and a "key:"-prefixed one in the
// restore path would be two different counters for the same key, so a budget
// would check against an empty one and survive every restart — which is how
// this was wrong when it was first written.
func counterFor(o store.LimitOwner, keyID string) string {
if k := o.CounterKey(); k != "" {
return k
}
return keyCounter(keyID)
}
// keyCounter is a key's own counter, in the same namespace as every other.
func or(v, fallback string) string {
if v == "" {
return fallback
}
return v
}
func keyCounter(keyID string) string {
if keyID == "" {
return ""
}
return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey()
}