Files
llmgw/internal/accounting/accounting.go
T
Ilker ManapandClaude Opus 5 0b93fb2883
CI / test (push) Successful in 1m7s
Enforce budgets and rate limits, and write usage off the request path
The second half of phase 5. A key now has a cost, a quota and a bill.

Rate limits use a sliding window rather than a fixed one. A fixed window lets
a caller send its whole minute's allowance in the last second of one window
and again in the first second of the next, so the observed rate is twice the
configured limit exactly when the system is least able to take it. Checking
and charging are one operation, or two concurrent requests both see room and
both take it.

Token limits are checked against an estimate, because the real count only
exists once the request has run; the difference is charged afterwards, so a
caller who consistently underestimates is still bounded, one window late. A
request with no max_tokens gets a modest assumption rather than zero —
assuming zero would let an unbounded request through every token limit.

Limits are checked before admission, not after: a request over its quota
should be refused rather than queued, where it would occupy a band's capacity
to no purpose. All refusals are 429, but a spent budget carries no Retry-After
because waiting does not help until the period rolls over, and saying
otherwise turns one client into a retry loop.

Usage goes to a buffered channel and is written in batches. Record never
blocks: if the buffer fills — a database stalled long enough to lose eight
thousand records — records are dropped and counted, because losing accounting
is bad and turning a stalled database into an outage is worse. A clean
shutdown drains what is queued; those are requests that were already served.
Failures are logged at error, since silently losing billing data is the kind
of thing discovered at the end of a month.

Requests are accounted whatever their outcome. A request that used a GPU and
then failed still used it, and a key that could spend nothing by failing would
be a way around a budget.

Budgets are counted in memory and restored from the store at startup, so a
restart does not hand every key a fresh allowance. The budget period travels
with the reservation rather than being re-derived, because a monthly budget
credited to a daily counter is a budget that never binds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 08:52:36 +02:00

162 lines
5.2 KiB
Go

package accounting
import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Refusals a key can hit. They are distinguished so the client is told which
// limit it hit, and so retrying is only suggested where retrying helps.
var (
ErrRateLimited = errors.New("request rate limit exceeded")
ErrTokenLimited = errors.New("token rate limit exceeded")
ErrBudgetSpent = errors.New("budget exhausted")
)
// LimitError carries which limit was hit and, where it makes sense, when to
// come back.
type LimitError struct {
Err error
// RetryAfter is zero when waiting does not help, which is the case for a
// spent budget: it will not refill until the period rolls over, and telling
// a client to retry in a second would be a lie.
RetryAfter time.Duration
}
func (e *LimitError) Error() string { return e.Err.Error() }
func (e *LimitError) Unwrap() error { return e.Err }
// Accountant enforces limits and records what was used.
//
// One object because the three halves are not independent: a budget check needs
// the price, the price needs the model, and the record closes the loop by
// feeding the counters that the next check reads.
type Accountant struct {
pricing *Pricing
budgets *Budgets
requests *Limiter
tokens *Limiter
recorder *Recorder
log *slog.Logger
}
// New builds an accountant.
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
if log == nil {
log = slog.Default()
}
return &Accountant{
pricing: pricing,
budgets: NewBudgets(),
requests: NewLimiter(),
tokens: NewLimiter(),
recorder: recorder,
log: log,
}
}
// RestoreBudgets seeds the in-memory counters from the store.
//
// Called at startup. A restart must not reset a budget, which it would if the
// counters only ever lived in memory.
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
start := PeriodStart(period, time.Now())
if start.IsZero() {
return nil
}
spend, err := s.SpendSince(ctx, start)
if err != nil {
return fmt.Errorf("restoring budgets: %w", err)
}
a.budgets.Restore(period, spend)
a.log.Info("restored budgets from the store", "keys", len(spend), "since", start)
return nil
}
// Reservation is what Check returns: the estimate it charged, so Record can
// settle the difference against the real figure.
type Reservation struct {
KeyID string
EstimatedTokens int64
Price Price
// Period is the key's budget period, carried from the check so the record
// counts against the same window the check read. Deriving it later would
// mean guessing, and a monthly budget credited to a daily counter is a
// budget that never binds.
Period store.Period
}
// Check decides whether a request may proceed.
//
// It runs before scheduling and touches no database: every counter it reads is
// in memory. The token limit is checked against an estimate, because the real
// figure only exists once the request has run; Record settles the difference.
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
price := a.pricing.For(model)
res := Reservation{
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
Price: price, Period: p.Limits.BudgetPeriod,
}
if p.KeyID == "" {
return res, nil
}
lim := p.Limits
// Budget first. It is the limit whose refusal cannot be retried away, so
// telling the client about it first is more useful than reporting a rate
// limit it would hit again in a second.
//
// A model that costs nothing is exempt, which is what lets a key with no
// budget left still be served a cheap fallback (plan §7.7).
if lim.Budget != nil && !price.Zero() &&
a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) {
return res, &LimitError{Err: ErrBudgetSpent}
}
if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) {
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
}
if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) {
// The request was not charged against the request-rate limit in vain:
// it was, and that is correct — it did arrive.
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
}
return res, nil
}
// Record settles a finished request: it reconciles the token estimate, adds to
// the budget, and queues the usage row.
func (a *Accountant) Record(res Reservation, u store.Usage) {
if res.KeyID == "" {
return
}
actual := u.PromptTokens + u.CompletionTokens
// The estimate was charged up front; charge the difference now. A caller
// whose requests are consistently larger than estimated is still bounded,
// one window later than with a perfect estimate.
a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens)
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
a.budgets.Add(res.KeyID, res.Period, u.CostMicros)
if a.recorder != nil {
a.recorder.Record(u)
}
}
// Status returns what a key has used, for headers and `ctl`.
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
if p.KeyID == "" {
return 0, 0, 0
}
return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod),
a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID)
}