CI / test (push) Successful in 1m7s
The second half of phase 5. A key now has a cost, a quota and a bill. Rate limits use a sliding window rather than a fixed one. A fixed window lets a caller send its whole minute's allowance in the last second of one window and again in the first second of the next, so the observed rate is twice the configured limit exactly when the system is least able to take it. Checking and charging are one operation, or two concurrent requests both see room and both take it. Token limits are checked against an estimate, because the real count only exists once the request has run; the difference is charged afterwards, so a caller who consistently underestimates is still bounded, one window late. A request with no max_tokens gets a modest assumption rather than zero — assuming zero would let an unbounded request through every token limit. Limits are checked before admission, not after: a request over its quota should be refused rather than queued, where it would occupy a band's capacity to no purpose. All refusals are 429, but a spent budget carries no Retry-After because waiting does not help until the period rolls over, and saying otherwise turns one client into a retry loop. Usage goes to a buffered channel and is written in batches. Record never blocks: if the buffer fills — a database stalled long enough to lose eight thousand records — records are dropped and counted, because losing accounting is bad and turning a stalled database into an outage is worse. A clean shutdown drains what is queued; those are requests that were already served. Failures are logged at error, since silently losing billing data is the kind of thing discovered at the end of a month. Requests are accounted whatever their outcome. A request that used a GPU and then failed still used it, and a key that could spend nothing by failing would be a way around a budget. Budgets are counted in memory and restored from the store at startup, so a restart does not hand every key a fresh allowance. The budget period travels with the reservation rather than being re-derived, because a monthly budget credited to a daily counter is a budget that never binds. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
162 lines
5.2 KiB
Go
162 lines
5.2 KiB
Go
package accounting
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Refusals a key can hit. They are distinguished so the client is told which
|
|
// limit it hit, and so retrying is only suggested where retrying helps.
|
|
var (
|
|
ErrRateLimited = errors.New("request rate limit exceeded")
|
|
ErrTokenLimited = errors.New("token rate limit exceeded")
|
|
ErrBudgetSpent = errors.New("budget exhausted")
|
|
)
|
|
|
|
// LimitError carries which limit was hit and, where it makes sense, when to
|
|
// come back.
|
|
type LimitError struct {
|
|
Err error
|
|
// RetryAfter is zero when waiting does not help, which is the case for a
|
|
// spent budget: it will not refill until the period rolls over, and telling
|
|
// a client to retry in a second would be a lie.
|
|
RetryAfter time.Duration
|
|
}
|
|
|
|
func (e *LimitError) Error() string { return e.Err.Error() }
|
|
func (e *LimitError) Unwrap() error { return e.Err }
|
|
|
|
// Accountant enforces limits and records what was used.
|
|
//
|
|
// One object because the three halves are not independent: a budget check needs
|
|
// the price, the price needs the model, and the record closes the loop by
|
|
// feeding the counters that the next check reads.
|
|
type Accountant struct {
|
|
pricing *Pricing
|
|
budgets *Budgets
|
|
requests *Limiter
|
|
tokens *Limiter
|
|
recorder *Recorder
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New builds an accountant.
|
|
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Accountant{
|
|
pricing: pricing,
|
|
budgets: NewBudgets(),
|
|
requests: NewLimiter(),
|
|
tokens: NewLimiter(),
|
|
recorder: recorder,
|
|
log: log,
|
|
}
|
|
}
|
|
|
|
// RestoreBudgets seeds the in-memory counters from the store.
|
|
//
|
|
// Called at startup. A restart must not reset a budget, which it would if the
|
|
// counters only ever lived in memory.
|
|
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
|
|
start := PeriodStart(period, time.Now())
|
|
if start.IsZero() {
|
|
return nil
|
|
}
|
|
spend, err := s.SpendSince(ctx, start)
|
|
if err != nil {
|
|
return fmt.Errorf("restoring budgets: %w", err)
|
|
}
|
|
a.budgets.Restore(period, spend)
|
|
a.log.Info("restored budgets from the store", "keys", len(spend), "since", start)
|
|
return nil
|
|
}
|
|
|
|
// Reservation is what Check returns: the estimate it charged, so Record can
|
|
// settle the difference against the real figure.
|
|
type Reservation struct {
|
|
KeyID string
|
|
EstimatedTokens int64
|
|
Price Price
|
|
// Period is the key's budget period, carried from the check so the record
|
|
// counts against the same window the check read. Deriving it later would
|
|
// mean guessing, and a monthly budget credited to a daily counter is a
|
|
// budget that never binds.
|
|
Period store.Period
|
|
}
|
|
|
|
// Check decides whether a request may proceed.
|
|
//
|
|
// It runs before scheduling and touches no database: every counter it reads is
|
|
// in memory. The token limit is checked against an estimate, because the real
|
|
// figure only exists once the request has run; Record settles the difference.
|
|
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
|
|
price := a.pricing.For(model)
|
|
res := Reservation{
|
|
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
|
|
Price: price, Period: p.Limits.BudgetPeriod,
|
|
}
|
|
if p.KeyID == "" {
|
|
return res, nil
|
|
}
|
|
lim := p.Limits
|
|
|
|
// Budget first. It is the limit whose refusal cannot be retried away, so
|
|
// telling the client about it first is more useful than reporting a rate
|
|
// limit it would hit again in a second.
|
|
//
|
|
// A model that costs nothing is exempt, which is what lets a key with no
|
|
// budget left still be served a cheap fallback (plan §7.7).
|
|
if lim.Budget != nil && !price.Zero() &&
|
|
a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) {
|
|
return res, &LimitError{Err: ErrBudgetSpent}
|
|
}
|
|
|
|
if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) {
|
|
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
|
|
}
|
|
if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) {
|
|
// The request was not charged against the request-rate limit in vain:
|
|
// it was, and that is correct — it did arrive.
|
|
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// Record settles a finished request: it reconciles the token estimate, adds to
|
|
// the budget, and queues the usage row.
|
|
func (a *Accountant) Record(res Reservation, u store.Usage) {
|
|
if res.KeyID == "" {
|
|
return
|
|
}
|
|
actual := u.PromptTokens + u.CompletionTokens
|
|
|
|
// The estimate was charged up front; charge the difference now. A caller
|
|
// whose requests are consistently larger than estimated is still bounded,
|
|
// one window later than with a perfect estimate.
|
|
a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens)
|
|
|
|
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
|
|
a.budgets.Add(res.KeyID, res.Period, u.CostMicros)
|
|
|
|
if a.recorder != nil {
|
|
a.recorder.Record(u)
|
|
}
|
|
}
|
|
|
|
// Status returns what a key has used, for headers and `ctl`.
|
|
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
|
|
if p.KeyID == "" {
|
|
return 0, 0, 0
|
|
}
|
|
return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod),
|
|
a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID)
|
|
}
|