Resolution took the most restrictive value along the chain and enforced only that one. For a single caller's ceiling that is right: a team cannot raise what its organization set. For a pool it is not, because two levels are two pools. An organization given 1000 with four teams given 500 each spent 2000. Each team filled its own counter, the organization's value lost to the tighter one and was never looked at again. The number was written, displayed, reported against — and enforced by nothing. Every level that sets a limit is now checked against its own counter, and settled onto all of them, so an organization's pool fills from its teams' spending rather than only from requests that somehow named it. A team still stops before its organization when its own pool runs out first, which is what a sub-allocation is for. Effective carries the chain as well as the collapsed values. The collapsed ones are still what a header or a profile page should show — they are the ceiling a single caller faces — they are just not enough to enforce with. Rate limits charge as they check, deliberately, so that two concurrent requests cannot both see room. With several levels that means a refusal at one has already taken from the levels before it, so those are handed back. Without the refund a team sitting at its organization's ceiling would quietly eat its own rate limit with requests that were never served. Budgets are checked before any of that, and only read, so a refusal on a budget charges nothing at all. Status reports the fullest pool rather than the smallest number. A team inside an organization with the same budget is bounded by whichever is nearer its ceiling, and that is the organization as soon as a second team spends anything; a header showing the team's would disagree with the refusal after it. A cluster default joins the chain as a key-level entry. Left out, it would be reported as the ceiling and then never checked. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
365 lines
12 KiB
Go
365 lines
12 KiB
Go
package accounting
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Refusals a key can hit. They are distinguished so the client is told which
|
|
// limit it hit, and so retrying is only suggested where retrying helps.
|
|
var (
|
|
ErrRateLimited = errors.New("request rate limit exceeded")
|
|
ErrTokenLimited = errors.New("token rate limit exceeded")
|
|
ErrBudgetSpent = errors.New("budget exhausted")
|
|
)
|
|
|
|
// LimitError carries which limit was hit and, where it makes sense, when to
|
|
// come back.
|
|
type LimitError struct {
|
|
Err error
|
|
// RetryAfter is zero when waiting does not help, which is the case for a
|
|
// spent budget: it will not refill until the period rolls over, and telling
|
|
// a client to retry in a second would be a lie.
|
|
RetryAfter time.Duration
|
|
}
|
|
|
|
func (e *LimitError) Error() string { return e.Err.Error() }
|
|
func (e *LimitError) Unwrap() error { return e.Err }
|
|
|
|
// Accountant enforces limits and records what was used.
|
|
//
|
|
// One object because the three halves are not independent: a budget check needs
|
|
// the price, the price needs the model, and the record closes the loop by
|
|
// feeding the counters that the next check reads.
|
|
type Accountant struct {
|
|
pricing *Pricing
|
|
budgets *Budgets
|
|
requests *Limiter
|
|
tokens *Limiter
|
|
recorder *Recorder
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New builds an accountant.
|
|
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Accountant{
|
|
pricing: pricing,
|
|
budgets: NewBudgets(),
|
|
requests: NewLimiter(),
|
|
tokens: NewLimiter(),
|
|
recorder: recorder,
|
|
log: log,
|
|
}
|
|
}
|
|
|
|
// RestoreBudgets seeds the in-memory counters from the store.
|
|
//
|
|
// Called at startup. A restart must not reset a budget, which it would if the
|
|
// counters only ever lived in memory.
|
|
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
|
|
start := PeriodStart(period, time.Now())
|
|
if start.IsZero() {
|
|
return nil
|
|
}
|
|
rows, err := s.SpendSinceByScope(ctx, start)
|
|
if err != nil {
|
|
return fmt.Errorf("restoring budgets: %w", err)
|
|
}
|
|
// Every scope a key sits under, not just the key. A budget owned by a team
|
|
// is one counter fed by every key beneath it, and restoring only per-key
|
|
// totals would bring the control plane back up believing the team had spent
|
|
// nothing — which is a budget that resets on every restart, for exactly the
|
|
// scopes where it matters most.
|
|
spend := make(map[string]int64, len(rows)*4)
|
|
for _, r := range rows {
|
|
for _, o := range []store.LimitOwner{
|
|
{Scope: store.ScopeKey, ID: r.KeyID},
|
|
{Scope: store.ScopeProject, ID: r.ProjectID},
|
|
{Scope: store.ScopeTeam, ID: r.TeamID},
|
|
{Scope: store.ScopeOrg, ID: r.OrgID},
|
|
} {
|
|
if k := o.CounterKey(); k != "" {
|
|
spend[k] += r.Micros
|
|
}
|
|
}
|
|
}
|
|
a.budgets.Restore(period, spend)
|
|
a.log.Info("restored budgets from the store",
|
|
"keys", len(rows), "counters", len(spend), "since", start)
|
|
return nil
|
|
}
|
|
|
|
// Reservation is what Check returns: the estimate it charged, so Record can
|
|
// settle the difference against the real figure.
|
|
// pool is one level's counter and the period its budget runs over.
|
|
type pool struct {
|
|
Counter string
|
|
Period store.Period
|
|
}
|
|
|
|
type Reservation struct {
|
|
KeyID string
|
|
// Budgets and Tokens are every counter the check went through, carried so
|
|
// the settlement lands on the same ones. Every, not the tightest: an
|
|
// organization's pool only fills up if each team's spend is added to it
|
|
// as well as to the team's own.
|
|
//
|
|
// Derived again at settlement would mean re-resolving the limits, and a
|
|
// budget checked against the team and settled against the key is a budget
|
|
// that never fills up.
|
|
Budgets []pool
|
|
Tokens []string
|
|
EstimatedTokens int64
|
|
Price Price
|
|
// Period is the key's budget period, carried from the check so the record
|
|
// counts against the same window the check read. Deriving it later would
|
|
// mean guessing, and a monthly budget credited to a daily counter is a
|
|
// budget that never binds.
|
|
Period store.Period
|
|
}
|
|
|
|
// Check decides whether a request may proceed.
|
|
//
|
|
// It runs before scheduling and touches no database: every counter it reads is
|
|
// in memory. The token limit is checked against an estimate, because the real
|
|
// figure only exists once the request has run; Record settles the difference.
|
|
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
|
|
price := a.pricing.For(model)
|
|
lim := p.Limits
|
|
// Every level that set a limit, each against its own pool. A quota written
|
|
// on a team is one pool the team shares, and the organization above it is a
|
|
// second pool that the team's spending also fills — both have to hold, or
|
|
// an organization given 1000 with four teams given 500 spends 2000.
|
|
chain := levels(lim, p.KeyID)
|
|
|
|
res := Reservation{
|
|
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
|
|
Price: price, Period: lim.BudgetPeriod,
|
|
}
|
|
for _, l := range chain {
|
|
if l.Budget != nil {
|
|
res.Budgets = append(res.Budgets, pool{l.Counter, l.Period})
|
|
}
|
|
if l.TPM != nil {
|
|
res.Tokens = append(res.Tokens, l.Counter)
|
|
}
|
|
}
|
|
if p.KeyID == "" {
|
|
return res, nil
|
|
}
|
|
|
|
// Budget first. It is the limit whose refusal cannot be retried away, so
|
|
// telling the client about it first is more useful than reporting a rate
|
|
// limit it would hit again in a second.
|
|
//
|
|
// A model that costs nothing is exempt, which is what lets a key with no
|
|
// budget left still be served a cheap fallback (plan §7.7).
|
|
// Budgets first, and all of them. Exhausted only reads, so the order costs
|
|
// nothing and a refusal here charges no counter.
|
|
if !price.Zero() {
|
|
for _, l := range chain {
|
|
if l.Budget != nil && a.budgets.Exhausted(l.Counter, l.Period, *l.Budget) {
|
|
return res, &LimitError{Err: ErrBudgetSpent}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Rate limits charge as they check — deliberately, so two concurrent
|
|
// requests cannot both see room. With several levels that means a refusal
|
|
// at one has already taken from the levels before it, so those are handed
|
|
// back. Without the refund a team at its ceiling would quietly eat its
|
|
// organization's rate limit with requests that were never served.
|
|
if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
|
|
return l.Counter, valInt(l.RPM), 1, l.RPM != nil
|
|
}, a.requests); err != nil {
|
|
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
|
|
}
|
|
if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
|
|
return l.Counter, valInt(l.TPM), estimatedTokens, l.TPM != nil
|
|
}, a.tokens); err != nil {
|
|
// The request was not charged against the request-rate limit in vain:
|
|
// it was, and that is correct — it did arrive.
|
|
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// level is one scope's limits with the counter they are enforced against.
|
|
type level struct {
|
|
Counter string
|
|
Period store.Period
|
|
RPM *int
|
|
TPM *int
|
|
Budget *int64
|
|
}
|
|
|
|
// levels turns a resolved chain into the pools a request must pass.
|
|
//
|
|
// Falls back to the collapsed values when the chain is empty, which is what a
|
|
// principal assembled by hand looks like — in tests, and anywhere that builds
|
|
// limits without walking the tree.
|
|
func levels(e store.Effective, keyID string) []level {
|
|
if len(e.Chain) == 0 {
|
|
var out []level
|
|
if e.RPM != nil || e.TPM != nil || e.Budget != nil {
|
|
out = append(out, level{
|
|
Counter: counterFor(e.BudgetOwner, keyID),
|
|
Period: e.BudgetPeriod, Budget: e.Budget,
|
|
})
|
|
out[0].RPM, out[0].TPM = e.RPM, e.TPM
|
|
// One entry, so a hand-built principal with different owners per
|
|
// limit still counts each where it said to.
|
|
if r, b := counterFor(e.RPMOwner, keyID), out[0].Counter; r != b && e.RPM != nil {
|
|
out[0].RPM = nil
|
|
out = append(out, level{Counter: r, RPM: e.RPM})
|
|
}
|
|
if tp, b := counterFor(e.TPMOwner, keyID), out[0].Counter; tp != b && e.TPM != nil {
|
|
out[0].TPM = nil
|
|
out = append(out, level{Counter: tp, TPM: e.TPM})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
out := make([]level, 0, len(e.Chain))
|
|
for _, l := range e.Chain {
|
|
out = append(out, level{
|
|
Counter: store.LimitOwner{Scope: l.Scope, ID: l.ScopeID}.CounterKey(),
|
|
Period: l.BudgetPeriod, RPM: l.RPM, TPM: l.TPM, Budget: l.Budget,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// allowAll charges every level that has this limit, refunding on a refusal.
|
|
func (a *Accountant) allowAll(chain []level,
|
|
pick func(level) (key string, limit int, n int64, has bool), lim *Limiter,
|
|
) error {
|
|
var charged []struct {
|
|
key string
|
|
n int64
|
|
}
|
|
for _, l := range chain {
|
|
key, limit, n, has := pick(l)
|
|
if !has {
|
|
continue
|
|
}
|
|
if !lim.Allow(key, limit, n) {
|
|
for _, c := range charged {
|
|
lim.Charge(c.key, -c.n)
|
|
}
|
|
return errRefused
|
|
}
|
|
charged = append(charged, struct {
|
|
key string
|
|
n int64
|
|
}{key, n})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var errRefused = errors.New("refused")
|
|
|
|
func valInt(v *int) int {
|
|
if v == nil {
|
|
return 0
|
|
}
|
|
return *v
|
|
}
|
|
|
|
// Record settles a finished request: it reconciles the token estimate, adds to
|
|
// the budget, and queues the usage row.
|
|
func (a *Accountant) Record(res Reservation, u store.Usage) {
|
|
if res.KeyID == "" {
|
|
return
|
|
}
|
|
actual := u.PromptTokens + u.CompletionTokens
|
|
|
|
// The estimate was charged up front; charge the difference now. A caller
|
|
// whose requests are consistently larger than estimated is still bounded,
|
|
// one window later than with a perfect estimate.
|
|
// Every level the check went through, so an organization's pool fills up
|
|
// from its teams' spending rather than only from requests that named it.
|
|
for _, c := range res.Tokens {
|
|
a.tokens.Charge(c, actual-res.EstimatedTokens)
|
|
}
|
|
|
|
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
|
|
for _, b := range res.Budgets {
|
|
a.budgets.Add(b.Counter, b.Period, u.CostMicros)
|
|
}
|
|
|
|
if a.recorder != nil {
|
|
a.recorder.Record(u)
|
|
}
|
|
}
|
|
|
|
// Status returns what a key has used, for headers and `ctl`.
|
|
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
|
|
if p.KeyID == "" {
|
|
return 0, 0, 0
|
|
}
|
|
// The pool that will refuse them, which is not always the one with the
|
|
// smallest number on it. A team given 2000 inside an organization given the
|
|
// same is bounded by whichever is nearer its ceiling, and that is the
|
|
// organization as soon as a second team spends anything. A header that
|
|
// reported the team's would disagree with the refusal that follows it.
|
|
for _, l := range levels(p.Limits, p.KeyID) {
|
|
if l.Budget != nil {
|
|
if v := a.budgets.Spent(l.Counter, l.Period); v > spent {
|
|
spent = v
|
|
}
|
|
}
|
|
if l.RPM != nil {
|
|
if v := a.requests.Rate(l.Counter); v > rpm {
|
|
rpm = v
|
|
}
|
|
}
|
|
if l.TPM != nil {
|
|
if v := a.tokens.Rate(l.Counter); v > tpm {
|
|
tpm = v
|
|
}
|
|
}
|
|
}
|
|
return spent, rpm, tpm
|
|
}
|
|
|
|
// counterFor is the counter a limit owned by this scope should use.
|
|
//
|
|
// Falls back to the key's own counter when no scope owns the limit, which
|
|
// happens when nothing in the chain set one.
|
|
//
|
|
// Namespaced even then. A bare key id here and a "key:"-prefixed one in the
|
|
// restore path would be two different counters for the same key, so a budget
|
|
// would check against an empty one and survive every restart — which is how
|
|
// this was wrong when it was first written.
|
|
func counterFor(o store.LimitOwner, keyID string) string {
|
|
if k := o.CounterKey(); k != "" {
|
|
return k
|
|
}
|
|
return keyCounter(keyID)
|
|
}
|
|
|
|
// keyCounter is a key's own counter, in the same namespace as every other.
|
|
func or(v, fallback string) string {
|
|
if v == "" {
|
|
return fallback
|
|
}
|
|
return v
|
|
}
|
|
|
|
func keyCounter(keyID string) string {
|
|
if keyID == "" {
|
|
return ""
|
|
}
|
|
return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey()
|
|
}
|