package accounting import ( "context" "errors" "fmt" "log/slog" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Refusals a key can hit. They are distinguished so the client is told which // limit it hit, and so retrying is only suggested where retrying helps. var ( ErrRateLimited = errors.New("request rate limit exceeded") ErrTokenLimited = errors.New("token rate limit exceeded") ErrBudgetSpent = errors.New("budget exhausted") ) // LimitError carries which limit was hit and, where it makes sense, when to // come back. type LimitError struct { Err error // RetryAfter is zero when waiting does not help, which is the case for a // spent budget: it will not refill until the period rolls over, and telling // a client to retry in a second would be a lie. RetryAfter time.Duration } func (e *LimitError) Error() string { return e.Err.Error() } func (e *LimitError) Unwrap() error { return e.Err } // Accountant enforces limits and records what was used. // // One object because the three halves are not independent: a budget check needs // the price, the price needs the model, and the record closes the loop by // feeding the counters that the next check reads. type Accountant struct { pricing *Pricing budgets *Budgets requests *Limiter tokens *Limiter recorder *Recorder log *slog.Logger } // New builds an accountant. func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant { if log == nil { log = slog.Default() } return &Accountant{ pricing: pricing, budgets: NewBudgets(), requests: NewLimiter(), tokens: NewLimiter(), recorder: recorder, log: log, } } // RestoreBudgets seeds the in-memory counters from the store. // // Called at startup. A restart must not reset a budget, which it would if the // counters only ever lived in memory. func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error { start := PeriodStart(period, time.Now()) if start.IsZero() { return nil } spend, err := s.SpendSince(ctx, start) if err != nil { return fmt.Errorf("restoring budgets: %w", err) } a.budgets.Restore(period, spend) a.log.Info("restored budgets from the store", "keys", len(spend), "since", start) return nil } // Reservation is what Check returns: the estimate it charged, so Record can // settle the difference against the real figure. type Reservation struct { KeyID string EstimatedTokens int64 Price Price // Period is the key's budget period, carried from the check so the record // counts against the same window the check read. Deriving it later would // mean guessing, and a monthly budget credited to a daily counter is a // budget that never binds. Period store.Period } // Check decides whether a request may proceed. // // It runs before scheduling and touches no database: every counter it reads is // in memory. The token limit is checked against an estimate, because the real // figure only exists once the request has run; Record settles the difference. func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) { price := a.pricing.For(model) res := Reservation{ KeyID: p.KeyID, EstimatedTokens: estimatedTokens, Price: price, Period: p.Limits.BudgetPeriod, } if p.KeyID == "" { return res, nil } lim := p.Limits // Budget first. It is the limit whose refusal cannot be retried away, so // telling the client about it first is more useful than reporting a rate // limit it would hit again in a second. // // A model that costs nothing is exempt, which is what lets a key with no // budget left still be served a cheap fallback (plan §7.7). if lim.Budget != nil && !price.Zero() && a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) { return res, &LimitError{Err: ErrBudgetSpent} } if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) { return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second} } if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) { // The request was not charged against the request-rate limit in vain: // it was, and that is correct — it did arrive. return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second} } return res, nil } // Record settles a finished request: it reconciles the token estimate, adds to // the budget, and queues the usage row. func (a *Accountant) Record(res Reservation, u store.Usage) { if res.KeyID == "" { return } actual := u.PromptTokens + u.CompletionTokens // The estimate was charged up front; charge the difference now. A caller // whose requests are consistently larger than estimated is still bounded, // one window later than with a perfect estimate. a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens) u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) a.budgets.Add(res.KeyID, res.Period, u.CostMicros) if a.recorder != nil { a.recorder.Record(u) } } // Status returns what a key has used, for headers and `ctl`. func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { return 0, 0, 0 } return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod), a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID) }