llmgw/internal/accounting/accounting.go

package accounting

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"sync/atomic"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// Refusals a key can hit. They are distinguished so the client is told which
// limit it hit, and so retrying is only suggested where retrying helps.
var (
	ErrRateLimited  = errors.New("request rate limit exceeded")
	ErrTokenLimited = errors.New("token rate limit exceeded")
	ErrBudgetSpent  = errors.New("budget exhausted")
)

// LimitError carries which limit was hit and, where it makes sense, when to
// come back.
type LimitError struct {
	Err error
	// RetryAfter is zero when waiting does not help, which is the case for a
	// spent budget: it will not refill until the period rolls over, and telling
	// a client to retry in a second would be a lie.
	RetryAfter time.Duration
}

func (e *LimitError) Error() string { return e.Err.Error() }
func (e *LimitError) Unwrap() error { return e.Err }

// Accountant enforces limits and records what was used.
//
// One object because the three halves are not independent: a budget check needs
// the price, the price needs the model, and the record closes the loop by
// feeding the counters that the next check reads.
type Accountant struct {
	pricing  *Pricing
	budgets  *Budgets
	requests *Limiter
	tokens   *Limiter
	recorder *Recorder
	log      *slog.Logger
	// notify is told when a budget runs out; nil tells no one.
	notify atomic.Pointer[notify.Notifier]
}

// SetNotifier gives the accountant somewhere to say a budget is spent.
func (a *Accountant) SetNotifier(n *notify.Notifier) { a.notify.Store(n) }

// New builds an accountant.
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
	if log == nil {
		log = slog.Default()
	}
	return &Accountant{
		pricing:  pricing,
		budgets:  NewBudgets(),
		requests: NewLimiter(),
		tokens:   NewLimiter(),
		recorder: recorder,
		log:      log,
	}
}

// RestoreBudgets seeds the in-memory counters from the store.
//
// Called at startup. A restart must not reset a budget, which it would if the
// counters only ever lived in memory.
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
	start := PeriodStart(period, time.Now())
	if start.IsZero() {
		return nil
	}
	spend, keys, err := storedSpend(ctx, s, start)
	if err != nil {
		return fmt.Errorf("restoring budgets: %w", err)
	}
	a.budgets.Restore(period, spend)
	a.log.Info("restored budgets from the store",
		"keys", keys, "counters", len(spend), "since", start)
	return nil
}

// CatchUpBudgets raises each counter to what the store has recorded, where
// that is more. Run every so often by a router that is one of several taking
// requests: the others' spend reaches it only through the store. Never
// lowers a counter, which leads the store by whatever this process has not
// written yet.
func (a *Accountant) CatchUpBudgets(ctx context.Context, s store.Store, period store.Period) error {
	start := PeriodStart(period, time.Now())
	if start.IsZero() {
		return nil
	}
	spend, _, err := storedSpend(ctx, s, start)
	if err != nil {
		return fmt.Errorf("reading budgets: %w", err)
	}
	a.budgets.Raise(period, spend)
	return nil
}

// storedSpend is the spend since start the store has recorded, by counter,
// and how many keys it came from.
func storedSpend(ctx context.Context, s store.Store, start time.Time) (map[string]int64, int, error) {
	rows, err := s.SpendSinceByScope(ctx, start)
	if err != nil {
		return nil, 0, err
	}
	// Every scope a key sits under, not just the key. A budget owned by a team
	// is one counter fed by every key beneath it, and restoring only per-key
	// totals would bring the control plane back up believing the team had spent
	// nothing — which is a budget that resets on every restart, for exactly the
	// scopes where it matters most.
	// And every pool a person is in: their own, and each per-member limit of
	// the project, team and organization they spent it in.
	spend := make(map[string]int64, len(rows)*8)
	for _, r := range rows {
		owners := []store.LimitOwner{
			{Scope: store.ScopeKey, ID: r.KeyID},
			{Scope: store.ScopeProject, ID: r.ProjectID},
			{Scope: store.ScopeTeam, ID: r.TeamID},
			{Scope: store.ScopeOrg, ID: r.OrgID},
			{Scope: store.ScopeUser, ID: r.UserID},
		}
		if r.UserID != "" && r.ProjectID != "" {
			owners = append(owners,
				store.MemberOwner(store.ScopeProjectMember, r.ProjectID, r.UserID),
				store.MemberOwner(store.ScopeTeamMember, r.TeamID, r.UserID),
				store.MemberOwner(store.ScopeOrgMember, r.OrgID, r.UserID))
		}
		for _, o := range owners {
			if o.ID == "" || strings.HasPrefix(o.ID, "/") {
				continue
			}
			if k := o.CounterKey(); k != "" {
				spend[k] += r.Micros
			}
		}
	}
	return spend, len(rows), nil
}

// Reservation is what Check returns: the estimate it charged, so Record can
// settle the difference against the real figure.
// pool is one level's counter and the period its budget runs over.
type pool struct {
	Counter string
	Period  store.Period
}

type Reservation struct {
	KeyID string
	// Budgets and Tokens are every counter the check went through, carried so
	// the settlement lands on the same ones. Every, not the tightest: an
	// organization's pool only fills up if each team's spend is added to it
	// as well as to the team's own.
	//
	// Derived again at settlement would mean re-resolving the limits, and a
	// budget checked against the team and settled against the key is a budget
	// that never fills up.
	Budgets         []pool
	Tokens          []string
	EstimatedTokens int64
	Price           Price
	// Period is the key's budget period, carried from the check so the record
	// counts against the same window the check read. Deriving it later would
	// mean guessing, and a monthly budget credited to a daily counter is a
	// budget that never binds.
	Period store.Period
}

// Check decides whether a request may proceed.
//
// It runs before scheduling and touches no database: every counter it reads is
// in memory. The token limit is checked against an estimate, because the real
// figure only exists once the request has run; Record settles the difference.
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
	price := a.pricing.For(model)
	lim := p.Limits
	// Every level that set a limit, each against its own pool. A quota written
	// on a team is one pool the team shares, and the organization above it is a
	// second pool that the team's spending also fills — both have to hold, or
	// an organization given 1000 with four teams given 500 spends 2000.
	chain := levels(lim, p.KeyID)

	res := Reservation{
		KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
		Price: price, Period: lim.BudgetPeriod,
	}
	for _, l := range chain {
		if l.Budget != nil {
			res.Budgets = append(res.Budgets, pool{l.Counter, l.Period})
		}
		if l.TPM != nil {
			res.Tokens = append(res.Tokens, l.Counter)
		}
	}
	if p.KeyID == "" {
		return res, nil
	}

	// Budget first. It is the limit whose refusal cannot be retried away, so
	// telling the client about it first is more useful than reporting a rate
	// limit it would hit again in a second.
	//
	// A model that costs nothing is exempt, which is what lets a key with no
	// budget left still be served a cheap fallback (plan §7.7).
	// Budgets first, and all of them. Exhausted only reads, so the order costs
	// nothing and a refusal here charges no counter.
	if !price.Zero() {
		for _, l := range chain {
			if l.Budget != nil && a.budgets.Exhausted(l.Counter, l.Period, *l.Budget) {
				a.notify.Load().Notify(notify.BudgetSpent, l.Counter,
					fmt.Sprintf("the %s budget of %s is spent: requests to priced models are refused until it renews",
						or(string(l.Period), "total"), l.Counter),
					map[string]any{"counter": l.Counter, "period": string(l.Period), "budget": *l.Budget})
				return res, &LimitError{Err: ErrBudgetSpent}
			}
		}
	}

	// Rate limits charge as they check — deliberately, so two concurrent
	// requests cannot both see room. With several levels that means a refusal
	// at one has already taken from the levels before it, so those are handed
	// back. Without the refund a team at its ceiling would quietly eat its
	// organization's rate limit with requests that were never served.
	if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
		return l.Counter, valInt(l.RPM), 1, l.RPM != nil
	}, a.requests); err != nil {
		return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
	}
	if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
		return l.Counter, valInt(l.TPM), estimatedTokens, l.TPM != nil
	}, a.tokens); err != nil {
		// The request was not charged against the request-rate limit in vain:
		// it was, and that is correct — it did arrive.
		return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
	}
	return res, nil
}

// level is one scope's limits with the counter they are enforced against.
type level struct {
	Counter string
	Period  store.Period
	RPM     *int
	TPM     *int
	Budget  *int64
}

// levels turns a resolved chain into the pools a request must pass.
//
// Falls back to the collapsed values when the chain is empty, which is what a
// principal assembled by hand looks like — in tests, and anywhere that builds
// limits without walking the tree.
func levels(e store.Effective, keyID string) []level {
	if len(e.Chain) == 0 {
		var out []level
		if e.RPM != nil || e.TPM != nil || e.Budget != nil {
			out = append(out, level{
				Counter: counterFor(e.BudgetOwner, keyID),
				Period:  e.BudgetPeriod, Budget: e.Budget,
			})
			out[0].RPM, out[0].TPM = e.RPM, e.TPM
			// One entry, so a hand-built principal with different owners per
			// limit still counts each where it said to.
			if r, b := counterFor(e.RPMOwner, keyID), out[0].Counter; r != b && e.RPM != nil {
				out[0].RPM = nil
				out = append(out, level{Counter: r, RPM: e.RPM})
			}
			if tp, b := counterFor(e.TPMOwner, keyID), out[0].Counter; tp != b && e.TPM != nil {
				out[0].TPM = nil
				out = append(out, level{Counter: tp, TPM: e.TPM})
			}
		}
		return out
	}
	out := make([]level, 0, len(e.Chain))
	for _, l := range e.Chain {
		out = append(out, level{
			Counter: store.LimitOwner{Scope: l.Scope, ID: l.ScopeID}.CounterKey(),
			Period:  l.BudgetPeriod, RPM: l.RPM, TPM: l.TPM, Budget: l.Budget,
		})
	}
	return out
}

// allowAll charges every level that has this limit, refunding on a refusal.
func (a *Accountant) allowAll(chain []level,
	pick func(level) (key string, limit int, n int64, has bool), lim *Limiter,
) error {
	var charged []struct {
		key string
		n   int64
	}
	for _, l := range chain {
		key, limit, n, has := pick(l)
		if !has {
			continue
		}
		if !lim.Allow(key, limit, n) {
			for _, c := range charged {
				lim.Charge(c.key, -c.n)
			}
			return errRefused
		}
		charged = append(charged, struct {
			key string
			n   int64
		}{key, n})
	}
	return nil
}

var errRefused = errors.New("refused")

func valInt(v *int) int {
	if v == nil {
		return 0
	}
	return *v
}

// Record settles a finished request: it reconciles the token estimate, adds to
// the budget, and queues the usage row.
func (a *Accountant) Record(res Reservation, u store.Usage) {
	if res.KeyID == "" {
		return
	}
	actual := u.PromptTokens + u.CompletionTokens

	// The estimate was charged up front; charge the difference now. A caller
	// whose requests are consistently larger than estimated is still bounded,
	// one window later than with a perfect estimate.
	// Every level the check went through, so an organization's pool fills up
	// from its teams' spending rather than only from requests that named it.
	for _, c := range res.Tokens {
		a.tokens.Charge(c, actual-res.EstimatedTokens)
	}

	// Priced by the model that answered: a fallback served by another model
	// costs what that one costs, not what the one asked for would have.
	price := res.Price
	if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil {
		price = a.pricing.For(u.ServedModel)
	}
	u.CostMicros = price.Cost(u.PromptTokens, u.CachedTokens, u.CompletionTokens, u.Images, u.Meshes)
	for _, b := range res.Budgets {
		a.budgets.Add(b.Counter, b.Period, u.CostMicros)
	}

	if a.recorder != nil {
		a.recorder.Record(u)
	}
}

// RecordRefused queues the row of a request a limit refused: nothing is
// charged — it used nothing — but it is counted, so a team running into its
// limits shows as one rather than as a team that went quiet.
func (a *Accountant) RecordRefused(u store.Usage) {
	if u.KeyID == "" || a.recorder == nil {
		return
	}
	u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0
	a.recorder.Record(u)
}

// Status returns what a key has used, for headers and `ctl`.
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
	if p.KeyID == "" {
		return 0, 0, 0
	}
	// The pool that will refuse them, which is not always the one with the
	// smallest number on it. A team given 2000 inside an organization given the
	// same is bounded by whichever is nearer its ceiling, and that is the
	// organization as soon as a second team spends anything. A header that
	// reported the team's would disagree with the refusal that follows it.
	for _, l := range levels(p.Limits, p.KeyID) {
		if l.Budget != nil {
			if v := a.budgets.Spent(l.Counter, l.Period); v > spent {
				spent = v
			}
		}
		if l.RPM != nil {
			if v := a.requests.Rate(l.Counter); v > rpm {
				rpm = v
			}
		}
		if l.TPM != nil {
			if v := a.tokens.Rate(l.Counter); v > tpm {
				tpm = v
			}
		}
	}
	return spent, rpm, tpm
}

// counterFor is the counter a limit owned by this scope should use.
//
// Falls back to the key's own counter when no scope owns the limit, which
// happens when nothing in the chain set one.
//
// Namespaced even then. A bare key id here and a "key:"-prefixed one in the
// restore path would be two different counters for the same key, so a budget
// would check against an empty one and survive every restart — which is how
// this was wrong when it was first written.
func counterFor(o store.LimitOwner, keyID string) string {
	if k := o.CounterKey(); k != "" {
		return k
	}
	return keyCounter(keyID)
}

// keyCounter is a key's own counter, in the same namespace as every other.
func or(v, fallback string) string {
	if v == "" {
		return fallback
	}
	return v
}

func keyCounter(keyID string) string {
	if keyID == "" {
		return ""
	}
	return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey()
}

// AnythingPriced reports whether a budget can ever be spent against.
//
// On the accountant rather than making callers reach for the price table: what
// they are asking is whether the budgets they are about to display mean
// anything, and the accountant is what would enforce them.
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }

// Headroom is the tightest of a principal's request and token limits: how
// much is left of each, and when the minute it is counted over rolls on.
type Headroom struct {
	RequestLimit, RequestsLeft int
	TokenLimit, TokensLeft     int
	Reset                      time.Duration
}

// Headroom reports what a principal has left, after whatever Check just
// charged. Of several levels the one with least left is the one that will
// refuse first, so that is the one reported. A limit of 0 means none is set.
func (a *Accountant) Headroom(p auth.Principal) Headroom {
	var h Headroom
	firstReq, firstTok := true, true
	for _, l := range levels(p.Limits, p.KeyID) {
		if l.RPM != nil && *l.RPM > 0 {
			left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
			if firstReq || left < h.RequestsLeft {
				h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
			}
		}
		if l.TPM != nil && *l.TPM > 0 {
			left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
			if firstTok || left < h.TokensLeft {
				h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
			}
		}
	}
	// The windows slide, so room comes back continuously; the next minute
	// boundary is when a whole window's worth has moved on, and is the figure
	// a client pacing itself can act on.
	now := time.Now()
	h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
	return h
}