diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 563a085..a1a2d2f 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -57,6 +57,25 @@ agent_listen: ":7443" # # How long a rejected key is remembered as rejected, so a client looping # # with a bad key cannot become a database read per request. # negative_ttl: 5s +# +# # What each model costs, in micro-units per thousand tokens. A unit is +# # whatever you decide: self-hosted models have no per-token price, so it can +# # be GPU-seconds, a notional rate, or 1000 everywhere — which turns budgets +# # into plain token quotas. +# # +# # A model priced at zero is exempt from budget checks, which is what lets a +# # key with nothing left still be served a cheap fallback. +# default_price: { prompt_per_1k: 0, completion_per_1k: 0 } +# pricing: +# llama-3.3-70b: { prompt_per_1k: 1000, completion_per_1k: 3000 } +# qwen2.5-32b: { prompt_per_1k: 400, completion_per_1k: 1200 } +# +# # Usage rows are written in batches off the request path. Beyond buffer_size +# # they are dropped rather than blocking a request: losing some accounting is +# # bad, turning a stalled database into an outage is worse. +# flush_interval: 200ms +# batch_size: 500 +# buffer_size: 8192 router: scrape_interval: 2s diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go new file mode 100644 index 0000000..b154035 --- /dev/null +++ b/internal/accounting/accounting.go @@ -0,0 +1,161 @@ +package accounting + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Refusals a key can hit. They are distinguished so the client is told which +// limit it hit, and so retrying is only suggested where retrying helps. +var ( + ErrRateLimited = errors.New("request rate limit exceeded") + ErrTokenLimited = errors.New("token rate limit exceeded") + ErrBudgetSpent = errors.New("budget exhausted") +) + +// LimitError carries which limit was hit and, where it makes sense, when to +// come back. +type LimitError struct { + Err error + // RetryAfter is zero when waiting does not help, which is the case for a + // spent budget: it will not refill until the period rolls over, and telling + // a client to retry in a second would be a lie. + RetryAfter time.Duration +} + +func (e *LimitError) Error() string { return e.Err.Error() } +func (e *LimitError) Unwrap() error { return e.Err } + +// Accountant enforces limits and records what was used. +// +// One object because the three halves are not independent: a budget check needs +// the price, the price needs the model, and the record closes the loop by +// feeding the counters that the next check reads. +type Accountant struct { + pricing *Pricing + budgets *Budgets + requests *Limiter + tokens *Limiter + recorder *Recorder + log *slog.Logger +} + +// New builds an accountant. +func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant { + if log == nil { + log = slog.Default() + } + return &Accountant{ + pricing: pricing, + budgets: NewBudgets(), + requests: NewLimiter(), + tokens: NewLimiter(), + recorder: recorder, + log: log, + } +} + +// RestoreBudgets seeds the in-memory counters from the store. +// +// Called at startup. A restart must not reset a budget, which it would if the +// counters only ever lived in memory. +func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error { + start := PeriodStart(period, time.Now()) + if start.IsZero() { + return nil + } + spend, err := s.SpendSince(ctx, start) + if err != nil { + return fmt.Errorf("restoring budgets: %w", err) + } + a.budgets.Restore(period, spend) + a.log.Info("restored budgets from the store", "keys", len(spend), "since", start) + return nil +} + +// Reservation is what Check returns: the estimate it charged, so Record can +// settle the difference against the real figure. +type Reservation struct { + KeyID string + EstimatedTokens int64 + Price Price + // Period is the key's budget period, carried from the check so the record + // counts against the same window the check read. Deriving it later would + // mean guessing, and a monthly budget credited to a daily counter is a + // budget that never binds. + Period store.Period +} + +// Check decides whether a request may proceed. +// +// It runs before scheduling and touches no database: every counter it reads is +// in memory. The token limit is checked against an estimate, because the real +// figure only exists once the request has run; Record settles the difference. +func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) { + price := a.pricing.For(model) + res := Reservation{ + KeyID: p.KeyID, EstimatedTokens: estimatedTokens, + Price: price, Period: p.Limits.BudgetPeriod, + } + if p.KeyID == "" { + return res, nil + } + lim := p.Limits + + // Budget first. It is the limit whose refusal cannot be retried away, so + // telling the client about it first is more useful than reporting a rate + // limit it would hit again in a second. + // + // A model that costs nothing is exempt, which is what lets a key with no + // budget left still be served a cheap fallback (plan §7.7). + if lim.Budget != nil && !price.Zero() && + a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) { + return res, &LimitError{Err: ErrBudgetSpent} + } + + if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) { + return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second} + } + if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) { + // The request was not charged against the request-rate limit in vain: + // it was, and that is correct — it did arrive. + return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second} + } + return res, nil +} + +// Record settles a finished request: it reconciles the token estimate, adds to +// the budget, and queues the usage row. +func (a *Accountant) Record(res Reservation, u store.Usage) { + if res.KeyID == "" { + return + } + actual := u.PromptTokens + u.CompletionTokens + + // The estimate was charged up front; charge the difference now. A caller + // whose requests are consistently larger than estimated is still bounded, + // one window later than with a perfect estimate. + a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens) + + u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) + a.budgets.Add(res.KeyID, res.Period, u.CostMicros) + + if a.recorder != nil { + a.recorder.Record(u) + } +} + +// Status returns what a key has used, for headers and `ctl`. +func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { + if p.KeyID == "" { + return 0, 0, 0 + } + return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod), + a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID) +} diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go new file mode 100644 index 0000000..327273e --- /dev/null +++ b/internal/accounting/accounting_test.go @@ -0,0 +1,290 @@ +package accounting + +import ( + "context" + "errors" + "log/slog" + "path/filepath" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func intp(v int) *int { return &v } +func i64p(v int64) *int64 { return &v } + +func principal(id string, tune func(*store.Effective)) auth.Principal { + e := store.Effective{KeyID: id, ProjectID: "proj"} + if tune != nil { + tune(&e) + } + return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e} +} + +func pricing(t *testing.T) *Pricing { + t.Helper() + return NewPricing(map[string]Price{ + "paid": {PromptPer1k: 1000, CompletionPer1k: 3000}, + "free": {}, + }, Price{}) +} + +func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) { + // Rounding down would make a stream of tiny requests cost nothing, which is + // exactly the traffic shape a quota exists to bound. + p := Price{PromptPer1k: 1000, CompletionPer1k: 3000} + if got := p.Cost(1, 0); got != 1 { + t.Errorf("one prompt token cost %d", got) + } + if got := p.Cost(1000, 1000); got != 4000 { + t.Errorf("a thousand of each cost %d, want 4000", got) + } + if got := p.Cost(0, 0); got != 0 { + t.Errorf("an empty request cost %d", got) + } +} + +func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) { + // This is what makes budget fallback work: a key with nothing left can + // still be served something that costs nothing. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(100) + e.BudgetPeriod = store.PeriodMonth + }) + + res, err := a.Check(p, "paid", 10) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0}) + + if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("a spent budget still served a paid model: %v", err) + } + if _, err := a.Check(p, "free", 10); err != nil { + t.Errorf("a spent budget refused a free model: %v", err) + } +} + +func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) { + // A budget does not refill until the period rolls over, so a Retry-After of + // a second would be a lie that turns one client into a retry loop. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(1) + e.BudgetPeriod = store.PeriodDay + }) + res, _ := a.Check(p, "paid", 10) + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000}) + + _, err := a.Check(p, "paid", 10) + var le *LimitError + if !errors.As(err, &le) { + t.Fatalf("expected a limit error, got %v", err) + } + if le.RetryAfter != 0 { + t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter) + } +} + +func TestRateLimitsBind(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) }) + + for i := range 3 { + if _, err := a.Check(p, "free", 1); err != nil { + t.Fatalf("request %d was refused: %v", i, err) + } + } + if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) { + t.Errorf("the fourth request was allowed: %v", err) + } +} + +func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) { + // The real token count only exists once the request has run, so the check + // has to work on an estimate. A caller that consistently underestimates + // must still end up bounded. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) }) + + // Claims 10 tokens, actually uses 900. + res, err := a.Check(p, "free", 10) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500}) + + // The settled figure is what the next check sees, so a second request of + // the same shape cannot fit. + if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) { + t.Errorf("the underestimate was never settled: %v", err) + } +} + +func TestNoLimitIsNotALimitOfZero(t *testing.T) { + // The distinction the store keeps with a nil pointer. Getting it wrong here + // would silently forbid every request for every key with no limits set. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", nil) + + for range 50 { + if _, err := a.Check(p, "paid", 10_000); err != nil { + t.Fatalf("a key with no limits was refused: %v", err) + } + } +} + +func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) { + // With auth off there is no key to charge, and Check must not refuse. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + res, err := a.Check(auth.Principal{}, "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{PromptTokens: 1000}) + if spent, _, _ := a.Status(auth.Principal{}); spent != 0 { + t.Errorf("an anonymous request was charged %d", spent) + } +} + +func TestBudgetSurvivesARestart(t *testing.T) { + // The counters live in memory; the store is what makes them survive. + ctx := context.Background() + path := filepath.Join(t.TempDir(), "acct.db") + s, err := store.OpenSQLite(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"})) + must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"})) + must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"})) + _, k, _ := store.NewKey("k1", "n", "proj") + k.CreatedAt = time.Now() + must(t, s.CreateKey(ctx, k)) + + must(t, s.WriteUsage(ctx, []store.Usage{ + {At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid", + ServedModel: "paid", CostMicros: 5000, Status: 200}, + })) + + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth)) + + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(5000) + e.BudgetPeriod = store.PeriodMonth + }) + if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("the budget was reset by the restart: %v", err) + } +} + +func TestBudgetsResetAtThePeriodBoundary(t *testing.T) { + b := NewBudgets() + now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC) + b.now = func() time.Time { return now } + + b.Add("k1", store.PeriodMonth, 900) + if got := b.Spent("k1", store.PeriodMonth); got != 900 { + t.Fatalf("spent %d", got) + } + + now = now.Add(2 * time.Minute) // into February + if got := b.Spent("k1", store.PeriodMonth); got != 0 { + t.Errorf("the counter carried %d across the month boundary", got) + } +} + +func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) { + b := NewBudgets() + now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC) + b.now = func() time.Time { return now } + + b.Add("daily", store.PeriodDay, 100) + b.Add("monthly", store.PeriodMonth, 100) + + now = now.Add(24 * time.Hour) + if got := b.Spent("daily", store.PeriodDay); got != 0 { + t.Errorf("the daily counter carried %d across a day", got) + } + if got := b.Spent("monthly", store.PeriodMonth); got != 100 { + t.Errorf("the monthly counter lost %d within a month", 100-got) + } +} + +func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) { + // The reason for a sliding window. With a fixed one, a caller sends its + // whole allowance at the end of a window and again at the start of the + // next, so the observed rate is double the limit exactly when the system is + // least able to take it. + l := NewLimiter() + base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + now := base.Add(59 * time.Second) + l.now = func() time.Time { return now } + + const limit = 60 + sent := 0 + for range limit { + if l.Allow("k", limit, 1) { + sent++ + } + } + if sent != limit { + t.Fatalf("the first burst only managed %d", sent) + } + + // One second later, in the next fixed window. + now = base.Add(61 * time.Second) + second := 0 + for range limit { + if l.Allow("k", limit, 1) { + second++ + } + } + // Some allowance returns as the old window slides out of view, but nowhere + // near the whole of it. + if second > limit/2 { + t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding", + second, sent) + } +} + +func TestLimiterIsSafeUnderConcurrency(t *testing.T) { + // Checking and charging have to be one operation, or two concurrent + // requests both see room and both take it. + l := NewLimiter() + const limit = 100 + + var wg sync.WaitGroup + var mu sync.Mutex + allowed := 0 + for range 500 { + wg.Add(1) + go func() { + defer wg.Done() + if l.Allow("k", limit, 1) { + mu.Lock() + allowed++ + mu.Unlock() + } + }() + } + wg.Wait() + if allowed != limit { + t.Errorf("%d requests passed a limit of %d", allowed, limit) + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/accounting/budget.go b/internal/accounting/budget.go new file mode 100644 index 0000000..0b47d79 --- /dev/null +++ b/internal/accounting/budget.go @@ -0,0 +1,101 @@ +package accounting + +import ( + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// PeriodStart is when the current budget period began. +// +// Periods are anchored to UTC rather than to a per-key start date, so every +// key's day and month boundary is the same instant. A key whose period started +// when it was created would make "spend this month" mean something different +// for every key, and make the restore query unanswerable in one pass. +func PeriodStart(p store.Period, now time.Time) time.Time { + u := now.UTC() + switch p { + case store.PeriodDay: + return time.Date(u.Year(), u.Month(), u.Day(), 0, 0, 0, 0, time.UTC) + case store.PeriodMonth: + return time.Date(u.Year(), u.Month(), 1, 0, 0, 0, 0, time.UTC) + default: + return time.Time{} + } +} + +// Budgets tracks spend per key in memory, so a budget check costs no query. +// +// The database catches up through the batch writer. That means the in-memory +// figure leads the stored one, which is the right direction: a budget that only +// counted what had already been written would let a burst overspend by however +// much the writer was behind. +type Budgets struct { + now func() time.Time + + mu sync.Mutex + spend map[string]counter +} + +type counter struct { + micros int64 + since time.Time +} + +// NewBudgets builds an empty tracker. +func NewBudgets() *Budgets { + return &Budgets{now: time.Now, spend: map[string]counter{}} +} + +// Restore seeds the counters from what the store already recorded. +// +// A restart must not reset a budget, which it would if the counter lived only +// in memory. The caller passes the spend since each period's start. +func (b *Budgets) Restore(period store.Period, spend map[string]int64) { + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + for key, micros := range spend { + b.spend[key] = counter{micros: micros, since: start} + } +} + +// Spent returns a key's spend in the current period, resetting it if the period +// has rolled over. +func (b *Budgets) Spent(keyID string, period store.Period) int64 { + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + return b.currentLocked(keyID, start).micros +} + +// Add records spend. +func (b *Budgets) Add(keyID string, period store.Period, micros int64) { + if micros == 0 { + return + } + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + c := b.currentLocked(keyID, start) + c.micros += micros + b.spend[keyID] = c +} + +// Exhausted reports whether a key has spent its budget. +func (b *Budgets) Exhausted(keyID string, period store.Period, budget int64) bool { + return b.Spent(keyID, period) >= budget +} + +// currentLocked returns the counter, rolled over if the period changed. +func (b *Budgets) currentLocked(keyID string, start time.Time) counter { + c, ok := b.spend[keyID] + if !ok || c.since.Before(start) { + // Either never seen, or the period rolled over while the process was + // running. Both start from zero. + c = counter{since: start} + b.spend[keyID] = c + } + return c +} diff --git a/internal/accounting/cost.go b/internal/accounting/cost.go new file mode 100644 index 0000000..34b00af --- /dev/null +++ b/internal/accounting/cost.go @@ -0,0 +1,70 @@ +// Package accounting turns finished requests into spend, enforces what a key +// is allowed, and gets both into the store without touching the request path +// (plan §7.9.3, §7.9.4). +package accounting + +import ( + "strings" + "sync" +) + +// Price is what one model costs, in micro-units per thousand tokens. +// +// Micro-units rather than a float because spend is accumulated by repeated +// addition, and repeated addition of floats drifts. What a unit means is the +// operator's: self-hosted models have no per-token price, so it may be +// GPU-seconds, a notional rate, or 1000 everywhere — which turns a budget into +// a token quota. +type Price struct { + PromptPer1k int64 `koanf:"prompt_per_1k"` + CompletionPer1k int64 `koanf:"completion_per_1k"` +} + +// Zero reports whether this model spends nothing. +// +// It matters beyond arithmetic: a zero-cost model stays available to a key +// whose budget is spent, which is what makes the plan's budget fallback work +// (§7.7). +func (p Price) Zero() bool { return p.PromptPer1k == 0 && p.CompletionPer1k == 0 } + +// Cost computes what a request spent, rounding up. +// +// Up rather than to nearest: rounding down would make a stream of tiny requests +// free, which is exactly the shape of traffic a quota exists to bound. +func (p Price) Cost(promptTokens, completionTokens int64) int64 { + return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000) +} + +func divUp(n, d int64) int64 { + if n <= 0 { + return 0 + } + return (n + d - 1) / d +} + +// Pricing resolves a model name to its price. +type Pricing struct { + mu sync.RWMutex + byModel map[string]Price + fallback Price +} + +// NewPricing builds a price table. Names are matched case-insensitively, +// because a model name reaches us from whatever the client typed. +func NewPricing(byModel map[string]Price, fallback Price) *Pricing { + table := make(map[string]Price, len(byModel)) + for name, p := range byModel { + table[strings.ToLower(name)] = p + } + return &Pricing{byModel: table, fallback: fallback} +} + +// For returns a model's price, or the fallback. +func (p *Pricing) For(model string) Price { + p.mu.RLock() + defer p.mu.RUnlock() + if v, ok := p.byModel[strings.ToLower(model)]; ok { + return v + } + return p.fallback +} diff --git a/internal/accounting/limiter.go b/internal/accounting/limiter.go new file mode 100644 index 0000000..e85efee --- /dev/null +++ b/internal/accounting/limiter.go @@ -0,0 +1,130 @@ +package accounting + +import ( + "sync" + "time" +) + +// Limiter enforces per-key rate limits. +// +// A sliding window rather than a fixed one. A fixed window lets a caller send +// its whole minute's allowance in the last second of one window and again in +// the first second of the next, so the observed rate is twice the configured +// limit exactly when the system is least able to take it. +// +// The window is approximated from two buckets, which is the usual compromise: +// exact enough that the doubling cannot happen, and bounded memory per key +// rather than a timestamp per request. +type Limiter struct { + now func() time.Time + + mu sync.Mutex + keys map[string]*window +} + +type window struct { + // current and previous are counts in this minute and the one before. + current, previous int64 + start time.Time +} + +// NewLimiter builds an empty limiter. +func NewLimiter() *Limiter { + return &Limiter{now: time.Now, keys: map[string]*window{}} +} + +const windowLen = time.Minute + +// Allow charges n against a key's limit and reports whether it fits. +// +// Charging and checking are one operation on purpose: doing them separately +// lets two concurrent requests both see room and both take it. +func (l *Limiter) Allow(key string, limit int, n int64) bool { + if limit <= 0 { + // No limit configured. Zero would be a limit of zero, which is why + // callers pass a nil pointer rather than 0 for "unset". + return true + } + now := l.now() + + l.mu.Lock() + defer l.mu.Unlock() + + w := l.windowLocked(key, now) + if w.rate(now)+float64(n) > float64(limit) { + return false + } + w.current += n + return true +} + +// Charge records usage that was already allowed, for reconciliation. +// +// Token limits are checked against an estimate before the request runs, because +// the real figure only exists once it has. The difference is charged here, so a +// caller whose requests are consistently larger than estimated is still bounded +// — one window later than it would be with a perfect estimate. +func (l *Limiter) Charge(key string, n int64) { + if n == 0 { + return + } + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + w := l.windowLocked(key, now) + w.current += n + if w.current < 0 { + w.current = 0 + } +} + +// Rate returns a key's current observed rate. +func (l *Limiter) Rate(key string) float64 { + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + return l.windowLocked(key, now).rate(now) +} + +func (l *Limiter) windowLocked(key string, now time.Time) *window { + w, ok := l.keys[key] + if !ok { + w = &window{start: now.Truncate(windowLen)} + l.keys[key] = w + } + w.roll(now) + return w +} + +// roll advances the window to now. +func (w *window) roll(now time.Time) { + start := now.Truncate(windowLen) + switch { + case start.Equal(w.start): + return + case start.Sub(w.start) == windowLen: + w.previous, w.current = w.current, 0 + default: + // More than one window has passed with no traffic. + w.previous, w.current = 0, 0 + } + w.start = start +} + +// rate weights the previous window by how much of it is still in view. +func (w *window) rate(now time.Time) float64 { + elapsed := now.Sub(w.start) + if elapsed >= windowLen { + return float64(w.current) + } + remaining := 1 - float64(elapsed)/float64(windowLen) + return float64(w.previous)*remaining + float64(w.current) +} + +// Forget drops a key's window. Used when a key is revoked; keeping it would +// leak memory for every key that ever called. +func (l *Limiter) Forget(key string) { + l.mu.Lock() + delete(l.keys, key) + l.mu.Unlock() +} diff --git a/internal/accounting/recorder.go b/internal/accounting/recorder.go new file mode 100644 index 0000000..6d50030 --- /dev/null +++ b/internal/accounting/recorder.go @@ -0,0 +1,189 @@ +package accounting + +import ( + "context" + "log/slog" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// RecorderConfig tunes the batch writer. +type RecorderConfig struct { + // FlushInterval and BatchSize decide when a batch is written. Whichever + // comes first. + FlushInterval time.Duration + BatchSize int + // Capacity bounds the buffer. Beyond it, records are dropped rather than + // blocking a request — see Record. + Capacity int +} + +// DefaultRecorderConfig returns sensible values. +func DefaultRecorderConfig() RecorderConfig { + return RecorderConfig{FlushInterval: 200 * time.Millisecond, BatchSize: 500, Capacity: 8192} +} + +func (c *RecorderConfig) normalize() { + d := DefaultRecorderConfig() + if c.FlushInterval <= 0 { + c.FlushInterval = d.FlushInterval + } + if c.BatchSize <= 0 { + c.BatchSize = d.BatchSize + } + if c.Capacity <= 0 { + c.Capacity = d.Capacity + } +} + +// Toucher is the part of the authenticator the recorder flushes alongside +// usage, so last-used timestamps ride the same transaction schedule. +type Toucher interface { + FlushTouches(ctx context.Context) error +} + +// Recorder batches usage records into the store. +// +// The buffered channel is the whole design. A request must never wait on a +// database write, so Record is a non-blocking send and the writer drains it on +// its own schedule. +type Recorder struct { + cfg RecorderConfig + store store.Store + touch Toucher + log *slog.Logger + + ch chan store.Usage + done chan struct{} + wg sync.WaitGroup + + mu sync.Mutex + dropped int64 + written int64 +} + +// NewRecorder builds a recorder. Start begins writing. +func NewRecorder(s store.Store, touch Toucher, cfg RecorderConfig, log *slog.Logger) *Recorder { + cfg.normalize() + if log == nil { + log = slog.Default() + } + return &Recorder{ + cfg: cfg, store: s, touch: touch, log: log, + ch: make(chan store.Usage, cfg.Capacity), + done: make(chan struct{}), + } +} + +// Record queues one request's accounting. +// +// Never blocks. If the buffer is full — a database stalled long enough to fill +// eight thousand records — the record is dropped and counted. Losing some +// accounting is bad; making every request wait on a stalled database is worse, +// because it converts a reporting problem into an outage. +func (r *Recorder) Record(u store.Usage) { + select { + case r.ch <- u: + default: + r.mu.Lock() + r.dropped++ + n := r.dropped + r.mu.Unlock() + if n == 1 || n%1000 == 0 { + r.log.Warn("dropping usage records: the store is not keeping up", + "dropped_total", n) + } + } +} + +// Start runs the writer until ctx ends. +func (r *Recorder) Start(ctx context.Context) { + r.wg.Add(1) + go func() { + defer r.wg.Done() + defer close(r.done) + r.run(ctx) + }() +} + +// Stop waits for the writer to drain and finish. +func (r *Recorder) Stop() { r.wg.Wait() } + +func (r *Recorder) run(ctx context.Context) { + t := time.NewTicker(r.cfg.FlushInterval) + defer t.Stop() + + batch := make([]store.Usage, 0, r.cfg.BatchSize) + for { + select { + case <-ctx.Done(): + // Drain what is already queued before leaving. These are completed + // requests that have already been served; throwing them away would + // mean a clean shutdown silently loses the last moments of billing. + r.drain(&batch) + r.flush(context.WithoutCancel(ctx), batch) + return + + case u := <-r.ch: + batch = append(batch, u) + if len(batch) >= r.cfg.BatchSize { + r.flush(ctx, batch) + batch = batch[:0] + } + + case <-t.C: + r.drain(&batch) + if len(batch) > 0 { + r.flush(ctx, batch) + batch = batch[:0] + } + if err := r.touchFlush(ctx); err != nil { + r.log.Warn("failed to write last-used timestamps", "err", err) + } + } + } +} + +// drain moves everything queued into the batch without blocking. +func (r *Recorder) drain(batch *[]store.Usage) { + for len(*batch) < r.cfg.BatchSize { + select { + case u := <-r.ch: + *batch = append(*batch, u) + default: + return + } + } +} + +func (r *Recorder) flush(ctx context.Context, batch []store.Usage) { + if len(batch) == 0 { + return + } + if err := r.store.WriteUsage(ctx, batch); err != nil { + // The records are gone either way; what matters is that this is loud. + // Silently losing billing data is the kind of thing discovered at the + // end of a month. + r.log.Error("failed to write usage records", "count", len(batch), "err", err) + return + } + r.mu.Lock() + r.written += int64(len(batch)) + r.mu.Unlock() +} + +func (r *Recorder) touchFlush(ctx context.Context) error { + if r.touch == nil { + return nil + } + return r.touch.FlushTouches(ctx) +} + +// Stats reports what the recorder has done, for metrics and tests. +func (r *Recorder) Stats() (written, dropped int64) { + r.mu.Lock() + defer r.mu.Unlock() + return r.written, r.dropped +} diff --git a/internal/accounting/recorder_test.go b/internal/accounting/recorder_test.go new file mode 100644 index 0000000..86f3cb5 --- /dev/null +++ b/internal/accounting/recorder_test.go @@ -0,0 +1,210 @@ +package accounting + +import ( + "context" + "errors" + "log/slog" + "sync" + "sync/atomic" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// fakeStore records what was written and can be made slow or broken. +type fakeStore struct { + store.Store + + mu sync.Mutex + batches [][]store.Usage + + block chan struct{} + fail atomic.Bool + touched atomic.Int64 +} + +func (f *fakeStore) WriteUsage(_ context.Context, rows []store.Usage) error { + if f.block != nil { + <-f.block + } + if f.fail.Load() { + return errors.New("store is down") + } + f.mu.Lock() + defer f.mu.Unlock() + f.batches = append(f.batches, append([]store.Usage(nil), rows...)) + return nil +} + +func (f *fakeStore) rows() []store.Usage { + f.mu.Lock() + defer f.mu.Unlock() + var out []store.Usage + for _, b := range f.batches { + out = append(out, b...) + } + return out +} + +func (f *fakeStore) batchCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.batches) +} + +type fakeToucher struct{ n *atomic.Int64 } + +func (f fakeToucher) FlushTouches(context.Context) error { + f.n.Add(1) + return nil +} + +func TestRecordsAreBatchedNotWrittenOneByOne(t *testing.T) { + // The whole point of the buffer: a hundred requests must not be a hundred + // transactions. + f := &fakeStore{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 20 * time.Millisecond, BatchSize: 500}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + + for i := range 100 { + r.Record(store.Usage{KeyID: "k", PromptTokens: int64(i)}) + } + waitFor(t, time.Second, func() bool { return len(f.rows()) == 100 }) + + if n := f.batchCount(); n > 5 { + t.Errorf("100 records took %d transactions", n) + } + cancel() + r.Stop() +} + +func TestRecordNeverBlocksARequest(t *testing.T) { + // A stalled database must not become an outage. Losing accounting is bad; + // making every request wait on the store is worse. + f := &fakeStore{block: make(chan struct{})} + r := NewRecorder(f, nil, RecorderConfig{ + FlushInterval: time.Millisecond, BatchSize: 1, Capacity: 8, + }, slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + defer func() { close(f.block); cancel(); r.Stop() }() + + done := make(chan struct{}) + go func() { + defer close(done) + for range 1000 { + r.Record(store.Usage{KeyID: "k"}) + } + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Record blocked while the store was stalled") + } + + _, dropped := r.Stats() + if dropped == 0 { + t.Error("the overflow was not counted, so nobody would ever know") + } +} + +func TestAShutdownDoesNotLoseWhatIsAlreadyQueued(t *testing.T) { + // Those are completed requests that were already served. Throwing them away + // means a clean shutdown silently loses the last moments of billing. + f := &fakeStore{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: time.Hour, BatchSize: 10_000}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + + for range 50 { + r.Record(store.Usage{KeyID: "k"}) + } + cancel() + r.Stop() + + if got := len(f.rows()); got != 50 { + t.Errorf("a clean shutdown wrote %d of 50 queued records", got) + } +} + +func TestAFailedWriteIsLoudRatherThanSilent(t *testing.T) { + // Silently losing billing data is the kind of thing discovered at the end + // of a month. + f := &fakeStore{} + f.fail.Store(true) + logged := &countingHandler{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 10 * time.Millisecond, BatchSize: 1}, + slog.New(logged)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + r.Record(store.Usage{KeyID: "k"}) + + waitFor(t, time.Second, func() bool { return logged.count(slog.LevelError) > 0 }) + cancel() + r.Stop() + + if written, _ := r.Stats(); written != 0 { + t.Errorf("a failed write was counted as %d written", written) + } +} + +func TestLastUsedRidesTheSameSchedule(t *testing.T) { + var n atomic.Int64 + f := &fakeStore{} + r := NewRecorder(f, fakeToucher{n: &n}, RecorderConfig{FlushInterval: 10 * time.Millisecond}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + waitFor(t, time.Second, func() bool { return n.Load() > 0 }) + cancel() + r.Stop() +} + +type countingHandler struct { + mu sync.Mutex + levels []slog.Level +} + +func (c *countingHandler) Enabled(context.Context, slog.Level) bool { return true } +func (c *countingHandler) WithAttrs([]slog.Attr) slog.Handler { return c } +func (c *countingHandler) WithGroup(string) slog.Handler { return c } +func (c *countingHandler) Handle(_ context.Context, r slog.Record) error { + c.mu.Lock() + c.levels = append(c.levels, r.Level) + c.mu.Unlock() + return nil +} + +func (c *countingHandler) count(l slog.Level) int { + c.mu.Lock() + defer c.mu.Unlock() + n := 0 + for _, got := range c.levels { + if got == l { + n++ + } + } + return n +} + +func waitFor(t *testing.T, limit time.Duration, pred func() bool) { + t.Helper() + deadline := time.Now().Add(limit) + for time.Now().Before(deadline) { + if pred() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition never became true") +} diff --git a/internal/config/config.go b/internal/config/config.go index a0663c2..eb0c50c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,29 @@ type Auth struct { // NegativeTTL is how long a rejected key is remembered as rejected, so a // client looping with a bad key cannot become a database read per request. NegativeTTL time.Duration `koanf:"negative_ttl"` + + // Pricing is what each model costs, for budgets. A model absent from the + // table costs whatever DefaultPrice says, which is nothing unless set. + Pricing map[string]Price `koanf:"pricing"` + DefaultPrice Price `koanf:"default_price"` + + // FlushInterval and BatchSize decide when usage records are written. + FlushInterval time.Duration `koanf:"flush_interval"` + BatchSize int `koanf:"batch_size"` + // BufferSize bounds the queue. Beyond it records are dropped rather than + // blocking a request: losing accounting is bad, turning a stalled database + // into an outage is worse. + BufferSize int `koanf:"buffer_size"` +} + +// Price is a model's cost in micro-units per thousand tokens. +// +// What a unit means is the operator's choice — self-hosted models have no +// per-token price, so it may be GPU-seconds, a notional rate, or 1000 +// everywhere, which turns a budget into a token quota (plan §7.9.4). +type Price struct { + PromptPer1k int64 `koanf:"prompt_per_1k"` + CompletionPer1k int64 `koanf:"completion_per_1k"` } // Control configures the control plane. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index ae01bae..07784d8 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -12,9 +12,11 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // admit waits for the flow control layer to let the request through. With @@ -111,6 +113,11 @@ type parsedRequest struct { PromptPrefix string PromptTokens int MaxTokens int + + // reservation is carried from the limit check to the point where the + // request is settled, so the record counts against the same budget window + // the check read. + reservation accounting.Reservation } // inferenceBody covers the fields the router reads across all inference @@ -143,10 +150,23 @@ func (r *Router) handleInference(path string) http.HandlerFunc { return } - if p, ok := auth.FromContext(req.Context()); ok && !r.permits(p, parsed.Model) { - auth.WriteForbiddenModel(w, parsed.Model) - r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) - return + if p, ok := auth.FromContext(req.Context()); ok { + if !r.permits(p, parsed.Model) { + auth.WriteForbiddenModel(w, parsed.Model) + r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) + return + } + // Limits are checked before admission, not after: a request that is + // over its quota should be refused rather than queued, and queueing + // it would have it occupy a band's capacity to no purpose. + if r.acct != nil { + reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens()) + if err != nil { + r.rejectLimit(w, parsed, err, start) + return + } + parsed.reservation = reservation + } } // Step 5 of the lifecycle: "now?" before "where?" (plan §7.1). @@ -217,6 +237,53 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR } r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) + r.account(req, res, status, time.Since(start)) +} + +// account settles a finished request against its key. +// +// It runs for every outcome, not only success: a request that used a GPU and +// then failed still used it, and a key that could spend nothing by failing +// would be a way around a budget. +func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) { + if r.acct == nil || req.KeyID == "" { + return + } + u := store.Usage{ + At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID, + Model: req.Model, ServedModel: res.model, Status: status, + LatencyMs: took.Milliseconds(), + } + if u.ServedModel == "" { + u.ServedModel = req.Model + } + if usage := res.outcome.Usage; usage != nil { + u.PromptTokens = int64(usage.PromptTokens) + u.CompletionTokens = int64(usage.CompletionTokens) + } else { + // No usage came back — a failure before the engine answered, or an + // engine that does not report it. Charging the estimate is closer to + // the truth than charging nothing. + u.PromptTokens = int64(req.PromptTokens) + } + r.acct.Record(req.reservation, u) +} + +// rejectLimit turns a quota refusal into a status the client can act on. +func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { + var le *accounting.LimitError + if !errors.As(err, &le) { + writeError(w, http.StatusInternalServerError, "server_error", err.Error()) + return + } + if le.RetryAfter > 0 { + w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds()))) + } + // 429 for all of them: a spent budget is a quota, and OpenAI clients + // already know how to surface that code. What differs is Retry-After, + // which is absent when waiting does not help. + writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) + r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0) } // parse reads the body and extracts the routing-relevant fields. @@ -416,3 +483,18 @@ func writeError(w http.ResponseWriter, code int, kind, msg string) { "message": msg, "type": kind, "code": nil, }}) } + +// estimatedTokens is what a token-rate limit is checked against, before the +// real figure exists. +// +// The prompt estimate plus whatever the client said it wanted back. A client +// that names no max_tokens gets a modest assumption rather than zero: assuming +// zero would let an unbounded request through every token limit. +func (p *parsedRequest) estimatedTokens() int64 { + const assumedCompletion = 256 + completion := p.MaxTokens + if completion <= 0 { + completion = assumedCompletion + } + return int64(p.PromptTokens + completion) +} diff --git a/internal/router/router.go b/internal/router/router.go index f8945e8..facd01d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" @@ -24,7 +25,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape" - "gitea.mynodes.xyz/ilker/llmgw/internal/store" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Router serves the OpenAI API in front of a set of engine replicas. @@ -44,9 +45,11 @@ type Router struct { // onto its catalog entry. models map[string]catalog.Model - // auth and store are nil when authentication is disabled. - auth *auth.Authenticator - store *store.SQLite + // auth, store, acct and recorder are nil when authentication is disabled. + auth *auth.Authenticator + store *storepkg.SQLite + acct *accounting.Accountant + recorder *accounting.Recorder } // Auth exposes the authenticator so ctl and the batch writer can reach it. @@ -54,7 +57,7 @@ type Router struct { func (r *Router) Auth() *auth.Authenticator { return r.auth } // Store exposes the key store, or nil when authentication is off. -func (r *Router) Store() *store.SQLite { return r.store } +func (r *Router) Store() *storepkg.SQLite { return r.store } // New builds a router from configuration. Start must be called to begin scraping. func New(cfg config.Server, log *slog.Logger) (*Router, error) { @@ -115,7 +118,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { }, log) if cfg.Auth.Enabled { - st, err := store.OpenSQLite(context.Background(), cfg.Auth.Store) + st, err := storepkg.OpenSQLite(context.Background(), cfg.Auth.Store) if err != nil { return nil, fmt.Errorf("auth store: %w", err) } @@ -124,6 +127,24 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { TTL: cfg.Auth.CacheTTL, NegativeTTL: cfg.Auth.NegativeTTL, }) + + prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing)) + for name, p := range cfg.Auth.Pricing { + prices[name] = accounting.Price{ + PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, + } + } + r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{ + FlushInterval: cfg.Auth.FlushInterval, + BatchSize: cfg.Auth.BatchSize, + Capacity: cfg.Auth.BufferSize, + }, log) + r.acct = accounting.New( + accounting.NewPricing(prices, accounting.Price{ + PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k, + CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k, + }), + r.recorder, log) } if cfg.Router.Admission.Enabled { @@ -209,6 +230,18 @@ func (r *Router) Start(ctx context.Context) { if r.adm != nil { r.adm.Start() } + if r.recorder != nil { + r.recorder.Start(ctx) + } + if r.acct != nil && r.store != nil { + // Budgets are counted in memory; the store is what makes them survive a + // restart. A failure here is not fatal — refusing to start because last + // month's spend could not be read would be worse than starting with + // counters that are low. + if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil { + r.log.Warn("could not restore budgets; they start from zero this period", "err", err) + } + } } // Stop halts scraping and admission. @@ -217,6 +250,13 @@ func (r *Router) Stop() { if r.adm != nil { r.adm.Close() } + // Drain before closing the database, or the last batch has nowhere to go. + if r.recorder != nil { + r.recorder.Stop() + } + if r.store != nil { + _ = r.store.Close() + } } // Registry exposes the endpoint registry, for tests and admin endpoints. diff --git a/llmgw-plan.md b/llmgw-plan.md index 40ddda8..1ce464f 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1407,7 +1407,11 @@ The launch path is now built too: designing a schema against guesses (§6.5). - [x] Tenancy: org/team/project/key hierarchy, key storage and authentication, limit resolution (most restrictive along the chain), `llmgw keys` -- [ ] Budgets, RPM/TPM enforcement and the usage batch writer (§7.9.3, §7.9.4) +- [x] Budgets, RPM/TPM enforcement and the usage batch writer (§7.9.3, §7.9.4). + Rate limits are a sliding window, not a fixed one: a fixed window lets a + caller send its whole allowance at the end of one window and again at the + start of the next, so the observed rate doubles exactly when the system is + least able to take it. - [x] Model allowlists and key→band mapping. With auth on, the band comes from the key and the header is ignored — it was trusted only while there was no identity to consult, which let any client promote itself. diff --git a/test/e2e/accounting_test.go b/test/e2e/accounting_test.go new file mode 100644 index 0000000..3672da1 --- /dev/null +++ b/test/e2e/accounting_test.go @@ -0,0 +1,185 @@ +package e2e + +import ( + "context" + "net/http" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// paidConfig turns on auth with a price attached to the test model, so spend +// is a number rather than always zero. +func paidConfig(path string) func(*config.Server) { + return func(c *config.Server) { + c.Auth = config.Auth{ + Enabled: true, Store: path, + CacheTTL: 50 * time.Millisecond, + FlushInterval: 20 * time.Millisecond, + BatchSize: 8, + Pricing: map[string]config.Price{ + testModel: {PromptPer1k: 1_000_000, CompletionPer1k: 1_000_000}, + }, + } + } +} + +func usageRows(t *testing.T, path string) []store.Usage { + t.Helper() + s, err := store.OpenSQLite(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + rows, err := s.SpendSince(context.Background(), time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + out := make([]store.Usage, 0, len(rows)) + for key, micros := range rows { + out = append(out, store.Usage{KeyID: key, CostMicros: micros}) + } + return out +} + +func TestUsageReachesTheStoreWithoutBlockingRequests(t *testing.T) { + path, mint := provision(t) + key := mint(t, "spender") + h := start(t, 2, paidConfig(path)) + + for range 12 { + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("%d %s", code, msg) + } + } + + // The writer is asynchronous, so this is the one place a wait is correct. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("no usage reached the store") +} + +func TestRequestRateLimitsBindThroughTheGateway(t *testing.T) { + path, mint := provision(t) + key := mint(t, "burst") + withStore(t, path, func(s *store.SQLite) { + rpm := 5 + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-burst", RPM: &rpm, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, paidConfig(path)) + + ok, limited := 0, 0 + for range 20 { + switch code, _ := h.call(t, key, testModel); code { + case http.StatusOK: + ok++ + case http.StatusTooManyRequests: + limited++ + default: + t.Fatalf("unexpected status %d", code) + } + } + if ok != 5 { + t.Errorf("a limit of 5 let %d requests through", ok) + } + if limited != 15 { + t.Errorf("%d requests were refused, expected 15", limited) + } +} + +func TestASpentBudgetRefusesAndDoesNotSuggestRetrying(t *testing.T) { + path, mint := provision(t) + key := mint(t, "broke") + withStore(t, path, func(s *store.SQLite) { + // One micro-unit: the first request exhausts it. + budget := int64(1) + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-broke", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, paidConfig(path)) + + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("the first request should go through: %d %s", code, msg) + } + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("a key kept spending past its budget") +} + +func TestBudgetSurvivesAGatewayRestart(t *testing.T) { + // The counters live in memory. If a restart reset them, a budget would be a + // suggestion that anyone could clear by bouncing the process. + path, mint := provision(t) + key := mint(t, "persistent") + + h := start(t, 2, paidConfig(path)) + for range 3 { + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("%d %s", code, msg) + } + } + // Let the batch writer land before the process goes away. + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 { + break + } + time.Sleep(25 * time.Millisecond) + } + + spent := usageRows(t, path) + if len(spent) == 0 || spent[0].CostMicros == 0 { + t.Fatal("nothing was recorded to restore from") + } + + // A second gateway over the same store, with a budget just under what was + // already spent: it must refuse immediately rather than starting fresh. + withStore(t, path, func(s *store.SQLite) { + budget := spent[0].CostMicros + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-persistent", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h2 := start(t, 2, paidConfig(path)) + if code, _ := h2.call(t, key, testModel); code != http.StatusTooManyRequests { + t.Errorf("the restarted gateway gave the key a fresh budget: %d", code) + } +} + +func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) { + // Accounting only exists where there is an identity to charge. + h := start(t, 2) + for range 30 { + if code, msg := h.call(t, "", testModel); code != http.StatusOK { + t.Fatalf("an unauthenticated gateway rate-limited a request: %d %s", code, msg) + } + } +}