From 0b93fb2883bf2cf972f68b705749e8854d506a7f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 08:52:36 +0200 Subject: [PATCH] Enforce budgets and rate limits, and write usage off the request path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second half of phase 5. A key now has a cost, a quota and a bill. Rate limits use a sliding window rather than a fixed one. A fixed window lets a caller send its whole minute's allowance in the last second of one window and again in the first second of the next, so the observed rate is twice the configured limit exactly when the system is least able to take it. Checking and charging are one operation, or two concurrent requests both see room and both take it. Token limits are checked against an estimate, because the real count only exists once the request has run; the difference is charged afterwards, so a caller who consistently underestimates is still bounded, one window late. A request with no max_tokens gets a modest assumption rather than zero — assuming zero would let an unbounded request through every token limit. Limits are checked before admission, not after: a request over its quota should be refused rather than queued, where it would occupy a band's capacity to no purpose. All refusals are 429, but a spent budget carries no Retry-After because waiting does not help until the period rolls over, and saying otherwise turns one client into a retry loop. Usage goes to a buffered channel and is written in batches. Record never blocks: if the buffer fills — a database stalled long enough to lose eight thousand records — records are dropped and counted, because losing accounting is bad and turning a stalled database into an outage is worse. A clean shutdown drains what is queued; those are requests that were already served. Failures are logged at error, since silently losing billing data is the kind of thing discovered at the end of a month. Requests are accounted whatever their outcome. A request that used a GPU and then failed still used it, and a key that could spend nothing by failing would be a way around a budget. Budgets are counted in memory and restored from the store at startup, so a restart does not hand every key a fresh allowance. The budget period travels with the reservation rather than being re-derived, because a monthly budget credited to a daily counter is a budget that never binds. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/examples/server.yaml | 19 ++ internal/accounting/accounting.go | 161 ++++++++++++++ internal/accounting/accounting_test.go | 290 +++++++++++++++++++++++++ internal/accounting/budget.go | 101 +++++++++ internal/accounting/cost.go | 70 ++++++ internal/accounting/limiter.go | 130 +++++++++++ internal/accounting/recorder.go | 189 ++++++++++++++++ internal/accounting/recorder_test.go | 210 ++++++++++++++++++ internal/config/config.go | 23 ++ internal/router/handlers.go | 90 +++++++- internal/router/router.go | 52 ++++- llmgw-plan.md | 6 +- test/e2e/accounting_test.go | 185 ++++++++++++++++ 13 files changed, 1515 insertions(+), 11 deletions(-) create mode 100644 internal/accounting/accounting.go create mode 100644 internal/accounting/accounting_test.go create mode 100644 internal/accounting/budget.go create mode 100644 internal/accounting/cost.go create mode 100644 internal/accounting/limiter.go create mode 100644 internal/accounting/recorder.go create mode 100644 internal/accounting/recorder_test.go create mode 100644 test/e2e/accounting_test.go diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 563a0856..a1a2d2fa 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -57,6 +57,25 @@ agent_listen: ":7443" # # How long a rejected key is remembered as rejected, so a client looping # # with a bad key cannot become a database read per request. # negative_ttl: 5s +# +# # What each model costs, in micro-units per thousand tokens. A unit is +# # whatever you decide: self-hosted models have no per-token price, so it can +# # be GPU-seconds, a notional rate, or 1000 everywhere — which turns budgets +# # into plain token quotas. +# # +# # A model priced at zero is exempt from budget checks, which is what lets a +# # key with nothing left still be served a cheap fallback. +# default_price: { prompt_per_1k: 0, completion_per_1k: 0 } +# pricing: +# llama-3.3-70b: { prompt_per_1k: 1000, completion_per_1k: 3000 } +# qwen2.5-32b: { prompt_per_1k: 400, completion_per_1k: 1200 } +# +# # Usage rows are written in batches off the request path. Beyond buffer_size +# # they are dropped rather than blocking a request: losing some accounting is +# # bad, turning a stalled database into an outage is worse. +# flush_interval: 200ms +# batch_size: 500 +# buffer_size: 8192 router: scrape_interval: 2s diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go new file mode 100644 index 00000000..b1540359 --- /dev/null +++ b/internal/accounting/accounting.go @@ -0,0 +1,161 @@ +package accounting + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Refusals a key can hit. They are distinguished so the client is told which +// limit it hit, and so retrying is only suggested where retrying helps. +var ( + ErrRateLimited = errors.New("request rate limit exceeded") + ErrTokenLimited = errors.New("token rate limit exceeded") + ErrBudgetSpent = errors.New("budget exhausted") +) + +// LimitError carries which limit was hit and, where it makes sense, when to +// come back. +type LimitError struct { + Err error + // RetryAfter is zero when waiting does not help, which is the case for a + // spent budget: it will not refill until the period rolls over, and telling + // a client to retry in a second would be a lie. + RetryAfter time.Duration +} + +func (e *LimitError) Error() string { return e.Err.Error() } +func (e *LimitError) Unwrap() error { return e.Err } + +// Accountant enforces limits and records what was used. +// +// One object because the three halves are not independent: a budget check needs +// the price, the price needs the model, and the record closes the loop by +// feeding the counters that the next check reads. +type Accountant struct { + pricing *Pricing + budgets *Budgets + requests *Limiter + tokens *Limiter + recorder *Recorder + log *slog.Logger +} + +// New builds an accountant. +func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant { + if log == nil { + log = slog.Default() + } + return &Accountant{ + pricing: pricing, + budgets: NewBudgets(), + requests: NewLimiter(), + tokens: NewLimiter(), + recorder: recorder, + log: log, + } +} + +// RestoreBudgets seeds the in-memory counters from the store. +// +// Called at startup. A restart must not reset a budget, which it would if the +// counters only ever lived in memory. +func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error { + start := PeriodStart(period, time.Now()) + if start.IsZero() { + return nil + } + spend, err := s.SpendSince(ctx, start) + if err != nil { + return fmt.Errorf("restoring budgets: %w", err) + } + a.budgets.Restore(period, spend) + a.log.Info("restored budgets from the store", "keys", len(spend), "since", start) + return nil +} + +// Reservation is what Check returns: the estimate it charged, so Record can +// settle the difference against the real figure. +type Reservation struct { + KeyID string + EstimatedTokens int64 + Price Price + // Period is the key's budget period, carried from the check so the record + // counts against the same window the check read. Deriving it later would + // mean guessing, and a monthly budget credited to a daily counter is a + // budget that never binds. + Period store.Period +} + +// Check decides whether a request may proceed. +// +// It runs before scheduling and touches no database: every counter it reads is +// in memory. The token limit is checked against an estimate, because the real +// figure only exists once the request has run; Record settles the difference. +func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) { + price := a.pricing.For(model) + res := Reservation{ + KeyID: p.KeyID, EstimatedTokens: estimatedTokens, + Price: price, Period: p.Limits.BudgetPeriod, + } + if p.KeyID == "" { + return res, nil + } + lim := p.Limits + + // Budget first. It is the limit whose refusal cannot be retried away, so + // telling the client about it first is more useful than reporting a rate + // limit it would hit again in a second. + // + // A model that costs nothing is exempt, which is what lets a key with no + // budget left still be served a cheap fallback (plan §7.7). + if lim.Budget != nil && !price.Zero() && + a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) { + return res, &LimitError{Err: ErrBudgetSpent} + } + + if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) { + return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second} + } + if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) { + // The request was not charged against the request-rate limit in vain: + // it was, and that is correct — it did arrive. + return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second} + } + return res, nil +} + +// Record settles a finished request: it reconciles the token estimate, adds to +// the budget, and queues the usage row. +func (a *Accountant) Record(res Reservation, u store.Usage) { + if res.KeyID == "" { + return + } + actual := u.PromptTokens + u.CompletionTokens + + // The estimate was charged up front; charge the difference now. A caller + // whose requests are consistently larger than estimated is still bounded, + // one window later than with a perfect estimate. + a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens) + + u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) + a.budgets.Add(res.KeyID, res.Period, u.CostMicros) + + if a.recorder != nil { + a.recorder.Record(u) + } +} + +// Status returns what a key has used, for headers and `ctl`. +func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { + if p.KeyID == "" { + return 0, 0, 0 + } + return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod), + a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID) +} diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go new file mode 100644 index 00000000..327273e0 --- /dev/null +++ b/internal/accounting/accounting_test.go @@ -0,0 +1,290 @@ +package accounting + +import ( + "context" + "errors" + "log/slog" + "path/filepath" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func intp(v int) *int { return &v } +func i64p(v int64) *int64 { return &v } + +func principal(id string, tune func(*store.Effective)) auth.Principal { + e := store.Effective{KeyID: id, ProjectID: "proj"} + if tune != nil { + tune(&e) + } + return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e} +} + +func pricing(t *testing.T) *Pricing { + t.Helper() + return NewPricing(map[string]Price{ + "paid": {PromptPer1k: 1000, CompletionPer1k: 3000}, + "free": {}, + }, Price{}) +} + +func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) { + // Rounding down would make a stream of tiny requests cost nothing, which is + // exactly the traffic shape a quota exists to bound. + p := Price{PromptPer1k: 1000, CompletionPer1k: 3000} + if got := p.Cost(1, 0); got != 1 { + t.Errorf("one prompt token cost %d", got) + } + if got := p.Cost(1000, 1000); got != 4000 { + t.Errorf("a thousand of each cost %d, want 4000", got) + } + if got := p.Cost(0, 0); got != 0 { + t.Errorf("an empty request cost %d", got) + } +} + +func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) { + // This is what makes budget fallback work: a key with nothing left can + // still be served something that costs nothing. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(100) + e.BudgetPeriod = store.PeriodMonth + }) + + res, err := a.Check(p, "paid", 10) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0}) + + if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("a spent budget still served a paid model: %v", err) + } + if _, err := a.Check(p, "free", 10); err != nil { + t.Errorf("a spent budget refused a free model: %v", err) + } +} + +func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) { + // A budget does not refill until the period rolls over, so a Retry-After of + // a second would be a lie that turns one client into a retry loop. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(1) + e.BudgetPeriod = store.PeriodDay + }) + res, _ := a.Check(p, "paid", 10) + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000}) + + _, err := a.Check(p, "paid", 10) + var le *LimitError + if !errors.As(err, &le) { + t.Fatalf("expected a limit error, got %v", err) + } + if le.RetryAfter != 0 { + t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter) + } +} + +func TestRateLimitsBind(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) }) + + for i := range 3 { + if _, err := a.Check(p, "free", 1); err != nil { + t.Fatalf("request %d was refused: %v", i, err) + } + } + if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) { + t.Errorf("the fourth request was allowed: %v", err) + } +} + +func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) { + // The real token count only exists once the request has run, so the check + // has to work on an estimate. A caller that consistently underestimates + // must still end up bounded. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) }) + + // Claims 10 tokens, actually uses 900. + res, err := a.Check(p, "free", 10) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500}) + + // The settled figure is what the next check sees, so a second request of + // the same shape cannot fit. + if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) { + t.Errorf("the underestimate was never settled: %v", err) + } +} + +func TestNoLimitIsNotALimitOfZero(t *testing.T) { + // The distinction the store keeps with a nil pointer. Getting it wrong here + // would silently forbid every request for every key with no limits set. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + p := principal("k1", nil) + + for range 50 { + if _, err := a.Check(p, "paid", 10_000); err != nil { + t.Fatalf("a key with no limits was refused: %v", err) + } + } +} + +func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) { + // With auth off there is no key to charge, and Check must not refuse. + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + res, err := a.Check(auth.Principal{}, "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{PromptTokens: 1000}) + if spent, _, _ := a.Status(auth.Principal{}); spent != 0 { + t.Errorf("an anonymous request was charged %d", spent) + } +} + +func TestBudgetSurvivesARestart(t *testing.T) { + // The counters live in memory; the store is what makes them survive. + ctx := context.Background() + path := filepath.Join(t.TempDir(), "acct.db") + s, err := store.OpenSQLite(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"})) + must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"})) + must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"})) + _, k, _ := store.NewKey("k1", "n", "proj") + k.CreatedAt = time.Now() + must(t, s.CreateKey(ctx, k)) + + must(t, s.WriteUsage(ctx, []store.Usage{ + {At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid", + ServedModel: "paid", CostMicros: 5000, Status: 200}, + })) + + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth)) + + p := principal("k1", func(e *store.Effective) { + e.Budget = i64p(5000) + e.BudgetPeriod = store.PeriodMonth + }) + if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("the budget was reset by the restart: %v", err) + } +} + +func TestBudgetsResetAtThePeriodBoundary(t *testing.T) { + b := NewBudgets() + now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC) + b.now = func() time.Time { return now } + + b.Add("k1", store.PeriodMonth, 900) + if got := b.Spent("k1", store.PeriodMonth); got != 900 { + t.Fatalf("spent %d", got) + } + + now = now.Add(2 * time.Minute) // into February + if got := b.Spent("k1", store.PeriodMonth); got != 0 { + t.Errorf("the counter carried %d across the month boundary", got) + } +} + +func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) { + b := NewBudgets() + now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC) + b.now = func() time.Time { return now } + + b.Add("daily", store.PeriodDay, 100) + b.Add("monthly", store.PeriodMonth, 100) + + now = now.Add(24 * time.Hour) + if got := b.Spent("daily", store.PeriodDay); got != 0 { + t.Errorf("the daily counter carried %d across a day", got) + } + if got := b.Spent("monthly", store.PeriodMonth); got != 100 { + t.Errorf("the monthly counter lost %d within a month", 100-got) + } +} + +func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) { + // The reason for a sliding window. With a fixed one, a caller sends its + // whole allowance at the end of a window and again at the start of the + // next, so the observed rate is double the limit exactly when the system is + // least able to take it. + l := NewLimiter() + base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + now := base.Add(59 * time.Second) + l.now = func() time.Time { return now } + + const limit = 60 + sent := 0 + for range limit { + if l.Allow("k", limit, 1) { + sent++ + } + } + if sent != limit { + t.Fatalf("the first burst only managed %d", sent) + } + + // One second later, in the next fixed window. + now = base.Add(61 * time.Second) + second := 0 + for range limit { + if l.Allow("k", limit, 1) { + second++ + } + } + // Some allowance returns as the old window slides out of view, but nowhere + // near the whole of it. + if second > limit/2 { + t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding", + second, sent) + } +} + +func TestLimiterIsSafeUnderConcurrency(t *testing.T) { + // Checking and charging have to be one operation, or two concurrent + // requests both see room and both take it. + l := NewLimiter() + const limit = 100 + + var wg sync.WaitGroup + var mu sync.Mutex + allowed := 0 + for range 500 { + wg.Add(1) + go func() { + defer wg.Done() + if l.Allow("k", limit, 1) { + mu.Lock() + allowed++ + mu.Unlock() + } + }() + } + wg.Wait() + if allowed != limit { + t.Errorf("%d requests passed a limit of %d", allowed, limit) + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/accounting/budget.go b/internal/accounting/budget.go new file mode 100644 index 00000000..0b47d798 --- /dev/null +++ b/internal/accounting/budget.go @@ -0,0 +1,101 @@ +package accounting + +import ( + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// PeriodStart is when the current budget period began. +// +// Periods are anchored to UTC rather than to a per-key start date, so every +// key's day and month boundary is the same instant. A key whose period started +// when it was created would make "spend this month" mean something different +// for every key, and make the restore query unanswerable in one pass. +func PeriodStart(p store.Period, now time.Time) time.Time { + u := now.UTC() + switch p { + case store.PeriodDay: + return time.Date(u.Year(), u.Month(), u.Day(), 0, 0, 0, 0, time.UTC) + case store.PeriodMonth: + return time.Date(u.Year(), u.Month(), 1, 0, 0, 0, 0, time.UTC) + default: + return time.Time{} + } +} + +// Budgets tracks spend per key in memory, so a budget check costs no query. +// +// The database catches up through the batch writer. That means the in-memory +// figure leads the stored one, which is the right direction: a budget that only +// counted what had already been written would let a burst overspend by however +// much the writer was behind. +type Budgets struct { + now func() time.Time + + mu sync.Mutex + spend map[string]counter +} + +type counter struct { + micros int64 + since time.Time +} + +// NewBudgets builds an empty tracker. +func NewBudgets() *Budgets { + return &Budgets{now: time.Now, spend: map[string]counter{}} +} + +// Restore seeds the counters from what the store already recorded. +// +// A restart must not reset a budget, which it would if the counter lived only +// in memory. The caller passes the spend since each period's start. +func (b *Budgets) Restore(period store.Period, spend map[string]int64) { + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + for key, micros := range spend { + b.spend[key] = counter{micros: micros, since: start} + } +} + +// Spent returns a key's spend in the current period, resetting it if the period +// has rolled over. +func (b *Budgets) Spent(keyID string, period store.Period) int64 { + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + return b.currentLocked(keyID, start).micros +} + +// Add records spend. +func (b *Budgets) Add(keyID string, period store.Period, micros int64) { + if micros == 0 { + return + } + start := PeriodStart(period, b.now()) + b.mu.Lock() + defer b.mu.Unlock() + c := b.currentLocked(keyID, start) + c.micros += micros + b.spend[keyID] = c +} + +// Exhausted reports whether a key has spent its budget. +func (b *Budgets) Exhausted(keyID string, period store.Period, budget int64) bool { + return b.Spent(keyID, period) >= budget +} + +// currentLocked returns the counter, rolled over if the period changed. +func (b *Budgets) currentLocked(keyID string, start time.Time) counter { + c, ok := b.spend[keyID] + if !ok || c.since.Before(start) { + // Either never seen, or the period rolled over while the process was + // running. Both start from zero. + c = counter{since: start} + b.spend[keyID] = c + } + return c +} diff --git a/internal/accounting/cost.go b/internal/accounting/cost.go new file mode 100644 index 00000000..34b00af7 --- /dev/null +++ b/internal/accounting/cost.go @@ -0,0 +1,70 @@ +// Package accounting turns finished requests into spend, enforces what a key +// is allowed, and gets both into the store without touching the request path +// (plan §7.9.3, §7.9.4). +package accounting + +import ( + "strings" + "sync" +) + +// Price is what one model costs, in micro-units per thousand tokens. +// +// Micro-units rather than a float because spend is accumulated by repeated +// addition, and repeated addition of floats drifts. What a unit means is the +// operator's: self-hosted models have no per-token price, so it may be +// GPU-seconds, a notional rate, or 1000 everywhere — which turns a budget into +// a token quota. +type Price struct { + PromptPer1k int64 `koanf:"prompt_per_1k"` + CompletionPer1k int64 `koanf:"completion_per_1k"` +} + +// Zero reports whether this model spends nothing. +// +// It matters beyond arithmetic: a zero-cost model stays available to a key +// whose budget is spent, which is what makes the plan's budget fallback work +// (§7.7). +func (p Price) Zero() bool { return p.PromptPer1k == 0 && p.CompletionPer1k == 0 } + +// Cost computes what a request spent, rounding up. +// +// Up rather than to nearest: rounding down would make a stream of tiny requests +// free, which is exactly the shape of traffic a quota exists to bound. +func (p Price) Cost(promptTokens, completionTokens int64) int64 { + return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000) +} + +func divUp(n, d int64) int64 { + if n <= 0 { + return 0 + } + return (n + d - 1) / d +} + +// Pricing resolves a model name to its price. +type Pricing struct { + mu sync.RWMutex + byModel map[string]Price + fallback Price +} + +// NewPricing builds a price table. Names are matched case-insensitively, +// because a model name reaches us from whatever the client typed. +func NewPricing(byModel map[string]Price, fallback Price) *Pricing { + table := make(map[string]Price, len(byModel)) + for name, p := range byModel { + table[strings.ToLower(name)] = p + } + return &Pricing{byModel: table, fallback: fallback} +} + +// For returns a model's price, or the fallback. +func (p *Pricing) For(model string) Price { + p.mu.RLock() + defer p.mu.RUnlock() + if v, ok := p.byModel[strings.ToLower(model)]; ok { + return v + } + return p.fallback +} diff --git a/internal/accounting/limiter.go b/internal/accounting/limiter.go new file mode 100644 index 00000000..e85efee6 --- /dev/null +++ b/internal/accounting/limiter.go @@ -0,0 +1,130 @@ +package accounting + +import ( + "sync" + "time" +) + +// Limiter enforces per-key rate limits. +// +// A sliding window rather than a fixed one. A fixed window lets a caller send +// its whole minute's allowance in the last second of one window and again in +// the first second of the next, so the observed rate is twice the configured +// limit exactly when the system is least able to take it. +// +// The window is approximated from two buckets, which is the usual compromise: +// exact enough that the doubling cannot happen, and bounded memory per key +// rather than a timestamp per request. +type Limiter struct { + now func() time.Time + + mu sync.Mutex + keys map[string]*window +} + +type window struct { + // current and previous are counts in this minute and the one before. + current, previous int64 + start time.Time +} + +// NewLimiter builds an empty limiter. +func NewLimiter() *Limiter { + return &Limiter{now: time.Now, keys: map[string]*window{}} +} + +const windowLen = time.Minute + +// Allow charges n against a key's limit and reports whether it fits. +// +// Charging and checking are one operation on purpose: doing them separately +// lets two concurrent requests both see room and both take it. +func (l *Limiter) Allow(key string, limit int, n int64) bool { + if limit <= 0 { + // No limit configured. Zero would be a limit of zero, which is why + // callers pass a nil pointer rather than 0 for "unset". + return true + } + now := l.now() + + l.mu.Lock() + defer l.mu.Unlock() + + w := l.windowLocked(key, now) + if w.rate(now)+float64(n) > float64(limit) { + return false + } + w.current += n + return true +} + +// Charge records usage that was already allowed, for reconciliation. +// +// Token limits are checked against an estimate before the request runs, because +// the real figure only exists once it has. The difference is charged here, so a +// caller whose requests are consistently larger than estimated is still bounded +// — one window later than it would be with a perfect estimate. +func (l *Limiter) Charge(key string, n int64) { + if n == 0 { + return + } + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + w := l.windowLocked(key, now) + w.current += n + if w.current < 0 { + w.current = 0 + } +} + +// Rate returns a key's current observed rate. +func (l *Limiter) Rate(key string) float64 { + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + return l.windowLocked(key, now).rate(now) +} + +func (l *Limiter) windowLocked(key string, now time.Time) *window { + w, ok := l.keys[key] + if !ok { + w = &window{start: now.Truncate(windowLen)} + l.keys[key] = w + } + w.roll(now) + return w +} + +// roll advances the window to now. +func (w *window) roll(now time.Time) { + start := now.Truncate(windowLen) + switch { + case start.Equal(w.start): + return + case start.Sub(w.start) == windowLen: + w.previous, w.current = w.current, 0 + default: + // More than one window has passed with no traffic. + w.previous, w.current = 0, 0 + } + w.start = start +} + +// rate weights the previous window by how much of it is still in view. +func (w *window) rate(now time.Time) float64 { + elapsed := now.Sub(w.start) + if elapsed >= windowLen { + return float64(w.current) + } + remaining := 1 - float64(elapsed)/float64(windowLen) + return float64(w.previous)*remaining + float64(w.current) +} + +// Forget drops a key's window. Used when a key is revoked; keeping it would +// leak memory for every key that ever called. +func (l *Limiter) Forget(key string) { + l.mu.Lock() + delete(l.keys, key) + l.mu.Unlock() +} diff --git a/internal/accounting/recorder.go b/internal/accounting/recorder.go new file mode 100644 index 00000000..6d500304 --- /dev/null +++ b/internal/accounting/recorder.go @@ -0,0 +1,189 @@ +package accounting + +import ( + "context" + "log/slog" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// RecorderConfig tunes the batch writer. +type RecorderConfig struct { + // FlushInterval and BatchSize decide when a batch is written. Whichever + // comes first. + FlushInterval time.Duration + BatchSize int + // Capacity bounds the buffer. Beyond it, records are dropped rather than + // blocking a request — see Record. + Capacity int +} + +// DefaultRecorderConfig returns sensible values. +func DefaultRecorderConfig() RecorderConfig { + return RecorderConfig{FlushInterval: 200 * time.Millisecond, BatchSize: 500, Capacity: 8192} +} + +func (c *RecorderConfig) normalize() { + d := DefaultRecorderConfig() + if c.FlushInterval <= 0 { + c.FlushInterval = d.FlushInterval + } + if c.BatchSize <= 0 { + c.BatchSize = d.BatchSize + } + if c.Capacity <= 0 { + c.Capacity = d.Capacity + } +} + +// Toucher is the part of the authenticator the recorder flushes alongside +// usage, so last-used timestamps ride the same transaction schedule. +type Toucher interface { + FlushTouches(ctx context.Context) error +} + +// Recorder batches usage records into the store. +// +// The buffered channel is the whole design. A request must never wait on a +// database write, so Record is a non-blocking send and the writer drains it on +// its own schedule. +type Recorder struct { + cfg RecorderConfig + store store.Store + touch Toucher + log *slog.Logger + + ch chan store.Usage + done chan struct{} + wg sync.WaitGroup + + mu sync.Mutex + dropped int64 + written int64 +} + +// NewRecorder builds a recorder. Start begins writing. +func NewRecorder(s store.Store, touch Toucher, cfg RecorderConfig, log *slog.Logger) *Recorder { + cfg.normalize() + if log == nil { + log = slog.Default() + } + return &Recorder{ + cfg: cfg, store: s, touch: touch, log: log, + ch: make(chan store.Usage, cfg.Capacity), + done: make(chan struct{}), + } +} + +// Record queues one request's accounting. +// +// Never blocks. If the buffer is full — a database stalled long enough to fill +// eight thousand records — the record is dropped and counted. Losing some +// accounting is bad; making every request wait on a stalled database is worse, +// because it converts a reporting problem into an outage. +func (r *Recorder) Record(u store.Usage) { + select { + case r.ch <- u: + default: + r.mu.Lock() + r.dropped++ + n := r.dropped + r.mu.Unlock() + if n == 1 || n%1000 == 0 { + r.log.Warn("dropping usage records: the store is not keeping up", + "dropped_total", n) + } + } +} + +// Start runs the writer until ctx ends. +func (r *Recorder) Start(ctx context.Context) { + r.wg.Add(1) + go func() { + defer r.wg.Done() + defer close(r.done) + r.run(ctx) + }() +} + +// Stop waits for the writer to drain and finish. +func (r *Recorder) Stop() { r.wg.Wait() } + +func (r *Recorder) run(ctx context.Context) { + t := time.NewTicker(r.cfg.FlushInterval) + defer t.Stop() + + batch := make([]store.Usage, 0, r.cfg.BatchSize) + for { + select { + case <-ctx.Done(): + // Drain what is already queued before leaving. These are completed + // requests that have already been served; throwing them away would + // mean a clean shutdown silently loses the last moments of billing. + r.drain(&batch) + r.flush(context.WithoutCancel(ctx), batch) + return + + case u := <-r.ch: + batch = append(batch, u) + if len(batch) >= r.cfg.BatchSize { + r.flush(ctx, batch) + batch = batch[:0] + } + + case <-t.C: + r.drain(&batch) + if len(batch) > 0 { + r.flush(ctx, batch) + batch = batch[:0] + } + if err := r.touchFlush(ctx); err != nil { + r.log.Warn("failed to write last-used timestamps", "err", err) + } + } + } +} + +// drain moves everything queued into the batch without blocking. +func (r *Recorder) drain(batch *[]store.Usage) { + for len(*batch) < r.cfg.BatchSize { + select { + case u := <-r.ch: + *batch = append(*batch, u) + default: + return + } + } +} + +func (r *Recorder) flush(ctx context.Context, batch []store.Usage) { + if len(batch) == 0 { + return + } + if err := r.store.WriteUsage(ctx, batch); err != nil { + // The records are gone either way; what matters is that this is loud. + // Silently losing billing data is the kind of thing discovered at the + // end of a month. + r.log.Error("failed to write usage records", "count", len(batch), "err", err) + return + } + r.mu.Lock() + r.written += int64(len(batch)) + r.mu.Unlock() +} + +func (r *Recorder) touchFlush(ctx context.Context) error { + if r.touch == nil { + return nil + } + return r.touch.FlushTouches(ctx) +} + +// Stats reports what the recorder has done, for metrics and tests. +func (r *Recorder) Stats() (written, dropped int64) { + r.mu.Lock() + defer r.mu.Unlock() + return r.written, r.dropped +} diff --git a/internal/accounting/recorder_test.go b/internal/accounting/recorder_test.go new file mode 100644 index 00000000..86f3cb5f --- /dev/null +++ b/internal/accounting/recorder_test.go @@ -0,0 +1,210 @@ +package accounting + +import ( + "context" + "errors" + "log/slog" + "sync" + "sync/atomic" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// fakeStore records what was written and can be made slow or broken. +type fakeStore struct { + store.Store + + mu sync.Mutex + batches [][]store.Usage + + block chan struct{} + fail atomic.Bool + touched atomic.Int64 +} + +func (f *fakeStore) WriteUsage(_ context.Context, rows []store.Usage) error { + if f.block != nil { + <-f.block + } + if f.fail.Load() { + return errors.New("store is down") + } + f.mu.Lock() + defer f.mu.Unlock() + f.batches = append(f.batches, append([]store.Usage(nil), rows...)) + return nil +} + +func (f *fakeStore) rows() []store.Usage { + f.mu.Lock() + defer f.mu.Unlock() + var out []store.Usage + for _, b := range f.batches { + out = append(out, b...) + } + return out +} + +func (f *fakeStore) batchCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.batches) +} + +type fakeToucher struct{ n *atomic.Int64 } + +func (f fakeToucher) FlushTouches(context.Context) error { + f.n.Add(1) + return nil +} + +func TestRecordsAreBatchedNotWrittenOneByOne(t *testing.T) { + // The whole point of the buffer: a hundred requests must not be a hundred + // transactions. + f := &fakeStore{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 20 * time.Millisecond, BatchSize: 500}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + + for i := range 100 { + r.Record(store.Usage{KeyID: "k", PromptTokens: int64(i)}) + } + waitFor(t, time.Second, func() bool { return len(f.rows()) == 100 }) + + if n := f.batchCount(); n > 5 { + t.Errorf("100 records took %d transactions", n) + } + cancel() + r.Stop() +} + +func TestRecordNeverBlocksARequest(t *testing.T) { + // A stalled database must not become an outage. Losing accounting is bad; + // making every request wait on the store is worse. + f := &fakeStore{block: make(chan struct{})} + r := NewRecorder(f, nil, RecorderConfig{ + FlushInterval: time.Millisecond, BatchSize: 1, Capacity: 8, + }, slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + defer func() { close(f.block); cancel(); r.Stop() }() + + done := make(chan struct{}) + go func() { + defer close(done) + for range 1000 { + r.Record(store.Usage{KeyID: "k"}) + } + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Record blocked while the store was stalled") + } + + _, dropped := r.Stats() + if dropped == 0 { + t.Error("the overflow was not counted, so nobody would ever know") + } +} + +func TestAShutdownDoesNotLoseWhatIsAlreadyQueued(t *testing.T) { + // Those are completed requests that were already served. Throwing them away + // means a clean shutdown silently loses the last moments of billing. + f := &fakeStore{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: time.Hour, BatchSize: 10_000}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + + for range 50 { + r.Record(store.Usage{KeyID: "k"}) + } + cancel() + r.Stop() + + if got := len(f.rows()); got != 50 { + t.Errorf("a clean shutdown wrote %d of 50 queued records", got) + } +} + +func TestAFailedWriteIsLoudRatherThanSilent(t *testing.T) { + // Silently losing billing data is the kind of thing discovered at the end + // of a month. + f := &fakeStore{} + f.fail.Store(true) + logged := &countingHandler{} + r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 10 * time.Millisecond, BatchSize: 1}, + slog.New(logged)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + r.Record(store.Usage{KeyID: "k"}) + + waitFor(t, time.Second, func() bool { return logged.count(slog.LevelError) > 0 }) + cancel() + r.Stop() + + if written, _ := r.Stats(); written != 0 { + t.Errorf("a failed write was counted as %d written", written) + } +} + +func TestLastUsedRidesTheSameSchedule(t *testing.T) { + var n atomic.Int64 + f := &fakeStore{} + r := NewRecorder(f, fakeToucher{n: &n}, RecorderConfig{FlushInterval: 10 * time.Millisecond}, + slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(context.Background()) + r.Start(ctx) + waitFor(t, time.Second, func() bool { return n.Load() > 0 }) + cancel() + r.Stop() +} + +type countingHandler struct { + mu sync.Mutex + levels []slog.Level +} + +func (c *countingHandler) Enabled(context.Context, slog.Level) bool { return true } +func (c *countingHandler) WithAttrs([]slog.Attr) slog.Handler { return c } +func (c *countingHandler) WithGroup(string) slog.Handler { return c } +func (c *countingHandler) Handle(_ context.Context, r slog.Record) error { + c.mu.Lock() + c.levels = append(c.levels, r.Level) + c.mu.Unlock() + return nil +} + +func (c *countingHandler) count(l slog.Level) int { + c.mu.Lock() + defer c.mu.Unlock() + n := 0 + for _, got := range c.levels { + if got == l { + n++ + } + } + return n +} + +func waitFor(t *testing.T, limit time.Duration, pred func() bool) { + t.Helper() + deadline := time.Now().Add(limit) + for time.Now().Before(deadline) { + if pred() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition never became true") +} diff --git a/internal/config/config.go b/internal/config/config.go index a0663c2c..eb0c50cc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,29 @@ type Auth struct { // NegativeTTL is how long a rejected key is remembered as rejected, so a // client looping with a bad key cannot become a database read per request. NegativeTTL time.Duration `koanf:"negative_ttl"` + + // Pricing is what each model costs, for budgets. A model absent from the + // table costs whatever DefaultPrice says, which is nothing unless set. + Pricing map[string]Price `koanf:"pricing"` + DefaultPrice Price `koanf:"default_price"` + + // FlushInterval and BatchSize decide when usage records are written. + FlushInterval time.Duration `koanf:"flush_interval"` + BatchSize int `koanf:"batch_size"` + // BufferSize bounds the queue. Beyond it records are dropped rather than + // blocking a request: losing accounting is bad, turning a stalled database + // into an outage is worse. + BufferSize int `koanf:"buffer_size"` +} + +// Price is a model's cost in micro-units per thousand tokens. +// +// What a unit means is the operator's choice — self-hosted models have no +// per-token price, so it may be GPU-seconds, a notional rate, or 1000 +// everywhere, which turns a budget into a token quota (plan §7.9.4). +type Price struct { + PromptPer1k int64 `koanf:"prompt_per_1k"` + CompletionPer1k int64 `koanf:"completion_per_1k"` } // Control configures the control plane. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index ae01bae6..07784d82 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -12,9 +12,11 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // admit waits for the flow control layer to let the request through. With @@ -111,6 +113,11 @@ type parsedRequest struct { PromptPrefix string PromptTokens int MaxTokens int + + // reservation is carried from the limit check to the point where the + // request is settled, so the record counts against the same budget window + // the check read. + reservation accounting.Reservation } // inferenceBody covers the fields the router reads across all inference @@ -143,10 +150,23 @@ func (r *Router) handleInference(path string) http.HandlerFunc { return } - if p, ok := auth.FromContext(req.Context()); ok && !r.permits(p, parsed.Model) { - auth.WriteForbiddenModel(w, parsed.Model) - r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) - return + if p, ok := auth.FromContext(req.Context()); ok { + if !r.permits(p, parsed.Model) { + auth.WriteForbiddenModel(w, parsed.Model) + r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) + return + } + // Limits are checked before admission, not after: a request that is + // over its quota should be refused rather than queued, and queueing + // it would have it occupy a band's capacity to no purpose. + if r.acct != nil { + reservation, err := r.acct.Check(p, parsed.Model, parsed.estimatedTokens()) + if err != nil { + r.rejectLimit(w, parsed, err, start) + return + } + parsed.reservation = reservation + } } // Step 5 of the lifecycle: "now?" before "where?" (plan §7.1). @@ -217,6 +237,53 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR } r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) + r.account(req, res, status, time.Since(start)) +} + +// account settles a finished request against its key. +// +// It runs for every outcome, not only success: a request that used a GPU and +// then failed still used it, and a key that could spend nothing by failing +// would be a way around a budget. +func (r *Router) account(req *parsedRequest, res dispatchResult, status int, took time.Duration) { + if r.acct == nil || req.KeyID == "" { + return + } + u := store.Usage{ + At: time.Now(), KeyID: req.KeyID, ProjectID: req.ProjectID, + Model: req.Model, ServedModel: res.model, Status: status, + LatencyMs: took.Milliseconds(), + } + if u.ServedModel == "" { + u.ServedModel = req.Model + } + if usage := res.outcome.Usage; usage != nil { + u.PromptTokens = int64(usage.PromptTokens) + u.CompletionTokens = int64(usage.CompletionTokens) + } else { + // No usage came back — a failure before the engine answered, or an + // engine that does not report it. Charging the estimate is closer to + // the truth than charging nothing. + u.PromptTokens = int64(req.PromptTokens) + } + r.acct.Record(req.reservation, u) +} + +// rejectLimit turns a quota refusal into a status the client can act on. +func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err error, start time.Time) { + var le *accounting.LimitError + if !errors.As(err, &le) { + writeError(w, http.StatusInternalServerError, "server_error", err.Error()) + return + } + if le.RetryAfter > 0 { + w.Header().Set("Retry-After", strconv.Itoa(int(le.RetryAfter.Seconds()))) + } + // 429 for all of them: a spent budget is a quota, and OpenAI clients + // already know how to surface that code. What differs is Retry-After, + // which is absent when waiting does not help. + writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) + r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0) } // parse reads the body and extracts the routing-relevant fields. @@ -416,3 +483,18 @@ func writeError(w http.ResponseWriter, code int, kind, msg string) { "message": msg, "type": kind, "code": nil, }}) } + +// estimatedTokens is what a token-rate limit is checked against, before the +// real figure exists. +// +// The prompt estimate plus whatever the client said it wanted back. A client +// that names no max_tokens gets a modest assumption rather than zero: assuming +// zero would let an unbounded request through every token limit. +func (p *parsedRequest) estimatedTokens() int64 { + const assumedCompletion = 256 + completion := p.MaxTokens + if completion <= 0 { + completion = assumedCompletion + } + return int64(p.PromptTokens + completion) +} diff --git a/internal/router/router.go b/internal/router/router.go index f8945e80..facd01de 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" @@ -24,7 +25,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape" - "gitea.mynodes.xyz/ilker/llmgw/internal/store" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Router serves the OpenAI API in front of a set of engine replicas. @@ -44,9 +45,11 @@ type Router struct { // onto its catalog entry. models map[string]catalog.Model - // auth and store are nil when authentication is disabled. - auth *auth.Authenticator - store *store.SQLite + // auth, store, acct and recorder are nil when authentication is disabled. + auth *auth.Authenticator + store *storepkg.SQLite + acct *accounting.Accountant + recorder *accounting.Recorder } // Auth exposes the authenticator so ctl and the batch writer can reach it. @@ -54,7 +57,7 @@ type Router struct { func (r *Router) Auth() *auth.Authenticator { return r.auth } // Store exposes the key store, or nil when authentication is off. -func (r *Router) Store() *store.SQLite { return r.store } +func (r *Router) Store() *storepkg.SQLite { return r.store } // New builds a router from configuration. Start must be called to begin scraping. func New(cfg config.Server, log *slog.Logger) (*Router, error) { @@ -115,7 +118,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { }, log) if cfg.Auth.Enabled { - st, err := store.OpenSQLite(context.Background(), cfg.Auth.Store) + st, err := storepkg.OpenSQLite(context.Background(), cfg.Auth.Store) if err != nil { return nil, fmt.Errorf("auth store: %w", err) } @@ -124,6 +127,24 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { TTL: cfg.Auth.CacheTTL, NegativeTTL: cfg.Auth.NegativeTTL, }) + + prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing)) + for name, p := range cfg.Auth.Pricing { + prices[name] = accounting.Price{ + PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, + } + } + r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{ + FlushInterval: cfg.Auth.FlushInterval, + BatchSize: cfg.Auth.BatchSize, + Capacity: cfg.Auth.BufferSize, + }, log) + r.acct = accounting.New( + accounting.NewPricing(prices, accounting.Price{ + PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k, + CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k, + }), + r.recorder, log) } if cfg.Router.Admission.Enabled { @@ -209,6 +230,18 @@ func (r *Router) Start(ctx context.Context) { if r.adm != nil { r.adm.Start() } + if r.recorder != nil { + r.recorder.Start(ctx) + } + if r.acct != nil && r.store != nil { + // Budgets are counted in memory; the store is what makes them survive a + // restart. A failure here is not fatal — refusing to start because last + // month's spend could not be read would be worse than starting with + // counters that are low. + if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil { + r.log.Warn("could not restore budgets; they start from zero this period", "err", err) + } + } } // Stop halts scraping and admission. @@ -217,6 +250,13 @@ func (r *Router) Stop() { if r.adm != nil { r.adm.Close() } + // Drain before closing the database, or the last batch has nowhere to go. + if r.recorder != nil { + r.recorder.Stop() + } + if r.store != nil { + _ = r.store.Close() + } } // Registry exposes the endpoint registry, for tests and admin endpoints. diff --git a/llmgw-plan.md b/llmgw-plan.md index 40ddda89..1ce464f9 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1407,7 +1407,11 @@ The launch path is now built too: designing a schema against guesses (§6.5). - [x] Tenancy: org/team/project/key hierarchy, key storage and authentication, limit resolution (most restrictive along the chain), `llmgw keys` -- [ ] Budgets, RPM/TPM enforcement and the usage batch writer (§7.9.3, §7.9.4) +- [x] Budgets, RPM/TPM enforcement and the usage batch writer (§7.9.3, §7.9.4). + Rate limits are a sliding window, not a fixed one: a fixed window lets a + caller send its whole allowance at the end of one window and again at the + start of the next, so the observed rate doubles exactly when the system is + least able to take it. - [x] Model allowlists and key→band mapping. With auth on, the band comes from the key and the header is ignored — it was trusted only while there was no identity to consult, which let any client promote itself. diff --git a/test/e2e/accounting_test.go b/test/e2e/accounting_test.go new file mode 100644 index 00000000..3672da10 --- /dev/null +++ b/test/e2e/accounting_test.go @@ -0,0 +1,185 @@ +package e2e + +import ( + "context" + "net/http" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// paidConfig turns on auth with a price attached to the test model, so spend +// is a number rather than always zero. +func paidConfig(path string) func(*config.Server) { + return func(c *config.Server) { + c.Auth = config.Auth{ + Enabled: true, Store: path, + CacheTTL: 50 * time.Millisecond, + FlushInterval: 20 * time.Millisecond, + BatchSize: 8, + Pricing: map[string]config.Price{ + testModel: {PromptPer1k: 1_000_000, CompletionPer1k: 1_000_000}, + }, + } + } +} + +func usageRows(t *testing.T, path string) []store.Usage { + t.Helper() + s, err := store.OpenSQLite(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + rows, err := s.SpendSince(context.Background(), time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + out := make([]store.Usage, 0, len(rows)) + for key, micros := range rows { + out = append(out, store.Usage{KeyID: key, CostMicros: micros}) + } + return out +} + +func TestUsageReachesTheStoreWithoutBlockingRequests(t *testing.T) { + path, mint := provision(t) + key := mint(t, "spender") + h := start(t, 2, paidConfig(path)) + + for range 12 { + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("%d %s", code, msg) + } + } + + // The writer is asynchronous, so this is the one place a wait is correct. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("no usage reached the store") +} + +func TestRequestRateLimitsBindThroughTheGateway(t *testing.T) { + path, mint := provision(t) + key := mint(t, "burst") + withStore(t, path, func(s *store.SQLite) { + rpm := 5 + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-burst", RPM: &rpm, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, paidConfig(path)) + + ok, limited := 0, 0 + for range 20 { + switch code, _ := h.call(t, key, testModel); code { + case http.StatusOK: + ok++ + case http.StatusTooManyRequests: + limited++ + default: + t.Fatalf("unexpected status %d", code) + } + } + if ok != 5 { + t.Errorf("a limit of 5 let %d requests through", ok) + } + if limited != 15 { + t.Errorf("%d requests were refused, expected 15", limited) + } +} + +func TestASpentBudgetRefusesAndDoesNotSuggestRetrying(t *testing.T) { + path, mint := provision(t) + key := mint(t, "broke") + withStore(t, path, func(s *store.SQLite) { + // One micro-unit: the first request exhausts it. + budget := int64(1) + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-broke", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h := start(t, 2, paidConfig(path)) + + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("the first request should go through: %d %s", code, msg) + } + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("a key kept spending past its budget") +} + +func TestBudgetSurvivesAGatewayRestart(t *testing.T) { + // The counters live in memory. If a restart reset them, a budget would be a + // suggestion that anyone could clear by bouncing the process. + path, mint := provision(t) + key := mint(t, "persistent") + + h := start(t, 2, paidConfig(path)) + for range 3 { + if code, msg := h.call(t, key, testModel); code != http.StatusOK { + t.Fatalf("%d %s", code, msg) + } + } + // Let the batch writer land before the process goes away. + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 { + break + } + time.Sleep(25 * time.Millisecond) + } + + spent := usageRows(t, path) + if len(spent) == 0 || spent[0].CostMicros == 0 { + t.Fatal("nothing was recorded to restore from") + } + + // A second gateway over the same store, with a budget just under what was + // already spent: it must refuse immediately rather than starting fresh. + withStore(t, path, func(s *store.SQLite) { + budget := spent[0].CostMicros + if err := s.PutLimits(context.Background(), store.Limits{ + Scope: store.ScopeKey, ScopeID: "key-persistent", + Budget: &budget, BudgetPeriod: store.PeriodMonth, + }); err != nil { + t.Fatal(err) + } + }) + + h2 := start(t, 2, paidConfig(path)) + if code, _ := h2.call(t, key, testModel); code != http.StatusTooManyRequests { + t.Errorf("the restarted gateway gave the key a fresh budget: %d", code) + } +} + +func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) { + // Accounting only exists where there is an identity to charge. + h := start(t, 2) + for range 30 { + if code, msg := h.call(t, "", testModel); code != http.StatusOK { + t.Fatalf("an unauthenticated gateway rate-limited a request: %d %s", code, msg) + } + } +}