CI / test (push) Successful in 1m7s
The second half of phase 5. A key now has a cost, a quota and a bill. Rate limits use a sliding window rather than a fixed one. A fixed window lets a caller send its whole minute's allowance in the last second of one window and again in the first second of the next, so the observed rate is twice the configured limit exactly when the system is least able to take it. Checking and charging are one operation, or two concurrent requests both see room and both take it. Token limits are checked against an estimate, because the real count only exists once the request has run; the difference is charged afterwards, so a caller who consistently underestimates is still bounded, one window late. A request with no max_tokens gets a modest assumption rather than zero — assuming zero would let an unbounded request through every token limit. Limits are checked before admission, not after: a request over its quota should be refused rather than queued, where it would occupy a band's capacity to no purpose. All refusals are 429, but a spent budget carries no Retry-After because waiting does not help until the period rolls over, and saying otherwise turns one client into a retry loop. Usage goes to a buffered channel and is written in batches. Record never blocks: if the buffer fills — a database stalled long enough to lose eight thousand records — records are dropped and counted, because losing accounting is bad and turning a stalled database into an outage is worse. A clean shutdown drains what is queued; those are requests that were already served. Failures are logged at error, since silently losing billing data is the kind of thing discovered at the end of a month. Requests are accounted whatever their outcome. A request that used a GPU and then failed still used it, and a key that could spend nothing by failing would be a way around a budget. Budgets are counted in memory and restored from the store at startup, so a restart does not hand every key a fresh allowance. The budget period travels with the reservation rather than being re-derived, because a monthly budget credited to a daily counter is a budget that never binds. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
291 lines
8.4 KiB
Go
291 lines
8.4 KiB
Go
package accounting
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func intp(v int) *int { return &v }
|
|
func i64p(v int64) *int64 { return &v }
|
|
|
|
func principal(id string, tune func(*store.Effective)) auth.Principal {
|
|
e := store.Effective{KeyID: id, ProjectID: "proj"}
|
|
if tune != nil {
|
|
tune(&e)
|
|
}
|
|
return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e}
|
|
}
|
|
|
|
func pricing(t *testing.T) *Pricing {
|
|
t.Helper()
|
|
return NewPricing(map[string]Price{
|
|
"paid": {PromptPer1k: 1000, CompletionPer1k: 3000},
|
|
"free": {},
|
|
}, Price{})
|
|
}
|
|
|
|
func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
|
|
// Rounding down would make a stream of tiny requests cost nothing, which is
|
|
// exactly the traffic shape a quota exists to bound.
|
|
p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
|
|
if got := p.Cost(1, 0); got != 1 {
|
|
t.Errorf("one prompt token cost %d", got)
|
|
}
|
|
if got := p.Cost(1000, 1000); got != 4000 {
|
|
t.Errorf("a thousand of each cost %d, want 4000", got)
|
|
}
|
|
if got := p.Cost(0, 0); got != 0 {
|
|
t.Errorf("an empty request cost %d", got)
|
|
}
|
|
}
|
|
|
|
func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) {
|
|
// This is what makes budget fallback work: a key with nothing left can
|
|
// still be served something that costs nothing.
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
p := principal("k1", func(e *store.Effective) {
|
|
e.Budget = i64p(100)
|
|
e.BudgetPeriod = store.PeriodMonth
|
|
})
|
|
|
|
res, err := a.Check(p, "paid", 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0})
|
|
|
|
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
|
t.Errorf("a spent budget still served a paid model: %v", err)
|
|
}
|
|
if _, err := a.Check(p, "free", 10); err != nil {
|
|
t.Errorf("a spent budget refused a free model: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) {
|
|
// A budget does not refill until the period rolls over, so a Retry-After of
|
|
// a second would be a lie that turns one client into a retry loop.
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
p := principal("k1", func(e *store.Effective) {
|
|
e.Budget = i64p(1)
|
|
e.BudgetPeriod = store.PeriodDay
|
|
})
|
|
res, _ := a.Check(p, "paid", 10)
|
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000})
|
|
|
|
_, err := a.Check(p, "paid", 10)
|
|
var le *LimitError
|
|
if !errors.As(err, &le) {
|
|
t.Fatalf("expected a limit error, got %v", err)
|
|
}
|
|
if le.RetryAfter != 0 {
|
|
t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter)
|
|
}
|
|
}
|
|
|
|
func TestRateLimitsBind(t *testing.T) {
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) })
|
|
|
|
for i := range 3 {
|
|
if _, err := a.Check(p, "free", 1); err != nil {
|
|
t.Fatalf("request %d was refused: %v", i, err)
|
|
}
|
|
}
|
|
if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) {
|
|
t.Errorf("the fourth request was allowed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) {
|
|
// The real token count only exists once the request has run, so the check
|
|
// has to work on an estimate. A caller that consistently underestimates
|
|
// must still end up bounded.
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) })
|
|
|
|
// Claims 10 tokens, actually uses 900.
|
|
res, err := a.Check(p, "free", 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500})
|
|
|
|
// The settled figure is what the next check sees, so a second request of
|
|
// the same shape cannot fit.
|
|
if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) {
|
|
t.Errorf("the underestimate was never settled: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestNoLimitIsNotALimitOfZero(t *testing.T) {
|
|
// The distinction the store keeps with a nil pointer. Getting it wrong here
|
|
// would silently forbid every request for every key with no limits set.
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
p := principal("k1", nil)
|
|
|
|
for range 50 {
|
|
if _, err := a.Check(p, "paid", 10_000); err != nil {
|
|
t.Fatalf("a key with no limits was refused: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) {
|
|
// With auth off there is no key to charge, and Check must not refuse.
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
res, err := a.Check(auth.Principal{}, "paid", 1000)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Record(res, store.Usage{PromptTokens: 1000})
|
|
if spent, _, _ := a.Status(auth.Principal{}); spent != 0 {
|
|
t.Errorf("an anonymous request was charged %d", spent)
|
|
}
|
|
}
|
|
|
|
func TestBudgetSurvivesARestart(t *testing.T) {
|
|
// The counters live in memory; the store is what makes them survive.
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "acct.db")
|
|
s, err := store.OpenSQLite(ctx, path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
|
|
must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
|
|
must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"}))
|
|
must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"}))
|
|
_, k, _ := store.NewKey("k1", "n", "proj")
|
|
k.CreatedAt = time.Now()
|
|
must(t, s.CreateKey(ctx, k))
|
|
|
|
must(t, s.WriteUsage(ctx, []store.Usage{
|
|
{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
|
|
ServedModel: "paid", CostMicros: 5000, Status: 200},
|
|
}))
|
|
|
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
|
must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
|
|
|
|
p := principal("k1", func(e *store.Effective) {
|
|
e.Budget = i64p(5000)
|
|
e.BudgetPeriod = store.PeriodMonth
|
|
})
|
|
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
|
t.Errorf("the budget was reset by the restart: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
|
|
b := NewBudgets()
|
|
now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
|
|
b.now = func() time.Time { return now }
|
|
|
|
b.Add("k1", store.PeriodMonth, 900)
|
|
if got := b.Spent("k1", store.PeriodMonth); got != 900 {
|
|
t.Fatalf("spent %d", got)
|
|
}
|
|
|
|
now = now.Add(2 * time.Minute) // into February
|
|
if got := b.Spent("k1", store.PeriodMonth); got != 0 {
|
|
t.Errorf("the counter carried %d across the month boundary", got)
|
|
}
|
|
}
|
|
|
|
func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) {
|
|
b := NewBudgets()
|
|
now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
|
|
b.now = func() time.Time { return now }
|
|
|
|
b.Add("daily", store.PeriodDay, 100)
|
|
b.Add("monthly", store.PeriodMonth, 100)
|
|
|
|
now = now.Add(24 * time.Hour)
|
|
if got := b.Spent("daily", store.PeriodDay); got != 0 {
|
|
t.Errorf("the daily counter carried %d across a day", got)
|
|
}
|
|
if got := b.Spent("monthly", store.PeriodMonth); got != 100 {
|
|
t.Errorf("the monthly counter lost %d within a month", 100-got)
|
|
}
|
|
}
|
|
|
|
func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) {
|
|
// The reason for a sliding window. With a fixed one, a caller sends its
|
|
// whole allowance at the end of a window and again at the start of the
|
|
// next, so the observed rate is double the limit exactly when the system is
|
|
// least able to take it.
|
|
l := NewLimiter()
|
|
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
now := base.Add(59 * time.Second)
|
|
l.now = func() time.Time { return now }
|
|
|
|
const limit = 60
|
|
sent := 0
|
|
for range limit {
|
|
if l.Allow("k", limit, 1) {
|
|
sent++
|
|
}
|
|
}
|
|
if sent != limit {
|
|
t.Fatalf("the first burst only managed %d", sent)
|
|
}
|
|
|
|
// One second later, in the next fixed window.
|
|
now = base.Add(61 * time.Second)
|
|
second := 0
|
|
for range limit {
|
|
if l.Allow("k", limit, 1) {
|
|
second++
|
|
}
|
|
}
|
|
// Some allowance returns as the old window slides out of view, but nowhere
|
|
// near the whole of it.
|
|
if second > limit/2 {
|
|
t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding",
|
|
second, sent)
|
|
}
|
|
}
|
|
|
|
func TestLimiterIsSafeUnderConcurrency(t *testing.T) {
|
|
// Checking and charging have to be one operation, or two concurrent
|
|
// requests both see room and both take it.
|
|
l := NewLimiter()
|
|
const limit = 100
|
|
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
allowed := 0
|
|
for range 500 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if l.Allow("k", limit, 1) {
|
|
mu.Lock()
|
|
allowed++
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if allowed != limit {
|
|
t.Errorf("%d requests passed a limit of %d", allowed, limit)
|
|
}
|
|
}
|
|
|
|
func must(t *testing.T, err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|