Files
llmgw/internal/store/store_test.go
T
Ilker ManapandClaude Opus 5 29617f8cb1 Add the store: tenancy, keys and usage on SQLite
Phase 5 begins. This is the persistence layer for the things the cluster
cannot reconstruct — who may call, what they are allowed, what they used.
Everything else llmgw knows is still derived from the fleet, which cannot be
wrong about itself the way a database can (§6.5).

SQLite behind an interface. The plan justified that by write volume, which was
the wrong argument: the write path is batched anyway, and a 500-row
transaction every 200ms is nothing for either engine. The real argument is the
failure mode. Postgres is a second service, and a gateway that stops serving
because its accounting database is unreachable is worse than one that buffers
some accounting. Embedded means the store shares the gateway's fate instead of
adding an independent way to be down. That reasoning stops holding at two
instances, where budgets have to be counted somewhere shared — so the
interface is here from the start, the SQL is portable, and migrations are per
dialect with shared version numbers.

Keys are stored as sha256 and nothing else, so a database leak hands over no
working credential. The fast hash is deliberate: the input is 256 random bits
with no dictionary to attack, so argon2 would buy nothing and charge tens of
milliseconds per request. That assumption is written down next to the code
that depends on it.

Two things the schema now says that it did not: budgets are integer
micro-units, because repeated addition of floats drifts; and an empty model
allowlist forbids everything while an absent one permits everything, which the
resolver has to keep distinct.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 08:37:40 +02:00

406 lines
11 KiB
Go

package store
import (
"context"
"errors"
"path/filepath"
"slices"
"strings"
"testing"
"time"
)
func open(t *testing.T) *SQLite {
t.Helper()
s, err := OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "llmgw.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
// tree builds org → team → project and returns the project id.
func tree(t *testing.T, s *SQLite) string {
t.Helper()
ctx := context.Background()
if err := s.PutOrg(ctx, Org{ID: "org1", Name: "Acme"}); err != nil {
t.Fatal(err)
}
if err := s.PutTeam(ctx, Team{ID: "team1", OrgID: "org1", Name: "Research"}); err != nil {
t.Fatal(err)
}
if err := s.PutProject(ctx, Project{ID: "proj1", TeamID: "team1", Name: "Chat"}); err != nil {
t.Fatal(err)
}
return "proj1"
}
func mustKey(t *testing.T, s *SQLite, id, project string) (string, Key) {
t.Helper()
secret, k, err := NewKey(id, id, project)
if err != nil {
t.Fatal(err)
}
k.CreatedAt = time.Now()
if err := s.CreateKey(context.Background(), k); err != nil {
t.Fatal(err)
}
return secret, k
}
func intp(v int) *int { return &v }
func i64p(v int64) *int64 { return &v }
func TestMigrationsAreIdempotent(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "llmgw.db")
s, err := OpenSQLite(context.Background(), path)
if err != nil {
t.Fatal(err)
}
tree(t, s)
_ = s.Close()
// Reopening must not try to create what is already there, and must not lose
// what was written.
s2, err := OpenSQLite(context.Background(), path)
if err != nil {
t.Fatalf("reopening ran the migrations again: %v", err)
}
defer s2.Close()
orgs, err := s2.Orgs(context.Background())
if err != nil || len(orgs) != 1 {
t.Fatalf("orgs after reopen: %v %v", orgs, err)
}
}
func TestTheKeyItselfIsNeverStored(t *testing.T) {
// The whole point of hashing: a database leak hands over no working
// credential.
s := open(t)
project := tree(t, s)
secret, _ := mustKey(t, s, "key1", project)
rows, err := s.db.Query(`SELECT id, key_hash, key_prefix, name FROM api_keys`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var id, prefix, name string
var hash []byte
if err := rows.Scan(&id, &hash, &prefix, &name); err != nil {
t.Fatal(err)
}
for _, field := range []string{id, prefix, name, string(hash)} {
if strings.Contains(field, strings.TrimPrefix(secret, KeyPrefix)) {
t.Fatal("the key itself reached the database")
}
}
if len(hash) != 32 {
t.Errorf("hash is %d bytes", len(hash))
}
}
}
func TestAuthenticationFindsAKeyByWhatWasPresented(t *testing.T) {
s := open(t)
project := tree(t, s)
secret, created := mustKey(t, s, "key1", project)
got, err := s.KeyByHash(context.Background(), HashKey(secret))
if err != nil {
t.Fatal(err)
}
if got.ID != created.ID || got.ProjectID != project {
t.Errorf("found %+v", got)
}
if !got.Usable(time.Now()) {
t.Error("a fresh key is not usable")
}
if _, err := s.KeyByHash(context.Background(), HashKey(secret+"x")); !errors.Is(err, ErrNotFound) {
t.Errorf("a wrong key was accepted: %v", err)
}
}
func TestRevocationKeepsTheAuditTrail(t *testing.T) {
// A revoked key must stop working and must still exist: usage rows
// reference it, and an audit trail that loses its subject is not one.
s := open(t)
ctx := context.Background()
project := tree(t, s)
secret, k := mustKey(t, s, "key1", project)
if err := s.RevokeKey(ctx, k.ID, time.Now()); err != nil {
t.Fatal(err)
}
got, err := s.KeyByHash(ctx, HashKey(secret))
if err != nil {
t.Fatalf("a revoked key vanished from the database: %v", err)
}
if got.Usable(time.Now()) {
t.Error("a revoked key still authenticates")
}
if err := s.RevokeKey(ctx, "nope", time.Now()); !errors.Is(err, ErrNotFound) {
t.Errorf("revoking a key that does not exist: %v", err)
}
}
func TestExpiryIsCheckedAgainstTheClockNotTheDatabase(t *testing.T) {
s := open(t)
project := tree(t, s)
secret, k := mustKey(t, s, "key1", project)
k.ExpiresAt = time.Now().Add(-time.Hour)
if err := s.CreateKey(context.Background(), Key{
ID: "key2", Hash: HashKey("other"), Prefix: "x", Name: "n",
ProjectID: project, CreatedAt: time.Now(), ExpiresAt: k.ExpiresAt,
}); err != nil {
t.Fatal(err)
}
expired, err := s.KeyByHash(context.Background(), HashKey("other"))
if err != nil {
t.Fatal(err)
}
if expired.Usable(time.Now()) {
t.Error("an expired key is usable")
}
fresh, _ := s.KeyByHash(context.Background(), HashKey(secret))
if !fresh.Usable(time.Now()) {
t.Error("an unexpired key is not usable")
}
}
func TestOrphansAreRefused(t *testing.T) {
// SQLite leaves foreign keys off by default, which would let the tenancy
// tree grow keys belonging to no project.
s := open(t)
_, k, _ := NewKey("key1", "n", "no-such-project")
k.CreatedAt = time.Now()
if err := s.CreateKey(context.Background(), k); err == nil {
t.Fatal("a key was created under a project that does not exist")
}
}
func TestLimitsResolveToTheMostRestrictive(t *testing.T) {
s := open(t)
ctx := context.Background()
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
// The organization is generous, the team less so, the key least. Whichever
// is tightest has to win at every field independently.
must(t, s.PutLimits(ctx, Limits{Scope: ScopeOrg, ScopeID: "org1", RPM: intp(1000), TPM: intp(100000)}))
must(t, s.PutLimits(ctx, Limits{Scope: ScopeTeam, ScopeID: "team1", RPM: intp(100)}))
must(t, s.PutLimits(ctx, Limits{Scope: ScopeKey, ScopeID: k.ID, TPM: intp(500)}))
e, err := s.Resolve(ctx, k.ID)
if err != nil {
t.Fatal(err)
}
if e.RPM == nil || *e.RPM != 100 {
t.Errorf("rpm=%v, the team's limit should win", e.RPM)
}
if e.TPM == nil || *e.TPM != 500 {
t.Errorf("tpm=%v, the key's limit should win", e.TPM)
}
if e.OrgID != "org1" || e.TeamID != "team1" || e.ProjectID != project {
t.Errorf("chain resolved to %+v", e)
}
}
func TestANarrowerScopeCannotRaiseALimit(t *testing.T) {
// The only composition rule under which a budget means anything.
s := open(t)
ctx := context.Background()
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
must(t, s.PutLimits(ctx, Limits{Scope: ScopeOrg, ScopeID: "org1", RPM: intp(10),
Budget: i64p(1_000_000), BudgetPeriod: PeriodMonth}))
must(t, s.PutLimits(ctx, Limits{Scope: ScopeKey, ScopeID: k.ID, RPM: intp(9999),
Budget: i64p(999_000_000), BudgetPeriod: PeriodDay}))
e, err := s.Resolve(ctx, k.ID)
if err != nil {
t.Fatal(err)
}
if *e.RPM != 10 {
t.Errorf("a key raised its organization's rate limit to %d", *e.RPM)
}
if *e.Budget != 1_000_000 || e.BudgetPeriod != PeriodMonth {
t.Errorf("a key raised its organization's budget to %d/%s", *e.Budget, e.BudgetPeriod)
}
}
func TestAllowlistsIntersectRatherThanReplace(t *testing.T) {
s := open(t)
ctx := context.Background()
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
must(t, s.PutLimits(ctx, Limits{Scope: ScopeTeam, ScopeID: "team1",
AllowedModels: []string{"qwen", "llama"}}))
must(t, s.PutLimits(ctx, Limits{Scope: ScopeKey, ScopeID: k.ID,
AllowedModels: []string{"llama", "gpt-4"}}))
e, err := s.Resolve(ctx, k.ID)
if err != nil {
t.Fatal(err)
}
if !slices.Equal(e.AllowedModels, []string{"llama"}) {
t.Fatalf("allowed=%v; a key must not add a model its team withheld", e.AllowedModels)
}
if e.Allows("gpt-4") {
t.Error("a model the team never allowed is served")
}
if !e.Allows("llama") {
t.Error("the intersection is not served")
}
}
func TestNoAllowlistMeansEveryModel(t *testing.T) {
s := open(t)
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
e, err := s.Resolve(context.Background(), k.ID)
if err != nil {
t.Fatal(err)
}
if e.AllowedModels != nil {
t.Errorf("allowed=%v", e.AllowedModels)
}
if !e.Allows("anything") {
t.Error("an unrestricted key was refused a model")
}
}
func TestAnEmptyAllowlistForbidsEverything(t *testing.T) {
// Empty is not the same as absent, and the store must not conflate them:
// one is "no restriction", the other is "nothing".
s := open(t)
ctx := context.Background()
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
must(t, s.PutLimits(ctx, Limits{Scope: ScopeKey, ScopeID: k.ID, AllowedModels: []string{}}))
e, err := s.Resolve(ctx, k.ID)
if err != nil {
t.Fatal(err)
}
if e.AllowedModels == nil {
t.Fatal("an empty allowlist was read back as no allowlist")
}
if e.Allows("qwen") {
t.Error("an empty allowlist served a model")
}
}
func TestBudgetSurvivesARestart(t *testing.T) {
// A counter that lives only in memory resets on restart, which would make a
// budget a suggestion.
dir := t.TempDir()
path := filepath.Join(dir, "llmgw.db")
ctx := context.Background()
s, err := OpenSQLite(ctx, path)
if err != nil {
t.Fatal(err)
}
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
now := time.Now()
must(t, s.WriteUsage(ctx, []Usage{
{At: now, KeyID: k.ID, ProjectID: project, Model: "qwen", ServedModel: "qwen",
PromptTokens: 100, CompletionTokens: 50, CostMicros: 1500, Status: 200},
{At: now, KeyID: k.ID, ProjectID: project, Model: "qwen", ServedModel: "qwen",
PromptTokens: 200, CompletionTokens: 80, CostMicros: 2800, Status: 200},
// Last period: must not count towards this one.
{At: now.Add(-48 * time.Hour), KeyID: k.ID, ProjectID: project, Model: "qwen",
ServedModel: "qwen", CostMicros: 999999, Status: 200},
}))
_ = s.Close()
s2, err := OpenSQLite(ctx, path)
if err != nil {
t.Fatal(err)
}
defer s2.Close()
spend, err := s2.SpendSince(ctx, now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if spend[k.ID] != 4300 {
t.Errorf("spend after restart = %d, want 4300", spend[k.ID])
}
}
func TestLastUsedIsOnlyEverMovedForward(t *testing.T) {
// The batch writer sees keys out of order; an older timestamp must not
// rewind the newer one.
s := open(t)
ctx := context.Background()
project := tree(t, s)
_, k := mustKey(t, s, "key1", project)
newer := time.Now().Truncate(time.Millisecond)
older := newer.Add(-time.Hour)
must(t, s.TouchKeys(ctx, map[string]time.Time{k.ID: newer}))
must(t, s.TouchKeys(ctx, map[string]time.Time{k.ID: older}))
got, err := s.KeyByHash(ctx, k.Hash)
if err != nil {
t.Fatal(err)
}
if !got.LastUsedAt.Equal(newer.UTC()) {
t.Errorf("last used = %v, want %v", got.LastUsedAt, newer.UTC())
}
}
func TestUpsertsAreIdempotent(t *testing.T) {
// Applying a configuration twice must not double the tree or fail.
s := open(t)
ctx := context.Background()
tree(t, s)
tree(t, s)
orgs, _ := s.Orgs(ctx)
teams, _ := s.Teams(ctx, "org1")
projects, _ := s.Projects(ctx, "team1")
if len(orgs) != 1 || len(teams) != 1 || len(projects) != 1 {
t.Errorf("tree duplicated: %d orgs, %d teams, %d projects", len(orgs), len(teams), len(projects))
}
must(t, s.PutOrg(ctx, Org{ID: "org1", Name: "Renamed"}))
orgs, _ = s.Orgs(ctx)
if orgs[0].Name != "Renamed" {
t.Errorf("upsert did not update: %q", orgs[0].Name)
}
}
func TestResolveRejectsAnUnknownKey(t *testing.T) {
s := open(t)
tree(t, s)
if _, err := s.Resolve(context.Background(), "nope"); !errors.Is(err, ErrNotFound) {
t.Errorf("resolving an unknown key: %v", err)
}
}
func must(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}