Files
llmgw/internal/store/postgres_test.go
T
Ilker ManapandClaude Opus 5 5b5e61cb97
CI / test (push) Successful in 1m35s
Stop a second control plane from placing work behind the first's back
The first thing HA needs that a shared store does not give. Two control planes
against one store both reconcile: both see a model wanting two replicas and
none running, and both place. The model ends up with twice what it asked for,
each replica holding GPUs the other's inventory believes are free — and nothing
crashes to say so.

A lease in a table rather than a lock held by a connection. PostgreSQL has
advisory locks and SQLite has nothing like them, and this store keeps one set
of queries for both; a row with an expiry works everywhere, and it survives a
leader being powered off rather than asked to stop, which is the case that
matters. Taking and renewing are the same statement, so the two cannot disagree
about what a held lease looks like, and it is one statement, so two processes
racing cannot both believe they won.

The owner id is random rather than the hostname. A hostname would let a
restarted process match its own stale lease and believe it had been leading all
along — including through the gap that made it restart, during which somebody
else may have taken over and started placing.

A store that cannot be reached is not evidence that somebody else is leading,
so an unreachable store keeps the claim. Two leaders is worse than one, but no
leader means nothing is placed and nothing is evicted, and a brief database
blip should not stop a cluster healing itself.

Unset means leader. Every deployment that existed before this, every
single-control-plane install and every test acts unconditionally, with nothing
to configure.

The same seven assertions run against both backends, which is how the test
harness was caught inheriting a lease between subtests — it passed on SQLite,
where each subtest gets a fresh file, and failed only on PostgreSQL.

What this does not yet do is move agents. A standby holding an agent's stream
is one the leader cannot command, so failover is degraded rather than clean
until agents know more than one address.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-25 18:24:06 +02:00

178 lines
5.1 KiB
Go

package store
import (
"context"
"errors"
"os"
"testing"
"time"
)
// openPG opens the test database named by LLMGW_TEST_POSTGRES, or skips.
//
// Env-gated rather than spun up here, because a store test that needs a
// container is a test nobody runs. Pointed at a database, it runs the same
// assertions the SQLite tests make — which is the only thing that can catch
// the two backends drifting apart.
//
// Every table is emptied first. The alternative is a test database that
// accumulates rows until an assertion about counts starts failing for reasons
// that have nothing to do with the change being tested.
func openPG(t *testing.T) *SQLite {
t.Helper()
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
if dsn == "" {
t.Skip("set LLMGW_TEST_POSTGRES to a database DSN to run these")
}
s, err := OpenPostgres(context.Background(), dsn)
if err != nil {
t.Fatalf("opening the test database: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
// CASCADE because the tables reference each other, and TRUNCATE rather
// than DELETE so identity columns restart too.
_, err = s.db.ExecContext(context.Background(),
// leadership included, and it is not in copyOrder: a subtest that
// inherits the previous one's lease is refused for reasons that have
// nothing to do with what it is testing. It passed on SQLite, where
// every subtest gets a fresh file, and failed only here.
`TRUNCATE thread_messages, threads, memory_notes, usage, api_keys,
key_claims, limits, models, sessions, users, projects, teams,
organizations, leadership RESTART IDENTITY CASCADE`)
if err != nil {
t.Fatalf("clearing the test database: %v", err)
}
return s
}
func TestPostgresCarriesTheWholeTenancyTree(t *testing.T) {
ctx := context.Background()
s := openPG(t)
proj := tree(t, s)
secret, k := mustKey(t, s, "k1", proj)
_ = secret
got, err := s.KeyByHash(ctx, k.Hash)
if err != nil {
t.Fatalf("reading the key back: %v", err)
}
if got.ID != k.ID || got.ProjectID != proj {
t.Errorf("key came back as %+v", got)
}
}
func TestPostgresKeepsUsageAndSpend(t *testing.T) {
// The accounting path, which is the one that decides whether a budget
// means anything after a restart.
ctx := context.Background()
s := openPG(t)
proj := tree(t, s)
_, k := mustKey(t, s, "k1", proj)
now := time.Now()
err := s.WriteUsage(ctx, []Usage{
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m",
PromptTokens: 10, CompletionTokens: 20, CostMicros: 500, Status: 200, LatencyMs: 42},
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m",
PromptTokens: 1, CompletionTokens: 2, CostMicros: 250, Status: 200, LatencyMs: 7},
})
if err != nil {
t.Fatalf("writing usage: %v", err)
}
spend, err := s.SpendSince(ctx, now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if spend[k.ID] != 750 {
t.Errorf("spend = %d, want 750", spend[k.ID])
}
}
func TestPostgresKeepsThreadsInOrderAndScoped(t *testing.T) {
// The newest tables, and the ones with the most SQL per operation —
// a transaction, a MAX(seq), an insert per turn.
ctx := context.Background()
s := openPG(t)
proj := tree(t, s)
if err := s.SetThreadRetention(ctx, proj, 24); err != nil {
t.Fatal(err)
}
for _, m := range []Message{
{Role: "user", Content: "first"},
{Role: "assistant", Content: "answer", Model: "small"},
{Role: "user", Content: "second"},
} {
if err := s.AppendThread(ctx, proj, "th1", []Message{m}); err != nil {
t.Fatal(err)
}
}
got, err := s.Thread(ctx, proj, "th1")
if err != nil {
t.Fatal(err)
}
if len(got) != 3 || got[0].Content != "first" || got[2].Content != "second" {
t.Fatalf("thread came back as %+v", got)
}
if got[1].Model != "small" {
t.Errorf("the model that answered was lost: %+v", got[1])
}
// And the scope holds here exactly as it does on SQLite.
if err := s.PutProject(ctx, Project{ID: "proj2", TeamID: "team1", Name: "Other"}); err != nil {
t.Fatal(err)
}
if _, err := s.Thread(ctx, "proj2", "th1"); !errors.Is(err, ErrNoThread) {
t.Errorf("reading across projects returned %v", err)
}
}
func TestPostgresEnforcesLimitsTheSameWay(t *testing.T) {
ctx := context.Background()
s := openPG(t)
proj := tree(t, s)
_, k := mustKey(t, s, "k1", proj)
rpm := 7
budget := int64(9000)
if err := s.PutLimits(ctx, Limits{
Scope: ScopeProject, ScopeID: proj, RPM: &rpm,
Budget: &budget, BudgetPeriod: PeriodMonth,
}); err != nil {
t.Fatal(err)
}
eff, err := s.Resolve(ctx, k.ID)
if err != nil {
t.Fatal(err)
}
if eff.RPM == nil || *eff.RPM != 7 {
t.Errorf("rpm did not resolve down the tree: %+v", eff.RPM)
}
if eff.Budget == nil || *eff.Budget != 9000 {
t.Errorf("budget did not resolve down the tree: %+v", eff.Budget)
}
}
func TestPostgresUpsertsRatherThanDuplicating(t *testing.T) {
// ON CONFLICT is the one non-trivial construct shared by both dialects.
// If it behaved differently here, applying a configuration twice would
// start failing instead of being a no-op.
ctx := context.Background()
s := openPG(t)
tree(t, s)
tree(t, s)
orgs, err := s.Orgs(ctx)
if err != nil {
t.Fatal(err)
}
if len(orgs) != 1 {
t.Errorf("applying the tree twice produced %d orgs", len(orgs))
}
}