Files
llmgw/internal/store/leadership_test.go
T
Ilker ManapandClaude Opus 5 ebce41e22d Stop calling the PostgreSQL store SQLite, and stop guessing where it is
Two things that were only ever confusing, and one of them stopped a command
working on the cluster.

The store type was named SQLite and kept the name when PostgreSQL was added, so
OpenPostgres returned a *SQLite and all seventy-odd methods on the production
store were spelled after the database it was not running on. Nothing broke;
people did. Earlier today I added three methods to *SQLite for a table that on
this cluster only ever lives in PostgreSQL, and the name did not once look
wrong. It is a field, which is all it ever was.

`llmgw users` and `llmgw keys` defaulted --store to a SQLite path. On a
deployment whose store is PostgreSQL every one of them failed with "unable to
open database file" while the right answer sat in auth.store in the config file
beside them; the conclusion an operator draws is that the tool is broken, and
they are not far off. They now ask the server's own configuration before
falling back, and the error names which of the three sources chose the address
— the failure was never the path, it was which one picked it.

The single-machine default still applies last, so a deployment with no config
file keeps working with no flags.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 12:20:40 +02:00

152 lines
4.2 KiB
Go

package store
import (
"context"
"errors"
"sync"
"testing"
"time"
)
// leadershipSuite is the same assertions against whichever store it is given,
// because the whole point of the lease is that it behaves identically on both
// — and the one that matters for HA is the one it was never tested on.
func leadershipSuite(t *testing.T, open func(*testing.T) *DB) {
ctx := context.Background()
t.Run("the first to ask gets it", func(t *testing.T) {
s := open(t)
lease, err := s.AcquireLeadership(ctx, "control", "a", time.Minute)
if err != nil {
t.Fatal(err)
}
if lease.Owner != "a" {
t.Errorf("owner = %q", lease.Owner)
}
})
t.Run("a second is refused and told who holds it", func(t *testing.T) {
s := open(t)
if _, err := s.AcquireLeadership(ctx, "control", "a", time.Minute); err != nil {
t.Fatal(err)
}
held, err := s.AcquireLeadership(ctx, "control", "b", time.Minute)
if !errors.Is(err, ErrNotLeader) {
t.Fatalf("a second control plane got %v", err)
}
if held.Owner != "a" {
t.Errorf("refused without naming the holder: %+v", held)
}
})
t.Run("the holder renews without losing when it started", func(t *testing.T) {
s := open(t)
first, err := s.AcquireLeadership(ctx, "control", "a", time.Minute)
if err != nil {
t.Fatal(err)
}
time.Sleep(5 * time.Millisecond)
again, err := s.AcquireLeadership(ctx, "control", "a", time.Minute)
if err != nil {
t.Fatalf("the holder could not renew: %v", err)
}
if !again.ExpiresAt.After(first.ExpiresAt) {
t.Error("renewing did not move the expiry")
}
if !again.AcquiredAt.Equal(first.AcquiredAt) {
t.Error("renewing reset when this leader took over")
}
})
t.Run("a lapsed lease is taken", func(t *testing.T) {
// The reason a lease and not a lock: a leader that was powered off
// cannot release anything, and nobody should have to be paged to make
// a standby take over.
s := open(t)
if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil {
t.Fatal(err)
}
time.Sleep(40 * time.Millisecond)
lease, err := s.AcquireLeadership(ctx, "control", "b", time.Minute)
if err != nil {
t.Fatalf("a lapsed lease was not takeable: %v", err)
}
if lease.Owner != "b" {
t.Errorf("owner after takeover = %q", lease.Owner)
}
})
t.Run("releasing hands over at once", func(t *testing.T) {
s := open(t)
if _, err := s.AcquireLeadership(ctx, "control", "a", time.Hour); err != nil {
t.Fatal(err)
}
if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil {
t.Fatal(err)
}
if _, err := s.AcquireLeadership(ctx, "control", "b", time.Minute); err != nil {
t.Errorf("a released lease was not free: %v", err)
}
})
t.Run("a former leader cannot release the current one", func(t *testing.T) {
// A process that lost the lease and is shutting down slowly must not
// take it from whoever has it now — which is exactly what an
// unconditional delete would do.
s := open(t)
if _, err := s.AcquireLeadership(ctx, "control", "a", 20*time.Millisecond); err != nil {
t.Fatal(err)
}
time.Sleep(40 * time.Millisecond)
if _, err := s.AcquireLeadership(ctx, "control", "b", time.Hour); err != nil {
t.Fatal(err)
}
if err := s.ReleaseLeadership(ctx, "control", "a"); err != nil {
t.Fatal(err)
}
held, err := s.Leadership(ctx, "control")
if err != nil {
t.Fatal(err)
}
if held.Owner != "b" {
t.Errorf("the old leader released the new one's lease: %+v", held)
}
})
t.Run("only one of many wins", func(t *testing.T) {
// The case the whole table exists for. Two control planes starting
// together is the ordinary way this gets tested in production.
s := open(t)
const racers = 8
var wg sync.WaitGroup
won := make([]bool, racers)
for i := range racers {
wg.Add(1)
go func() {
defer wg.Done()
_, err := s.AcquireLeadership(ctx, "control", string(rune('a'+i)), time.Minute)
won[i] = err == nil
}()
}
wg.Wait()
n := 0
for _, w := range won {
if w {
n++
}
}
if n != 1 {
t.Errorf("%d of %d processes believed they were leader", n, racers)
}
})
}
func TestLeadershipOnSQLite(t *testing.T) {
leadershipSuite(t, func(t *testing.T) *DB { return openDB(t) })
}
func TestLeadershipOnPostgres(t *testing.T) {
leadershipSuite(t, openPG)
}