Files
llmgw/internal/control/leadership_test.go
T
Ilker ManapandClaude Opus 5.5 b09922b29b
CI / test (push) Failing after 30m3s
Leadership stays put while the store's primary is replaced
Measured again after the store fixes. Cutting the store primary's power no
longer hung requests, but leadership moved anyway: the lease had lapsed for
everyone while Patroni promoted the standby, and a standby asked first. The
new leader then spent about 25 s without its nodes, and so without its
judges.

A standby that saw the store out of reach now leaves the lease alone for one
renew round after the store returns. The default lease is 10 s, and a node
detects a control plane that died silently in 8 s (5 s idle, 3 s ping)
rather than 15.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 23:17:58 +02:00

157 lines
4.1 KiB
Go

package control
import (
"context"
"errors"
"log/slog"
"path/filepath"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
log := slog.New(slog.DiscardHandler)
a := NewLeadership(st, time.Minute, log)
b := NewLeadership(st, time.Minute, log)
a.attempt(t.Context())
b.attempt(t.Context())
if !a.IsLeader() || b.IsLeader() {
t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader())
}
// Stepping down gives the lease back, rather than leaving b to wait out
// the minute it was taken for.
a.StepDown()
if a.IsLeader() {
t.Error("still leading after stepping down")
}
// A renewal already on its way when the process began to stop must not
// take it back.
a.attempt(t.Context())
if a.IsLeader() {
t.Error("took the lease again after stepping down")
}
b.attempt(t.Context())
if !b.IsLeader() {
t.Error("the standby could not take the lease that was given up")
}
lease, err := st.Leadership(t.Context(), ControlRole)
if err != nil || lease.Owner != b.Owner() {
t.Errorf("the lease is %q (%v), not b's", lease.Owner, err)
}
if got := HolderName(b.Owner()); got == "" || got == b.Owner() {
t.Errorf("holder name %q from %q", got, b.Owner())
}
}
func TestAStandbyIsToldWhenItBecomesTheLeader(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
log := slog.New(slog.DiscardHandler)
a := NewLeadership(st, time.Minute, log)
b := NewLeadership(st, time.Minute, log)
led := make(chan struct{}, 4)
b.OnLead(func() { led <- struct{}{} })
ctx := t.Context()
a.attempt(ctx)
b.attempt(ctx)
if !a.IsLeader() || b.IsLeader() {
t.Fatal("a should lead, b stand by")
}
a.StepDown()
b.attempt(ctx)
select {
case <-led:
case <-time.After(time.Second):
t.Fatal("b became the leader and was not told")
}
}
// cuttable is a store that can be out of reach, as one whose primary was lost.
type cuttable struct {
*store.DB
down bool
}
func (c *cuttable) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) {
if c.down {
return store.Lease{}, errors.New("dial error: timeout")
}
return c.DB.AcquireLeadership(ctx, role, owner, ttl)
}
func (c *cuttable) Leadership(ctx context.Context, role string) (store.Lease, error) {
if c.down {
return store.Lease{}, errors.New("dial error: timeout")
}
return c.DB.Leadership(ctx, role)
}
// The store out of reach for longer than the lease lives, as while Patroni
// promotes a standby: when it answers again the lease has lapsed for all,
// and the leader keeps it rather than a standby taking it for nothing. A
// leader that is truly gone is still replaced, a round later.
func TestAStoreThatReturnsLeavesTheLeaderLeading(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
c := &cuttable{DB: st}
log := slog.New(slog.DiscardHandler)
ctx := t.Context()
const ttl = 150 * time.Millisecond
a, b := NewLeadership(c, ttl, log), NewLeadership(c, ttl, log)
a.attempt(ctx)
b.attempt(ctx)
if !a.IsLeader() || b.IsLeader() {
t.Fatal("a should lead, b stand by")
}
c.down = true
a.attempt(ctx)
b.attempt(ctx)
time.Sleep(ttl + 20*time.Millisecond) // the lease lapses while nobody reaches the store
c.down = false
b.attempt(ctx) // the standby is first to ask
if b.IsLeader() {
t.Fatal("a standby took the lease the moment the store came back")
}
a.attempt(ctx)
if !a.IsLeader() {
t.Fatal("the leader did not keep the lease")
}
time.Sleep(a.renew)
b.attempt(ctx)
if b.IsLeader() {
t.Error("the standby took the lease from a leader renewing it")
}
// Now the leader is gone: the standby takes over after its round.
c.down = true
b.attempt(ctx)
time.Sleep(ttl + 20*time.Millisecond)
c.down = false
b.attempt(ctx)
if b.IsLeader() {
t.Fatal("took it at once")
}
time.Sleep(b.renew)
b.attempt(ctx)
if !b.IsLeader() {
t.Error("a leader that was gone was not replaced")
}
}