CI / test (push) Failing after 30m3s
Measured again after the store fixes. Cutting the store primary's power no longer hung requests, but leadership moved anyway: the lease had lapsed for everyone while Patroni promoted the standby, and a standby asked first. The new leader then spent about 25 s without its nodes, and so without its judges. A standby that saw the store out of reach now leaves the lease alone for one renew round after the store returns. The default lease is 10 s, and a node detects a control plane that died silently in 8 s (5 s idle, 3 s ping) rather than 15. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
157 lines
4.1 KiB
Go
157 lines
4.1 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func TestAStoppingLeaderHandsOverAtOnceAndNeverTakesItBack(t *testing.T) {
|
|
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
log := slog.New(slog.DiscardHandler)
|
|
a := NewLeadership(st, time.Minute, log)
|
|
b := NewLeadership(st, time.Minute, log)
|
|
|
|
a.attempt(t.Context())
|
|
b.attempt(t.Context())
|
|
if !a.IsLeader() || b.IsLeader() {
|
|
t.Fatalf("a %v, b %v", a.IsLeader(), b.IsLeader())
|
|
}
|
|
|
|
// Stepping down gives the lease back, rather than leaving b to wait out
|
|
// the minute it was taken for.
|
|
a.StepDown()
|
|
if a.IsLeader() {
|
|
t.Error("still leading after stepping down")
|
|
}
|
|
// A renewal already on its way when the process began to stop must not
|
|
// take it back.
|
|
a.attempt(t.Context())
|
|
if a.IsLeader() {
|
|
t.Error("took the lease again after stepping down")
|
|
}
|
|
b.attempt(t.Context())
|
|
if !b.IsLeader() {
|
|
t.Error("the standby could not take the lease that was given up")
|
|
}
|
|
lease, err := st.Leadership(t.Context(), ControlRole)
|
|
if err != nil || lease.Owner != b.Owner() {
|
|
t.Errorf("the lease is %q (%v), not b's", lease.Owner, err)
|
|
}
|
|
if got := HolderName(b.Owner()); got == "" || got == b.Owner() {
|
|
t.Errorf("holder name %q from %q", got, b.Owner())
|
|
}
|
|
}
|
|
|
|
func TestAStandbyIsToldWhenItBecomesTheLeader(t *testing.T) {
|
|
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
log := slog.New(slog.DiscardHandler)
|
|
a := NewLeadership(st, time.Minute, log)
|
|
b := NewLeadership(st, time.Minute, log)
|
|
led := make(chan struct{}, 4)
|
|
b.OnLead(func() { led <- struct{}{} })
|
|
ctx := t.Context()
|
|
a.attempt(ctx)
|
|
b.attempt(ctx)
|
|
if !a.IsLeader() || b.IsLeader() {
|
|
t.Fatal("a should lead, b stand by")
|
|
}
|
|
a.StepDown()
|
|
b.attempt(ctx)
|
|
select {
|
|
case <-led:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("b became the leader and was not told")
|
|
}
|
|
}
|
|
|
|
// cuttable is a store that can be out of reach, as one whose primary was lost.
|
|
type cuttable struct {
|
|
*store.DB
|
|
down bool
|
|
}
|
|
|
|
func (c *cuttable) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) {
|
|
if c.down {
|
|
return store.Lease{}, errors.New("dial error: timeout")
|
|
}
|
|
return c.DB.AcquireLeadership(ctx, role, owner, ttl)
|
|
}
|
|
|
|
func (c *cuttable) Leadership(ctx context.Context, role string) (store.Lease, error) {
|
|
if c.down {
|
|
return store.Lease{}, errors.New("dial error: timeout")
|
|
}
|
|
return c.DB.Leadership(ctx, role)
|
|
}
|
|
|
|
// The store out of reach for longer than the lease lives, as while Patroni
|
|
// promotes a standby: when it answers again the lease has lapsed for all,
|
|
// and the leader keeps it rather than a standby taking it for nothing. A
|
|
// leader that is truly gone is still replaced, a round later.
|
|
func TestAStoreThatReturnsLeavesTheLeaderLeading(t *testing.T) {
|
|
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer st.Close()
|
|
c := &cuttable{DB: st}
|
|
log := slog.New(slog.DiscardHandler)
|
|
ctx := t.Context()
|
|
const ttl = 150 * time.Millisecond
|
|
a, b := NewLeadership(c, ttl, log), NewLeadership(c, ttl, log)
|
|
a.attempt(ctx)
|
|
b.attempt(ctx)
|
|
if !a.IsLeader() || b.IsLeader() {
|
|
t.Fatal("a should lead, b stand by")
|
|
}
|
|
|
|
c.down = true
|
|
a.attempt(ctx)
|
|
b.attempt(ctx)
|
|
time.Sleep(ttl + 20*time.Millisecond) // the lease lapses while nobody reaches the store
|
|
c.down = false
|
|
|
|
b.attempt(ctx) // the standby is first to ask
|
|
if b.IsLeader() {
|
|
t.Fatal("a standby took the lease the moment the store came back")
|
|
}
|
|
a.attempt(ctx)
|
|
if !a.IsLeader() {
|
|
t.Fatal("the leader did not keep the lease")
|
|
}
|
|
time.Sleep(a.renew)
|
|
b.attempt(ctx)
|
|
if b.IsLeader() {
|
|
t.Error("the standby took the lease from a leader renewing it")
|
|
}
|
|
|
|
// Now the leader is gone: the standby takes over after its round.
|
|
c.down = true
|
|
b.attempt(ctx)
|
|
time.Sleep(ttl + 20*time.Millisecond)
|
|
c.down = false
|
|
b.attempt(ctx)
|
|
if b.IsLeader() {
|
|
t.Fatal("took it at once")
|
|
}
|
|
time.Sleep(b.renew)
|
|
b.attempt(ctx)
|
|
if !b.IsLeader() {
|
|
t.Error("a leader that was gone was not replaced")
|
|
}
|
|
}
|