Leadership stays put while the store's primary is replaced
CI / test (push) Failing after 30m3s

Measured again after the store fixes. Cutting the store primary's power no
longer hung requests, but leadership moved anyway: the lease had lapsed for
everyone while Patroni promoted the standby, and a standby asked first. The
new leader then spent about 25 s without its nodes, and so without its
judges.

A standby that saw the store out of reach now leaves the lease alone for one
renew round after the store returns. The default lease is 10 s, and a node
detects a control plane that died silently in 8 s (5 s idle, 3 s ping)
rather than 15.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 23:17:58 +02:00
1 parent 5a4b022dcd
commit b09922b29b
7 files changed
+151 -9

No files matched your search

+56 -2
View File
@@ -3,6 +3,7 @@ package control
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"log/slog"
@@ -59,6 +60,17 @@ type Leadership struct {
hookMu sync.Mutex
onLead []func()
// cut is a store this process could not reach, and back when it was
// reached again: Run's alone, so unlocked.
cut bool
back time.Time
}
// LeaseReader reads who holds a lease without competing for it: a store
// that has it lets a standby hold back after the store returns.
type LeaseReader interface {
Leadership(ctx context.Context, role string) (store.Lease, error)
}
// OnLead is called each time this process becomes the leader, after
@@ -91,8 +103,9 @@ func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadersh
if ttl <= 0 {
// Long enough to survive a slow database or a brief network stall,
// short enough that a standby takes over before anyone has finished
// reading the alert.
ttl = 15 * time.Second
// reading the alert. A store out of reach for longer — its primary
// being replaced — does not move it: see holdBack.
ttl = 10 * time.Second
}
l := &Leadership{
store: st,
@@ -146,6 +159,9 @@ func (l *Leadership) attempt(ctx context.Context) {
if l.stopped.Load() {
return
}
if l.holdBack(ctx) {
return
}
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
switch {
case err == nil && l.stopped.Load():
@@ -177,9 +193,47 @@ func (l *Leadership) attempt(ctx context.Context) {
// stop a cluster from healing itself.
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
"err", err)
l.cut, l.back = true, time.Time{}
}
}
// holdBack is a standby leaving a lapsed lease alone for one round after the
// store returns from being out of reach.
//
// A store's primary that is lost takes longer to replace than the lease
// lives, so the lease has lapsed by the time the store answers again — for
// the leader as for the standbys, which all failed to reach it alike. The
// first to ask then took it, and it was as often a standby as the leader:
// leadership moved for nothing, and the new leader spent half a minute
// learning from the agents where the judges were, refusing requests
// meanwhile. The leader renews within a round of the store's return, so a
// standby that waits that round takes the lease only from a leader that is
// gone.
func (l *Leadership) holdBack(ctx context.Context) bool {
if l.leader.Load() || !l.cut {
return false
}
r, ok := l.store.(LeaseReader)
if !ok {
l.cut = false
return false
}
if _, err := r.Leadership(ctx, controlRole); err != nil && !errors.Is(err, sql.ErrNoRows) {
l.back = time.Time{}
return true
}
if l.back.IsZero() {
l.back = time.Now()
l.log.Info("the store answers again; leaving the lease to its holder for a round", "owner", l.owner)
}
// A round less a tenth, for the ticker's own jitter.
if time.Since(l.back) < l.renew-l.renew/10 {
return true
}
l.cut, l.back = false, time.Time{}
return false
}
// StepDown gives the lease up now, for a process on its way out: called
// before the agents are let go, so the standby they move to is already the
// leader, and before the process exits, which would otherwise race the
+79
View File
@@ -1,6 +1,8 @@
package control
import (
"context"
"errors"
"log/slog"
"path/filepath"
"testing"
@@ -75,3 +77,80 @@ func TestAStandbyIsToldWhenItBecomesTheLeader(t *testing.T) {
t.Fatal("b became the leader and was not told")
}
}
// cuttable is a store that can be out of reach, as one whose primary was lost.
type cuttable struct {
*store.DB
down bool
}
func (c *cuttable) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) {
if c.down {
return store.Lease{}, errors.New("dial error: timeout")
}
return c.DB.AcquireLeadership(ctx, role, owner, ttl)
}
func (c *cuttable) Leadership(ctx context.Context, role string) (store.Lease, error) {
if c.down {
return store.Lease{}, errors.New("dial error: timeout")
}
return c.DB.Leadership(ctx, role)
}
// The store out of reach for longer than the lease lives, as while Patroni
// promotes a standby: when it answers again the lease has lapsed for all,
// and the leader keeps it rather than a standby taking it for nothing. A
// leader that is truly gone is still replaced, a round later.
func TestAStoreThatReturnsLeavesTheLeaderLeading(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer st.Close()
c := &cuttable{DB: st}
log := slog.New(slog.DiscardHandler)
ctx := t.Context()
const ttl = 150 * time.Millisecond
a, b := NewLeadership(c, ttl, log), NewLeadership(c, ttl, log)
a.attempt(ctx)
b.attempt(ctx)
if !a.IsLeader() || b.IsLeader() {
t.Fatal("a should lead, b stand by")
}
c.down = true
a.attempt(ctx)
b.attempt(ctx)
time.Sleep(ttl + 20*time.Millisecond) // the lease lapses while nobody reaches the store
c.down = false
b.attempt(ctx) // the standby is first to ask
if b.IsLeader() {
t.Fatal("a standby took the lease the moment the store came back")
}
a.attempt(ctx)
if !a.IsLeader() {
t.Fatal("the leader did not keep the lease")
}
time.Sleep(a.renew)
b.attempt(ctx)
if b.IsLeader() {
t.Error("the standby took the lease from a leader renewing it")
}
// Now the leader is gone: the standby takes over after its round.
c.down = true
b.attempt(ctx)
time.Sleep(ttl + 20*time.Millisecond)
c.down = false
b.attempt(ctx)
if b.IsLeader() {
t.Fatal("took it at once")
}
time.Sleep(b.renew)
b.attempt(ctx)
if !b.IsLeader() {
t.Error("a leader that was gone was not replaced")
}
}