Measured again after the store fixes. Cutting the store primary's power no longer hung requests, but leadership moved anyway: the lease had lapsed for everyone while Patroni promoted the standby, and a standby asked first. The new leader then spent about 25 s without its nodes, and so without its judges. A standby that saw the store out of reach now leaves the lease alone for one renew round after the store returns. The default lease is 10 s, and a node detects a control plane that died silently in 8 s (5 s idle, 3 s ping) rather than 15. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5a4b022dcd
commit
b09922b29b
7 files changed
+151
-9
No files matched your search
@@ -3,6 +3,7 @@ package control
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log/slog"
|
||||
@@ -59,6 +60,17 @@ type Leadership struct {
|
||||
|
||||
hookMu sync.Mutex
|
||||
onLead []func()
|
||||
|
||||
// cut is a store this process could not reach, and back when it was
|
||||
// reached again: Run's alone, so unlocked.
|
||||
cut bool
|
||||
back time.Time
|
||||
}
|
||||
|
||||
// LeaseReader reads who holds a lease without competing for it: a store
|
||||
// that has it lets a standby hold back after the store returns.
|
||||
type LeaseReader interface {
|
||||
Leadership(ctx context.Context, role string) (store.Lease, error)
|
||||
}
|
||||
|
||||
// OnLead is called each time this process becomes the leader, after
|
||||
@@ -91,8 +103,9 @@ func NewLeadership(st LeaseStore, ttl time.Duration, log *slog.Logger) *Leadersh
|
||||
if ttl <= 0 {
|
||||
// Long enough to survive a slow database or a brief network stall,
|
||||
// short enough that a standby takes over before anyone has finished
|
||||
// reading the alert.
|
||||
ttl = 15 * time.Second
|
||||
// reading the alert. A store out of reach for longer — its primary
|
||||
// being replaced — does not move it: see holdBack.
|
||||
ttl = 10 * time.Second
|
||||
}
|
||||
l := &Leadership{
|
||||
store: st,
|
||||
@@ -146,6 +159,9 @@ func (l *Leadership) attempt(ctx context.Context) {
|
||||
if l.stopped.Load() {
|
||||
return
|
||||
}
|
||||
if l.holdBack(ctx) {
|
||||
return
|
||||
}
|
||||
lease, err := l.store.AcquireLeadership(ctx, controlRole, l.owner, l.ttl)
|
||||
switch {
|
||||
case err == nil && l.stopped.Load():
|
||||
@@ -177,9 +193,47 @@ func (l *Leadership) attempt(ctx context.Context) {
|
||||
// stop a cluster from healing itself.
|
||||
l.log.Warn("could not reach the store to renew leadership; keeping it for now",
|
||||
"err", err)
|
||||
l.cut, l.back = true, time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// holdBack is a standby leaving a lapsed lease alone for one round after the
|
||||
// store returns from being out of reach.
|
||||
//
|
||||
// A store's primary that is lost takes longer to replace than the lease
|
||||
// lives, so the lease has lapsed by the time the store answers again — for
|
||||
// the leader as for the standbys, which all failed to reach it alike. The
|
||||
// first to ask then took it, and it was as often a standby as the leader:
|
||||
// leadership moved for nothing, and the new leader spent half a minute
|
||||
// learning from the agents where the judges were, refusing requests
|
||||
// meanwhile. The leader renews within a round of the store's return, so a
|
||||
// standby that waits that round takes the lease only from a leader that is
|
||||
// gone.
|
||||
func (l *Leadership) holdBack(ctx context.Context) bool {
|
||||
if l.leader.Load() || !l.cut {
|
||||
return false
|
||||
}
|
||||
r, ok := l.store.(LeaseReader)
|
||||
if !ok {
|
||||
l.cut = false
|
||||
return false
|
||||
}
|
||||
if _, err := r.Leadership(ctx, controlRole); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
l.back = time.Time{}
|
||||
return true
|
||||
}
|
||||
if l.back.IsZero() {
|
||||
l.back = time.Now()
|
||||
l.log.Info("the store answers again; leaving the lease to its holder for a round", "owner", l.owner)
|
||||
}
|
||||
// A round less a tenth, for the ticker's own jitter.
|
||||
if time.Since(l.back) < l.renew-l.renew/10 {
|
||||
return true
|
||||
}
|
||||
l.cut, l.back = false, time.Time{}
|
||||
return false
|
||||
}
|
||||
|
||||
// StepDown gives the lease up now, for a process on its way out: called
|
||||
// before the agents are let go, so the standby they move to is already the
|
||||
// leader, and before the process exits, which would otherwise race the
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -75,3 +77,80 @@ func TestAStandbyIsToldWhenItBecomesTheLeader(t *testing.T) {
|
||||
t.Fatal("b became the leader and was not told")
|
||||
}
|
||||
}
|
||||
|
||||
// cuttable is a store that can be out of reach, as one whose primary was lost.
|
||||
type cuttable struct {
|
||||
*store.DB
|
||||
down bool
|
||||
}
|
||||
|
||||
func (c *cuttable) AcquireLeadership(ctx context.Context, role, owner string, ttl time.Duration) (store.Lease, error) {
|
||||
if c.down {
|
||||
return store.Lease{}, errors.New("dial error: timeout")
|
||||
}
|
||||
return c.DB.AcquireLeadership(ctx, role, owner, ttl)
|
||||
}
|
||||
|
||||
func (c *cuttable) Leadership(ctx context.Context, role string) (store.Lease, error) {
|
||||
if c.down {
|
||||
return store.Lease{}, errors.New("dial error: timeout")
|
||||
}
|
||||
return c.DB.Leadership(ctx, role)
|
||||
}
|
||||
|
||||
// The store out of reach for longer than the lease lives, as while Patroni
|
||||
// promotes a standby: when it answers again the lease has lapsed for all,
|
||||
// and the leader keeps it rather than a standby taking it for nothing. A
|
||||
// leader that is truly gone is still replaced, a round later.
|
||||
func TestAStoreThatReturnsLeavesTheLeaderLeading(t *testing.T) {
|
||||
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
c := &cuttable{DB: st}
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
ctx := t.Context()
|
||||
const ttl = 150 * time.Millisecond
|
||||
a, b := NewLeadership(c, ttl, log), NewLeadership(c, ttl, log)
|
||||
a.attempt(ctx)
|
||||
b.attempt(ctx)
|
||||
if !a.IsLeader() || b.IsLeader() {
|
||||
t.Fatal("a should lead, b stand by")
|
||||
}
|
||||
|
||||
c.down = true
|
||||
a.attempt(ctx)
|
||||
b.attempt(ctx)
|
||||
time.Sleep(ttl + 20*time.Millisecond) // the lease lapses while nobody reaches the store
|
||||
c.down = false
|
||||
|
||||
b.attempt(ctx) // the standby is first to ask
|
||||
if b.IsLeader() {
|
||||
t.Fatal("a standby took the lease the moment the store came back")
|
||||
}
|
||||
a.attempt(ctx)
|
||||
if !a.IsLeader() {
|
||||
t.Fatal("the leader did not keep the lease")
|
||||
}
|
||||
time.Sleep(a.renew)
|
||||
b.attempt(ctx)
|
||||
if b.IsLeader() {
|
||||
t.Error("the standby took the lease from a leader renewing it")
|
||||
}
|
||||
|
||||
// Now the leader is gone: the standby takes over after its round.
|
||||
c.down = true
|
||||
b.attempt(ctx)
|
||||
time.Sleep(ttl + 20*time.Millisecond)
|
||||
c.down = false
|
||||
b.attempt(ctx)
|
||||
if b.IsLeader() {
|
||||
t.Fatal("took it at once")
|
||||
}
|
||||
time.Sleep(b.renew)
|
||||
b.attempt(ctx)
|
||||
if !b.IsLeader() {
|
||||
t.Error("a leader that was gone was not replaced")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user