A lost store machine is given up on in seconds, and a new leader waits for its judge
Measured on three VMs: cutting the store primary's power cost about 45 s of hanging requests. New connections tried the dead member first with no connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of both cost a further 17 s of 503s while the new leader knew no judge. The store dials each member for 3 s and closes connections to a machine that is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing clusters through its API. Judges that all fail at once are asked again every 250 ms while the policy's budget lasts, and reported once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
eeaffd764d
commit
5a4b022dcd
12 files changed
+313
-23
No files matched your search
@@ -169,9 +169,9 @@ etcd3:
|
||||
%[14]s
|
||||
bootstrap:
|
||||
dcs:
|
||||
ttl: 30
|
||||
loop_wait: 10
|
||||
retry_timeout: 10
|
||||
ttl: %[15]d
|
||||
loop_wait: %[16]d
|
||||
retry_timeout: %[17]d
|
||||
maximum_lag_on_failover: 1048576
|
||||
postgresql:
|
||||
use_pg_rewind: true
|
||||
@@ -211,7 +211,7 @@ postgresql:
|
||||
parameters:
|
||||
unix_socket_directories: /var/run/postgresql
|
||||
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines,
|
||||
s.RestAPI, RestUser, sslLines, restTLSLines(tls), etcdTLSLines(tls))
|
||||
s.RestAPI, RestUser, sslLines, restTLSLines(tls), etcdTLSLines(tls), leaderTTL, loopWait, retryTimeout)
|
||||
b.WriteString(`
|
||||
tags:
|
||||
nofailover: false
|
||||
|
||||
@@ -559,3 +559,42 @@ func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) {
|
||||
t.Errorf("restarted %v; only the member pending", restarted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALostPrimaryIsReplacedWithinTwentySeconds(t *testing.T) {
|
||||
cfg := map[string]any{"ttl": 30, "loop_wait": 10, "retry_timeout": 10}
|
||||
patches := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
_ = json.NewEncoder(w).Encode(cfg)
|
||||
case http.MethodPatch:
|
||||
patches++
|
||||
_ = json.NewDecoder(r.Body).Decode(&cfg)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
u, _ := url.Parse(srv.URL)
|
||||
old := restPort
|
||||
restPort, _ = strconv.Atoi(u.Port())
|
||||
defer func() { restPort = old }()
|
||||
ca, err := tlsconf.NewCA("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
|
||||
for range 2 {
|
||||
if err := s.timings(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if patches != 1 || cfg["ttl"].(float64) != 20 || cfg["loop_wait"].(float64) != 5 || cfg["retry_timeout"].(float64) != 5 {
|
||||
t.Errorf("patched %d times to %v", patches, cfg)
|
||||
}
|
||||
// Patroni's own rule, or it refuses them.
|
||||
if loopWait+2*retryTimeout > leaderTTL {
|
||||
t.Error("loop_wait + 2*retry_timeout is more than ttl")
|
||||
}
|
||||
if c := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, nil, "16", Secrets{}, false); !strings.Contains(c, "ttl: 20\n loop_wait: 5\n retry_timeout: 5\n") {
|
||||
t.Errorf("a new cluster bootstraps with others:\n%s", c)
|
||||
}
|
||||
}
|
||||
@@ -131,6 +131,9 @@ func (s *Setup) Run(ctx context.Context) (string, error) {
|
||||
if err := s.memory(ctx, sec); err != nil {
|
||||
return "", fmt.Errorf("memory: %w", err)
|
||||
}
|
||||
if err := s.timings(ctx, sec); err != nil {
|
||||
return "", fmt.Errorf("failover: %w", err)
|
||||
}
|
||||
if err := s.database(ctx); err != nil {
|
||||
return "", fmt.Errorf("database: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package pgha
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// How soon a lost primary is replaced. Patroni's own defaults — the leader
|
||||
// key lives 30 seconds, checked every 10 — had the control planes wait the
|
||||
// better part of a minute for a store after a database machine lost its
|
||||
// power. Patroni asks that loop_wait + 2*retry_timeout <= ttl, so a lost
|
||||
// primary is replaced within ttl and a slow etcd round is not mistaken for
|
||||
// one.
|
||||
const (
|
||||
leaderTTL = 20
|
||||
loopWait = 5
|
||||
retryTimeout = 5
|
||||
)
|
||||
|
||||
// timingSettings are the shared configuration's keys for them.
|
||||
func timingSettings() map[string]any {
|
||||
return map[string]any{"ttl": leaderTTL, "loop_wait": loopWait, "retry_timeout": retryTimeout}
|
||||
}
|
||||
|
||||
// timings puts them in the cluster's shared configuration, through
|
||||
// Patroni's API: a cluster set up with the old ones gets these in place.
|
||||
func (s *Setup) timings(ctx context.Context, sec Secrets) error {
|
||||
// As with TLS, changed in place by the machine with the authority: init
|
||||
// has it, and a new cluster has these from its bootstrap already.
|
||||
if s.CA == nil {
|
||||
return nil
|
||||
}
|
||||
var lastErr error
|
||||
for _, d := range s.DB {
|
||||
cfg, err := getConfig(ctx, d.IP)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
want := timingSettings()
|
||||
same := true
|
||||
for k, v := range want {
|
||||
// JSON numbers come back as float64.
|
||||
if n, ok := cfg[k].(float64); !ok || int(n) != v.(int) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
if same {
|
||||
s.emit("failover", false, "a lost primary is replaced within %d seconds", leaderTTL)
|
||||
return nil
|
||||
}
|
||||
if err := patchConfig(ctx, d.IP, sec.RestAPI, want); err != nil {
|
||||
return fmt.Errorf("Patroni on %s: %w", d.IP, err)
|
||||
}
|
||||
s.emit("failover", true, "a lost primary is replaced within %d seconds (ttl %d, loop_wait %d, retry_timeout %d)",
|
||||
leaderTTL, leaderTTL, loopWait, retryTimeout)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
|
||||
}
|
||||
Reference in new issue
Block a user