CI / test (push) Failing after 30m6s
Measured on three VMs: cutting the store primary's power cost about 45 s of hanging requests. New connections tried the dead member first with no connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of both cost a further 17 s of 503s while the new leader knew no judge. The store dials each member for 3 s and closes connections to a machine that is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing clusters through its API. Judges that all fail at once are asked again every 250 ms while the policy's budget lasts, and reported once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
61 lines
1.9 KiB
Go
61 lines
1.9 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
)
|
|
|
|
// How soon a lost primary is replaced. Patroni's own defaults — the leader
|
|
// key lives 30 seconds, checked every 10 — had the control planes wait the
|
|
// better part of a minute for a store after a database machine lost its
|
|
// power. Patroni asks that loop_wait + 2*retry_timeout <= ttl, so a lost
|
|
// primary is replaced within ttl and a slow etcd round is not mistaken for
|
|
// one.
|
|
const (
|
|
leaderTTL = 20
|
|
loopWait = 5
|
|
retryTimeout = 5
|
|
)
|
|
|
|
// timingSettings are the shared configuration's keys for them.
|
|
func timingSettings() map[string]any {
|
|
return map[string]any{"ttl": leaderTTL, "loop_wait": loopWait, "retry_timeout": retryTimeout}
|
|
}
|
|
|
|
// timings puts them in the cluster's shared configuration, through
|
|
// Patroni's API: a cluster set up with the old ones gets these in place.
|
|
func (s *Setup) timings(ctx context.Context, sec Secrets) error {
|
|
// As with TLS, changed in place by the machine with the authority: init
|
|
// has it, and a new cluster has these from its bootstrap already.
|
|
if s.CA == nil {
|
|
return nil
|
|
}
|
|
var lastErr error
|
|
for _, d := range s.DB {
|
|
cfg, err := getConfig(ctx, d.IP)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
want := timingSettings()
|
|
same := true
|
|
for k, v := range want {
|
|
// JSON numbers come back as float64.
|
|
if n, ok := cfg[k].(float64); !ok || int(n) != v.(int) {
|
|
same = false
|
|
}
|
|
}
|
|
if same {
|
|
s.emit("failover", false, "a lost primary is replaced within %d seconds", leaderTTL)
|
|
return nil
|
|
}
|
|
if err := patchConfig(ctx, d.IP, sec.RestAPI, want); err != nil {
|
|
return fmt.Errorf("Patroni on %s: %w", d.IP, err)
|
|
}
|
|
s.emit("failover", true, "a lost primary is replaced within %d seconds (ttl %d, loop_wait %d, retry_timeout %d)",
|
|
leaderTTL, leaderTTL, loopWait, retryTimeout)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
|
|
}
|