Files
llmgw/internal/pgha/timings.go
T
Ilker ManapandClaude Opus 5.5 5a4b022dcd
CI / test (push) Failing after 30m6s
A lost store machine is given up on in seconds, and a new leader waits for its judge
Measured on three VMs: cutting the store primary's power cost about 45 s of
hanging requests. New connections tried the dead member first with no
connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of
both cost a further 17 s of 503s while the new leader knew no judge.

The store dials each member for 3 s and closes connections to a machine that
is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with
ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing
clusters through its API. Judges that all fail at once are asked again
every 250 ms while the policy's budget lasts, and reported once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 20:33:13 +02:00

61 lines
1.9 KiB
Go

package pgha
import (
"context"
"fmt"
)
// How soon a lost primary is replaced. Patroni's own defaults — the leader
// key lives 30 seconds, checked every 10 — had the control planes wait the
// better part of a minute for a store after a database machine lost its
// power. Patroni asks that loop_wait + 2*retry_timeout <= ttl, so a lost
// primary is replaced within ttl and a slow etcd round is not mistaken for
// one.
const (
leaderTTL = 20
loopWait = 5
retryTimeout = 5
)
// timingSettings are the shared configuration's keys for them.
func timingSettings() map[string]any {
return map[string]any{"ttl": leaderTTL, "loop_wait": loopWait, "retry_timeout": retryTimeout}
}
// timings puts them in the cluster's shared configuration, through
// Patroni's API: a cluster set up with the old ones gets these in place.
func (s *Setup) timings(ctx context.Context, sec Secrets) error {
// As with TLS, changed in place by the machine with the authority: init
// has it, and a new cluster has these from its bootstrap already.
if s.CA == nil {
return nil
}
var lastErr error
for _, d := range s.DB {
cfg, err := getConfig(ctx, d.IP)
if err != nil {
lastErr = err
continue
}
want := timingSettings()
same := true
for k, v := range want {
// JSON numbers come back as float64.
if n, ok := cfg[k].(float64); !ok || int(n) != v.(int) {
same = false
}
}
if same {
s.emit("failover", false, "a lost primary is replaced within %d seconds", leaderTTL)
return nil
}
if err := patchConfig(ctx, d.IP, sec.RestAPI, want); err != nil {
return fmt.Errorf("Patroni on %s: %w", d.IP, err)
}
s.emit("failover", true, "a lost primary is replaced within %d seconds (ttl %d, loop_wait %d, retry_timeout %d)",
leaderTTL, leaderTTL, loopWait, retryTimeout)
return nil
}
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
}