CI / test (push) Failing after 30m6s
Measured on three VMs: cutting the store primary's power cost about 45 s of hanging requests. New connections tried the dead member first with no connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of both cost a further 17 s of 503s while the new leader knew no judge. The store dials each member for 3 s and closes connections to a machine that is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing clusters through its API. Judges that all fail at once are asked again every 250 ms while the policy's budget lasts, and reported once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
269 lines
8.2 KiB
Go
269 lines
8.2 KiB
Go
// Package pgha sets up the store as PostgreSQL under Patroni: two or more
|
|
// machines, one primary streaming to the others, and a standby promoted when
|
|
// the primary is lost. Patroni keeps its leader lock in etcd, which decides
|
|
// only with a majority, so etcd runs on three machines — the database
|
|
// machines and, when they are two, the control plane as the third vote.
|
|
//
|
|
// Everything is done over the same ssh the rest of onboarding uses, and every
|
|
// step looks before it acts: run against a cluster that is already up, it
|
|
// changes nothing.
|
|
package pgha
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
)
|
|
|
|
// Scope is the Patroni cluster's name, and the etcd cluster token.
|
|
const Scope = "llmgw"
|
|
|
|
// PatroniPort is where Patroni's REST API listens on every member: what the
|
|
// control plane reads the cluster's state from.
|
|
const PatroniPort = 8008
|
|
|
|
// restPort is where this package asks Patroni; a test moves it.
|
|
var restPort = PatroniPort
|
|
|
|
// Role and Database are what the control plane connects as.
|
|
const (
|
|
Role = "llmgw"
|
|
Database = "llmgw"
|
|
)
|
|
|
|
// Member is one machine of the cluster, by the address the others reach it on.
|
|
type Member struct {
|
|
Name string
|
|
IP string
|
|
}
|
|
|
|
// Secrets are the cluster's passwords: PostgreSQL's three, and the one
|
|
// Patroni's REST API asks for before a change.
|
|
type Secrets struct {
|
|
Superuser, Replicator, App, RestAPI string
|
|
}
|
|
|
|
// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
|
|
//
|
|
// With tls, members speak TLS to each other and to clients, with a
|
|
// certificate from the cluster's authority (EtcdTLSDir), and a client must
|
|
// present one too: without, anyone reaching :2379 could take the leader lock
|
|
// or rewrite Patroni's configuration.
|
|
func Etcd(m Member, all []Member, tls bool) string {
|
|
scheme := "http"
|
|
if tls {
|
|
scheme = "https"
|
|
}
|
|
var peers []string
|
|
for _, p := range all {
|
|
peers = append(peers, fmt.Sprintf("%s=%s://%s:2380", p.Name, scheme, p.IP))
|
|
}
|
|
client := fmt.Sprintf("http://%s:2379,http://127.0.0.1:2379", m.IP)
|
|
if tls {
|
|
client = fmt.Sprintf("https://%s:2379,https://127.0.0.1:2379", m.IP)
|
|
}
|
|
conf := fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock.
|
|
ETCD_NAME=%[1]s
|
|
ETCD_DATA_DIR=/var/lib/etcd/default
|
|
ETCD_LISTEN_PEER_URLS=%[5]s://%[2]s:2380
|
|
ETCD_LISTEN_CLIENT_URLS=%[6]s
|
|
ETCD_INITIAL_ADVERTISE_PEER_URLS=%[5]s://%[2]s:2380
|
|
ETCD_ADVERTISE_CLIENT_URLS=%[5]s://%[2]s:2379
|
|
ETCD_INITIAL_CLUSTER=%[3]s
|
|
ETCD_INITIAL_CLUSTER_STATE=new
|
|
ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
|
|
`, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni", scheme, client)
|
|
if tls {
|
|
conf += `ETCD_CERT_FILE=` + EtcdCert + `
|
|
ETCD_KEY_FILE=` + EtcdKey + `
|
|
ETCD_TRUSTED_CA_FILE=` + EtcdCA + `
|
|
ETCD_CLIENT_CERT_AUTH=true
|
|
ETCD_PEER_CERT_FILE=` + EtcdCert + `
|
|
ETCD_PEER_KEY_FILE=` + EtcdKey + `
|
|
ETCD_PEER_TRUSTED_CA_FILE=` + EtcdCA + `
|
|
ETCD_PEER_CLIENT_CERT_AUTH=true
|
|
`
|
|
}
|
|
return conf
|
|
}
|
|
|
|
// EtcdTLSDir holds an etcd member's certificate, which it presents both as
|
|
// a server and as a client of the other members, and the authority's.
|
|
const EtcdTLSDir = "/etc/etcd/tls"
|
|
|
|
// The files in EtcdTLSDir.
|
|
const (
|
|
EtcdCA = EtcdTLSDir + "/ca.crt"
|
|
EtcdCert = EtcdTLSDir + "/peer.crt"
|
|
EtcdKey = EtcdTLSDir + "/peer.key"
|
|
)
|
|
|
|
// etcdctl is the command that asks the cluster, over TLS with a member's
|
|
// certificate when tls; run on a member, as root, which reads it.
|
|
func etcdctl(tls bool, endpoints []string) string {
|
|
c := "sudo ETCDCTL_API=3 etcdctl --dial-timeout=3s --endpoints=" + strings.Join(endpoints, ",")
|
|
if tls {
|
|
c += " --cacert=" + EtcdCA + " --cert=" + EtcdCert + " --key=" + EtcdKey
|
|
}
|
|
return c
|
|
}
|
|
|
|
// Patroni is a database member's /etc/patroni/config.yml.
|
|
//
|
|
// clients are the addresses the control planes connect from; the members
|
|
// themselves are always allowed, since a second control plane usually runs on
|
|
// one of them.
|
|
//
|
|
// With tls, PostgreSQL is given the member's certificate (certs puts it in
|
|
// TLSDir) and every remote rule is hostssl.
|
|
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string {
|
|
var hosts []string
|
|
for _, e := range etcd {
|
|
hosts = append(hosts, e.IP+":2379")
|
|
}
|
|
var hba []string
|
|
hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256")
|
|
for _, d := range db {
|
|
hba = append(hba,
|
|
"host replication replicator "+d.IP+"/32 scram-sha-256",
|
|
"host all postgres "+d.IP+"/32 scram-sha-256")
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, c := range append(append([]string{}, clients...), ips(db)...) {
|
|
if c == "" || seen[c] {
|
|
continue
|
|
}
|
|
seen[c] = true
|
|
hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
|
|
}
|
|
var hbaLines string
|
|
for _, h := range hba {
|
|
if tls {
|
|
h = hostssl(h)
|
|
}
|
|
hbaLines += " - " + h + "\n"
|
|
}
|
|
var sslLines string
|
|
if tls {
|
|
for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} {
|
|
sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k])
|
|
}
|
|
}
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
|
|
scope: %[1]s
|
|
namespace: /service/
|
|
name: %[2]s
|
|
|
|
restapi:
|
|
listen: %[3]s:%[4]d
|
|
connect_address: %[3]s:%[4]d
|
|
# Asked for before a change: anyone reaching this port could otherwise
|
|
# rewrite the access rules or move the leader.
|
|
authentication:
|
|
username: %[11]s
|
|
password: "%[10]s"
|
|
%[13]s
|
|
etcd3:
|
|
hosts: %[5]s
|
|
%[14]s
|
|
bootstrap:
|
|
dcs:
|
|
ttl: %[15]d
|
|
loop_wait: %[16]d
|
|
retry_timeout: %[17]d
|
|
maximum_lag_on_failover: 1048576
|
|
postgresql:
|
|
use_pg_rewind: true
|
|
use_slots: true
|
|
parameters:
|
|
wal_level: replica
|
|
hot_standby: "on"
|
|
max_wal_senders: 10
|
|
max_replication_slots: 10
|
|
wal_keep_size: 1GB
|
|
wal_log_hints: "on"
|
|
password_encryption: scram-sha-256
|
|
%[12]s # In the cluster's shared configuration rather than each member's, so a
|
|
# control plane added later is let in by one change through Patroni's
|
|
# API (llmgw server add) instead of an edit on every member.
|
|
pg_hba:
|
|
%[9]s initdb:
|
|
- encoding: UTF8
|
|
- data-checksums
|
|
|
|
postgresql:
|
|
listen: 127.0.0.1,%[3]s:5432
|
|
connect_address: %[3]s:5432
|
|
data_dir: /var/lib/postgresql/%[6]s/llmgw
|
|
bin_dir: /usr/lib/postgresql/%[6]s/bin
|
|
pgpass: /var/lib/postgresql/.pgpass_patroni
|
|
authentication:
|
|
superuser:
|
|
username: postgres
|
|
password: "%[7]s"
|
|
replication:
|
|
username: replicator
|
|
password: "%[8]s"
|
|
rewind:
|
|
username: postgres
|
|
password: "%[7]s"
|
|
parameters:
|
|
unix_socket_directories: /var/run/postgresql
|
|
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines,
|
|
s.RestAPI, RestUser, sslLines, restTLSLines(tls), etcdTLSLines(tls), leaderTTL, loopWait, retryTimeout)
|
|
b.WriteString(`
|
|
tags:
|
|
nofailover: false
|
|
noloadbalance: false
|
|
clonefrom: false
|
|
nosync: false
|
|
`)
|
|
return b.String()
|
|
}
|
|
|
|
// restTLSLines put Patroni's API on TLS, with the member's certificate, and
|
|
// have patronictl verify it.
|
|
func restTLSLines(tls bool) string {
|
|
if !tls {
|
|
return ""
|
|
}
|
|
return " certfile: " + ServerCert + "\n keyfile: " + ServerKey + "\n cafile: " + CACert + "\n" +
|
|
"\nctl:\n cacert: " + CACert + "\n"
|
|
}
|
|
|
|
// etcdTLSLines have Patroni reach etcd over TLS, presenting the member's
|
|
// certificate.
|
|
func etcdTLSLines(tls bool) string {
|
|
if !tls {
|
|
return ""
|
|
}
|
|
return " protocol: https\n cacert: " + CACert + "\n cert: " + ServerCert + "\n key: " + ServerKey + "\n"
|
|
}
|
|
|
|
// DSN is the store address the control planes are given: every member, and
|
|
// the driver told to use whichever accepts writes, so a failover is followed
|
|
// without anything in front of the database. With caFile — the cluster's
|
|
// authority, on the control plane — the connection is TLS and the member it
|
|
// reaches is verified against it.
|
|
func DSN(db []Member, password, caFile string) string {
|
|
var hosts []string
|
|
for _, d := range db {
|
|
hosts = append(hosts, d.IP+":5432")
|
|
}
|
|
ssl := "sslmode=disable"
|
|
if caFile != "" {
|
|
ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile)
|
|
}
|
|
return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write",
|
|
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl)
|
|
}
|
|
|
|
func ips(ms []Member) []string {
|
|
out := make([]string, len(ms))
|
|
for i, m := range ms {
|
|
out[i] = m.IP
|
|
}
|
|
return out
|
|
}
|