Files
llmgw/internal/pgha/config.go
T
Ilker ManapandClaude Opus 5.5 2b29016d5b Ask for a password on Patroni's API and give the store TLS, new clusters and existing ones in place
Patroni's REST API accepted changes from anyone on port 8008, and
PostgreSQL ran with sslmode=disable between machines. Setup now keeps a
fourth secret for the API (carried to members of older clusters), issues
each member a certificate from the cluster's authority, writes
restapi.authentication into each member's config and reloads it, and
turns ssl on with hostssl remote rules through an authenticated PATCH of
the shared configuration. The DSN becomes sslmode=verify-full against the
authority's certificate whenever the cluster has TLS, so a control plane
without the authority's key reconnects too. server add authenticates its
PATCH and lets the new control plane in by hostssl. Tried on the e2e
cluster: unauthenticated PATCH 401, plaintext refused, replication
streaming, the control plane on verify-full.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 10:11:28 +02:00

204 lines
6.1 KiB
Go

// Package pgha sets up the store as PostgreSQL under Patroni: two or more
// machines, one primary streaming to the others, and a standby promoted when
// the primary is lost. Patroni keeps its leader lock in etcd, which decides
// only with a majority, so etcd runs on three machines — the database
// machines and, when they are two, the control plane as the third vote.
//
// Everything is done over the same ssh the rest of onboarding uses, and every
// step looks before it acts: run against a cluster that is already up, it
// changes nothing.
package pgha
import (
"fmt"
"net/url"
"strings"
)
// Scope is the Patroni cluster's name, and the etcd cluster token.
const Scope = "llmgw"
// PatroniPort is where Patroni's REST API listens on every member: what the
// control plane reads the cluster's state from.
const PatroniPort = 8008
// restPort is where this package asks Patroni; a test moves it.
var restPort = PatroniPort
// Role and Database are what the control plane connects as.
const (
Role = "llmgw"
Database = "llmgw"
)
// Member is one machine of the cluster, by the address the others reach it on.
type Member struct {
Name string
IP string
}
// Secrets are the cluster's passwords: PostgreSQL's three, and the one
// Patroni's REST API asks for before a change.
type Secrets struct {
Superuser, Replicator, App, RestAPI string
}
// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
func Etcd(m Member, all []Member) string {
var peers []string
for _, p := range all {
peers = append(peers, fmt.Sprintf("%s=http://%s:2380", p.Name, p.IP))
}
return fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock.
ETCD_NAME=%[1]s
ETCD_DATA_DIR=/var/lib/etcd/default
ETCD_LISTEN_PEER_URLS=http://%[2]s:2380
ETCD_LISTEN_CLIENT_URLS=http://%[2]s:2379,http://127.0.0.1:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%[2]s:2380
ETCD_ADVERTISE_CLIENT_URLS=http://%[2]s:2379
ETCD_INITIAL_CLUSTER=%[3]s
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
`, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni")
}
// Patroni is a database member's /etc/patroni/config.yml.
//
// clients are the addresses the control planes connect from; the members
// themselves are always allowed, since a second control plane usually runs on
// one of them.
//
// With tls, PostgreSQL is given the member's certificate (certs puts it in
// TLSDir) and every remote rule is hostssl.
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string {
var hosts []string
for _, e := range etcd {
hosts = append(hosts, e.IP+":2379")
}
var hba []string
hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256")
for _, d := range db {
hba = append(hba,
"host replication replicator "+d.IP+"/32 scram-sha-256",
"host all postgres "+d.IP+"/32 scram-sha-256")
}
seen := map[string]bool{}
for _, c := range append(append([]string{}, clients...), ips(db)...) {
if c == "" || seen[c] {
continue
}
seen[c] = true
hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
}
var hbaLines string
for _, h := range hba {
if tls {
h = hostssl(h)
}
hbaLines += " - " + h + "\n"
}
var sslLines string
if tls {
for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} {
sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k])
}
}
var b strings.Builder
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
scope: %[1]s
namespace: /service/
name: %[2]s
restapi:
listen: %[3]s:%[4]d
connect_address: %[3]s:%[4]d
# Asked for before a change: anyone reaching this port could otherwise
# rewrite the access rules or move the leader.
authentication:
username: %[11]s
password: "%[10]s"
etcd3:
hosts: %[5]s
bootstrap:
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
postgresql:
use_pg_rewind: true
use_slots: true
parameters:
wal_level: replica
hot_standby: "on"
max_wal_senders: 10
max_replication_slots: 10
wal_keep_size: 1GB
wal_log_hints: "on"
password_encryption: scram-sha-256
%[12]s # In the cluster's shared configuration rather than each member's, so a
# control plane added later is let in by one change through Patroni's
# API (llmgw server add) instead of an edit on every member.
pg_hba:
%[9]s initdb:
- encoding: UTF8
- data-checksums
postgresql:
listen: 127.0.0.1,%[3]s:5432
connect_address: %[3]s:5432
data_dir: /var/lib/postgresql/%[6]s/llmgw
bin_dir: /usr/lib/postgresql/%[6]s/bin
pgpass: /var/lib/postgresql/.pgpass_patroni
authentication:
superuser:
username: postgres
password: "%[7]s"
replication:
username: replicator
password: "%[8]s"
rewind:
username: postgres
password: "%[7]s"
parameters:
unix_socket_directories: /var/run/postgresql
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines,
s.RestAPI, RestUser, sslLines)
b.WriteString(`
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
`)
return b.String()
}
// DSN is the store address the control planes are given: every member, and
// the driver told to use whichever accepts writes, so a failover is followed
// without anything in front of the database. With caFile — the cluster's
// authority, on the control plane — the connection is TLS and the member it
// reaches is verified against it.
func DSN(db []Member, password, caFile string) string {
var hosts []string
for _, d := range db {
hosts = append(hosts, d.IP+":5432")
}
ssl := "sslmode=disable"
if caFile != "" {
ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile)
}
return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write",
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl)
}
func ips(ms []Member) []string {
out := make([]string, len(ms))
for i, m := range ms {
out[i] = m.IP
}
return out
}