// Package pgha sets up the store as PostgreSQL under Patroni: two or more // machines, one primary streaming to the others, and a standby promoted when // the primary is lost. Patroni keeps its leader lock in etcd, which decides // only with a majority, so etcd runs on three machines — the database // machines and, when they are two, the control plane as the third vote. // // Everything is done over the same ssh the rest of onboarding uses, and every // step looks before it acts: run against a cluster that is already up, it // changes nothing. package pgha import ( "fmt" "net/url" "strings" ) // Scope is the Patroni cluster's name, and the etcd cluster token. const Scope = "llmgw" // PatroniPort is where Patroni's REST API listens on every member: what the // control plane reads the cluster's state from. const PatroniPort = 8008 // restPort is where this package asks Patroni; a test moves it. var restPort = PatroniPort // Role and Database are what the control plane connects as. const ( Role = "llmgw" Database = "llmgw" ) // Member is one machine of the cluster, by the address the others reach it on. type Member struct { Name string IP string } // Secrets are the cluster's three passwords. type Secrets struct { Superuser, Replicator, App string } // Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server. func Etcd(m Member, all []Member) string { var peers []string for _, p := range all { peers = append(peers, fmt.Sprintf("%s=http://%s:2380", p.Name, p.IP)) } return fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock. ETCD_NAME=%[1]s ETCD_DATA_DIR=/var/lib/etcd/default ETCD_LISTEN_PEER_URLS=http://%[2]s:2380 ETCD_LISTEN_CLIENT_URLS=http://%[2]s:2379,http://127.0.0.1:2379 ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%[2]s:2380 ETCD_ADVERTISE_CLIENT_URLS=http://%[2]s:2379 ETCD_INITIAL_CLUSTER=%[3]s ETCD_INITIAL_CLUSTER_STATE=new ETCD_INITIAL_CLUSTER_TOKEN=%[4]s `, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni") } // Patroni is a database member's /etc/patroni/config.yml. // // clients are the addresses the control planes connect from; the members // themselves are always allowed, since a second control plane usually runs on // one of them. func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string { var hosts []string for _, e := range etcd { hosts = append(hosts, e.IP+":2379") } var hba []string hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256") for _, d := range db { hba = append(hba, "host replication replicator "+d.IP+"/32 scram-sha-256", "host all postgres "+d.IP+"/32 scram-sha-256") } seen := map[string]bool{} for _, c := range append(append([]string{}, clients...), ips(db)...) { if c == "" || seen[c] { continue } seen[c] = true hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c)) } var hbaLines string for _, h := range hba { hbaLines += " - " + h + "\n" } var b strings.Builder fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd. scope: %[1]s namespace: /service/ name: %[2]s restapi: listen: %[3]s:%[4]d connect_address: %[3]s:%[4]d etcd3: hosts: %[5]s bootstrap: dcs: ttl: 30 loop_wait: 10 retry_timeout: 10 maximum_lag_on_failover: 1048576 postgresql: use_pg_rewind: true use_slots: true parameters: wal_level: replica hot_standby: "on" max_wal_senders: 10 max_replication_slots: 10 wal_keep_size: 1GB wal_log_hints: "on" password_encryption: scram-sha-256 # In the cluster's shared configuration rather than each member's, so a # control plane added later is let in by one change through Patroni's # API (llmgw server add) instead of an edit on every member. pg_hba: %[9]s initdb: - encoding: UTF8 - data-checksums postgresql: listen: 127.0.0.1,%[3]s:5432 connect_address: %[3]s:5432 data_dir: /var/lib/postgresql/%[6]s/llmgw bin_dir: /usr/lib/postgresql/%[6]s/bin pgpass: /var/lib/postgresql/.pgpass_patroni authentication: superuser: username: postgres password: "%[7]s" replication: username: replicator password: "%[8]s" rewind: username: postgres password: "%[7]s" parameters: unix_socket_directories: /var/run/postgresql `, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines) b.WriteString(` tags: nofailover: false noloadbalance: false clonefrom: false nosync: false `) return b.String() } // DSN is the store address the control planes are given: every member, and // the driver told to use whichever accepts writes, so a failover is followed // without anything in front of the database. func DSN(db []Member, password string) string { var hosts []string for _, d := range db { hosts = append(hosts, d.IP+":5432") } return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write", Role, url.QueryEscape(password), strings.Join(hosts, ","), Database) } func ips(ms []Member) []string { out := make([]string, len(ms)) for i, m := range ms { out[i] = m.IP } return out }