// Package pgha sets up the store as PostgreSQL under Patroni: two or more // machines, one primary streaming to the others, and a standby promoted when // the primary is lost. Patroni keeps its leader lock in etcd, which decides // only with a majority, so etcd runs on three machines — the database // machines and, when they are two, the control plane as the third vote. // // Everything is done over the same ssh the rest of onboarding uses, and every // step looks before it acts: run against a cluster that is already up, it // changes nothing. package pgha import ( "fmt" "net/url" "strings" ) // Scope is the Patroni cluster's name, and the etcd cluster token. const Scope = "llmgw" // PatroniPort is where Patroni's REST API listens on every member: what the // control plane reads the cluster's state from. const PatroniPort = 8008 // restPort is where this package asks Patroni; a test moves it. var restPort = PatroniPort // Role and Database are what the control plane connects as. const ( Role = "llmgw" Database = "llmgw" ) // Member is one machine of the cluster, by the address the others reach it on. type Member struct { Name string IP string } // Secrets are the cluster's passwords: PostgreSQL's three, and the one // Patroni's REST API asks for before a change. type Secrets struct { Superuser, Replicator, App, RestAPI string } // Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server. // // With tls, members speak TLS to each other and to clients, with a // certificate from the cluster's authority (EtcdTLSDir), and a client must // present one too: without, anyone reaching :2379 could take the leader lock // or rewrite Patroni's configuration. func Etcd(m Member, all []Member, tls bool) string { scheme := "http" if tls { scheme = "https" } var peers []string for _, p := range all { peers = append(peers, fmt.Sprintf("%s=%s://%s:2380", p.Name, scheme, p.IP)) } client := fmt.Sprintf("http://%s:2379,http://127.0.0.1:2379", m.IP) if tls { client = fmt.Sprintf("https://%s:2379,https://127.0.0.1:2379", m.IP) } conf := fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock. ETCD_NAME=%[1]s ETCD_DATA_DIR=/var/lib/etcd/default ETCD_LISTEN_PEER_URLS=%[5]s://%[2]s:2380 ETCD_LISTEN_CLIENT_URLS=%[6]s ETCD_INITIAL_ADVERTISE_PEER_URLS=%[5]s://%[2]s:2380 ETCD_ADVERTISE_CLIENT_URLS=%[5]s://%[2]s:2379 ETCD_INITIAL_CLUSTER=%[3]s ETCD_INITIAL_CLUSTER_STATE=new ETCD_INITIAL_CLUSTER_TOKEN=%[4]s `, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni", scheme, client) if tls { conf += `ETCD_CERT_FILE=` + EtcdCert + ` ETCD_KEY_FILE=` + EtcdKey + ` ETCD_TRUSTED_CA_FILE=` + EtcdCA + ` ETCD_CLIENT_CERT_AUTH=true ETCD_PEER_CERT_FILE=` + EtcdCert + ` ETCD_PEER_KEY_FILE=` + EtcdKey + ` ETCD_PEER_TRUSTED_CA_FILE=` + EtcdCA + ` ETCD_PEER_CLIENT_CERT_AUTH=true ` } return conf } // EtcdTLSDir holds an etcd member's certificate, which it presents both as // a server and as a client of the other members, and the authority's. const EtcdTLSDir = "/etc/etcd/tls" // The files in EtcdTLSDir. const ( EtcdCA = EtcdTLSDir + "/ca.crt" EtcdCert = EtcdTLSDir + "/peer.crt" EtcdKey = EtcdTLSDir + "/peer.key" ) // etcdctl is the command that asks the cluster, over TLS with a member's // certificate when tls; run on a member, as root, which reads it. func etcdctl(tls bool, endpoints []string) string { c := "sudo ETCDCTL_API=3 etcdctl --dial-timeout=3s --endpoints=" + strings.Join(endpoints, ",") if tls { c += " --cacert=" + EtcdCA + " --cert=" + EtcdCert + " --key=" + EtcdKey } return c } // Patroni is a database member's /etc/patroni/config.yml. // // clients are the addresses the control planes connect from; the members // themselves are always allowed, since a second control plane usually runs on // one of them. // // With tls, PostgreSQL is given the member's certificate (certs puts it in // TLSDir) and every remote rule is hostssl. func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string { var hosts []string for _, e := range etcd { hosts = append(hosts, e.IP+":2379") } var hba []string hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256") for _, d := range db { hba = append(hba, "host replication replicator "+d.IP+"/32 scram-sha-256", "host all postgres "+d.IP+"/32 scram-sha-256") } seen := map[string]bool{} for _, c := range append(append([]string{}, clients...), ips(db)...) { if c == "" || seen[c] { continue } seen[c] = true hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c)) } var hbaLines string for _, h := range hba { if tls { h = hostssl(h) } hbaLines += " - " + h + "\n" } var sslLines string if tls { for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} { sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k]) } } var b strings.Builder fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd. scope: %[1]s namespace: /service/ name: %[2]s restapi: listen: %[3]s:%[4]d connect_address: %[3]s:%[4]d # Asked for before a change: anyone reaching this port could otherwise # rewrite the access rules or move the leader. authentication: username: %[11]s password: "%[10]s" %[13]s etcd3: hosts: %[5]s %[14]s bootstrap: dcs: ttl: 30 loop_wait: 10 retry_timeout: 10 maximum_lag_on_failover: 1048576 postgresql: use_pg_rewind: true use_slots: true parameters: wal_level: replica hot_standby: "on" max_wal_senders: 10 max_replication_slots: 10 wal_keep_size: 1GB wal_log_hints: "on" password_encryption: scram-sha-256 %[12]s # In the cluster's shared configuration rather than each member's, so a # control plane added later is let in by one change through Patroni's # API (llmgw server add) instead of an edit on every member. pg_hba: %[9]s initdb: - encoding: UTF8 - data-checksums postgresql: listen: 127.0.0.1,%[3]s:5432 connect_address: %[3]s:5432 data_dir: /var/lib/postgresql/%[6]s/llmgw bin_dir: /usr/lib/postgresql/%[6]s/bin pgpass: /var/lib/postgresql/.pgpass_patroni authentication: superuser: username: postgres password: "%[7]s" replication: username: replicator password: "%[8]s" rewind: username: postgres password: "%[7]s" parameters: unix_socket_directories: /var/run/postgresql `, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines, s.RestAPI, RestUser, sslLines, restTLSLines(tls), etcdTLSLines(tls)) b.WriteString(` tags: nofailover: false noloadbalance: false clonefrom: false nosync: false `) return b.String() } // restTLSLines put Patroni's API on TLS, with the member's certificate, and // have patronictl verify it. func restTLSLines(tls bool) string { if !tls { return "" } return " certfile: " + ServerCert + "\n keyfile: " + ServerKey + "\n cafile: " + CACert + "\n" + "\nctl:\n cacert: " + CACert + "\n" } // etcdTLSLines have Patroni reach etcd over TLS, presenting the member's // certificate. func etcdTLSLines(tls bool) string { if !tls { return "" } return " protocol: https\n cacert: " + CACert + "\n cert: " + ServerCert + "\n key: " + ServerKey + "\n" } // DSN is the store address the control planes are given: every member, and // the driver told to use whichever accepts writes, so a failover is followed // without anything in front of the database. With caFile — the cluster's // authority, on the control plane — the connection is TLS and the member it // reaches is verified against it. func DSN(db []Member, password, caFile string) string { var hosts []string for _, d := range db { hosts = append(hosts, d.IP+":5432") } ssl := "sslmode=disable" if caFile != "" { ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile) } return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write", Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl) } func ips(ms []Member) []string { out := make([]string, len(ms)) for i, m := range ms { out[i] = m.IP } return out }