llmgw/internal/pgha/config.go

// Package pgha sets up the store as PostgreSQL under Patroni: two or more
// machines, one primary streaming to the others, and a standby promoted when
// the primary is lost. Patroni keeps its leader lock in etcd, which decides
// only with a majority, so etcd runs on three machines — the database
// machines and, when they are two, the control plane as the third vote.
//
// Everything is done over the same ssh the rest of onboarding uses, and every
// step looks before it acts: run against a cluster that is already up, it
// changes nothing.
package pgha

import (
	"fmt"
	"net/url"
	"strings"
)

// Scope is the Patroni cluster's name, and the etcd cluster token.
const Scope = "llmgw"

// PatroniPort is where Patroni's REST API listens on every member: what the
// control plane reads the cluster's state from.
const PatroniPort = 8008

// restPort is where this package asks Patroni; a test moves it.
var restPort = PatroniPort

// Role and Database are what the control plane connects as.
const (
	Role     = "llmgw"
	Database = "llmgw"
)

// Member is one machine of the cluster, by the address the others reach it on.
type Member struct {
	Name string
	IP   string
}

// Secrets are the cluster's passwords: PostgreSQL's three, and the one
// Patroni's REST API asks for before a change.
type Secrets struct {
	Superuser, Replicator, App, RestAPI string
}

// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
//
// With tls, members speak TLS to each other and to clients, with a
// certificate from the cluster's authority (EtcdTLSDir), and a client must
// present one too: without, anyone reaching :2379 could take the leader lock
// or rewrite Patroni's configuration.
func Etcd(m Member, all []Member, tls bool) string {
	scheme := "http"
	if tls {
		scheme = "https"
	}
	var peers []string
	for _, p := range all {
		peers = append(peers, fmt.Sprintf("%s=%s://%s:2380", p.Name, scheme, p.IP))
	}
	client := fmt.Sprintf("http://%s:2379,http://127.0.0.1:2379", m.IP)
	if tls {
		client = fmt.Sprintf("https://%s:2379,https://127.0.0.1:2379", m.IP)
	}
	conf := fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock.
ETCD_NAME=%[1]s
ETCD_DATA_DIR=/var/lib/etcd/default
ETCD_LISTEN_PEER_URLS=%[5]s://%[2]s:2380
ETCD_LISTEN_CLIENT_URLS=%[6]s
ETCD_INITIAL_ADVERTISE_PEER_URLS=%[5]s://%[2]s:2380
ETCD_ADVERTISE_CLIENT_URLS=%[5]s://%[2]s:2379
ETCD_INITIAL_CLUSTER=%[3]s
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
`, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni", scheme, client)
	if tls {
		conf += `ETCD_CERT_FILE=` + EtcdCert + `
ETCD_KEY_FILE=` + EtcdKey + `
ETCD_TRUSTED_CA_FILE=` + EtcdCA + `
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_CERT_FILE=` + EtcdCert + `
ETCD_PEER_KEY_FILE=` + EtcdKey + `
ETCD_PEER_TRUSTED_CA_FILE=` + EtcdCA + `
ETCD_PEER_CLIENT_CERT_AUTH=true
`
	}
	return conf
}

// EtcdTLSDir holds an etcd member's certificate, which it presents both as
// a server and as a client of the other members, and the authority's.
const EtcdTLSDir = "/etc/etcd/tls"

// The files in EtcdTLSDir.
const (
	EtcdCA   = EtcdTLSDir + "/ca.crt"
	EtcdCert = EtcdTLSDir + "/peer.crt"
	EtcdKey  = EtcdTLSDir + "/peer.key"
)

// etcdctl is the command that asks the cluster, over TLS with a member's
// certificate when tls; run on a member, as root, which reads it.
func etcdctl(tls bool, endpoints []string) string {
	c := "sudo ETCDCTL_API=3 etcdctl --dial-timeout=3s --endpoints=" + strings.Join(endpoints, ",")
	if tls {
		c += " --cacert=" + EtcdCA + " --cert=" + EtcdCert + " --key=" + EtcdKey
	}
	return c
}

// Patroni is a database member's /etc/patroni/config.yml.
//
// clients are the addresses the control planes connect from; the members
// themselves are always allowed, since a second control plane usually runs on
// one of them.
//
// With tls, PostgreSQL is given the member's certificate (certs puts it in
// TLSDir) and every remote rule is hostssl.
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string {
	var hosts []string
	for _, e := range etcd {
		hosts = append(hosts, e.IP+":2379")
	}
	var hba []string
	hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256")
	for _, d := range db {
		hba = append(hba,
			"host replication replicator "+d.IP+"/32 scram-sha-256",
			"host all postgres "+d.IP+"/32 scram-sha-256")
	}
	seen := map[string]bool{}
	for _, c := range append(append([]string{}, clients...), ips(db)...) {
		if c == "" || seen[c] {
			continue
		}
		seen[c] = true
		hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
	}
	var hbaLines string
	for _, h := range hba {
		if tls {
			h = hostssl(h)
		}
		hbaLines += "        - " + h + "\n"
	}
	var sslLines string
	if tls {
		for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} {
			sslLines += fmt.Sprintf("        %s: %q\n", k, sslParameters()[k])
		}
	}
	var b strings.Builder
	fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
scope: %[1]s
namespace: /service/
name: %[2]s

restapi:
  listen: %[3]s:%[4]d
  connect_address: %[3]s:%[4]d
  # Asked for before a change: anyone reaching this port could otherwise
  # rewrite the access rules or move the leader.
  authentication:
    username: %[11]s
    password: "%[10]s"
%[13]s
etcd3:
  hosts: %[5]s
%[14]s
bootstrap:
  dcs:
    ttl: 30
    loop_wait: 10
    retry_timeout: 10
    maximum_lag_on_failover: 1048576
    postgresql:
      use_pg_rewind: true
      use_slots: true
      parameters:
        wal_level: replica
        hot_standby: "on"
        max_wal_senders: 10
        max_replication_slots: 10
        wal_keep_size: 1GB
        wal_log_hints: "on"
        password_encryption: scram-sha-256
%[12]s      # In the cluster's shared configuration rather than each member's, so a
      # control plane added later is let in by one change through Patroni's
      # API (llmgw server add) instead of an edit on every member.
      pg_hba:
%[9]s  initdb:
    - encoding: UTF8
    - data-checksums

postgresql:
  listen: 127.0.0.1,%[3]s:5432
  connect_address: %[3]s:5432
  data_dir: /var/lib/postgresql/%[6]s/llmgw
  bin_dir: /usr/lib/postgresql/%[6]s/bin
  pgpass: /var/lib/postgresql/.pgpass_patroni
  authentication:
    superuser:
      username: postgres
      password: "%[7]s"
    replication:
      username: replicator
      password: "%[8]s"
    rewind:
      username: postgres
      password: "%[7]s"
  parameters:
    unix_socket_directories: /var/run/postgresql
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines,
		s.RestAPI, RestUser, sslLines, restTLSLines(tls), etcdTLSLines(tls))
	b.WriteString(`
tags:
  nofailover: false
  noloadbalance: false
  clonefrom: false
  nosync: false
`)
	return b.String()
}

// restTLSLines put Patroni's API on TLS, with the member's certificate, and
// have patronictl verify it.
func restTLSLines(tls bool) string {
	if !tls {
		return ""
	}
	return "  certfile: " + ServerCert + "\n  keyfile: " + ServerKey + "\n  cafile: " + CACert + "\n" +
		"\nctl:\n  cacert: " + CACert + "\n"
}

// etcdTLSLines have Patroni reach etcd over TLS, presenting the member's
// certificate.
func etcdTLSLines(tls bool) string {
	if !tls {
		return ""
	}
	return "  protocol: https\n  cacert: " + CACert + "\n  cert: " + ServerCert + "\n  key: " + ServerKey + "\n"
}

// DSN is the store address the control planes are given: every member, and
// the driver told to use whichever accepts writes, so a failover is followed
// without anything in front of the database. With caFile — the cluster's
// authority, on the control plane — the connection is TLS and the member it
// reaches is verified against it.
func DSN(db []Member, password, caFile string) string {
	var hosts []string
	for _, d := range db {
		hosts = append(hosts, d.IP+":5432")
	}
	ssl := "sslmode=disable"
	if caFile != "" {
		ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile)
	}
	return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write",
		Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl)
}

func ips(ms []Member) []string {
	out := make([]string, len(ms))
	for i, m := range ms {
		out[i] = m.IP
	}
	return out
}