Patroni's REST API accepted changes from anyone on port 8008, and PostgreSQL ran with sslmode=disable between machines. Setup now keeps a fourth secret for the API (carried to members of older clusters), issues each member a certificate from the cluster's authority, writes restapi.authentication into each member's config and reloads it, and turns ssl on with hostssl remote rules through an authenticated PATCH of the shared configuration. The DSN becomes sslmode=verify-full against the authority's certificate whenever the cluster has TLS, so a control plane without the authority's key reconnects too. server add authenticates its PATCH and lets the new control plane in by hostssl. Tried on the e2e cluster: unauthenticated PATCH 401, plaintext refused, replication streaming, the control plane on verify-full. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
248 lines
8.0 KiB
Go
248 lines
8.0 KiB
Go
package pgha
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Hardening: what a cluster set up before this lacked, applied to it in
|
|
// place, and to a new one from the start.
|
|
//
|
|
// - Patroni's REST API asks for a password before a change. Without one,
|
|
// anyone who reached port 8008 could rewrite pg_hba through PATCH /config
|
|
// — `host all all 0.0.0.0/0 trust` — or switch the leader over.
|
|
// - PostgreSQL speaks TLS, with a certificate from the cluster's authority
|
|
// on each member, and every remote rule is hostssl: prompts kept in
|
|
// threads, key and session hashes, and the engines' secret no longer
|
|
// cross the network in clear. The control planes verify the member they
|
|
// reach (sslmode=verify-full against the cluster's CA).
|
|
//
|
|
// Each step looks first and changes only what differs, so running it on a
|
|
// hardened cluster does nothing, and each is applied by reload, not restart:
|
|
// the cluster keeps serving.
|
|
|
|
// RestUser is the user Patroni's REST API asks for before a change.
|
|
const RestUser = "patroni"
|
|
|
|
// TLSDir holds each member's certificate, key and the cluster's CA.
|
|
const TLSDir = "/etc/patroni/tls"
|
|
|
|
// ServerCert, ServerKey and CACert are the files in TLSDir.
|
|
const (
|
|
ServerCert = TLSDir + "/server.crt"
|
|
ServerKey = TLSDir + "/server.key"
|
|
CACert = TLSDir + "/ca.crt"
|
|
)
|
|
|
|
// sslParameters are what turn TLS on, in the cluster's shared parameters.
|
|
func sslParameters() map[string]any {
|
|
return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert}
|
|
}
|
|
|
|
// hostssl rewrites a remote rule to accept only TLS; local and loopback
|
|
// rules are left alone.
|
|
func hostssl(line string) string {
|
|
f := strings.Fields(line)
|
|
if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" {
|
|
return line
|
|
}
|
|
return "hostssl" + strings.TrimPrefix(line, "host")
|
|
}
|
|
|
|
// certs gives each database member a certificate from the cluster's
|
|
// authority, for its address and name — unless it has one for them already.
|
|
func (s *Setup) certs(ctx context.Context) error {
|
|
if s.CA == nil {
|
|
s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS")
|
|
return nil
|
|
}
|
|
for _, d := range s.DB {
|
|
have, _ := d.Remote.Run(ctx, "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName 2>/dev/null", nil)
|
|
if strings.Contains(string(have), "IP Address:"+d.IP) {
|
|
continue
|
|
}
|
|
leaf, err := s.CA.IssueServer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var body bytes.Buffer
|
|
body.Write(s.CA.PEM)
|
|
body.WriteString("\x00")
|
|
body.Write(leaf.CertPEM)
|
|
body.WriteString("\x00")
|
|
body.Write(leaf.KeyPEM)
|
|
if out, err := d.Remote.Run(ctx, `set -e
|
|
sudo install -d -o postgres -g postgres -m 750 `+TLSDir+`
|
|
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
|
cat > "$t/all"
|
|
python3 - "$t" <<'PY'
|
|
import sys
|
|
d = sys.argv[1]
|
|
ca, crt, key = open(d + "/all", "rb").read().split(b"\x00")
|
|
for name, data in (("ca.crt", ca), ("server.crt", crt), ("server.key", key)):
|
|
open(d + "/" + name, "wb").write(data)
|
|
PY
|
|
sudo install -o postgres -g postgres -m 644 "$t/ca.crt" `+CACert+`
|
|
sudo install -o postgres -g postgres -m 644 "$t/server.crt" `+ServerCert+`
|
|
sudo install -o postgres -g postgres -m 600 "$t/server.key" `+ServerKey, body.Bytes()); err != nil {
|
|
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// restAuth makes each member's Patroni ask for the password before a change,
|
|
// and reloads it where that was not so.
|
|
func (s *Setup) restAuth(ctx context.Context, sec Secrets) error {
|
|
for _, d := range s.DB {
|
|
out, err := d.Remote.Run(ctx, `set -e
|
|
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
|
|
# The password from stdin into a file only this user reads: the program
|
|
# itself comes in on python's stdin.
|
|
f=$(mktemp); trap 'rm -f "$f"' EXIT
|
|
cat > "$f"
|
|
r=$(sudo python3 - "$f" <<'PY'
|
|
import sys, yaml
|
|
pw = open(sys.argv[1]).read().strip()
|
|
p = "/etc/patroni/config.yml"
|
|
c = yaml.safe_load(open(p))
|
|
want = {"username": "`+RestUser+`", "password": pw}
|
|
api = c.setdefault("restapi", {})
|
|
if api.get("authentication") == want:
|
|
print("same")
|
|
else:
|
|
api["authentication"] = want
|
|
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
|
|
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
|
|
print("changed")
|
|
PY
|
|
)
|
|
if [ "$r" = changed ]; then
|
|
sudo chown root:postgres /etc/patroni/config.yml.new
|
|
sudo chmod 640 /etc/patroni/config.yml.new
|
|
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
|
|
sudo systemctl reload patroni
|
|
fi
|
|
echo "$r"`, []byte(sec.RestAPI+"\n"))
|
|
if err != nil {
|
|
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
switch strings.TrimSpace(lastLine(out)) {
|
|
case "changed":
|
|
s.emit("patroni api", true, "%s asks for a password before a change", d.Name)
|
|
case "missing":
|
|
return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// tlsInShared turns TLS on in the cluster's shared configuration, and every
|
|
// remote access rule to hostssl, through Patroni's API — which reloads
|
|
// PostgreSQL on every member.
|
|
func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error {
|
|
if s.CA == nil {
|
|
return nil
|
|
}
|
|
hosts := make([]string, len(s.DB))
|
|
for i, d := range s.DB {
|
|
hosts[i] = d.IP
|
|
}
|
|
client := &http.Client{Timeout: 15 * time.Second}
|
|
var lastErr error
|
|
for _, h := range hosts {
|
|
base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort))
|
|
cfg, err := getConfig(ctx, client, base)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
pg, _ := cfg["postgresql"].(map[string]any)
|
|
params, _ := pg["parameters"].(map[string]any)
|
|
var hba, next []string
|
|
if list, ok := pg["pg_hba"].([]any); ok {
|
|
for _, l := range list {
|
|
if line, ok := l.(string); ok {
|
|
hba = append(hba, line)
|
|
next = append(next, hostssl(line))
|
|
}
|
|
}
|
|
}
|
|
want := sslParameters()
|
|
same := slices.Equal(hba, next)
|
|
for k, v := range want {
|
|
if params[k] != v {
|
|
same = false
|
|
}
|
|
}
|
|
if same {
|
|
s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl")
|
|
return nil
|
|
}
|
|
patch := map[string]any{"parameters": want}
|
|
if len(hba) > 0 {
|
|
patch["pg_hba"] = next
|
|
}
|
|
if err := patchConfig(ctx, client, base, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil {
|
|
return fmt.Errorf("Patroni on %s: %w", h, err)
|
|
}
|
|
s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl")
|
|
return nil
|
|
}
|
|
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
|
|
}
|
|
|
|
// patchConfig changes the cluster's shared configuration, with the REST
|
|
// password.
|
|
func patchConfig(ctx context.Context, client *http.Client, base, password string, body map[string]any) error {
|
|
b, _ := json.Marshal(body)
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(b))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
if password != "" {
|
|
req.SetBasicAuth(RestUser, password)
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("refused the change: %s", resp.Status)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sharedTLS reports whether the cluster's shared configuration has TLS on:
|
|
// for a control plane without the authority's key, which signs nothing but
|
|
// must still connect by TLS once the cluster asks for it.
|
|
func (s *Setup) sharedTLS(ctx context.Context) bool {
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
for _, d := range s.DB {
|
|
cfg, err := getConfig(ctx, client, "http://"+net.JoinHostPort(d.IP, strconv.Itoa(restPort)))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
pg, _ := cfg["postgresql"].(map[string]any)
|
|
params, _ := pg["parameters"].(map[string]any)
|
|
return params["ssl"] == "on"
|
|
}
|
|
return false
|
|
}
|
|
|
|
func lastLine(b []byte) string {
|
|
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
|
return lines[len(lines)-1]
|
|
}
|