Files
llmgw/internal/pgha/harden.go
T
Ilker ManapandClaude Opus 5.5 2b29016d5b Ask for a password on Patroni's API and give the store TLS, new clusters and existing ones in place
Patroni's REST API accepted changes from anyone on port 8008, and
PostgreSQL ran with sslmode=disable between machines. Setup now keeps a
fourth secret for the API (carried to members of older clusters), issues
each member a certificate from the cluster's authority, writes
restapi.authentication into each member's config and reloads it, and
turns ssl on with hostssl remote rules through an authenticated PATCH of
the shared configuration. The DSN becomes sslmode=verify-full against the
authority's certificate whenever the cluster has TLS, so a control plane
without the authority's key reconnects too. server add authenticates its
PATCH and lets the new control plane in by hostssl. Tried on the e2e
cluster: unauthenticated PATCH 401, plaintext refused, replication
streaming, the control plane on verify-full.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 10:11:28 +02:00

248 lines
8.0 KiB
Go

package pgha
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
)
// Hardening: what a cluster set up before this lacked, applied to it in
// place, and to a new one from the start.
//
// - Patroni's REST API asks for a password before a change. Without one,
// anyone who reached port 8008 could rewrite pg_hba through PATCH /config
// — `host all all 0.0.0.0/0 trust` — or switch the leader over.
// - PostgreSQL speaks TLS, with a certificate from the cluster's authority
// on each member, and every remote rule is hostssl: prompts kept in
// threads, key and session hashes, and the engines' secret no longer
// cross the network in clear. The control planes verify the member they
// reach (sslmode=verify-full against the cluster's CA).
//
// Each step looks first and changes only what differs, so running it on a
// hardened cluster does nothing, and each is applied by reload, not restart:
// the cluster keeps serving.
// RestUser is the user Patroni's REST API asks for before a change.
const RestUser = "patroni"
// TLSDir holds each member's certificate, key and the cluster's CA.
const TLSDir = "/etc/patroni/tls"
// ServerCert, ServerKey and CACert are the files in TLSDir.
const (
ServerCert = TLSDir + "/server.crt"
ServerKey = TLSDir + "/server.key"
CACert = TLSDir + "/ca.crt"
)
// sslParameters are what turn TLS on, in the cluster's shared parameters.
func sslParameters() map[string]any {
return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert}
}
// hostssl rewrites a remote rule to accept only TLS; local and loopback
// rules are left alone.
func hostssl(line string) string {
f := strings.Fields(line)
if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" {
return line
}
return "hostssl" + strings.TrimPrefix(line, "host")
}
// certs gives each database member a certificate from the cluster's
// authority, for its address and name — unless it has one for them already.
func (s *Setup) certs(ctx context.Context) error {
if s.CA == nil {
s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS")
return nil
}
for _, d := range s.DB {
have, _ := d.Remote.Run(ctx, "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName 2>/dev/null", nil)
if strings.Contains(string(have), "IP Address:"+d.IP) {
continue
}
leaf, err := s.CA.IssueServer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"})
if err != nil {
return err
}
var body bytes.Buffer
body.Write(s.CA.PEM)
body.WriteString("\x00")
body.Write(leaf.CertPEM)
body.WriteString("\x00")
body.Write(leaf.KeyPEM)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -o postgres -g postgres -m 750 `+TLSDir+`
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
cat > "$t/all"
python3 - "$t" <<'PY'
import sys
d = sys.argv[1]
ca, crt, key = open(d + "/all", "rb").read().split(b"\x00")
for name, data in (("ca.crt", ca), ("server.crt", crt), ("server.key", key)):
open(d + "/" + name, "wb").write(data)
PY
sudo install -o postgres -g postgres -m 644 "$t/ca.crt" `+CACert+`
sudo install -o postgres -g postgres -m 644 "$t/server.crt" `+ServerCert+`
sudo install -o postgres -g postgres -m 600 "$t/server.key" `+ServerKey, body.Bytes()); err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP)
}
return nil
}
// restAuth makes each member's Patroni ask for the password before a change,
// and reloads it where that was not so.
func (s *Setup) restAuth(ctx context.Context, sec Secrets) error {
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
# The password from stdin into a file only this user reads: the program
# itself comes in on python's stdin.
f=$(mktemp); trap 'rm -f "$f"' EXIT
cat > "$f"
r=$(sudo python3 - "$f" <<'PY'
import sys, yaml
pw = open(sys.argv[1]).read().strip()
p = "/etc/patroni/config.yml"
c = yaml.safe_load(open(p))
want = {"username": "`+RestUser+`", "password": pw}
api = c.setdefault("restapi", {})
if api.get("authentication") == want:
print("same")
else:
api["authentication"] = want
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
print("changed")
PY
)
if [ "$r" = changed ]; then
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl reload patroni
fi
echo "$r"`, []byte(sec.RestAPI+"\n"))
if err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
switch strings.TrimSpace(lastLine(out)) {
case "changed":
s.emit("patroni api", true, "%s asks for a password before a change", d.Name)
case "missing":
return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
}
}
return nil
}
// tlsInShared turns TLS on in the cluster's shared configuration, and every
// remote access rule to hostssl, through Patroni's API — which reloads
// PostgreSQL on every member.
func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error {
if s.CA == nil {
return nil
}
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
client := &http.Client{Timeout: 15 * time.Second}
var lastErr error
for _, h := range hosts {
base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort))
cfg, err := getConfig(ctx, client, base)
if err != nil {
lastErr = err
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
var hba, next []string
if list, ok := pg["pg_hba"].([]any); ok {
for _, l := range list {
if line, ok := l.(string); ok {
hba = append(hba, line)
next = append(next, hostssl(line))
}
}
}
want := sslParameters()
same := slices.Equal(hba, next)
for k, v := range want {
if params[k] != v {
same = false
}
}
if same {
s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl")
return nil
}
patch := map[string]any{"parameters": want}
if len(hba) > 0 {
patch["pg_hba"] = next
}
if err := patchConfig(ctx, client, base, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil {
return fmt.Errorf("Patroni on %s: %w", h, err)
}
s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl")
return nil
}
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
}
// patchConfig changes the cluster's shared configuration, with the REST
// password.
func patchConfig(ctx context.Context, client *http.Client, base, password string, body map[string]any) error {
b, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(b))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if password != "" {
req.SetBasicAuth(RestUser, password)
}
resp, err := client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("refused the change: %s", resp.Status)
}
return nil
}
// sharedTLS reports whether the cluster's shared configuration has TLS on:
// for a control plane without the authority's key, which signs nothing but
// must still connect by TLS once the cluster asks for it.
func (s *Setup) sharedTLS(ctx context.Context) bool {
client := &http.Client{Timeout: 10 * time.Second}
for _, d := range s.DB {
cfg, err := getConfig(ctx, client, "http://"+net.JoinHostPort(d.IP, strconv.Itoa(restPort)))
if err != nil {
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
return params["ssl"] == "on"
}
return false
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return lines[len(lines)-1]
}