Files
llmgw/internal/pgha/harden.go
T
Ilker ManapandClaude Opus 5.5 43d3c12302 Make a member's own access rules hostssl when the cluster keeps them there
A cluster set up before the rules moved into the shared configuration kept
them in each member's config.yml, which the TLS step never looked at — and
then said its remote rules were hostssl. Found on the PoC: plaintext
connections were still accepted.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 12:11:05 +02:00

273 lines
9.0 KiB
Go

package pgha
import (
"context"
"encoding/json"
"fmt"
"net/http"
"slices"
"strings"
"time"
)
// Hardening: what a cluster set up before this lacked, applied to it in
// place, and to a new one from the start.
//
// - Patroni's REST API asks for a password before a change. Without one,
// anyone who reached port 8008 could rewrite pg_hba through PATCH /config
// — `host all all 0.0.0.0/0 trust` — or switch the leader over.
// - PostgreSQL speaks TLS, with a certificate from the cluster's authority
// on each member, and every remote rule is hostssl: prompts kept in
// threads, key and session hashes, and the engines' secret no longer
// cross the network in clear. The control planes verify the member they
// reach (sslmode=verify-full against the cluster's CA).
//
// Each step looks first and changes only what differs, so running it on a
// hardened cluster does nothing, and each is applied by reload, not restart:
// the cluster keeps serving.
// RestUser is the user Patroni's REST API asks for before a change.
const RestUser = "patroni"
// TLSDir holds each member's certificate, key and the cluster's CA.
const TLSDir = "/etc/patroni/tls"
// ServerCert, ServerKey and CACert are the files in TLSDir.
const (
ServerCert = TLSDir + "/server.crt"
ServerKey = TLSDir + "/server.key"
CACert = TLSDir + "/ca.crt"
)
// sslParameters are what turn TLS on, in the cluster's shared parameters.
func sslParameters() map[string]any {
return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert}
}
// hostssl rewrites a remote rule to accept only TLS; local and loopback
// rules are left alone.
func hostssl(line string) string {
f := strings.Fields(line)
if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" {
return line
}
return "hostssl" + strings.TrimPrefix(line, "host")
}
// certs gives each database member a certificate from the cluster's
// authority, for its address and name — unless it has one for them already.
// It serves PostgreSQL and Patroni's API, and is what Patroni presents to
// etcd, so it identifies a client too. A member without PostgreSQL's user
// yet is skipped: the Patroni step calls this again once it is installed.
func (s *Setup) certs(ctx context.Context) error {
if s.CA == nil {
s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS")
return nil
}
for _, d := range s.DB {
have, _ := d.Remote.Run(ctx, "id postgres >/dev/null 2>&1 || echo no-postgres; "+
"sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName,extendedKeyUsage 2>/dev/null", nil)
if strings.Contains(string(have), "no-postgres") {
continue
}
if strings.Contains(string(have), "IP Address:"+d.IP) && strings.Contains(string(have), "Client Authentication") {
continue
}
leaf, err := s.CA.IssuePeer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"})
if err != nil {
return err
}
if err := putTLS(ctx, d, "postgres", TLSDir, CACert, ServerCert, ServerKey, s.CA.PEM, leaf.CertPEM, leaf.KeyPEM); err != nil {
return err
}
s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP)
}
return nil
}
// restAuth makes each member's Patroni ask for the password before a change,
// and reloads it where that was not so.
func (s *Setup) restAuth(ctx context.Context, sec Secrets) error {
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
# The password from stdin into a file only this user reads: the program
# itself comes in on python's stdin.
f=$(mktemp); trap 'rm -f "$f"' EXIT
cat > "$f"
r=$(sudo python3 - "$f" <<'PY'
import sys, yaml
pw = open(sys.argv[1]).read().strip()
p = "/etc/patroni/config.yml"
c = yaml.safe_load(open(p))
want = {"username": "`+RestUser+`", "password": pw}
api = c.setdefault("restapi", {})
if api.get("authentication") == want:
print("same")
else:
api["authentication"] = want
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
print("changed")
PY
)
if [ "$r" = changed ]; then
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl reload patroni
fi
echo "$r"`, []byte(sec.RestAPI+"\n"))
if err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
switch strings.TrimSpace(lastLine(out)) {
case "changed":
s.emit("patroni api", true, "%s asks for a password before a change", d.Name)
case "missing":
return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
}
}
return nil
}
// tlsInShared turns TLS on in the cluster's shared configuration, and every
// remote access rule to hostssl, through Patroni's API — which reloads
// PostgreSQL on every member.
func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error {
if s.CA == nil {
return nil
}
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
var lastErr error
for _, h := range hosts {
cfg, err := getConfig(ctx, h)
if err != nil {
lastErr = err
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
var hba, next []string
if list, ok := pg["pg_hba"].([]any); ok {
for _, l := range list {
if line, ok := l.(string); ok {
hba = append(hba, line)
next = append(next, hostssl(line))
}
}
}
if len(hba) == 0 {
// Kept in each member's own file, by a cluster set up before
// they moved here: made hostssl there.
if err := s.memberHostssl(ctx); err != nil {
return err
}
}
want := sslParameters()
same := slices.Equal(hba, next)
for k, v := range want {
if params[k] != v {
same = false
}
}
if same {
s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl")
return nil
}
patch := map[string]any{"parameters": want}
if len(hba) > 0 {
patch["pg_hba"] = next
}
if err := patchConfig(ctx, h, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil {
return fmt.Errorf("Patroni on %s: %w", h, err)
}
s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl")
return nil
}
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
}
// memberHostssl makes the remote rules in each member's own
// /etc/patroni/config.yml hostssl, and reloads Patroni there, which rewrites
// pg_hba.conf and reloads PostgreSQL without restarting it.
func (s *Setup) memberHostssl(ctx context.Context) error {
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
r=$(sudo python3 - <<'PY'
import yaml
p = "/etc/patroni/config.yml"
c = yaml.safe_load(open(p))
rules = c.get("postgresql", {}).get("pg_hba") or []
def hostssl(line):
f = line.split()
if len(f) < 4 or f[0] != "host" or f[3].startswith("127.") or f[3] == "::1/128":
return line
return "hostssl" + line[len("host"):]
new = [hostssl(l) for l in rules]
if new == rules:
print("same")
else:
c["postgresql"]["pg_hba"] = new
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
print("changed")
PY
)
if [ "$r" = changed ]; then
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl reload patroni
fi
echo "$r"`, nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
if strings.TrimSpace(lastLine(out)) == "changed" {
s.emit("tls", true, "%s's own access rules are hostssl", d.Name)
}
}
return nil
}
// patchConfig changes the cluster's shared configuration, with the REST
// password.
func patchConfig(ctx context.Context, host, password string, body map[string]any) error {
b, _ := json.Marshal(body)
resp, err := restRequest(ctx, host, http.MethodPatch, "/config", b, password, 15*time.Second)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("refused the change: %s", resp.Status)
}
return nil
}
// sharedTLS reports whether the cluster's shared configuration has TLS on:
// for a control plane without the authority's key, which signs nothing but
// must still connect by TLS once the cluster asks for it.
func (s *Setup) sharedTLS(ctx context.Context) bool {
for _, d := range s.DB {
cfg, err := getConfig(ctx, d.IP)
if err != nil {
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
return params["ssl"] == "on"
}
return false
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return lines[len(lines)-1]
}