A cluster set up before the rules moved into the shared configuration kept them in each member's config.yml, which the TLS step never looked at — and then said its remote rules were hostssl. Found on the PoC: plaintext connections were still accepted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
273 lines
9.0 KiB
Go
273 lines
9.0 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Hardening: what a cluster set up before this lacked, applied to it in
|
|
// place, and to a new one from the start.
|
|
//
|
|
// - Patroni's REST API asks for a password before a change. Without one,
|
|
// anyone who reached port 8008 could rewrite pg_hba through PATCH /config
|
|
// — `host all all 0.0.0.0/0 trust` — or switch the leader over.
|
|
// - PostgreSQL speaks TLS, with a certificate from the cluster's authority
|
|
// on each member, and every remote rule is hostssl: prompts kept in
|
|
// threads, key and session hashes, and the engines' secret no longer
|
|
// cross the network in clear. The control planes verify the member they
|
|
// reach (sslmode=verify-full against the cluster's CA).
|
|
//
|
|
// Each step looks first and changes only what differs, so running it on a
|
|
// hardened cluster does nothing, and each is applied by reload, not restart:
|
|
// the cluster keeps serving.
|
|
|
|
// RestUser is the user Patroni's REST API asks for before a change.
|
|
const RestUser = "patroni"
|
|
|
|
// TLSDir holds each member's certificate, key and the cluster's CA.
|
|
const TLSDir = "/etc/patroni/tls"
|
|
|
|
// ServerCert, ServerKey and CACert are the files in TLSDir.
|
|
const (
|
|
ServerCert = TLSDir + "/server.crt"
|
|
ServerKey = TLSDir + "/server.key"
|
|
CACert = TLSDir + "/ca.crt"
|
|
)
|
|
|
|
// sslParameters are what turn TLS on, in the cluster's shared parameters.
|
|
func sslParameters() map[string]any {
|
|
return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert}
|
|
}
|
|
|
|
// hostssl rewrites a remote rule to accept only TLS; local and loopback
|
|
// rules are left alone.
|
|
func hostssl(line string) string {
|
|
f := strings.Fields(line)
|
|
if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" {
|
|
return line
|
|
}
|
|
return "hostssl" + strings.TrimPrefix(line, "host")
|
|
}
|
|
|
|
// certs gives each database member a certificate from the cluster's
|
|
// authority, for its address and name — unless it has one for them already.
|
|
// It serves PostgreSQL and Patroni's API, and is what Patroni presents to
|
|
// etcd, so it identifies a client too. A member without PostgreSQL's user
|
|
// yet is skipped: the Patroni step calls this again once it is installed.
|
|
func (s *Setup) certs(ctx context.Context) error {
|
|
if s.CA == nil {
|
|
s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS")
|
|
return nil
|
|
}
|
|
for _, d := range s.DB {
|
|
have, _ := d.Remote.Run(ctx, "id postgres >/dev/null 2>&1 || echo no-postgres; "+
|
|
"sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName,extendedKeyUsage 2>/dev/null", nil)
|
|
if strings.Contains(string(have), "no-postgres") {
|
|
continue
|
|
}
|
|
if strings.Contains(string(have), "IP Address:"+d.IP) && strings.Contains(string(have), "Client Authentication") {
|
|
continue
|
|
}
|
|
leaf, err := s.CA.IssuePeer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := putTLS(ctx, d, "postgres", TLSDir, CACert, ServerCert, ServerKey, s.CA.PEM, leaf.CertPEM, leaf.KeyPEM); err != nil {
|
|
return err
|
|
}
|
|
s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// restAuth makes each member's Patroni ask for the password before a change,
|
|
// and reloads it where that was not so.
|
|
func (s *Setup) restAuth(ctx context.Context, sec Secrets) error {
|
|
for _, d := range s.DB {
|
|
out, err := d.Remote.Run(ctx, `set -e
|
|
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
|
|
# The password from stdin into a file only this user reads: the program
|
|
# itself comes in on python's stdin.
|
|
f=$(mktemp); trap 'rm -f "$f"' EXIT
|
|
cat > "$f"
|
|
r=$(sudo python3 - "$f" <<'PY'
|
|
import sys, yaml
|
|
pw = open(sys.argv[1]).read().strip()
|
|
p = "/etc/patroni/config.yml"
|
|
c = yaml.safe_load(open(p))
|
|
want = {"username": "`+RestUser+`", "password": pw}
|
|
api = c.setdefault("restapi", {})
|
|
if api.get("authentication") == want:
|
|
print("same")
|
|
else:
|
|
api["authentication"] = want
|
|
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
|
|
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
|
|
print("changed")
|
|
PY
|
|
)
|
|
if [ "$r" = changed ]; then
|
|
sudo chown root:postgres /etc/patroni/config.yml.new
|
|
sudo chmod 640 /etc/patroni/config.yml.new
|
|
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
|
|
sudo systemctl reload patroni
|
|
fi
|
|
echo "$r"`, []byte(sec.RestAPI+"\n"))
|
|
if err != nil {
|
|
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
switch strings.TrimSpace(lastLine(out)) {
|
|
case "changed":
|
|
s.emit("patroni api", true, "%s asks for a password before a change", d.Name)
|
|
case "missing":
|
|
return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// tlsInShared turns TLS on in the cluster's shared configuration, and every
|
|
// remote access rule to hostssl, through Patroni's API — which reloads
|
|
// PostgreSQL on every member.
|
|
func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error {
|
|
if s.CA == nil {
|
|
return nil
|
|
}
|
|
hosts := make([]string, len(s.DB))
|
|
for i, d := range s.DB {
|
|
hosts[i] = d.IP
|
|
}
|
|
var lastErr error
|
|
for _, h := range hosts {
|
|
cfg, err := getConfig(ctx, h)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
pg, _ := cfg["postgresql"].(map[string]any)
|
|
params, _ := pg["parameters"].(map[string]any)
|
|
var hba, next []string
|
|
if list, ok := pg["pg_hba"].([]any); ok {
|
|
for _, l := range list {
|
|
if line, ok := l.(string); ok {
|
|
hba = append(hba, line)
|
|
next = append(next, hostssl(line))
|
|
}
|
|
}
|
|
}
|
|
if len(hba) == 0 {
|
|
// Kept in each member's own file, by a cluster set up before
|
|
// they moved here: made hostssl there.
|
|
if err := s.memberHostssl(ctx); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
want := sslParameters()
|
|
same := slices.Equal(hba, next)
|
|
for k, v := range want {
|
|
if params[k] != v {
|
|
same = false
|
|
}
|
|
}
|
|
if same {
|
|
s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl")
|
|
return nil
|
|
}
|
|
patch := map[string]any{"parameters": want}
|
|
if len(hba) > 0 {
|
|
patch["pg_hba"] = next
|
|
}
|
|
if err := patchConfig(ctx, h, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil {
|
|
return fmt.Errorf("Patroni on %s: %w", h, err)
|
|
}
|
|
s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl")
|
|
return nil
|
|
}
|
|
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
|
|
}
|
|
|
|
// memberHostssl makes the remote rules in each member's own
|
|
// /etc/patroni/config.yml hostssl, and reloads Patroni there, which rewrites
|
|
// pg_hba.conf and reloads PostgreSQL without restarting it.
|
|
func (s *Setup) memberHostssl(ctx context.Context) error {
|
|
for _, d := range s.DB {
|
|
out, err := d.Remote.Run(ctx, `set -e
|
|
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
|
|
r=$(sudo python3 - <<'PY'
|
|
import yaml
|
|
p = "/etc/patroni/config.yml"
|
|
c = yaml.safe_load(open(p))
|
|
rules = c.get("postgresql", {}).get("pg_hba") or []
|
|
def hostssl(line):
|
|
f = line.split()
|
|
if len(f) < 4 or f[0] != "host" or f[3].startswith("127.") or f[3] == "::1/128":
|
|
return line
|
|
return "hostssl" + line[len("host"):]
|
|
new = [hostssl(l) for l in rules]
|
|
if new == rules:
|
|
print("same")
|
|
else:
|
|
c["postgresql"]["pg_hba"] = new
|
|
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
|
|
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
|
|
print("changed")
|
|
PY
|
|
)
|
|
if [ "$r" = changed ]; then
|
|
sudo chown root:postgres /etc/patroni/config.yml.new
|
|
sudo chmod 640 /etc/patroni/config.yml.new
|
|
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
|
|
sudo systemctl reload patroni
|
|
fi
|
|
echo "$r"`, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
if strings.TrimSpace(lastLine(out)) == "changed" {
|
|
s.emit("tls", true, "%s's own access rules are hostssl", d.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// patchConfig changes the cluster's shared configuration, with the REST
|
|
// password.
|
|
func patchConfig(ctx context.Context, host, password string, body map[string]any) error {
|
|
b, _ := json.Marshal(body)
|
|
resp, err := restRequest(ctx, host, http.MethodPatch, "/config", b, password, 15*time.Second)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("refused the change: %s", resp.Status)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sharedTLS reports whether the cluster's shared configuration has TLS on:
|
|
// for a control plane without the authority's key, which signs nothing but
|
|
// must still connect by TLS once the cluster asks for it.
|
|
func (s *Setup) sharedTLS(ctx context.Context) bool {
|
|
for _, d := range s.DB {
|
|
cfg, err := getConfig(ctx, d.IP)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
pg, _ := cfg["postgresql"].(map[string]any)
|
|
params, _ := pg["parameters"].(map[string]any)
|
|
return params["ssl"] == "on"
|
|
}
|
|
return false
|
|
}
|
|
|
|
func lastLine(b []byte) string {
|
|
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
|
return lines[len(lines)-1]
|
|
}
|