package pgha import ( "context" "encoding/json" "fmt" "net/http" "slices" "strings" "time" ) // Hardening: what a cluster set up before this lacked, applied to it in // place, and to a new one from the start. // // - Patroni's REST API asks for a password before a change. Without one, // anyone who reached port 8008 could rewrite pg_hba through PATCH /config // — `host all all 0.0.0.0/0 trust` — or switch the leader over. // - PostgreSQL speaks TLS, with a certificate from the cluster's authority // on each member, and every remote rule is hostssl: prompts kept in // threads, key and session hashes, and the engines' secret no longer // cross the network in clear. The control planes verify the member they // reach (sslmode=verify-full against the cluster's CA). // // Each step looks first and changes only what differs, so running it on a // hardened cluster does nothing, and each is applied by reload, not restart: // the cluster keeps serving. // RestUser is the user Patroni's REST API asks for before a change. const RestUser = "patroni" // TLSDir holds each member's certificate, key and the cluster's CA. const TLSDir = "/etc/patroni/tls" // ServerCert, ServerKey and CACert are the files in TLSDir. const ( ServerCert = TLSDir + "/server.crt" ServerKey = TLSDir + "/server.key" CACert = TLSDir + "/ca.crt" ) // sslParameters are what turn TLS on, in the cluster's shared parameters. func sslParameters() map[string]any { return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert} } // hostssl rewrites a remote rule to accept only TLS; local and loopback // rules are left alone. func hostssl(line string) string { f := strings.Fields(line) if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" { return line } return "hostssl" + strings.TrimPrefix(line, "host") } // certs gives each database member a certificate from the cluster's // authority, for its address and name — unless it has one for them already. // It serves PostgreSQL and Patroni's API, and is what Patroni presents to // etcd, so it identifies a client too. A member without PostgreSQL's user // yet is skipped: the Patroni step calls this again once it is installed. func (s *Setup) certs(ctx context.Context) error { if s.CA == nil { s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS") return nil } for _, d := range s.DB { have, _ := d.Remote.Run(ctx, "id postgres >/dev/null 2>&1 || echo no-postgres; "+ "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName,extendedKeyUsage 2>/dev/null", nil) if strings.Contains(string(have), "no-postgres") { continue } if strings.Contains(string(have), "IP Address:"+d.IP) && strings.Contains(string(have), "Client Authentication") { continue } leaf, err := s.CA.IssuePeer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"}) if err != nil { return err } if err := putTLS(ctx, d, "postgres", TLSDir, CACert, ServerCert, ServerKey, s.CA.PEM, leaf.CertPEM, leaf.KeyPEM); err != nil { return err } s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP) } return nil } // restAuth makes each member's Patroni ask for the password before a change, // and reloads it where that was not so. func (s *Setup) restAuth(ctx context.Context, sec Secrets) error { for _, d := range s.DB { out, err := d.Remote.Run(ctx, `set -e test -f /etc/patroni/config.yml || { echo missing; exit 0; } # The password from stdin into a file only this user reads: the program # itself comes in on python's stdin. f=$(mktemp); trap 'rm -f "$f"' EXIT cat > "$f" r=$(sudo python3 - "$f" <<'PY' import sys, yaml pw = open(sys.argv[1]).read().strip() p = "/etc/patroni/config.yml" c = yaml.safe_load(open(p)) want = {"username": "`+RestUser+`", "password": pw} api = c.setdefault("restapi", {}) if api.get("authentication") == want: print("same") else: api["authentication"] = want open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" + yaml.safe_dump(c, default_flow_style=False, sort_keys=False)) print("changed") PY ) if [ "$r" = changed ]; then sudo chown root:postgres /etc/patroni/config.yml.new sudo chmod 640 /etc/patroni/config.yml.new sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml sudo systemctl reload patroni fi echo "$r"`, []byte(sec.RestAPI+"\n")) if err != nil { return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) } switch strings.TrimSpace(lastLine(out)) { case "changed": s.emit("patroni api", true, "%s asks for a password before a change", d.Name) case "missing": return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name) } } return nil } // tlsInShared turns TLS on in the cluster's shared configuration, and every // remote access rule to hostssl, through Patroni's API — which reloads // PostgreSQL on every member. func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error { if s.CA == nil { return nil } hosts := make([]string, len(s.DB)) for i, d := range s.DB { hosts[i] = d.IP } var lastErr error for _, h := range hosts { cfg, err := getConfig(ctx, h) if err != nil { lastErr = err continue } pg, _ := cfg["postgresql"].(map[string]any) params, _ := pg["parameters"].(map[string]any) var hba, next []string if list, ok := pg["pg_hba"].([]any); ok { for _, l := range list { if line, ok := l.(string); ok { hba = append(hba, line) next = append(next, hostssl(line)) } } } if len(hba) == 0 { // Kept in each member's own file, by a cluster set up before // they moved here: made hostssl there. if err := s.memberHostssl(ctx); err != nil { return err } } want := sslParameters() same := slices.Equal(hba, next) for k, v := range want { if params[k] != v { same = false } } if same { s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl") return nil } patch := map[string]any{"parameters": want} if len(hba) > 0 { patch["pg_hba"] = next } if err := patchConfig(ctx, h, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil { return fmt.Errorf("Patroni on %s: %w", h, err) } s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl") return nil } return fmt.Errorf("no member's Patroni answered: %w", lastErr) } // memberHostssl makes the remote rules in each member's own // /etc/patroni/config.yml hostssl, and reloads Patroni there, which rewrites // pg_hba.conf and reloads PostgreSQL without restarting it. func (s *Setup) memberHostssl(ctx context.Context) error { for _, d := range s.DB { out, err := d.Remote.Run(ctx, `set -e test -f /etc/patroni/config.yml || { echo missing; exit 0; } r=$(sudo python3 - <<'PY' import yaml p = "/etc/patroni/config.yml" c = yaml.safe_load(open(p)) rules = c.get("postgresql", {}).get("pg_hba") or [] def hostssl(line): f = line.split() if len(f) < 4 or f[0] != "host" or f[3].startswith("127.") or f[3] == "::1/128": return line return "hostssl" + line[len("host"):] new = [hostssl(l) for l in rules] if new == rules: print("same") else: c["postgresql"]["pg_hba"] = new open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" + yaml.safe_dump(c, default_flow_style=False, sort_keys=False)) print("changed") PY ) if [ "$r" = changed ]; then sudo chown root:postgres /etc/patroni/config.yml.new sudo chmod 640 /etc/patroni/config.yml.new sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml sudo systemctl reload patroni fi echo "$r"`, nil) if err != nil { return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) } if strings.TrimSpace(lastLine(out)) == "changed" { s.emit("tls", true, "%s's own access rules are hostssl", d.Name) } } return nil } // patchConfig changes the cluster's shared configuration, with the REST // password. func patchConfig(ctx context.Context, host, password string, body map[string]any) error { b, _ := json.Marshal(body) resp, err := restRequest(ctx, host, http.MethodPatch, "/config", b, password, 15*time.Second) if err != nil { return err } resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("refused the change: %s", resp.Status) } return nil } // sharedTLS reports whether the cluster's shared configuration has TLS on: // for a control plane without the authority's key, which signs nothing but // must still connect by TLS once the cluster asks for it. func (s *Setup) sharedTLS(ctx context.Context) bool { for _, d := range s.DB { cfg, err := getConfig(ctx, d.IP) if err != nil { continue } pg, _ := cfg["postgresql"].(map[string]any) params, _ := pg["parameters"].(map[string]any) return params["ssl"] == "on" } return false } func lastLine(b []byte) string { lines := strings.Split(strings.TrimSpace(string(b)), "\n") return lines[len(lines)-1] }