Members' etcd speaks TLS to peers and clients with a certificate from the cluster's authority and asks clients for one; Patroni reaches it with the member's certificate and serves its API over TLS. A running plain-HTTP cluster is moved with Patroni paused, so PostgreSQL serves throughout: Patroni's lease-free keys are carried over to the new etcd. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
237 lines
7.7 KiB
Go
237 lines
7.7 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Moving a running cluster's etcd to TLS cannot be done member by member:
|
|
// a member's peers reach it at the address the cluster recorded, http, and
|
|
// changing that one member at a time loses the majority half way. So it is
|
|
// done the way a first install is — every member's configuration rewritten
|
|
// and started together, empty — with what Patroni keeps there carried over.
|
|
//
|
|
// While etcd is down Patroni would demote the primary. So the cluster is in
|
|
// maintenance mode (pause) throughout: Patroni stops managing PostgreSQL,
|
|
// which goes on serving, and a lost leader lock means nothing. Patroni is
|
|
// then restarted against the new etcd, which in maintenance mode leaves
|
|
// PostgreSQL running, and maintenance ends.
|
|
|
|
// etcdToTLS moves a healthy plain-HTTP etcd, and Patroni with it, to TLS.
|
|
func (s *Setup) etcdToTLS(ctx context.Context, sec Secrets) error {
|
|
nodes := s.etcdMembers()
|
|
if err := s.etcdCerts(ctx, nodes); err != nil {
|
|
return err
|
|
}
|
|
if err := s.certs(ctx); err != nil {
|
|
return err
|
|
}
|
|
if err := s.pause(ctx, sec, true); err != nil {
|
|
return fmt.Errorf("maintenance mode: %w", err)
|
|
}
|
|
s.emit("etcd", true, "Patroni in maintenance mode: PostgreSQL serves on while etcd moves to TLS")
|
|
|
|
kept, err := s.etcdDump(ctx, nodes[0])
|
|
if err != nil {
|
|
return s.stuck(fmt.Errorf("reading what Patroni keeps in etcd: %w", err))
|
|
}
|
|
if err := s.etcdBootstrap(ctx, true); err != nil {
|
|
return s.stuck(err)
|
|
}
|
|
if err := s.etcdRestore(ctx, nodes[0], kept); err != nil {
|
|
return s.stuck(fmt.Errorf("putting Patroni's keys back: %w", err))
|
|
}
|
|
s.emit("etcd", true, "%d of Patroni's keys carried over", len(kept))
|
|
|
|
if _, err := s.patroniOnTLS(ctx, true); err != nil {
|
|
return s.stuck(err)
|
|
}
|
|
hosts := s.dbHosts()
|
|
if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Leader() != "" }); err != nil {
|
|
return s.stuck(fmt.Errorf("Patroni against the new etcd: %w", err))
|
|
}
|
|
if err := s.pause(ctx, sec, false); err != nil {
|
|
return s.stuck(fmt.Errorf("ending maintenance mode: %w", err))
|
|
}
|
|
if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil {
|
|
return err
|
|
}
|
|
s.emit("etcd", true, "etcd and Patroni's API speak TLS; maintenance mode ended")
|
|
return nil
|
|
}
|
|
|
|
// stuck says where a move that failed half way leaves the cluster.
|
|
func (s *Setup) stuck(err error) error {
|
|
return fmt.Errorf("%w — the cluster is left in maintenance mode, PostgreSQL still serving; "+
|
|
"run this again, or end it with `patronictl -c /etc/patroni/config.yml resume` on a member", err)
|
|
}
|
|
|
|
func (s *Setup) dbHosts() []string {
|
|
hosts := make([]string, len(s.DB))
|
|
for i, d := range s.DB {
|
|
hosts[i] = d.IP
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// pause turns maintenance mode on or off, and waits until every member says
|
|
// so.
|
|
func (s *Setup) pause(ctx context.Context, sec Secrets, on bool) error {
|
|
var err error
|
|
for _, h := range s.dbHosts() {
|
|
if err = patchConfig(ctx, h, sec.RestAPI, map[string]any{"pause": on}); err == nil {
|
|
break
|
|
}
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
deadline := time.Now().Add(s.Wait)
|
|
for {
|
|
all := true
|
|
for _, h := range s.dbHosts() {
|
|
b, err := restJSON(ctx, h, http.MethodGet, "/patroni", nil, "", 5*time.Second)
|
|
var st struct {
|
|
Pause bool `json:"pause"`
|
|
}
|
|
if err != nil || json.Unmarshal(b, &st) != nil || st.Pause != on {
|
|
all = false
|
|
}
|
|
}
|
|
if all {
|
|
return nil
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("not every member was in maintenance mode=%v within %s", on, s.Wait)
|
|
}
|
|
if err := sleep(ctx, 2*time.Second); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
// etcdKV is one key Patroni keeps, and whether it is tied to a lease: a
|
|
// leader lock or a member's entry, which Patroni writes again itself and
|
|
// which, put back without its lease, would never expire.
|
|
type etcdKV struct {
|
|
Key, Value []byte
|
|
Lease int64
|
|
}
|
|
|
|
func (s *Setup) etcdDump(ctx context.Context, n Node) ([]etcdKV, error) {
|
|
out, err := n.Remote.Run(ctx, etcdctl(false, []string{"http://" + n.IP + ":2379"})+
|
|
" get --prefix /service/ -w json", nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
var got struct {
|
|
KVs []struct {
|
|
Key string `json:"key"`
|
|
Value string `json:"value"`
|
|
Lease int64 `json:"lease"`
|
|
} `json:"kvs"`
|
|
}
|
|
if err := json.Unmarshal(out, &got); err != nil {
|
|
return nil, err
|
|
}
|
|
var kvs []etcdKV
|
|
for _, kv := range got.KVs {
|
|
k, err1 := base64.StdEncoding.DecodeString(kv.Key)
|
|
v, err2 := base64.StdEncoding.DecodeString(kv.Value)
|
|
if err1 != nil || err2 != nil {
|
|
return nil, fmt.Errorf("an unreadable key in etcd's answer")
|
|
}
|
|
if kv.Lease == 0 {
|
|
kvs = append(kvs, etcdKV{Key: k, Value: v})
|
|
}
|
|
}
|
|
if len(kvs) == 0 {
|
|
return nil, fmt.Errorf("etcd holds nothing of Patroni's under /service/")
|
|
}
|
|
return kvs, nil
|
|
}
|
|
|
|
func (s *Setup) etcdRestore(ctx context.Context, n Node, kvs []etcdKV) error {
|
|
for _, kv := range kvs {
|
|
if strings.ContainsAny(string(kv.Key), "'\n") {
|
|
return fmt.Errorf("a key that cannot be written back: %q", kv.Key)
|
|
}
|
|
out, err := n.Remote.Run(ctx, etcdctl(true, []string{"https://" + n.IP + ":2379"})+
|
|
" put '"+string(kv.Key)+"'", kv.Value)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: %v: %s", kv.Key, err, strings.TrimSpace(string(out)))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// patroniOnTLS has each member's Patroni reach etcd over TLS and serve its
|
|
// API over TLS, restarting those whose configuration changed. With paused
|
|
// false, maintenance mode is entered for the restart and left after.
|
|
func (s *Setup) patroniOnTLS(ctx context.Context, paused bool) (bool, error) {
|
|
var todo []Node
|
|
for _, d := range s.DB {
|
|
out, err := d.Remote.Run(ctx, patroniTLSEdit("check"), nil)
|
|
if err != nil {
|
|
return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
switch strings.TrimSpace(lastLine(out)) {
|
|
case "changed":
|
|
todo = append(todo, d)
|
|
case "missing":
|
|
return false, fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
|
|
}
|
|
}
|
|
if len(todo) == 0 {
|
|
return false, nil
|
|
}
|
|
if !paused {
|
|
return false, fmt.Errorf("Patroni on %d member(s) is not yet on TLS, which needs maintenance mode", len(todo))
|
|
}
|
|
for _, d := range todo {
|
|
if out, err := d.Remote.Run(ctx, patroniTLSEdit("apply")+`
|
|
sudo systemctl restart patroni`, nil); err != nil {
|
|
return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
|
|
}
|
|
s.emit("patroni api", true, "%s reaches etcd over TLS and serves its API over TLS", d.Name)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// patroniTLSEdit edits /etc/patroni/config.yml: "check" says whether it
|
|
// would change, "apply" writes it.
|
|
func patroniTLSEdit(mode string) string {
|
|
return `set -e
|
|
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
|
|
sudo python3 - ` + mode + ` <<'PY'
|
|
import sys, yaml
|
|
p = "/etc/patroni/config.yml"
|
|
c = yaml.safe_load(open(p))
|
|
want = {
|
|
"restapi": {"certfile": "` + ServerCert + `", "keyfile": "` + ServerKey + `", "cafile": "` + CACert + `"},
|
|
"ctl": {"cacert": "` + CACert + `"},
|
|
"etcd3": {"protocol": "https", "cacert": "` + CACert + `", "cert": "` + ServerCert + `", "key": "` + ServerKey + `"},
|
|
}
|
|
changed = False
|
|
for section, kv in want.items():
|
|
sec = c.setdefault(section, {})
|
|
for k, v in kv.items():
|
|
if sec.get(k) != v:
|
|
sec[k] = v
|
|
changed = True
|
|
if sys.argv[1] == "apply" and changed:
|
|
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
|
|
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
|
|
import os, shutil
|
|
shutil.chown(p + ".new", "root", "postgres")
|
|
os.chmod(p + ".new", 0o640)
|
|
os.replace(p + ".new", p)
|
|
print("changed" if changed else "same")
|
|
PY`
|
|
}
|