Files
llmgw/internal/pgha/tlsmove.go
T
Ilker ManapandClaude Opus 5.5 191c65e3db Put etcd and Patroni's API on TLS, and move a running cluster there in maintenance mode
Members' etcd speaks TLS to peers and clients with a certificate from the
cluster's authority and asks clients for one; Patroni reaches it with the
member's certificate and serves its API over TLS. A running plain-HTTP
cluster is moved with Patroni paused, so PostgreSQL serves throughout:
Patroni's lease-free keys are carried over to the new etcd.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 12:07:17 +02:00

237 lines
7.7 KiB
Go

package pgha
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
)
// Moving a running cluster's etcd to TLS cannot be done member by member:
// a member's peers reach it at the address the cluster recorded, http, and
// changing that one member at a time loses the majority half way. So it is
// done the way a first install is — every member's configuration rewritten
// and started together, empty — with what Patroni keeps there carried over.
//
// While etcd is down Patroni would demote the primary. So the cluster is in
// maintenance mode (pause) throughout: Patroni stops managing PostgreSQL,
// which goes on serving, and a lost leader lock means nothing. Patroni is
// then restarted against the new etcd, which in maintenance mode leaves
// PostgreSQL running, and maintenance ends.
// etcdToTLS moves a healthy plain-HTTP etcd, and Patroni with it, to TLS.
func (s *Setup) etcdToTLS(ctx context.Context, sec Secrets) error {
nodes := s.etcdMembers()
if err := s.etcdCerts(ctx, nodes); err != nil {
return err
}
if err := s.certs(ctx); err != nil {
return err
}
if err := s.pause(ctx, sec, true); err != nil {
return fmt.Errorf("maintenance mode: %w", err)
}
s.emit("etcd", true, "Patroni in maintenance mode: PostgreSQL serves on while etcd moves to TLS")
kept, err := s.etcdDump(ctx, nodes[0])
if err != nil {
return s.stuck(fmt.Errorf("reading what Patroni keeps in etcd: %w", err))
}
if err := s.etcdBootstrap(ctx, true); err != nil {
return s.stuck(err)
}
if err := s.etcdRestore(ctx, nodes[0], kept); err != nil {
return s.stuck(fmt.Errorf("putting Patroni's keys back: %w", err))
}
s.emit("etcd", true, "%d of Patroni's keys carried over", len(kept))
if _, err := s.patroniOnTLS(ctx, true); err != nil {
return s.stuck(err)
}
hosts := s.dbHosts()
if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Leader() != "" }); err != nil {
return s.stuck(fmt.Errorf("Patroni against the new etcd: %w", err))
}
if err := s.pause(ctx, sec, false); err != nil {
return s.stuck(fmt.Errorf("ending maintenance mode: %w", err))
}
if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil {
return err
}
s.emit("etcd", true, "etcd and Patroni's API speak TLS; maintenance mode ended")
return nil
}
// stuck says where a move that failed half way leaves the cluster.
func (s *Setup) stuck(err error) error {
return fmt.Errorf("%w — the cluster is left in maintenance mode, PostgreSQL still serving; "+
"run this again, or end it with `patronictl -c /etc/patroni/config.yml resume` on a member", err)
}
func (s *Setup) dbHosts() []string {
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
return hosts
}
// pause turns maintenance mode on or off, and waits until every member says
// so.
func (s *Setup) pause(ctx context.Context, sec Secrets, on bool) error {
var err error
for _, h := range s.dbHosts() {
if err = patchConfig(ctx, h, sec.RestAPI, map[string]any{"pause": on}); err == nil {
break
}
}
if err != nil {
return err
}
deadline := time.Now().Add(s.Wait)
for {
all := true
for _, h := range s.dbHosts() {
b, err := restJSON(ctx, h, http.MethodGet, "/patroni", nil, "", 5*time.Second)
var st struct {
Pause bool `json:"pause"`
}
if err != nil || json.Unmarshal(b, &st) != nil || st.Pause != on {
all = false
}
}
if all {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("not every member was in maintenance mode=%v within %s", on, s.Wait)
}
if err := sleep(ctx, 2*time.Second); err != nil {
return err
}
}
}
// etcdKV is one key Patroni keeps, and whether it is tied to a lease: a
// leader lock or a member's entry, which Patroni writes again itself and
// which, put back without its lease, would never expire.
type etcdKV struct {
Key, Value []byte
Lease int64
}
func (s *Setup) etcdDump(ctx context.Context, n Node) ([]etcdKV, error) {
out, err := n.Remote.Run(ctx, etcdctl(false, []string{"http://" + n.IP + ":2379"})+
" get --prefix /service/ -w json", nil)
if err != nil {
return nil, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
}
var got struct {
KVs []struct {
Key string `json:"key"`
Value string `json:"value"`
Lease int64 `json:"lease"`
} `json:"kvs"`
}
if err := json.Unmarshal(out, &got); err != nil {
return nil, err
}
var kvs []etcdKV
for _, kv := range got.KVs {
k, err1 := base64.StdEncoding.DecodeString(kv.Key)
v, err2 := base64.StdEncoding.DecodeString(kv.Value)
if err1 != nil || err2 != nil {
return nil, fmt.Errorf("an unreadable key in etcd's answer")
}
if kv.Lease == 0 {
kvs = append(kvs, etcdKV{Key: k, Value: v})
}
}
if len(kvs) == 0 {
return nil, fmt.Errorf("etcd holds nothing of Patroni's under /service/")
}
return kvs, nil
}
func (s *Setup) etcdRestore(ctx context.Context, n Node, kvs []etcdKV) error {
for _, kv := range kvs {
if strings.ContainsAny(string(kv.Key), "'\n") {
return fmt.Errorf("a key that cannot be written back: %q", kv.Key)
}
out, err := n.Remote.Run(ctx, etcdctl(true, []string{"https://" + n.IP + ":2379"})+
" put '"+string(kv.Key)+"'", kv.Value)
if err != nil {
return fmt.Errorf("%s: %v: %s", kv.Key, err, strings.TrimSpace(string(out)))
}
}
return nil
}
// patroniOnTLS has each member's Patroni reach etcd over TLS and serve its
// API over TLS, restarting those whose configuration changed. With paused
// false, maintenance mode is entered for the restart and left after.
func (s *Setup) patroniOnTLS(ctx context.Context, paused bool) (bool, error) {
var todo []Node
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, patroniTLSEdit("check"), nil)
if err != nil {
return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
switch strings.TrimSpace(lastLine(out)) {
case "changed":
todo = append(todo, d)
case "missing":
return false, fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
}
}
if len(todo) == 0 {
return false, nil
}
if !paused {
return false, fmt.Errorf("Patroni on %d member(s) is not yet on TLS, which needs maintenance mode", len(todo))
}
for _, d := range todo {
if out, err := d.Remote.Run(ctx, patroniTLSEdit("apply")+`
sudo systemctl restart patroni`, nil); err != nil {
return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
s.emit("patroni api", true, "%s reaches etcd over TLS and serves its API over TLS", d.Name)
}
return true, nil
}
// patroniTLSEdit edits /etc/patroni/config.yml: "check" says whether it
// would change, "apply" writes it.
func patroniTLSEdit(mode string) string {
return `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
sudo python3 - ` + mode + ` <<'PY'
import sys, yaml
p = "/etc/patroni/config.yml"
c = yaml.safe_load(open(p))
want = {
"restapi": {"certfile": "` + ServerCert + `", "keyfile": "` + ServerKey + `", "cafile": "` + CACert + `"},
"ctl": {"cacert": "` + CACert + `"},
"etcd3": {"protocol": "https", "cacert": "` + CACert + `", "cert": "` + ServerCert + `", "key": "` + ServerKey + `"},
}
changed = False
for section, kv in want.items():
sec = c.setdefault(section, {})
for k, v in kv.items():
if sec.get(k) != v:
sec[k] = v
changed = True
if sys.argv[1] == "apply" and changed:
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
import os, shutil
shutil.chown(p + ".new", "root", "postgres")
os.chmod(p + ".new", 0o640)
os.replace(p + ".new", p)
print("changed" if changed else "same")
PY`
}