package pgha import ( "context" "encoding/base64" "encoding/json" "fmt" "net/http" "strings" "time" ) // Moving a running cluster's etcd to TLS cannot be done member by member: // a member's peers reach it at the address the cluster recorded, http, and // changing that one member at a time loses the majority half way. So it is // done the way a first install is — every member's configuration rewritten // and started together, empty — with what Patroni keeps there carried over. // // While etcd is down Patroni would demote the primary. So the cluster is in // maintenance mode (pause) throughout: Patroni stops managing PostgreSQL, // which goes on serving, and a lost leader lock means nothing. Patroni is // then restarted against the new etcd, which in maintenance mode leaves // PostgreSQL running, and maintenance ends. // etcdToTLS moves a healthy plain-HTTP etcd, and Patroni with it, to TLS. func (s *Setup) etcdToTLS(ctx context.Context, sec Secrets) error { nodes := s.etcdMembers() if err := s.etcdCerts(ctx, nodes); err != nil { return err } if err := s.certs(ctx); err != nil { return err } if err := s.pause(ctx, sec, true); err != nil { return fmt.Errorf("maintenance mode: %w", err) } s.emit("etcd", true, "Patroni in maintenance mode: PostgreSQL serves on while etcd moves to TLS") kept, err := s.etcdDump(ctx, nodes[0]) if err != nil { return s.stuck(fmt.Errorf("reading what Patroni keeps in etcd: %w", err)) } if err := s.etcdBootstrap(ctx, true); err != nil { return s.stuck(err) } if err := s.etcdRestore(ctx, nodes[0], kept); err != nil { return s.stuck(fmt.Errorf("putting Patroni's keys back: %w", err)) } s.emit("etcd", true, "%d of Patroni's keys carried over", len(kept)) if _, err := s.patroniOnTLS(ctx, true); err != nil { return s.stuck(err) } hosts := s.dbHosts() if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Leader() != "" }); err != nil { return s.stuck(fmt.Errorf("Patroni against the new etcd: %w", err)) } if err := s.pause(ctx, sec, false); err != nil { return s.stuck(fmt.Errorf("ending maintenance mode: %w", err)) } if err := s.waitFor(ctx, hosts, func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil { return err } s.emit("etcd", true, "etcd and Patroni's API speak TLS; maintenance mode ended") return nil } // stuck says where a move that failed half way leaves the cluster. func (s *Setup) stuck(err error) error { return fmt.Errorf("%w — the cluster is left in maintenance mode, PostgreSQL still serving; "+ "run this again, or end it with `patronictl -c /etc/patroni/config.yml resume` on a member", err) } func (s *Setup) dbHosts() []string { hosts := make([]string, len(s.DB)) for i, d := range s.DB { hosts[i] = d.IP } return hosts } // pause turns maintenance mode on or off, and waits until every member says // so. func (s *Setup) pause(ctx context.Context, sec Secrets, on bool) error { var err error for _, h := range s.dbHosts() { if err = patchConfig(ctx, h, sec.RestAPI, map[string]any{"pause": on}); err == nil { break } } if err != nil { return err } deadline := time.Now().Add(s.Wait) for { all := true for _, h := range s.dbHosts() { b, err := restJSON(ctx, h, http.MethodGet, "/patroni", nil, "", 5*time.Second) var st struct { Pause bool `json:"pause"` } if err != nil || json.Unmarshal(b, &st) != nil || st.Pause != on { all = false } } if all { return nil } if time.Now().After(deadline) { return fmt.Errorf("not every member was in maintenance mode=%v within %s", on, s.Wait) } if err := sleep(ctx, 2*time.Second); err != nil { return err } } } // etcdKV is one key Patroni keeps, and whether it is tied to a lease: a // leader lock or a member's entry, which Patroni writes again itself and // which, put back without its lease, would never expire. type etcdKV struct { Key, Value []byte Lease int64 } func (s *Setup) etcdDump(ctx context.Context, n Node) ([]etcdKV, error) { out, err := n.Remote.Run(ctx, etcdctl(false, []string{"http://" + n.IP + ":2379"})+ " get --prefix /service/ -w json", nil) if err != nil { return nil, fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) } var got struct { KVs []struct { Key string `json:"key"` Value string `json:"value"` Lease int64 `json:"lease"` } `json:"kvs"` } if err := json.Unmarshal(out, &got); err != nil { return nil, err } var kvs []etcdKV for _, kv := range got.KVs { k, err1 := base64.StdEncoding.DecodeString(kv.Key) v, err2 := base64.StdEncoding.DecodeString(kv.Value) if err1 != nil || err2 != nil { return nil, fmt.Errorf("an unreadable key in etcd's answer") } if kv.Lease == 0 { kvs = append(kvs, etcdKV{Key: k, Value: v}) } } if len(kvs) == 0 { return nil, fmt.Errorf("etcd holds nothing of Patroni's under /service/") } return kvs, nil } func (s *Setup) etcdRestore(ctx context.Context, n Node, kvs []etcdKV) error { for _, kv := range kvs { if strings.ContainsAny(string(kv.Key), "'\n") { return fmt.Errorf("a key that cannot be written back: %q", kv.Key) } out, err := n.Remote.Run(ctx, etcdctl(true, []string{"https://" + n.IP + ":2379"})+ " put '"+string(kv.Key)+"'", kv.Value) if err != nil { return fmt.Errorf("%s: %v: %s", kv.Key, err, strings.TrimSpace(string(out))) } } return nil } // patroniOnTLS has each member's Patroni reach etcd over TLS and serve its // API over TLS, restarting those whose configuration changed. With paused // false, maintenance mode is entered for the restart and left after. func (s *Setup) patroniOnTLS(ctx context.Context, paused bool) (bool, error) { var todo []Node for _, d := range s.DB { out, err := d.Remote.Run(ctx, patroniTLSEdit("check"), nil) if err != nil { return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) } switch strings.TrimSpace(lastLine(out)) { case "changed": todo = append(todo, d) case "missing": return false, fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name) } } if len(todo) == 0 { return false, nil } if !paused { return false, fmt.Errorf("Patroni on %d member(s) is not yet on TLS, which needs maintenance mode", len(todo)) } for _, d := range todo { if out, err := d.Remote.Run(ctx, patroniTLSEdit("apply")+` sudo systemctl restart patroni`, nil); err != nil { return false, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) } s.emit("patroni api", true, "%s reaches etcd over TLS and serves its API over TLS", d.Name) } return true, nil } // patroniTLSEdit edits /etc/patroni/config.yml: "check" says whether it // would change, "apply" writes it. func patroniTLSEdit(mode string) string { return `set -e test -f /etc/patroni/config.yml || { echo missing; exit 0; } sudo python3 - ` + mode + ` <<'PY' import sys, yaml p = "/etc/patroni/config.yml" c = yaml.safe_load(open(p)) want = { "restapi": {"certfile": "` + ServerCert + `", "keyfile": "` + ServerKey + `", "cafile": "` + CACert + `"}, "ctl": {"cacert": "` + CACert + `"}, "etcd3": {"protocol": "https", "cacert": "` + CACert + `", "cert": "` + ServerCert + `", "key": "` + ServerKey + `"}, } changed = False for section, kv in want.items(): sec = c.setdefault(section, {}) for k, v in kv.items(): if sec.get(k) != v: sec[k] = v changed = True if sys.argv[1] == "apply" and changed: open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" + yaml.safe_dump(c, default_flow_style=False, sort_keys=False)) import os, shutil shutil.chown(p + ".new", "root", "postgres") os.chmod(p + ".new", 0o640) os.replace(p + ".new", p) print("changed" if changed else "same") PY` }