Ask for a password on Patroni's API and give the store TLS, new clusters and existing ones in place

Patroni's REST API accepted changes from anyone on port 8008, and
PostgreSQL ran with sslmode=disable between machines. Setup now keeps a
fourth secret for the API (carried to members of older clusters), issues
each member a certificate from the cluster's authority, writes
restapi.authentication into each member's config and reloads it, and
turns ssl on with hostssl remote rules through an authenticated PATCH of
the shared configuration. The DSN becomes sslmode=verify-full against the
authority's certificate whenever the cluster has TLS, so a control plane
without the authority's key reconnects too. server add authenticates its
PATCH and lets the new control plane in by hostssl. Tried on the e2e
cluster: unauthenticated PATCH 401, plaintext refused, replication
streaming, the control plane on verify-full.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:11:28 +02:00
1 parent 9d961c3fc8
commit 2b29016d5b
8 files changed
+558 -51

No files matched your search

+17
View File
@@ -74,6 +74,23 @@ APIs change between minors without a deprecation period.
elsewhere. pprof is off unless `debug_pprof: true`. `admin_listen` off
loopback is refused unless `admin_remote: true`. The read-only dashboard
pages, `/metrics` and `/health` are unchanged.
- **The replicated store asks for a password and speaks TLS.** Patroni's
REST API took changes from anyone who reached port 8008 — rewriting the
access rules to `trust`, or moving the leader — and PostgreSQL traffic
between the machines was plaintext (`sslmode=disable`). Patroni now asks
for a password before a change (kept with the cluster's others in
`/root/llmgw-pg`, and on the control plane in
`/etc/llmgw/secrets/patroni-rest` for `server add`); each member has a
certificate from the cluster's authority, PostgreSQL has `ssl` on, every
remote access rule is `hostssl`, and the control planes connect with
`sslmode=verify-full` against the authority's certificate. A new cluster is
set up this way; an existing one is changed in place, by reload, the
cluster serving throughout, by running **Replicate the store** in the
console again — or `sudo --preserve-env=SSH_AUTH_SOCK llmgw server init
--patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node
--no-model` — on the control plane that holds the authority, then on each
other control plane so it reconnects by TLS. etcd is not yet given TLS:
keep ports 2379 and 2380 to the cluster's machines.
### Everything that can be changed on a running cluster, from the console
+5 -1
View File
@@ -307,7 +307,11 @@ func (sa *serverAdd) storeAccess(ctx context.Context) error {
case slices.Contains(hosts, sa.ip):
sa.kept("store: %s is one of its members", sa.ip)
case len(hosts) > 1:
changed, err := pgha.AllowClient(ctx, hosts, sa.ip)
rest := ""
if b, err := os.ReadFile(patroniRestFile(sa.etcDir)); err == nil { //nolint:gosec // our own secret
rest = strings.TrimSpace(string(b))
}
changed, err := pgha.AllowClient(ctx, hosts, sa.ip, rest)
if err != nil {
return err
}
+35 -3
View File
@@ -6,6 +6,7 @@ import (
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
@@ -58,13 +59,26 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) {
setup.Witness = &pgha.Node{Member: pgha.Member{Name: name, IP: self}, Remote: pgha.Local{}}
setup.Clients = []string{self}
cfg, err := config.Load(in.configPath())
if err != nil {
return err
}
// The cluster's authority signs each member's certificate, and the
// control plane verifies the member it reaches against it: where its
// key is on this machine, and the server has the certificate to read.
setup.CAFile = cfg.AgentTLS.CA
if ca, cerr := loadCA(in.pkiDir); cerr == nil && cfg.AgentTLS.CA != "" {
setup.CA = ca
} else {
in.kept("store: the authority's key is not here, so no member certificates are signed from this machine; " +
"run this on the control plane that holds it to turn TLS on")
}
dsn, err := setup.Run(ctx)
if err != nil {
return err
}
cfg, err := config.Load(in.configPath())
if err != nil {
if err := in.writeSecret(patroniRestFile(in.etcDir), setup.RestPassword); err != nil {
return err
}
current := cfg.Auth.Store
@@ -180,3 +194,21 @@ func ipv4Of(host string) (string, error) {
}
return "", fmt.Errorf("%s has no IPv4 address", host)
}
// patroniRestFile holds what Patroni's API asks for before a change, on the
// control plane: for `server add`, which lets a second one in.
func patroniRestFile(etcDir string) string { return filepath.Join(etcDir, "secrets", "patroni-rest") }
// writeSecret writes a secret root alone reads, unless it is there already.
func (in *serverInit) writeSecret(path, value string) error {
if value == "" {
return nil
}
if b, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(b)) == value { //nolint:gosec // our own secret
return nil
}
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return err
}
return os.WriteFile(path, []byte(value+"\n"), 0o600)
}
+34 -9
View File
@@ -37,9 +37,10 @@ type Member struct {
IP string
}
// Secrets are the cluster's three passwords.
// Secrets are the cluster's passwords: PostgreSQL's three, and the one
// Patroni's REST API asks for before a change.
type Secrets struct {
Superuser, Replicator, App string
Superuser, Replicator, App, RestAPI string
}
// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
@@ -66,7 +67,10 @@ ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
// clients are the addresses the control planes connect from; the members
// themselves are always allowed, since a second control plane usually runs on
// one of them.
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string {
//
// With tls, PostgreSQL is given the member's certificate (certs puts it in
// TLSDir) and every remote rule is hostssl.
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string {
var hosts []string
for _, e := range etcd {
hosts = append(hosts, e.IP+":2379")
@@ -88,8 +92,17 @@ func Patroni(m Member, db, etcd []Member, clients []string, version string, s Se
}
var hbaLines string
for _, h := range hba {
if tls {
h = hostssl(h)
}
hbaLines += " - " + h + "\n"
}
var sslLines string
if tls {
for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} {
sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k])
}
}
var b strings.Builder
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
scope: %[1]s
@@ -99,6 +112,11 @@ name: %[2]s
restapi:
listen: %[3]s:%[4]d
connect_address: %[3]s:%[4]d
# Asked for before a change: anyone reaching this port could otherwise
# rewrite the access rules or move the leader.
authentication:
username: %[11]s
password: "%[10]s"
etcd3:
hosts: %[5]s
@@ -120,7 +138,7 @@ bootstrap:
wal_keep_size: 1GB
wal_log_hints: "on"
password_encryption: scram-sha-256
# In the cluster's shared configuration rather than each member's, so a
%[12]s # In the cluster's shared configuration rather than each member's, so a
# control plane added later is let in by one change through Patroni's
# API (llmgw server add) instead of an edit on every member.
pg_hba:
@@ -146,7 +164,8 @@ postgresql:
password: "%[7]s"
parameters:
unix_socket_directories: /var/run/postgresql
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines)
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines,
s.RestAPI, RestUser, sslLines)
b.WriteString(`
tags:
nofailover: false
@@ -159,14 +178,20 @@ tags:
// DSN is the store address the control planes are given: every member, and
// the driver told to use whichever accepts writes, so a failover is followed
// without anything in front of the database.
func DSN(db []Member, password string) string {
// without anything in front of the database. With caFile — the cluster's
// authority, on the control plane — the connection is TLS and the member it
// reaches is verified against it.
func DSN(db []Member, password, caFile string) string {
var hosts []string
for _, d := range db {
hosts = append(hosts, d.IP+":5432")
}
return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write",
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database)
ssl := "sslmode=disable"
if caFile != "" {
ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile)
}
return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write",
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl)
}
func ips(ms []Member) []string {
+247
View File
@@ -0,0 +1,247 @@
package pgha
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
)
// Hardening: what a cluster set up before this lacked, applied to it in
// place, and to a new one from the start.
//
// - Patroni's REST API asks for a password before a change. Without one,
// anyone who reached port 8008 could rewrite pg_hba through PATCH /config
// — `host all all 0.0.0.0/0 trust` — or switch the leader over.
// - PostgreSQL speaks TLS, with a certificate from the cluster's authority
// on each member, and every remote rule is hostssl: prompts kept in
// threads, key and session hashes, and the engines' secret no longer
// cross the network in clear. The control planes verify the member they
// reach (sslmode=verify-full against the cluster's CA).
//
// Each step looks first and changes only what differs, so running it on a
// hardened cluster does nothing, and each is applied by reload, not restart:
// the cluster keeps serving.
// RestUser is the user Patroni's REST API asks for before a change.
const RestUser = "patroni"
// TLSDir holds each member's certificate, key and the cluster's CA.
const TLSDir = "/etc/patroni/tls"
// ServerCert, ServerKey and CACert are the files in TLSDir.
const (
ServerCert = TLSDir + "/server.crt"
ServerKey = TLSDir + "/server.key"
CACert = TLSDir + "/ca.crt"
)
// sslParameters are what turn TLS on, in the cluster's shared parameters.
func sslParameters() map[string]any {
return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert}
}
// hostssl rewrites a remote rule to accept only TLS; local and loopback
// rules are left alone.
func hostssl(line string) string {
f := strings.Fields(line)
if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" {
return line
}
return "hostssl" + strings.TrimPrefix(line, "host")
}
// certs gives each database member a certificate from the cluster's
// authority, for its address and name — unless it has one for them already.
func (s *Setup) certs(ctx context.Context) error {
if s.CA == nil {
s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS")
return nil
}
for _, d := range s.DB {
have, _ := d.Remote.Run(ctx, "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName 2>/dev/null", nil)
if strings.Contains(string(have), "IP Address:"+d.IP) {
continue
}
leaf, err := s.CA.IssueServer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"})
if err != nil {
return err
}
var body bytes.Buffer
body.Write(s.CA.PEM)
body.WriteString("\x00")
body.Write(leaf.CertPEM)
body.WriteString("\x00")
body.Write(leaf.KeyPEM)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -o postgres -g postgres -m 750 `+TLSDir+`
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
cat > "$t/all"
python3 - "$t" <<'PY'
import sys
d = sys.argv[1]
ca, crt, key = open(d + "/all", "rb").read().split(b"\x00")
for name, data in (("ca.crt", ca), ("server.crt", crt), ("server.key", key)):
open(d + "/" + name, "wb").write(data)
PY
sudo install -o postgres -g postgres -m 644 "$t/ca.crt" `+CACert+`
sudo install -o postgres -g postgres -m 644 "$t/server.crt" `+ServerCert+`
sudo install -o postgres -g postgres -m 600 "$t/server.key" `+ServerKey, body.Bytes()); err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP)
}
return nil
}
// restAuth makes each member's Patroni ask for the password before a change,
// and reloads it where that was not so.
func (s *Setup) restAuth(ctx context.Context, sec Secrets) error {
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
# The password from stdin into a file only this user reads: the program
# itself comes in on python's stdin.
f=$(mktemp); trap 'rm -f "$f"' EXIT
cat > "$f"
r=$(sudo python3 - "$f" <<'PY'
import sys, yaml
pw = open(sys.argv[1]).read().strip()
p = "/etc/patroni/config.yml"
c = yaml.safe_load(open(p))
want = {"username": "`+RestUser+`", "password": pw}
api = c.setdefault("restapi", {})
if api.get("authentication") == want:
print("same")
else:
api["authentication"] = want
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
print("changed")
PY
)
if [ "$r" = changed ]; then
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl reload patroni
fi
echo "$r"`, []byte(sec.RestAPI+"\n"))
if err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
switch strings.TrimSpace(lastLine(out)) {
case "changed":
s.emit("patroni api", true, "%s asks for a password before a change", d.Name)
case "missing":
return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name)
}
}
return nil
}
// tlsInShared turns TLS on in the cluster's shared configuration, and every
// remote access rule to hostssl, through Patroni's API — which reloads
// PostgreSQL on every member.
func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error {
if s.CA == nil {
return nil
}
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
client := &http.Client{Timeout: 15 * time.Second}
var lastErr error
for _, h := range hosts {
base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort))
cfg, err := getConfig(ctx, client, base)
if err != nil {
lastErr = err
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
var hba, next []string
if list, ok := pg["pg_hba"].([]any); ok {
for _, l := range list {
if line, ok := l.(string); ok {
hba = append(hba, line)
next = append(next, hostssl(line))
}
}
}
want := sslParameters()
same := slices.Equal(hba, next)
for k, v := range want {
if params[k] != v {
same = false
}
}
if same {
s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl")
return nil
}
patch := map[string]any{"parameters": want}
if len(hba) > 0 {
patch["pg_hba"] = next
}
if err := patchConfig(ctx, client, base, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil {
return fmt.Errorf("Patroni on %s: %w", h, err)
}
s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl")
return nil
}
return fmt.Errorf("no member's Patroni answered: %w", lastErr)
}
// patchConfig changes the cluster's shared configuration, with the REST
// password.
func patchConfig(ctx context.Context, client *http.Client, base, password string, body map[string]any) error {
b, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(b))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if password != "" {
req.SetBasicAuth(RestUser, password)
}
resp, err := client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("refused the change: %s", resp.Status)
}
return nil
}
// sharedTLS reports whether the cluster's shared configuration has TLS on:
// for a control plane without the authority's key, which signs nothing but
// must still connect by TLS once the cluster asks for it.
func (s *Setup) sharedTLS(ctx context.Context) bool {
client := &http.Client{Timeout: 10 * time.Second}
for _, d := range s.DB {
cfg, err := getConfig(ctx, client, "http://"+net.JoinHostPort(d.IP, strconv.Itoa(restPort)))
if err != nil {
continue
}
pg, _ := cfg["postgresql"].(map[string]any)
params, _ := pg["parameters"].(map[string]any)
return params["ssl"] == "on"
}
return false
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return lines[len(lines)-1]
}
+11 -16
View File
@@ -1,7 +1,6 @@
package pgha
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -9,6 +8,7 @@ import (
"net/http"
"slices"
"strconv"
"strings"
"time"
)
@@ -24,7 +24,10 @@ func ClientLine(ip string) string {
// A cluster set up before the rules moved into the shared configuration
// keeps them in each member's own file, which this cannot reach; that is
// said, with what to add, rather than half done.
func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) {
//
// password is what Patroni's API asks for before a change (RestUser); a
// cluster whose rules are hostssl lets the new control plane in by TLS too.
func AllowClient(ctx context.Context, hosts []string, ip, password string) (bool, error) {
client := &http.Client{Timeout: 5 * time.Second}
var lastErr error
for _, h := range hosts {
@@ -50,23 +53,15 @@ func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) {
"on every member, and run `patronictl reload llmgw`", ClientLine(ip))
}
line := ClientLine(ip)
if slices.ContainsFunc(hba, func(l string) bool { return strings.HasPrefix(l, "hostssl ") }) {
line = hostssl(line)
}
if slices.Contains(hba, line) {
return false, nil
}
body, _ := json.Marshal(map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}})
req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(body))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
lastErr = err
continue
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Errorf("Patroni on %s refused the change: %s", h, resp.Status)
if err := patchConfig(ctx, client, base, password,
map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}); err != nil {
return false, fmt.Errorf("Patroni on %s: %w", h, err)
}
return true, nil
}
+160 -9
View File
@@ -12,6 +12,7 @@ import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/expfmt"
"go.yaml.in/yaml/v3"
@@ -24,8 +25,8 @@ var (
)
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
sec := Secrets{"su", "rep", "app"}
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec)
sec := Secrets{"su", "rep", "app", "api"}
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false)
for _, want := range []string{
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
@@ -47,10 +48,14 @@ func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
t.Errorf("etcd defaults:\n%s", e)
}
if got := DSN([]Member{pg01, ctl02}, "pw"); got !=
if got := DSN([]Member{pg01, ctl02}, "pw", ""); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
t.Errorf("dsn %s", got)
}
if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" {
t.Errorf("dsn with tls %s", got)
}
}
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
@@ -102,7 +107,7 @@ func (m *machine) ranAny(frag string) bool {
return false
}
const pws = "superuser=a\nreplicator=b\nllmgw=c\n"
const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n"
func healthy(n int) string {
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
@@ -129,7 +134,8 @@ func TestARunningClusterIsLeftAsItIs(t *testing.T) {
t.Errorf("dsn %s", dsn)
}
for _, m := range []*machine{a, b, w} {
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} {
// Looked at, for the API's password, but not rewritten or restarted.
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} {
if m.ranAny(frag) {
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
}
@@ -216,7 +222,7 @@ func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
// Where `server add` can change them for every member at once.
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"})
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false)
var doc struct {
Bootstrap struct {
DCS struct {
@@ -262,7 +268,7 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150")
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "")
if err != nil || !changed {
t.Fatalf("%v %v", changed, err)
}
@@ -270,13 +276,158 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
t.Errorf("patched %q", patched)
}
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed {
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed {
t.Errorf("one already let in was changed again: %v %v", changed, err)
}
hba = nil // the rules are in each member's file
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil ||
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil ||
!strings.Contains(err.Error(), "each member") {
t.Errorf("rules kept per member: %v", err)
}
}
func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) {
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16",
Secrets{"a", "b", "c", "the-api-pw"}, true)
var doc struct {
RestAPI struct {
Authentication struct{ Username, Password string } `yaml:"authentication"`
} `yaml:"restapi"`
Bootstrap struct {
DCS struct {
PostgreSQL struct {
Parameters map[string]string `yaml:"parameters"`
PgHBA []string `yaml:"pg_hba"`
} `yaml:"postgresql"`
} `yaml:"dcs"`
} `yaml:"bootstrap"`
}
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
t.Fatalf("%v\n%s", err, cfg)
}
if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" {
t.Errorf("rest api authentication %+v", a)
}
p := doc.Bootstrap.DCS.PostgreSQL.Parameters
if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey {
t.Errorf("parameters %v", p)
}
for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA {
if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") {
t.Errorf("a remote rule without TLS: %q", l)
}
}
}
func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) {
for in, want := range map[string]string{
"host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256",
"host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256",
"host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256",
"local all all peer": "local all all peer",
"hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256",
} {
if got := hostssl(in); got != want {
t.Errorf("%q: %q", in, got)
}
}
}
func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) {
// The second member holds the three PostgreSQL passwords and no API one.
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
if _, err := s.Run(context.Background()); err != nil {
t.Fatal(err)
}
if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" {
t.Errorf("the API password was not carried: %q", s.RestPassword)
}
for _, m := range []*machine{a, b} {
if !m.ranAny(`"username": "patroni"`) {
t.Errorf("%s: the API was not given its password", m.name)
}
}
}
func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) {
var patched map[string]any
var user, pass string
hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}})
case http.MethodPatch:
user, pass, _ = r.BasicAuth()
_ = json.NewDecoder(r.Body).Decode(&patched)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
ca, err := tlsconf.NewCA("test")
if err != nil {
t.Fatal(err)
}
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
t.Fatal(err)
}
if user != RestUser || pass != "pw" {
t.Errorf("auth %q %q", user, pass)
}
pg := patched["postgresql"].(map[string]any)
if pg["parameters"].(map[string]any)["ssl"] != "on" {
t.Errorf("parameters %v", pg["parameters"])
}
got := pg["pg_hba"].([]any)
if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) {
t.Errorf("hba %v", got)
}
// A control plane let in afterwards is let in by TLS.
hba = got
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil {
t.Fatal(err)
}
last := patched["postgresql"].(map[string]any)["pg_hba"].([]any)
if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) {
t.Errorf("let in without TLS: %v", last)
}
}
func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) {
ssl := "on"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}})
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}},
Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem",
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
dsn, err := s.Run(context.Background())
if err != nil || !strings.Contains(dsn, "sslmode=verify-full") {
t.Fatalf("%s %v", dsn, err)
}
ssl = "off"
if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") {
t.Errorf("TLS asked of a cluster without it: %s", dsn)
}
}
+49 -13
View File
@@ -10,6 +10,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// Node is a machine the setup reaches, and how.
@@ -34,6 +35,15 @@ type Setup struct {
Emit func(step, detail string, changed bool)
// Wait bounds each wait for the cluster to come up.
Wait time.Duration
// CA signs each member's certificate; nil signs none and turns nothing
// on. CAFile is where this control plane finds the authority's
// certificate, for the DSN: used when TLS is on, whoever turned it on.
CA *tlsconf.CA
CAFile string
// RestPassword is what Patroni's API asks for before a change, once Run
// has found or made it: for `server add`, which changes the access rules.
RestPassword string
}
// Local is a Remote for the machine this runs on, which init runs as root.
@@ -87,12 +97,24 @@ func (s *Setup) Run(ctx context.Context) (string, error) {
if err != nil {
return "", fmt.Errorf("passwords: %w", err)
}
s.RestPassword = sec.RestAPI
if err := s.etcd(ctx); err != nil {
return "", fmt.Errorf("etcd: %w", err)
}
if err := s.certs(ctx); err != nil {
return "", fmt.Errorf("certificates: %w", err)
}
if err := s.patroni(ctx, sec); err != nil {
return "", fmt.Errorf("patroni: %w", err)
}
// What a cluster set up before them lacks, applied in place; a new one
// has them already and these find nothing to change.
if err := s.restAuth(ctx, sec); err != nil {
return "", fmt.Errorf("patroni api: %w", err)
}
if err := s.tlsInShared(ctx, sec); err != nil {
return "", fmt.Errorf("tls: %w", err)
}
if err := s.database(ctx); err != nil {
return "", fmt.Errorf("database: %w", err)
}
@@ -100,52 +122,64 @@ func (s *Setup) Run(ctx context.Context) (string, error) {
for i, d := range s.DB {
members[i] = d.Member
}
return DSN(members, sec.App), nil
// TLS when this setup turned it on, or found it on: then the member
// reached is verified against the authority's certificate, which every
// control plane has even where its key is not.
caFile := ""
if s.CAFile != "" && (s.CA != nil || s.sharedTLS(ctx)) {
caFile = s.CAFile
}
return DSN(members, sec.App, caFile), nil
}
// secretDir holds the passwords on every database machine, readable by root
// alone. Generated on the first, carried to the others.
const secretDir = "/root/llmgw-pg"
var secretNames = []string{"superuser", "replicator", "llmgw"}
var secretNames = []string{"superuser", "replicator", "llmgw", "restapi"}
func (s *Setup) secrets(ctx context.Context) (Secrets, error) {
script := `set -e
sudo install -d -m 700 ` + secretDir + `
for f in superuser replicator llmgw; do
for f in superuser replicator llmgw restapi; do
sudo test -s ` + secretDir + `/$f.pw || sudo sh -c "umask 077; openssl rand -hex 24 > ` + secretDir + `/$f.pw"
done
for f in superuser replicator llmgw; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done`
for f in superuser replicator llmgw restapi; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done`
out, err := s.DB[0].Remote.Run(ctx, script, nil)
if err != nil {
return Secrets{}, fmt.Errorf("on %s: %v: %s", s.DB[0].Name, err, strings.TrimSpace(string(out)))
}
got := parseSecrets(out)
if got.Superuser == "" || got.Replicator == "" || got.App == "" {
if got.Superuser == "" || got.Replicator == "" || got.App == "" || got.RestAPI == "" {
return Secrets{}, fmt.Errorf("could not read them back from %s", s.DB[0].Name)
}
for _, d := range s.DB[1:] {
have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw; do
have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw restapi; do
if sudo test -s `+secretDir+`/$f.pw; then printf '%s=' $f; sudo cat `+secretDir+`/$f.pw; fi
done`, nil)
if err != nil {
return Secrets{}, fmt.Errorf("on %s: %v", d.Name, err)
}
if h := parseSecrets(have); h != (Secrets{}) {
if h != got {
h := parseSecrets(have)
if h.Superuser != "" || h.Replicator != "" || h.App != "" {
if h.Superuser != got.Superuser || h.Replicator != got.Replicator || h.App != got.App {
// Not overwritten: a member holding other passwords belongs
// to another cluster, or to an earlier one somebody meant to
// keep, and replacing them would lock it out of itself.
return Secrets{}, fmt.Errorf("%s holds passwords in %s that differ from %s's; "+
"remove them there if that machine is meant to join this cluster", d.Name, secretDir, s.DB[0].Name)
}
continue
if h.RestAPI == got.RestAPI {
continue
}
// A cluster from before the API asked for a password: the
// member is given the one the first holds.
}
body := fmt.Sprintf("%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App)
body := fmt.Sprintf("%s\n%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App, got.RestAPI)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -m 700 `+secretDir+`
read su; read rep; read app
for kv in "superuser $su" "replicator $rep" "llmgw $app"; do
read su; read rep; read app; read api
for kv in "superuser $su" "replicator $rep" "llmgw $app" "restapi $api"; do
set -- $kv
printf '%s\n' "$2" | sudo sh -c "umask 077; cat > `+secretDir+`/$1.pw"
done`, []byte(body)); err != nil {
@@ -171,6 +205,8 @@ func parseSecrets(out []byte) Secrets {
s.Replicator = v
case "llmgw":
s.App = v
case "restapi":
s.RestAPI = v
}
}
return s
@@ -288,7 +324,7 @@ func (s *Setup) patroni(ctx context.Context, sec Secrets) error {
db = append(db, d.Member)
}
for i, d := range s.DB {
cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec)
cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec, s.CA != nil)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -m 755 /etc/patroni
sudo install -d -o postgres -g postgres -m 700 /var/lib/postgresql/`+version+`/llmgw