From 2b29016d5b21c2433f04163c98e9305dea47be52 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 10:11:28 +0200 Subject: [PATCH] Ask for a password on Patroni's API and give the store TLS, new clusters and existing ones in place Patroni's REST API accepted changes from anyone on port 8008, and PostgreSQL ran with sslmode=disable between machines. Setup now keeps a fourth secret for the API (carried to members of older clusters), issues each member a certificate from the cluster's authority, writes restapi.authentication into each member's config and reloads it, and turns ssl on with hostssl remote rules through an authenticated PATCH of the shared configuration. The DSN becomes sslmode=verify-full against the authority's certificate whenever the cluster has TLS, so a control plane without the authority's key reconnects too. server add authenticates its PATCH and lets the new control plane in by hostssl. Tried on the e2e cluster: unauthenticated PATCH 401, plaintext refused, replication streaming, the control plane on verify-full. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 17 ++ internal/cli/serveradd.go | 6 +- internal/cli/serverinit_patroni.go | 38 ++++- internal/pgha/config.go | 43 +++-- internal/pgha/harden.go | 247 +++++++++++++++++++++++++++++ internal/pgha/hba.go | 27 ++-- internal/pgha/pgha_test.go | 169 ++++++++++++++++++-- internal/pgha/setup.go | 62 ++++++-- 8 files changed, 558 insertions(+), 51 deletions(-) create mode 100644 internal/pgha/harden.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1470c..290d96f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,23 @@ APIs change between minors without a deprecation period. elsewhere. pprof is off unless `debug_pprof: true`. `admin_listen` off loopback is refused unless `admin_remote: true`. The read-only dashboard pages, `/metrics` and `/health` are unchanged. +- **The replicated store asks for a password and speaks TLS.** Patroni's + REST API took changes from anyone who reached port 8008 — rewriting the + access rules to `trust`, or moving the leader — and PostgreSQL traffic + between the machines was plaintext (`sslmode=disable`). Patroni now asks + for a password before a change (kept with the cluster's others in + `/root/llmgw-pg`, and on the control plane in + `/etc/llmgw/secrets/patroni-rest` for `server add`); each member has a + certificate from the cluster's authority, PostgreSQL has `ssl` on, every + remote access rule is `hostssl`, and the control planes connect with + `sslmode=verify-full` against the authority's certificate. A new cluster is + set up this way; an existing one is changed in place, by reload, the + cluster serving throughout, by running **Replicate the store** in the + console again — or `sudo --preserve-env=SSH_AUTH_SOCK llmgw server init + --patroni @ --patroni @ --no-mirror --no-node + --no-model` — on the control plane that holds the authority, then on each + other control plane so it reconnects by TLS. etcd is not yet given TLS: + keep ports 2379 and 2380 to the cluster's machines. ### Everything that can be changed on a running cluster, from the console diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 7dcbacf..7462a68 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -307,7 +307,11 @@ func (sa *serverAdd) storeAccess(ctx context.Context) error { case slices.Contains(hosts, sa.ip): sa.kept("store: %s is one of its members", sa.ip) case len(hosts) > 1: - changed, err := pgha.AllowClient(ctx, hosts, sa.ip) + rest := "" + if b, err := os.ReadFile(patroniRestFile(sa.etcDir)); err == nil { //nolint:gosec // our own secret + rest = strings.TrimSpace(string(b)) + } + changed, err := pgha.AllowClient(ctx, hosts, sa.ip, rest) if err != nil { return err } diff --git a/internal/cli/serverinit_patroni.go b/internal/cli/serverinit_patroni.go index 075434c..fc7aa7e 100644 --- a/internal/cli/serverinit_patroni.go +++ b/internal/cli/serverinit_patroni.go @@ -6,6 +6,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/config" @@ -58,13 +59,26 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) { setup.Witness = &pgha.Node{Member: pgha.Member{Name: name, IP: self}, Remote: pgha.Local{}} setup.Clients = []string{self} + cfg, err := config.Load(in.configPath()) + if err != nil { + return err + } + // The cluster's authority signs each member's certificate, and the + // control plane verifies the member it reaches against it: where its + // key is on this machine, and the server has the certificate to read. + setup.CAFile = cfg.AgentTLS.CA + if ca, cerr := loadCA(in.pkiDir); cerr == nil && cfg.AgentTLS.CA != "" { + setup.CA = ca + } else { + in.kept("store: the authority's key is not here, so no member certificates are signed from this machine; " + + "run this on the control plane that holds it to turn TLS on") + } + dsn, err := setup.Run(ctx) if err != nil { return err } - - cfg, err := config.Load(in.configPath()) - if err != nil { + if err := in.writeSecret(patroniRestFile(in.etcDir), setup.RestPassword); err != nil { return err } current := cfg.Auth.Store @@ -180,3 +194,21 @@ func ipv4Of(host string) (string, error) { } return "", fmt.Errorf("%s has no IPv4 address", host) } + +// patroniRestFile holds what Patroni's API asks for before a change, on the +// control plane: for `server add`, which lets a second one in. +func patroniRestFile(etcDir string) string { return filepath.Join(etcDir, "secrets", "patroni-rest") } + +// writeSecret writes a secret root alone reads, unless it is there already. +func (in *serverInit) writeSecret(path, value string) error { + if value == "" { + return nil + } + if b, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(b)) == value { //nolint:gosec // our own secret + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + return os.WriteFile(path, []byte(value+"\n"), 0o600) +} diff --git a/internal/pgha/config.go b/internal/pgha/config.go index d8edf3c..2cb0c90 100644 --- a/internal/pgha/config.go +++ b/internal/pgha/config.go @@ -37,9 +37,10 @@ type Member struct { IP string } -// Secrets are the cluster's three passwords. +// Secrets are the cluster's passwords: PostgreSQL's three, and the one +// Patroni's REST API asks for before a change. type Secrets struct { - Superuser, Replicator, App string + Superuser, Replicator, App, RestAPI string } // Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server. @@ -66,7 +67,10 @@ ETCD_INITIAL_CLUSTER_TOKEN=%[4]s // clients are the addresses the control planes connect from; the members // themselves are always allowed, since a second control plane usually runs on // one of them. -func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string { +// +// With tls, PostgreSQL is given the member's certificate (certs puts it in +// TLSDir) and every remote rule is hostssl. +func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string { var hosts []string for _, e := range etcd { hosts = append(hosts, e.IP+":2379") @@ -88,8 +92,17 @@ func Patroni(m Member, db, etcd []Member, clients []string, version string, s Se } var hbaLines string for _, h := range hba { + if tls { + h = hostssl(h) + } hbaLines += " - " + h + "\n" } + var sslLines string + if tls { + for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} { + sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k]) + } + } var b strings.Builder fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd. scope: %[1]s @@ -99,6 +112,11 @@ name: %[2]s restapi: listen: %[3]s:%[4]d connect_address: %[3]s:%[4]d + # Asked for before a change: anyone reaching this port could otherwise + # rewrite the access rules or move the leader. + authentication: + username: %[11]s + password: "%[10]s" etcd3: hosts: %[5]s @@ -120,7 +138,7 @@ bootstrap: wal_keep_size: 1GB wal_log_hints: "on" password_encryption: scram-sha-256 - # In the cluster's shared configuration rather than each member's, so a +%[12]s # In the cluster's shared configuration rather than each member's, so a # control plane added later is let in by one change through Patroni's # API (llmgw server add) instead of an edit on every member. pg_hba: @@ -146,7 +164,8 @@ postgresql: password: "%[7]s" parameters: unix_socket_directories: /var/run/postgresql -`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines) +`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines, + s.RestAPI, RestUser, sslLines) b.WriteString(` tags: nofailover: false @@ -159,14 +178,20 @@ tags: // DSN is the store address the control planes are given: every member, and // the driver told to use whichever accepts writes, so a failover is followed -// without anything in front of the database. -func DSN(db []Member, password string) string { +// without anything in front of the database. With caFile — the cluster's +// authority, on the control plane — the connection is TLS and the member it +// reaches is verified against it. +func DSN(db []Member, password, caFile string) string { var hosts []string for _, d := range db { hosts = append(hosts, d.IP+":5432") } - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write", - Role, url.QueryEscape(password), strings.Join(hosts, ","), Database) + ssl := "sslmode=disable" + if caFile != "" { + ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile) + } + return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write", + Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl) } func ips(ms []Member) []string { diff --git a/internal/pgha/harden.go b/internal/pgha/harden.go new file mode 100644 index 0000000..1f37e55 --- /dev/null +++ b/internal/pgha/harden.go @@ -0,0 +1,247 @@ +package pgha + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "slices" + "strconv" + "strings" + "time" +) + +// Hardening: what a cluster set up before this lacked, applied to it in +// place, and to a new one from the start. +// +// - Patroni's REST API asks for a password before a change. Without one, +// anyone who reached port 8008 could rewrite pg_hba through PATCH /config +// — `host all all 0.0.0.0/0 trust` — or switch the leader over. +// - PostgreSQL speaks TLS, with a certificate from the cluster's authority +// on each member, and every remote rule is hostssl: prompts kept in +// threads, key and session hashes, and the engines' secret no longer +// cross the network in clear. The control planes verify the member they +// reach (sslmode=verify-full against the cluster's CA). +// +// Each step looks first and changes only what differs, so running it on a +// hardened cluster does nothing, and each is applied by reload, not restart: +// the cluster keeps serving. + +// RestUser is the user Patroni's REST API asks for before a change. +const RestUser = "patroni" + +// TLSDir holds each member's certificate, key and the cluster's CA. +const TLSDir = "/etc/patroni/tls" + +// ServerCert, ServerKey and CACert are the files in TLSDir. +const ( + ServerCert = TLSDir + "/server.crt" + ServerKey = TLSDir + "/server.key" + CACert = TLSDir + "/ca.crt" +) + +// sslParameters are what turn TLS on, in the cluster's shared parameters. +func sslParameters() map[string]any { + return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert} +} + +// hostssl rewrites a remote rule to accept only TLS; local and loopback +// rules are left alone. +func hostssl(line string) string { + f := strings.Fields(line) + if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" { + return line + } + return "hostssl" + strings.TrimPrefix(line, "host") +} + +// certs gives each database member a certificate from the cluster's +// authority, for its address and name — unless it has one for them already. +func (s *Setup) certs(ctx context.Context) error { + if s.CA == nil { + s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS") + return nil + } + for _, d := range s.DB { + have, _ := d.Remote.Run(ctx, "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName 2>/dev/null", nil) + if strings.Contains(string(have), "IP Address:"+d.IP) { + continue + } + leaf, err := s.CA.IssueServer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"}) + if err != nil { + return err + } + var body bytes.Buffer + body.Write(s.CA.PEM) + body.WriteString("\x00") + body.Write(leaf.CertPEM) + body.WriteString("\x00") + body.Write(leaf.KeyPEM) + if out, err := d.Remote.Run(ctx, `set -e +sudo install -d -o postgres -g postgres -m 750 `+TLSDir+` +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT +cat > "$t/all" +python3 - "$t" <<'PY' +import sys +d = sys.argv[1] +ca, crt, key = open(d + "/all", "rb").read().split(b"\x00") +for name, data in (("ca.crt", ca), ("server.crt", crt), ("server.key", key)): + open(d + "/" + name, "wb").write(data) +PY +sudo install -o postgres -g postgres -m 644 "$t/ca.crt" `+CACert+` +sudo install -o postgres -g postgres -m 644 "$t/server.crt" `+ServerCert+` +sudo install -o postgres -g postgres -m 600 "$t/server.key" `+ServerKey, body.Bytes()); err != nil { + return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) + } + s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP) + } + return nil +} + +// restAuth makes each member's Patroni ask for the password before a change, +// and reloads it where that was not so. +func (s *Setup) restAuth(ctx context.Context, sec Secrets) error { + for _, d := range s.DB { + out, err := d.Remote.Run(ctx, `set -e +test -f /etc/patroni/config.yml || { echo missing; exit 0; } +# The password from stdin into a file only this user reads: the program +# itself comes in on python's stdin. +f=$(mktemp); trap 'rm -f "$f"' EXIT +cat > "$f" +r=$(sudo python3 - "$f" <<'PY' +import sys, yaml +pw = open(sys.argv[1]).read().strip() +p = "/etc/patroni/config.yml" +c = yaml.safe_load(open(p)) +want = {"username": "`+RestUser+`", "password": pw} +api = c.setdefault("restapi", {}) +if api.get("authentication") == want: + print("same") +else: + api["authentication"] = want + open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" + + yaml.safe_dump(c, default_flow_style=False, sort_keys=False)) + print("changed") +PY +) +if [ "$r" = changed ]; then + sudo chown root:postgres /etc/patroni/config.yml.new + sudo chmod 640 /etc/patroni/config.yml.new + sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml + sudo systemctl reload patroni +fi +echo "$r"`, []byte(sec.RestAPI+"\n")) + if err != nil { + return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) + } + switch strings.TrimSpace(lastLine(out)) { + case "changed": + s.emit("patroni api", true, "%s asks for a password before a change", d.Name) + case "missing": + return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name) + } + } + return nil +} + +// tlsInShared turns TLS on in the cluster's shared configuration, and every +// remote access rule to hostssl, through Patroni's API — which reloads +// PostgreSQL on every member. +func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error { + if s.CA == nil { + return nil + } + hosts := make([]string, len(s.DB)) + for i, d := range s.DB { + hosts[i] = d.IP + } + client := &http.Client{Timeout: 15 * time.Second} + var lastErr error + for _, h := range hosts { + base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort)) + cfg, err := getConfig(ctx, client, base) + if err != nil { + lastErr = err + continue + } + pg, _ := cfg["postgresql"].(map[string]any) + params, _ := pg["parameters"].(map[string]any) + var hba, next []string + if list, ok := pg["pg_hba"].([]any); ok { + for _, l := range list { + if line, ok := l.(string); ok { + hba = append(hba, line) + next = append(next, hostssl(line)) + } + } + } + want := sslParameters() + same := slices.Equal(hba, next) + for k, v := range want { + if params[k] != v { + same = false + } + } + if same { + s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl") + return nil + } + patch := map[string]any{"parameters": want} + if len(hba) > 0 { + patch["pg_hba"] = next + } + if err := patchConfig(ctx, client, base, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil { + return fmt.Errorf("Patroni on %s: %w", h, err) + } + s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl") + return nil + } + return fmt.Errorf("no member's Patroni answered: %w", lastErr) +} + +// patchConfig changes the cluster's shared configuration, with the REST +// password. +func patchConfig(ctx context.Context, client *http.Client, base, password string, body map[string]any) error { + b, _ := json.Marshal(body) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(b)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if password != "" { + req.SetBasicAuth(RestUser, password) + } + resp, err := client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("refused the change: %s", resp.Status) + } + return nil +} + +// sharedTLS reports whether the cluster's shared configuration has TLS on: +// for a control plane without the authority's key, which signs nothing but +// must still connect by TLS once the cluster asks for it. +func (s *Setup) sharedTLS(ctx context.Context) bool { + client := &http.Client{Timeout: 10 * time.Second} + for _, d := range s.DB { + cfg, err := getConfig(ctx, client, "http://"+net.JoinHostPort(d.IP, strconv.Itoa(restPort))) + if err != nil { + continue + } + pg, _ := cfg["postgresql"].(map[string]any) + params, _ := pg["parameters"].(map[string]any) + return params["ssl"] == "on" + } + return false +} + +func lastLine(b []byte) string { + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + return lines[len(lines)-1] +} diff --git a/internal/pgha/hba.go b/internal/pgha/hba.go index 3a4243b..1fe4242 100644 --- a/internal/pgha/hba.go +++ b/internal/pgha/hba.go @@ -1,7 +1,6 @@ package pgha import ( - "bytes" "context" "encoding/json" "fmt" @@ -9,6 +8,7 @@ import ( "net/http" "slices" "strconv" + "strings" "time" ) @@ -24,7 +24,10 @@ func ClientLine(ip string) string { // A cluster set up before the rules moved into the shared configuration // keeps them in each member's own file, which this cannot reach; that is // said, with what to add, rather than half done. -func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) { +// +// password is what Patroni's API asks for before a change (RestUser); a +// cluster whose rules are hostssl lets the new control plane in by TLS too. +func AllowClient(ctx context.Context, hosts []string, ip, password string) (bool, error) { client := &http.Client{Timeout: 5 * time.Second} var lastErr error for _, h := range hosts { @@ -50,23 +53,15 @@ func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) { "on every member, and run `patronictl reload llmgw`", ClientLine(ip)) } line := ClientLine(ip) + if slices.ContainsFunc(hba, func(l string) bool { return strings.HasPrefix(l, "hostssl ") }) { + line = hostssl(line) + } if slices.Contains(hba, line) { return false, nil } - body, _ := json.Marshal(map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}) - req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(body)) - if err != nil { - return false, err - } - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - lastErr = err - continue - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf("Patroni on %s refused the change: %s", h, resp.Status) + if err := patchConfig(ctx, client, base, password, + map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}); err != nil { + return false, fmt.Errorf("Patroni on %s: %w", h, err) } return true, nil } diff --git a/internal/pgha/pgha_test.go b/internal/pgha/pgha_test.go index b3029fe..90e4446 100644 --- a/internal/pgha/pgha_test.go +++ b/internal/pgha/pgha_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/expfmt" "go.yaml.in/yaml/v3" @@ -24,8 +25,8 @@ var ( ) func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { - sec := Secrets{"su", "rep", "app"} - cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec) + sec := Secrets{"su", "rep", "app", "api"} + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false) for _, want := range []string{ "name: llmgw-pg01", "listen: 10.0.0.149:8008", "hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379", @@ -47,10 +48,14 @@ func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { !strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") { t.Errorf("etcd defaults:\n%s", e) } - if got := DSN([]Member{pg01, ctl02}, "pw"); got != + if got := DSN([]Member{pg01, ctl02}, "pw", ""); got != "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" { t.Errorf("dsn %s", got) } + if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got != + "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" { + t.Errorf("dsn with tls %s", got) + } } func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) { @@ -102,7 +107,7 @@ func (m *machine) ranAny(frag string) bool { return false } -const pws = "superuser=a\nreplicator=b\nllmgw=c\n" +const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n" func healthy(n int) string { return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n) @@ -129,7 +134,8 @@ func TestARunningClusterIsLeftAsItIs(t *testing.T) { t.Errorf("dsn %s", dsn) } for _, m := range []*machine{a, b, w} { - for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} { + // Looked at, for the API's password, but not rewritten or restarted. + for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} { if m.ranAny(frag) { t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag) } @@ -216,7 +222,7 @@ func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) { func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) { // Where `server add` can change them for every member at once. - cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"}) + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false) var doc struct { Bootstrap struct { DCS struct { @@ -262,7 +268,7 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() - changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150") + changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "") if err != nil || !changed { t.Fatalf("%v %v", changed, err) } @@ -270,13 +276,158 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") { t.Errorf("patched %q", patched) } - if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed { + if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed { t.Errorf("one already let in was changed again: %v %v", changed, err) } hba = nil // the rules are in each member's file - if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil || + if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil || !strings.Contains(err.Error(), "each member") { t.Errorf("rules kept per member: %v", err) } } + +func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) { + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", + Secrets{"a", "b", "c", "the-api-pw"}, true) + var doc struct { + RestAPI struct { + Authentication struct{ Username, Password string } `yaml:"authentication"` + } `yaml:"restapi"` + Bootstrap struct { + DCS struct { + PostgreSQL struct { + Parameters map[string]string `yaml:"parameters"` + PgHBA []string `yaml:"pg_hba"` + } `yaml:"postgresql"` + } `yaml:"dcs"` + } `yaml:"bootstrap"` + } + if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { + t.Fatalf("%v\n%s", err, cfg) + } + if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" { + t.Errorf("rest api authentication %+v", a) + } + p := doc.Bootstrap.DCS.PostgreSQL.Parameters + if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey { + t.Errorf("parameters %v", p) + } + for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA { + if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") { + t.Errorf("a remote rule without TLS: %q", l) + } + } +} + +func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) { + for in, want := range map[string]string{ + "host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256", + "host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256", + "host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256", + "local all all peer": "local all all peer", + "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256", + } { + if got := hostssl(in); got != want { + t.Errorf("%q: %q", in, got) + } + } +} + +func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) { + // The second member holds the three PostgreSQL passwords and no API one. + a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} + b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}} + up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, + {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} + s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}, + Status: func(context.Context, []string) (Cluster, error) { return up, nil }} + if _, err := s.Run(context.Background()); err != nil { + t.Fatal(err) + } + if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" { + t.Errorf("the API password was not carried: %q", s.RestPassword) + } + for _, m := range []*machine{a, b} { + if !m.ranAny(`"username": "patroni"`) { + t.Errorf("%s: the API was not given its password", m.name) + } + } +} + +func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) { + var patched map[string]any + var user, pass string + hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}}) + case http.MethodPatch: + user, pass, _ = r.BasicAuth() + _ = json.NewDecoder(r.Body).Decode(&patched) + } + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + ca, err := tlsconf.NewCA("test") + if err != nil { + t.Fatal(err) + } + s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca} + if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil { + t.Fatal(err) + } + if user != RestUser || pass != "pw" { + t.Errorf("auth %q %q", user, pass) + } + pg := patched["postgresql"].(map[string]any) + if pg["parameters"].(map[string]any)["ssl"] != "on" { + t.Errorf("parameters %v", pg["parameters"]) + } + got := pg["pg_hba"].([]any) + if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) { + t.Errorf("hba %v", got) + } + // A control plane let in afterwards is let in by TLS. + hba = got + if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil { + t.Fatal(err) + } + last := patched["postgresql"].(map[string]any)["pg_hba"].([]any) + if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) { + t.Errorf("let in without TLS: %v", last) + } +} + +func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) { + ssl := "on" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}}) + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} + b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} + up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, + {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} + s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}}, + Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem", + Status: func(context.Context, []string) (Cluster, error) { return up, nil }} + dsn, err := s.Run(context.Background()) + if err != nil || !strings.Contains(dsn, "sslmode=verify-full") { + t.Fatalf("%s %v", dsn, err) + } + ssl = "off" + if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") { + t.Errorf("TLS asked of a cluster without it: %s", dsn) + } +} diff --git a/internal/pgha/setup.go b/internal/pgha/setup.go index 2dca4da..6e6bf07 100644 --- a/internal/pgha/setup.go +++ b/internal/pgha/setup.go @@ -10,6 +10,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Node is a machine the setup reaches, and how. @@ -34,6 +35,15 @@ type Setup struct { Emit func(step, detail string, changed bool) // Wait bounds each wait for the cluster to come up. Wait time.Duration + // CA signs each member's certificate; nil signs none and turns nothing + // on. CAFile is where this control plane finds the authority's + // certificate, for the DSN: used when TLS is on, whoever turned it on. + CA *tlsconf.CA + CAFile string + + // RestPassword is what Patroni's API asks for before a change, once Run + // has found or made it: for `server add`, which changes the access rules. + RestPassword string } // Local is a Remote for the machine this runs on, which init runs as root. @@ -87,12 +97,24 @@ func (s *Setup) Run(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf("passwords: %w", err) } + s.RestPassword = sec.RestAPI if err := s.etcd(ctx); err != nil { return "", fmt.Errorf("etcd: %w", err) } + if err := s.certs(ctx); err != nil { + return "", fmt.Errorf("certificates: %w", err) + } if err := s.patroni(ctx, sec); err != nil { return "", fmt.Errorf("patroni: %w", err) } + // What a cluster set up before them lacks, applied in place; a new one + // has them already and these find nothing to change. + if err := s.restAuth(ctx, sec); err != nil { + return "", fmt.Errorf("patroni api: %w", err) + } + if err := s.tlsInShared(ctx, sec); err != nil { + return "", fmt.Errorf("tls: %w", err) + } if err := s.database(ctx); err != nil { return "", fmt.Errorf("database: %w", err) } @@ -100,52 +122,64 @@ func (s *Setup) Run(ctx context.Context) (string, error) { for i, d := range s.DB { members[i] = d.Member } - return DSN(members, sec.App), nil + // TLS when this setup turned it on, or found it on: then the member + // reached is verified against the authority's certificate, which every + // control plane has even where its key is not. + caFile := "" + if s.CAFile != "" && (s.CA != nil || s.sharedTLS(ctx)) { + caFile = s.CAFile + } + return DSN(members, sec.App, caFile), nil } // secretDir holds the passwords on every database machine, readable by root // alone. Generated on the first, carried to the others. const secretDir = "/root/llmgw-pg" -var secretNames = []string{"superuser", "replicator", "llmgw"} +var secretNames = []string{"superuser", "replicator", "llmgw", "restapi"} func (s *Setup) secrets(ctx context.Context) (Secrets, error) { script := `set -e sudo install -d -m 700 ` + secretDir + ` -for f in superuser replicator llmgw; do +for f in superuser replicator llmgw restapi; do sudo test -s ` + secretDir + `/$f.pw || sudo sh -c "umask 077; openssl rand -hex 24 > ` + secretDir + `/$f.pw" done -for f in superuser replicator llmgw; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done` +for f in superuser replicator llmgw restapi; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done` out, err := s.DB[0].Remote.Run(ctx, script, nil) if err != nil { return Secrets{}, fmt.Errorf("on %s: %v: %s", s.DB[0].Name, err, strings.TrimSpace(string(out))) } got := parseSecrets(out) - if got.Superuser == "" || got.Replicator == "" || got.App == "" { + if got.Superuser == "" || got.Replicator == "" || got.App == "" || got.RestAPI == "" { return Secrets{}, fmt.Errorf("could not read them back from %s", s.DB[0].Name) } for _, d := range s.DB[1:] { - have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw; do + have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw restapi; do if sudo test -s `+secretDir+`/$f.pw; then printf '%s=' $f; sudo cat `+secretDir+`/$f.pw; fi done`, nil) if err != nil { return Secrets{}, fmt.Errorf("on %s: %v", d.Name, err) } - if h := parseSecrets(have); h != (Secrets{}) { - if h != got { + h := parseSecrets(have) + if h.Superuser != "" || h.Replicator != "" || h.App != "" { + if h.Superuser != got.Superuser || h.Replicator != got.Replicator || h.App != got.App { // Not overwritten: a member holding other passwords belongs // to another cluster, or to an earlier one somebody meant to // keep, and replacing them would lock it out of itself. return Secrets{}, fmt.Errorf("%s holds passwords in %s that differ from %s's; "+ "remove them there if that machine is meant to join this cluster", d.Name, secretDir, s.DB[0].Name) } - continue + if h.RestAPI == got.RestAPI { + continue + } + // A cluster from before the API asked for a password: the + // member is given the one the first holds. } - body := fmt.Sprintf("%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App) + body := fmt.Sprintf("%s\n%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App, got.RestAPI) if out, err := d.Remote.Run(ctx, `set -e sudo install -d -m 700 `+secretDir+` -read su; read rep; read app -for kv in "superuser $su" "replicator $rep" "llmgw $app"; do +read su; read rep; read app; read api +for kv in "superuser $su" "replicator $rep" "llmgw $app" "restapi $api"; do set -- $kv printf '%s\n' "$2" | sudo sh -c "umask 077; cat > `+secretDir+`/$1.pw" done`, []byte(body)); err != nil { @@ -171,6 +205,8 @@ func parseSecrets(out []byte) Secrets { s.Replicator = v case "llmgw": s.App = v + case "restapi": + s.RestAPI = v } } return s @@ -288,7 +324,7 @@ func (s *Setup) patroni(ctx context.Context, sec Secrets) error { db = append(db, d.Member) } for i, d := range s.DB { - cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec) + cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec, s.CA != nil) if out, err := d.Remote.Run(ctx, `set -e sudo install -d -m 755 /etc/patroni sudo install -d -o postgres -g postgres -m 700 /var/lib/postgresql/`+version+`/llmgw