diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1470c..290d96f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,23 @@ APIs change between minors without a deprecation period. elsewhere. pprof is off unless `debug_pprof: true`. `admin_listen` off loopback is refused unless `admin_remote: true`. The read-only dashboard pages, `/metrics` and `/health` are unchanged. +- **The replicated store asks for a password and speaks TLS.** Patroni's + REST API took changes from anyone who reached port 8008 — rewriting the + access rules to `trust`, or moving the leader — and PostgreSQL traffic + between the machines was plaintext (`sslmode=disable`). Patroni now asks + for a password before a change (kept with the cluster's others in + `/root/llmgw-pg`, and on the control plane in + `/etc/llmgw/secrets/patroni-rest` for `server add`); each member has a + certificate from the cluster's authority, PostgreSQL has `ssl` on, every + remote access rule is `hostssl`, and the control planes connect with + `sslmode=verify-full` against the authority's certificate. A new cluster is + set up this way; an existing one is changed in place, by reload, the + cluster serving throughout, by running **Replicate the store** in the + console again — or `sudo --preserve-env=SSH_AUTH_SOCK llmgw server init + --patroni @ --patroni @ --no-mirror --no-node + --no-model` — on the control plane that holds the authority, then on each + other control plane so it reconnects by TLS. etcd is not yet given TLS: + keep ports 2379 and 2380 to the cluster's machines. ### Everything that can be changed on a running cluster, from the console diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 7dcbacf..7462a68 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -307,7 +307,11 @@ func (sa *serverAdd) storeAccess(ctx context.Context) error { case slices.Contains(hosts, sa.ip): sa.kept("store: %s is one of its members", sa.ip) case len(hosts) > 1: - changed, err := pgha.AllowClient(ctx, hosts, sa.ip) + rest := "" + if b, err := os.ReadFile(patroniRestFile(sa.etcDir)); err == nil { //nolint:gosec // our own secret + rest = strings.TrimSpace(string(b)) + } + changed, err := pgha.AllowClient(ctx, hosts, sa.ip, rest) if err != nil { return err } diff --git a/internal/cli/serverinit_patroni.go b/internal/cli/serverinit_patroni.go index 075434c..fc7aa7e 100644 --- a/internal/cli/serverinit_patroni.go +++ b/internal/cli/serverinit_patroni.go @@ -6,6 +6,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/config" @@ -58,13 +59,26 @@ func (in *serverInit) ensurePatroni(ctx context.Context) (err error) { setup.Witness = &pgha.Node{Member: pgha.Member{Name: name, IP: self}, Remote: pgha.Local{}} setup.Clients = []string{self} + cfg, err := config.Load(in.configPath()) + if err != nil { + return err + } + // The cluster's authority signs each member's certificate, and the + // control plane verifies the member it reaches against it: where its + // key is on this machine, and the server has the certificate to read. + setup.CAFile = cfg.AgentTLS.CA + if ca, cerr := loadCA(in.pkiDir); cerr == nil && cfg.AgentTLS.CA != "" { + setup.CA = ca + } else { + in.kept("store: the authority's key is not here, so no member certificates are signed from this machine; " + + "run this on the control plane that holds it to turn TLS on") + } + dsn, err := setup.Run(ctx) if err != nil { return err } - - cfg, err := config.Load(in.configPath()) - if err != nil { + if err := in.writeSecret(patroniRestFile(in.etcDir), setup.RestPassword); err != nil { return err } current := cfg.Auth.Store @@ -180,3 +194,21 @@ func ipv4Of(host string) (string, error) { } return "", fmt.Errorf("%s has no IPv4 address", host) } + +// patroniRestFile holds what Patroni's API asks for before a change, on the +// control plane: for `server add`, which lets a second one in. +func patroniRestFile(etcDir string) string { return filepath.Join(etcDir, "secrets", "patroni-rest") } + +// writeSecret writes a secret root alone reads, unless it is there already. +func (in *serverInit) writeSecret(path, value string) error { + if value == "" { + return nil + } + if b, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(b)) == value { //nolint:gosec // our own secret + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + return os.WriteFile(path, []byte(value+"\n"), 0o600) +} diff --git a/internal/pgha/config.go b/internal/pgha/config.go index d8edf3c..2cb0c90 100644 --- a/internal/pgha/config.go +++ b/internal/pgha/config.go @@ -37,9 +37,10 @@ type Member struct { IP string } -// Secrets are the cluster's three passwords. +// Secrets are the cluster's passwords: PostgreSQL's three, and the one +// Patroni's REST API asks for before a change. type Secrets struct { - Superuser, Replicator, App string + Superuser, Replicator, App, RestAPI string } // Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server. @@ -66,7 +67,10 @@ ETCD_INITIAL_CLUSTER_TOKEN=%[4]s // clients are the addresses the control planes connect from; the members // themselves are always allowed, since a second control plane usually runs on // one of them. -func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string { +// +// With tls, PostgreSQL is given the member's certificate (certs puts it in +// TLSDir) and every remote rule is hostssl. +func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets, tls bool) string { var hosts []string for _, e := range etcd { hosts = append(hosts, e.IP+":2379") @@ -88,8 +92,17 @@ func Patroni(m Member, db, etcd []Member, clients []string, version string, s Se } var hbaLines string for _, h := range hba { + if tls { + h = hostssl(h) + } hbaLines += " - " + h + "\n" } + var sslLines string + if tls { + for _, k := range []string{"ssl", "ssl_cert_file", "ssl_key_file", "ssl_ca_file"} { + sslLines += fmt.Sprintf(" %s: %q\n", k, sslParameters()[k]) + } + } var b strings.Builder fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd. scope: %[1]s @@ -99,6 +112,11 @@ name: %[2]s restapi: listen: %[3]s:%[4]d connect_address: %[3]s:%[4]d + # Asked for before a change: anyone reaching this port could otherwise + # rewrite the access rules or move the leader. + authentication: + username: %[11]s + password: "%[10]s" etcd3: hosts: %[5]s @@ -120,7 +138,7 @@ bootstrap: wal_keep_size: 1GB wal_log_hints: "on" password_encryption: scram-sha-256 - # In the cluster's shared configuration rather than each member's, so a +%[12]s # In the cluster's shared configuration rather than each member's, so a # control plane added later is let in by one change through Patroni's # API (llmgw server add) instead of an edit on every member. pg_hba: @@ -146,7 +164,8 @@ postgresql: password: "%[7]s" parameters: unix_socket_directories: /var/run/postgresql -`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines) +`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines, + s.RestAPI, RestUser, sslLines) b.WriteString(` tags: nofailover: false @@ -159,14 +178,20 @@ tags: // DSN is the store address the control planes are given: every member, and // the driver told to use whichever accepts writes, so a failover is followed -// without anything in front of the database. -func DSN(db []Member, password string) string { +// without anything in front of the database. With caFile — the cluster's +// authority, on the control plane — the connection is TLS and the member it +// reaches is verified against it. +func DSN(db []Member, password, caFile string) string { var hosts []string for _, d := range db { hosts = append(hosts, d.IP+":5432") } - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write", - Role, url.QueryEscape(password), strings.Join(hosts, ","), Database) + ssl := "sslmode=disable" + if caFile != "" { + ssl = "sslmode=verify-full&sslrootcert=" + url.QueryEscape(caFile) + } + return fmt.Sprintf("postgres://%s:%s@%s/%s?%s&target_session_attrs=read-write", + Role, url.QueryEscape(password), strings.Join(hosts, ","), Database, ssl) } func ips(ms []Member) []string { diff --git a/internal/pgha/harden.go b/internal/pgha/harden.go new file mode 100644 index 0000000..1f37e55 --- /dev/null +++ b/internal/pgha/harden.go @@ -0,0 +1,247 @@ +package pgha + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "slices" + "strconv" + "strings" + "time" +) + +// Hardening: what a cluster set up before this lacked, applied to it in +// place, and to a new one from the start. +// +// - Patroni's REST API asks for a password before a change. Without one, +// anyone who reached port 8008 could rewrite pg_hba through PATCH /config +// — `host all all 0.0.0.0/0 trust` — or switch the leader over. +// - PostgreSQL speaks TLS, with a certificate from the cluster's authority +// on each member, and every remote rule is hostssl: prompts kept in +// threads, key and session hashes, and the engines' secret no longer +// cross the network in clear. The control planes verify the member they +// reach (sslmode=verify-full against the cluster's CA). +// +// Each step looks first and changes only what differs, so running it on a +// hardened cluster does nothing, and each is applied by reload, not restart: +// the cluster keeps serving. + +// RestUser is the user Patroni's REST API asks for before a change. +const RestUser = "patroni" + +// TLSDir holds each member's certificate, key and the cluster's CA. +const TLSDir = "/etc/patroni/tls" + +// ServerCert, ServerKey and CACert are the files in TLSDir. +const ( + ServerCert = TLSDir + "/server.crt" + ServerKey = TLSDir + "/server.key" + CACert = TLSDir + "/ca.crt" +) + +// sslParameters are what turn TLS on, in the cluster's shared parameters. +func sslParameters() map[string]any { + return map[string]any{"ssl": "on", "ssl_cert_file": ServerCert, "ssl_key_file": ServerKey, "ssl_ca_file": CACert} +} + +// hostssl rewrites a remote rule to accept only TLS; local and loopback +// rules are left alone. +func hostssl(line string) string { + f := strings.Fields(line) + if len(f) < 4 || f[0] != "host" || strings.HasPrefix(f[3], "127.") || f[3] == "::1/128" { + return line + } + return "hostssl" + strings.TrimPrefix(line, "host") +} + +// certs gives each database member a certificate from the cluster's +// authority, for its address and name — unless it has one for them already. +func (s *Setup) certs(ctx context.Context) error { + if s.CA == nil { + s.emit("tls", false, "no certificate authority on this machine: PostgreSQL stays without TLS") + return nil + } + for _, d := range s.DB { + have, _ := d.Remote.Run(ctx, "sudo openssl x509 -in "+ServerCert+" -noout -ext subjectAltName 2>/dev/null", nil) + if strings.Contains(string(have), "IP Address:"+d.IP) { + continue + } + leaf, err := s.CA.IssueServer(d.Name, []string{d.IP, d.Name, "127.0.0.1", "localhost"}) + if err != nil { + return err + } + var body bytes.Buffer + body.Write(s.CA.PEM) + body.WriteString("\x00") + body.Write(leaf.CertPEM) + body.WriteString("\x00") + body.Write(leaf.KeyPEM) + if out, err := d.Remote.Run(ctx, `set -e +sudo install -d -o postgres -g postgres -m 750 `+TLSDir+` +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT +cat > "$t/all" +python3 - "$t" <<'PY' +import sys +d = sys.argv[1] +ca, crt, key = open(d + "/all", "rb").read().split(b"\x00") +for name, data in (("ca.crt", ca), ("server.crt", crt), ("server.key", key)): + open(d + "/" + name, "wb").write(data) +PY +sudo install -o postgres -g postgres -m 644 "$t/ca.crt" `+CACert+` +sudo install -o postgres -g postgres -m 644 "$t/server.crt" `+ServerCert+` +sudo install -o postgres -g postgres -m 600 "$t/server.key" `+ServerKey, body.Bytes()); err != nil { + return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) + } + s.emit("tls", true, "certificate for %s (%s)", d.Name, d.IP) + } + return nil +} + +// restAuth makes each member's Patroni ask for the password before a change, +// and reloads it where that was not so. +func (s *Setup) restAuth(ctx context.Context, sec Secrets) error { + for _, d := range s.DB { + out, err := d.Remote.Run(ctx, `set -e +test -f /etc/patroni/config.yml || { echo missing; exit 0; } +# The password from stdin into a file only this user reads: the program +# itself comes in on python's stdin. +f=$(mktemp); trap 'rm -f "$f"' EXIT +cat > "$f" +r=$(sudo python3 - "$f" <<'PY' +import sys, yaml +pw = open(sys.argv[1]).read().strip() +p = "/etc/patroni/config.yml" +c = yaml.safe_load(open(p)) +want = {"username": "`+RestUser+`", "password": pw} +api = c.setdefault("restapi", {}) +if api.get("authentication") == want: + print("same") +else: + api["authentication"] = want + open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" + + yaml.safe_dump(c, default_flow_style=False, sort_keys=False)) + print("changed") +PY +) +if [ "$r" = changed ]; then + sudo chown root:postgres /etc/patroni/config.yml.new + sudo chmod 640 /etc/patroni/config.yml.new + sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml + sudo systemctl reload patroni +fi +echo "$r"`, []byte(sec.RestAPI+"\n")) + if err != nil { + return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out))) + } + switch strings.TrimSpace(lastLine(out)) { + case "changed": + s.emit("patroni api", true, "%s asks for a password before a change", d.Name) + case "missing": + return fmt.Errorf("%s has no /etc/patroni/config.yml", d.Name) + } + } + return nil +} + +// tlsInShared turns TLS on in the cluster's shared configuration, and every +// remote access rule to hostssl, through Patroni's API — which reloads +// PostgreSQL on every member. +func (s *Setup) tlsInShared(ctx context.Context, sec Secrets) error { + if s.CA == nil { + return nil + } + hosts := make([]string, len(s.DB)) + for i, d := range s.DB { + hosts[i] = d.IP + } + client := &http.Client{Timeout: 15 * time.Second} + var lastErr error + for _, h := range hosts { + base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort)) + cfg, err := getConfig(ctx, client, base) + if err != nil { + lastErr = err + continue + } + pg, _ := cfg["postgresql"].(map[string]any) + params, _ := pg["parameters"].(map[string]any) + var hba, next []string + if list, ok := pg["pg_hba"].([]any); ok { + for _, l := range list { + if line, ok := l.(string); ok { + hba = append(hba, line) + next = append(next, hostssl(line)) + } + } + } + want := sslParameters() + same := slices.Equal(hba, next) + for k, v := range want { + if params[k] != v { + same = false + } + } + if same { + s.emit("tls", false, "PostgreSQL speaks TLS, and remote rules are hostssl") + return nil + } + patch := map[string]any{"parameters": want} + if len(hba) > 0 { + patch["pg_hba"] = next + } + if err := patchConfig(ctx, client, base, sec.RestAPI, map[string]any{"postgresql": patch}); err != nil { + return fmt.Errorf("Patroni on %s: %w", h, err) + } + s.emit("tls", true, "PostgreSQL speaks TLS on every member; remote rules are hostssl") + return nil + } + return fmt.Errorf("no member's Patroni answered: %w", lastErr) +} + +// patchConfig changes the cluster's shared configuration, with the REST +// password. +func patchConfig(ctx context.Context, client *http.Client, base, password string, body map[string]any) error { + b, _ := json.Marshal(body) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(b)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if password != "" { + req.SetBasicAuth(RestUser, password) + } + resp, err := client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("refused the change: %s", resp.Status) + } + return nil +} + +// sharedTLS reports whether the cluster's shared configuration has TLS on: +// for a control plane without the authority's key, which signs nothing but +// must still connect by TLS once the cluster asks for it. +func (s *Setup) sharedTLS(ctx context.Context) bool { + client := &http.Client{Timeout: 10 * time.Second} + for _, d := range s.DB { + cfg, err := getConfig(ctx, client, "http://"+net.JoinHostPort(d.IP, strconv.Itoa(restPort))) + if err != nil { + continue + } + pg, _ := cfg["postgresql"].(map[string]any) + params, _ := pg["parameters"].(map[string]any) + return params["ssl"] == "on" + } + return false +} + +func lastLine(b []byte) string { + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + return lines[len(lines)-1] +} diff --git a/internal/pgha/hba.go b/internal/pgha/hba.go index 3a4243b..1fe4242 100644 --- a/internal/pgha/hba.go +++ b/internal/pgha/hba.go @@ -1,7 +1,6 @@ package pgha import ( - "bytes" "context" "encoding/json" "fmt" @@ -9,6 +8,7 @@ import ( "net/http" "slices" "strconv" + "strings" "time" ) @@ -24,7 +24,10 @@ func ClientLine(ip string) string { // A cluster set up before the rules moved into the shared configuration // keeps them in each member's own file, which this cannot reach; that is // said, with what to add, rather than half done. -func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) { +// +// password is what Patroni's API asks for before a change (RestUser); a +// cluster whose rules are hostssl lets the new control plane in by TLS too. +func AllowClient(ctx context.Context, hosts []string, ip, password string) (bool, error) { client := &http.Client{Timeout: 5 * time.Second} var lastErr error for _, h := range hosts { @@ -50,23 +53,15 @@ func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) { "on every member, and run `patronictl reload llmgw`", ClientLine(ip)) } line := ClientLine(ip) + if slices.ContainsFunc(hba, func(l string) bool { return strings.HasPrefix(l, "hostssl ") }) { + line = hostssl(line) + } if slices.Contains(hba, line) { return false, nil } - body, _ := json.Marshal(map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}) - req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(body)) - if err != nil { - return false, err - } - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - lastErr = err - continue - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf("Patroni on %s refused the change: %s", h, resp.Status) + if err := patchConfig(ctx, client, base, password, + map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}); err != nil { + return false, fmt.Errorf("Patroni on %s: %w", h, err) } return true, nil } diff --git a/internal/pgha/pgha_test.go b/internal/pgha/pgha_test.go index b3029fe..90e4446 100644 --- a/internal/pgha/pgha_test.go +++ b/internal/pgha/pgha_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/expfmt" "go.yaml.in/yaml/v3" @@ -24,8 +25,8 @@ var ( ) func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { - sec := Secrets{"su", "rep", "app"} - cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec) + sec := Secrets{"su", "rep", "app", "api"} + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false) for _, want := range []string{ "name: llmgw-pg01", "listen: 10.0.0.149:8008", "hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379", @@ -47,10 +48,14 @@ func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { !strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") { t.Errorf("etcd defaults:\n%s", e) } - if got := DSN([]Member{pg01, ctl02}, "pw"); got != + if got := DSN([]Member{pg01, ctl02}, "pw", ""); got != "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" { t.Errorf("dsn %s", got) } + if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got != + "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" { + t.Errorf("dsn with tls %s", got) + } } func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) { @@ -102,7 +107,7 @@ func (m *machine) ranAny(frag string) bool { return false } -const pws = "superuser=a\nreplicator=b\nllmgw=c\n" +const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n" func healthy(n int) string { return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n) @@ -129,7 +134,8 @@ func TestARunningClusterIsLeftAsItIs(t *testing.T) { t.Errorf("dsn %s", dsn) } for _, m := range []*machine{a, b, w} { - for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} { + // Looked at, for the API's password, but not rewritten or restarted. + for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} { if m.ranAny(frag) { t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag) } @@ -216,7 +222,7 @@ func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) { func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) { // Where `server add` can change them for every member at once. - cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"}) + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false) var doc struct { Bootstrap struct { DCS struct { @@ -262,7 +268,7 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() - changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150") + changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "") if err != nil || !changed { t.Fatalf("%v %v", changed, err) } @@ -270,13 +276,158 @@ func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") { t.Errorf("patched %q", patched) } - if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed { + if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed { t.Errorf("one already let in was changed again: %v %v", changed, err) } hba = nil // the rules are in each member's file - if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil || + if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil || !strings.Contains(err.Error(), "each member") { t.Errorf("rules kept per member: %v", err) } } + +func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) { + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", + Secrets{"a", "b", "c", "the-api-pw"}, true) + var doc struct { + RestAPI struct { + Authentication struct{ Username, Password string } `yaml:"authentication"` + } `yaml:"restapi"` + Bootstrap struct { + DCS struct { + PostgreSQL struct { + Parameters map[string]string `yaml:"parameters"` + PgHBA []string `yaml:"pg_hba"` + } `yaml:"postgresql"` + } `yaml:"dcs"` + } `yaml:"bootstrap"` + } + if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { + t.Fatalf("%v\n%s", err, cfg) + } + if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" { + t.Errorf("rest api authentication %+v", a) + } + p := doc.Bootstrap.DCS.PostgreSQL.Parameters + if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey { + t.Errorf("parameters %v", p) + } + for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA { + if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") { + t.Errorf("a remote rule without TLS: %q", l) + } + } +} + +func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) { + for in, want := range map[string]string{ + "host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256", + "host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256", + "host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256", + "local all all peer": "local all all peer", + "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256", + } { + if got := hostssl(in); got != want { + t.Errorf("%q: %q", in, got) + } + } +} + +func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) { + // The second member holds the three PostgreSQL passwords and no API one. + a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} + b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}} + up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, + {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} + s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}, + Status: func(context.Context, []string) (Cluster, error) { return up, nil }} + if _, err := s.Run(context.Background()); err != nil { + t.Fatal(err) + } + if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" { + t.Errorf("the API password was not carried: %q", s.RestPassword) + } + for _, m := range []*machine{a, b} { + if !m.ranAny(`"username": "patroni"`) { + t.Errorf("%s: the API was not given its password", m.name) + } + } +} + +func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) { + var patched map[string]any + var user, pass string + hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}}) + case http.MethodPatch: + user, pass, _ = r.BasicAuth() + _ = json.NewDecoder(r.Body).Decode(&patched) + } + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + ca, err := tlsconf.NewCA("test") + if err != nil { + t.Fatal(err) + } + s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca} + if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil { + t.Fatal(err) + } + if user != RestUser || pass != "pw" { + t.Errorf("auth %q %q", user, pass) + } + pg := patched["postgresql"].(map[string]any) + if pg["parameters"].(map[string]any)["ssl"] != "on" { + t.Errorf("parameters %v", pg["parameters"]) + } + got := pg["pg_hba"].([]any) + if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) { + t.Errorf("hba %v", got) + } + // A control plane let in afterwards is let in by TLS. + hba = got + if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil { + t.Fatal(err) + } + last := patched["postgresql"].(map[string]any)["pg_hba"].([]any) + if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) { + t.Errorf("let in without TLS: %v", last) + } +} + +func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) { + ssl := "on" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}}) + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} + b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} + up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, + {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} + s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}}, + Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem", + Status: func(context.Context, []string) (Cluster, error) { return up, nil }} + dsn, err := s.Run(context.Background()) + if err != nil || !strings.Contains(dsn, "sslmode=verify-full") { + t.Fatalf("%s %v", dsn, err) + } + ssl = "off" + if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") { + t.Errorf("TLS asked of a cluster without it: %s", dsn) + } +} diff --git a/internal/pgha/setup.go b/internal/pgha/setup.go index 2dca4da..6e6bf07 100644 --- a/internal/pgha/setup.go +++ b/internal/pgha/setup.go @@ -10,6 +10,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Node is a machine the setup reaches, and how. @@ -34,6 +35,15 @@ type Setup struct { Emit func(step, detail string, changed bool) // Wait bounds each wait for the cluster to come up. Wait time.Duration + // CA signs each member's certificate; nil signs none and turns nothing + // on. CAFile is where this control plane finds the authority's + // certificate, for the DSN: used when TLS is on, whoever turned it on. + CA *tlsconf.CA + CAFile string + + // RestPassword is what Patroni's API asks for before a change, once Run + // has found or made it: for `server add`, which changes the access rules. + RestPassword string } // Local is a Remote for the machine this runs on, which init runs as root. @@ -87,12 +97,24 @@ func (s *Setup) Run(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf("passwords: %w", err) } + s.RestPassword = sec.RestAPI if err := s.etcd(ctx); err != nil { return "", fmt.Errorf("etcd: %w", err) } + if err := s.certs(ctx); err != nil { + return "", fmt.Errorf("certificates: %w", err) + } if err := s.patroni(ctx, sec); err != nil { return "", fmt.Errorf("patroni: %w", err) } + // What a cluster set up before them lacks, applied in place; a new one + // has them already and these find nothing to change. + if err := s.restAuth(ctx, sec); err != nil { + return "", fmt.Errorf("patroni api: %w", err) + } + if err := s.tlsInShared(ctx, sec); err != nil { + return "", fmt.Errorf("tls: %w", err) + } if err := s.database(ctx); err != nil { return "", fmt.Errorf("database: %w", err) } @@ -100,52 +122,64 @@ func (s *Setup) Run(ctx context.Context) (string, error) { for i, d := range s.DB { members[i] = d.Member } - return DSN(members, sec.App), nil + // TLS when this setup turned it on, or found it on: then the member + // reached is verified against the authority's certificate, which every + // control plane has even where its key is not. + caFile := "" + if s.CAFile != "" && (s.CA != nil || s.sharedTLS(ctx)) { + caFile = s.CAFile + } + return DSN(members, sec.App, caFile), nil } // secretDir holds the passwords on every database machine, readable by root // alone. Generated on the first, carried to the others. const secretDir = "/root/llmgw-pg" -var secretNames = []string{"superuser", "replicator", "llmgw"} +var secretNames = []string{"superuser", "replicator", "llmgw", "restapi"} func (s *Setup) secrets(ctx context.Context) (Secrets, error) { script := `set -e sudo install -d -m 700 ` + secretDir + ` -for f in superuser replicator llmgw; do +for f in superuser replicator llmgw restapi; do sudo test -s ` + secretDir + `/$f.pw || sudo sh -c "umask 077; openssl rand -hex 24 > ` + secretDir + `/$f.pw" done -for f in superuser replicator llmgw; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done` +for f in superuser replicator llmgw restapi; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done` out, err := s.DB[0].Remote.Run(ctx, script, nil) if err != nil { return Secrets{}, fmt.Errorf("on %s: %v: %s", s.DB[0].Name, err, strings.TrimSpace(string(out))) } got := parseSecrets(out) - if got.Superuser == "" || got.Replicator == "" || got.App == "" { + if got.Superuser == "" || got.Replicator == "" || got.App == "" || got.RestAPI == "" { return Secrets{}, fmt.Errorf("could not read them back from %s", s.DB[0].Name) } for _, d := range s.DB[1:] { - have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw; do + have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw restapi; do if sudo test -s `+secretDir+`/$f.pw; then printf '%s=' $f; sudo cat `+secretDir+`/$f.pw; fi done`, nil) if err != nil { return Secrets{}, fmt.Errorf("on %s: %v", d.Name, err) } - if h := parseSecrets(have); h != (Secrets{}) { - if h != got { + h := parseSecrets(have) + if h.Superuser != "" || h.Replicator != "" || h.App != "" { + if h.Superuser != got.Superuser || h.Replicator != got.Replicator || h.App != got.App { // Not overwritten: a member holding other passwords belongs // to another cluster, or to an earlier one somebody meant to // keep, and replacing them would lock it out of itself. return Secrets{}, fmt.Errorf("%s holds passwords in %s that differ from %s's; "+ "remove them there if that machine is meant to join this cluster", d.Name, secretDir, s.DB[0].Name) } - continue + if h.RestAPI == got.RestAPI { + continue + } + // A cluster from before the API asked for a password: the + // member is given the one the first holds. } - body := fmt.Sprintf("%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App) + body := fmt.Sprintf("%s\n%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App, got.RestAPI) if out, err := d.Remote.Run(ctx, `set -e sudo install -d -m 700 `+secretDir+` -read su; read rep; read app -for kv in "superuser $su" "replicator $rep" "llmgw $app"; do +read su; read rep; read app; read api +for kv in "superuser $su" "replicator $rep" "llmgw $app" "restapi $api"; do set -- $kv printf '%s\n' "$2" | sudo sh -c "umask 077; cat > `+secretDir+`/$1.pw" done`, []byte(body)); err != nil { @@ -171,6 +205,8 @@ func parseSecrets(out []byte) Secrets { s.Replicator = v case "llmgw": s.App = v + case "restapi": + s.RestAPI = v } } return s @@ -288,7 +324,7 @@ func (s *Setup) patroni(ctx context.Context, sec Secrets) error { db = append(db, d.Member) } for i, d := range s.DB { - cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec) + cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec, s.CA != nil) if out, err := d.Remote.Run(ctx, `set -e sudo install -d -m 755 /etc/patroni sudo install -d -o postgres -g postgres -m 700 /var/lib/postgresql/`+version+`/llmgw