Patroni's REST API accepted changes from anyone on port 8008, and PostgreSQL ran with sslmode=disable between machines. Setup now keeps a fourth secret for the API (carried to members of older clusters), issues each member a certificate from the cluster's authority, writes restapi.authentication into each member's config and reloads it, and turns ssl on with hostssl remote rules through an authenticated PATCH of the shared configuration. The DSN becomes sslmode=verify-full against the authority's certificate whenever the cluster has TLS, so a control plane without the authority's key reconnects too. server add authenticates its PATCH and lets the new control plane in by hostssl. Tried on the e2e cluster: unauthenticated PATCH 401, plaintext refused, replication streaming, the control plane on verify-full. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
434 lines
16 KiB
Go
434 lines
16 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/prometheus/common/expfmt"
|
|
"go.yaml.in/yaml/v3"
|
|
)
|
|
|
|
var (
|
|
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
|
|
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
|
|
ctl = Member{"llmgw-ctl", "10.0.0.140"}
|
|
)
|
|
|
|
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
|
|
sec := Secrets{"su", "rep", "app", "api"}
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false)
|
|
for _, want := range []string{
|
|
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
|
|
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
|
|
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
|
|
"- host replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
// A member can run a control plane too.
|
|
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
|
|
} {
|
|
if !strings.Contains(cfg, want) {
|
|
t.Errorf("patroni config lacks %q", want)
|
|
}
|
|
}
|
|
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
|
|
t.Error("a member's address is allowed twice")
|
|
}
|
|
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
|
|
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
|
|
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
|
|
t.Errorf("etcd defaults:\n%s", e)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw", ""); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
|
|
t.Errorf("dsn %s", got)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" {
|
|
t.Errorf("dsn with tls %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
|
|
var c Cluster
|
|
if err := json.Unmarshal([]byte(`{"members":[
|
|
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
|
|
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
|
|
t.Errorf("%s: leader %q", c, c.Leader())
|
|
}
|
|
c.Members[0].State = "starting"
|
|
if c.Complete(2) {
|
|
t.Error("a replica still starting counted as streaming")
|
|
}
|
|
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
|
|
t.Error("lag not read")
|
|
}
|
|
}
|
|
|
|
// machine is a fake host: it records what it was asked to run and answers
|
|
// from a table of script fragments.
|
|
type machine struct {
|
|
name string
|
|
mu sync.Mutex
|
|
ran []string
|
|
answers map[string]string
|
|
}
|
|
|
|
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.ran = append(m.ran, script)
|
|
for frag, out := range m.answers {
|
|
if strings.Contains(script, frag) {
|
|
return []byte(out), nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *machine) ranAny(frag string) bool {
|
|
for _, s := range m.ran {
|
|
if strings.Contains(s, frag) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n"
|
|
|
|
func healthy(n int) string {
|
|
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
|
|
}
|
|
|
|
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
|
|
// What init does when run again, or against the reference cluster: find
|
|
// it up, change nothing, and give the address.
|
|
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
w := &machine{name: "ctl"}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{
|
|
DB: []Node{{pg01, a}, {ctl02, b}},
|
|
Witness: &Node{ctl, w},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
|
|
t.Errorf("dsn %s", dsn)
|
|
}
|
|
for _, m := range []*machine{a, b, w} {
|
|
// Looked at, for the API's password, but not rewritten or restarted.
|
|
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} {
|
|
if m.ranAny(frag) {
|
|
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
|
|
}
|
|
}
|
|
}
|
|
if !a.ranAny("CREATE ROLE") {
|
|
t.Error("the role was not made sure of on the leader")
|
|
}
|
|
}
|
|
|
|
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
|
|
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
|
|
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "differ") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if b.ranAny("cat > /root/llmgw-pg") {
|
|
t.Error("the other cluster's passwords were replaced")
|
|
}
|
|
}
|
|
|
|
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
|
|
"pg_dropcluster": "holds-data\n"}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
|
|
Wait: time.Second}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "databases in it") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if a.ranAny("systemctl restart patroni") {
|
|
t.Error("patroni was started over somebody's data")
|
|
}
|
|
}
|
|
|
|
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.149"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
|
|
}
|
|
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"llmgw_store_patroni_up 1",
|
|
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
|
|
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
|
|
} {
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("missing %s in\n%s", want, got)
|
|
}
|
|
}
|
|
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
|
|
t.Error("the leader was given a lag")
|
|
}
|
|
}
|
|
|
|
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
|
|
}
|
|
cl, err := c.Read()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
|
|
t.Errorf("%s", cl)
|
|
}
|
|
}
|
|
|
|
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
|
|
// Where `server add` can change them for every member at once.
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false)
|
|
var doc struct {
|
|
Bootstrap struct {
|
|
DCS struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `yaml:"pg_hba"`
|
|
} `yaml:"postgresql"`
|
|
} `yaml:"dcs"`
|
|
} `yaml:"bootstrap"`
|
|
PostgreSQL map[string]any `yaml:"postgresql"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
|
|
t.Fatalf("not YAML: %v\n%s", err, cfg)
|
|
}
|
|
hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
|
|
if len(hba) < 5 || hba[0] != "local all all peer" {
|
|
t.Errorf("shared pg_hba: %q", hba)
|
|
}
|
|
if _, local := doc.PostgreSQL["pg_hba"]; local {
|
|
t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
|
|
}
|
|
}
|
|
|
|
func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
|
|
var patched []string
|
|
hba := []any{"local all all peer", ClientLine("10.0.0.140")}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
|
|
case http.MethodPatch:
|
|
var body struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `json:"pg_hba"`
|
|
} `json:"postgresql"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
patched = body.PostgreSQL.PgHBA
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "")
|
|
if err != nil || !changed {
|
|
t.Fatalf("%v %v", changed, err)
|
|
}
|
|
// Everything that was there is kept, and the new line added once.
|
|
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
|
|
t.Errorf("patched %q", patched)
|
|
}
|
|
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed {
|
|
t.Errorf("one already let in was changed again: %v %v", changed, err)
|
|
}
|
|
|
|
hba = nil // the rules are in each member's file
|
|
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil ||
|
|
!strings.Contains(err.Error(), "each member") {
|
|
t.Errorf("rules kept per member: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) {
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16",
|
|
Secrets{"a", "b", "c", "the-api-pw"}, true)
|
|
var doc struct {
|
|
RestAPI struct {
|
|
Authentication struct{ Username, Password string } `yaml:"authentication"`
|
|
} `yaml:"restapi"`
|
|
Bootstrap struct {
|
|
DCS struct {
|
|
PostgreSQL struct {
|
|
Parameters map[string]string `yaml:"parameters"`
|
|
PgHBA []string `yaml:"pg_hba"`
|
|
} `yaml:"postgresql"`
|
|
} `yaml:"dcs"`
|
|
} `yaml:"bootstrap"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
|
|
t.Fatalf("%v\n%s", err, cfg)
|
|
}
|
|
if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" {
|
|
t.Errorf("rest api authentication %+v", a)
|
|
}
|
|
p := doc.Bootstrap.DCS.PostgreSQL.Parameters
|
|
if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey {
|
|
t.Errorf("parameters %v", p)
|
|
}
|
|
for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA {
|
|
if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") {
|
|
t.Errorf("a remote rule without TLS: %q", l)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
"host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256",
|
|
"local all all peer": "local all all peer",
|
|
"hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256",
|
|
} {
|
|
if got := hostssl(in); got != want {
|
|
t.Errorf("%q: %q", in, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) {
|
|
// The second member holds the three PostgreSQL passwords and no API one.
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
|
|
if _, err := s.Run(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" {
|
|
t.Errorf("the API password was not carried: %q", s.RestPassword)
|
|
}
|
|
for _, m := range []*machine{a, b} {
|
|
if !m.ranAny(`"username": "patroni"`) {
|
|
t.Errorf("%s: the API was not given its password", m.name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) {
|
|
var patched map[string]any
|
|
var user, pass string
|
|
hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}})
|
|
case http.MethodPatch:
|
|
user, pass, _ = r.BasicAuth()
|
|
_ = json.NewDecoder(r.Body).Decode(&patched)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
ca, err := tlsconf.NewCA("test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
|
|
if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if user != RestUser || pass != "pw" {
|
|
t.Errorf("auth %q %q", user, pass)
|
|
}
|
|
pg := patched["postgresql"].(map[string]any)
|
|
if pg["parameters"].(map[string]any)["ssl"] != "on" {
|
|
t.Errorf("parameters %v", pg["parameters"])
|
|
}
|
|
got := pg["pg_hba"].([]any)
|
|
if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) {
|
|
t.Errorf("hba %v", got)
|
|
}
|
|
// A control plane let in afterwards is let in by TLS.
|
|
hba = got
|
|
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
last := patched["postgresql"].(map[string]any)["pg_hba"].([]any)
|
|
if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) {
|
|
t.Errorf("let in without TLS: %v", last)
|
|
}
|
|
}
|
|
|
|
func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) {
|
|
ssl := "on"
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}})
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}},
|
|
Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem",
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil || !strings.Contains(dsn, "sslmode=verify-full") {
|
|
t.Fatalf("%s %v", dsn, err)
|
|
}
|
|
ssl = "off"
|
|
if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") {
|
|
t.Errorf("TLS asked of a cluster without it: %s", dsn)
|
|
}
|
|
}
|