package pgha import ( "context" "encoding/json" "net/http" "net/http/httptest" "net/url" "strconv" "strings" "sync" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/expfmt" "go.yaml.in/yaml/v3" ) var ( pg01 = Member{"llmgw-pg01", "10.0.0.149"} ctl02 = Member{"llmgw-ctl02", "10.0.0.150"} ctl = Member{"llmgw-ctl", "10.0.0.140"} ) func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { sec := Secrets{"su", "rep", "app", "api"} cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false) for _, want := range []string{ "name: llmgw-pg01", "listen: 10.0.0.149:8008", "hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379", "data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`, "- host replication replicator 10.0.0.150/32 scram-sha-256", "- host llmgw llmgw 10.0.0.140/32 scram-sha-256", // A member can run a control plane too. "- host llmgw llmgw 10.0.0.150/32 scram-sha-256", } { if !strings.Contains(cfg, want) { t.Errorf("patroni config lacks %q", want) } } if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 { t.Error("a member's address is allowed twice") } e := Etcd(ctl, []Member{pg01, ctl02, ctl}, false) if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") || !strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") { t.Errorf("etcd defaults:\n%s", e) } e = Etcd(ctl, []Member{pg01, ctl02, ctl}, true) for _, want := range []string{ "llmgw-pg01=https://10.0.0.149:2380", "ETCD_LISTEN_CLIENT_URLS=https://10.0.0.140:2379,https://127.0.0.1:2379\n", "ETCD_ADVERTISE_CLIENT_URLS=https://10.0.0.140:2379\n", "ETCD_CLIENT_CERT_AUTH=true\n", "ETCD_PEER_CLIENT_CERT_AUTH=true\n", "ETCD_TRUSTED_CA_FILE=/etc/etcd/tls/ca.crt\n", } { if !strings.Contains(e, want) { t.Errorf("etcd over TLS lacks %q:\n%s", want, e) } } if strings.Contains(e, "http://") { t.Errorf("etcd over TLS still listens in the clear:\n%s", e) } if got := DSN([]Member{pg01, ctl02}, "pw", ""); got != "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" { t.Errorf("dsn %s", got) } if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got != "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" { t.Errorf("dsn with tls %s", got) } } func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) { var c Cluster if err := json.Unmarshal([]byte(`{"members":[ {"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0}, {"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil { t.Fatal(err) } if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) { t.Errorf("%s: leader %q", c, c.Leader()) } c.Members[0].State = "starting" if c.Complete(2) { t.Error("a replica still starting counted as streaming") } if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 { t.Error("lag not read") } } // machine is a fake host: it records what it was asked to run and answers // from a table of script fragments. type machine struct { name string mu sync.Mutex ran []string answers map[string]string } func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) { m.mu.Lock() defer m.mu.Unlock() m.ran = append(m.ran, script) for frag, out := range m.answers { if strings.Contains(script, frag) { return []byte(out), nil } } return nil, nil } func (m *machine) ranAny(frag string) bool { for _, s := range m.ran { if strings.Contains(s, frag) { return true } } return false } const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n" func healthy(n int) string { return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n) } func TestARunningClusterIsLeftAsItIs(t *testing.T) { // What init does when run again, or against the reference cluster: find // it up, change nothing, and give the address. a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} w := &machine{name: "ctl"} up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} s := Setup{ DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, w}, Status: func(context.Context, []string) (Cluster, error) { return up, nil }, } dsn, err := s.Run(context.Background()) if err != nil { t.Fatal(err) } if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") { t.Errorf("dsn %s", dsn) } for _, m := range []*machine{a, b, w} { // Looked at, for the API's password, but not rewritten or restarted. for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} { if m.ranAny(frag) { t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag) } } } if !a.ranAny("CREATE ROLE") { t.Error("the role was not made sure of on the leader") } // Enabled where the package is there, which on a running cluster is // all a rerun does about it: it installs nothing. if !a.ranAny("CREATE EXTENSION IF NOT EXISTS vector") { t.Error("pgvector was not enabled in llmgw's database on the leader") } } func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) { s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}} if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") { t.Errorf("got %v", err) } } func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) { a := &machine{answers: map[string]string{"openssl rand": pws}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}} s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}} _, err := s.Run(context.Background()) if err == nil || !strings.Contains(err.Error(), "differ") { t.Fatalf("got %v", err) } if b.ranAny("cat > /root/llmgw-pg") { t.Error("the other cluster's passwords were replaced") } } func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) { a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3), "pg_dropcluster": "holds-data\n"}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}, Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil }, Wait: time.Second} _, err := s.Run(context.Background()) if err == nil || !strings.Contains(err.Error(), "databases in it") { t.Fatalf("got %v", err) } if a.ranAny("systemctl restart patroni") { t.Error("patroni was started over somebody's data") } } func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) { c := NewCollector([]string{"10.0.0.149"}) c.status = func(context.Context, []string) (Cluster, error) { return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil } got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes") if err != nil { t.Fatal(err) } for _, want := range []string{ "llmgw_store_patroni_up 1", `llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`, `llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`, } { if !strings.Contains(string(got), want) { t.Errorf("missing %s in\n%s", want, got) } } if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) { t.Error("the leader was given a lag") } } func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) { c := NewCollector([]string{"10.0.0.153", "10.0.0.154"}) c.status = func(context.Context, []string) (Cluster, error) { return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil } cl, err := c.Read() if err != nil { t.Fatal(err) } if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" { t.Errorf("%s", cl) } } func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) { // Where `server add` can change them for every member at once. cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false) var doc struct { Bootstrap struct { DCS struct { PostgreSQL struct { PgHBA []string `yaml:"pg_hba"` } `yaml:"postgresql"` } `yaml:"dcs"` } `yaml:"bootstrap"` PostgreSQL map[string]any `yaml:"postgresql"` } if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { t.Fatalf("not YAML: %v\n%s", err, cfg) } hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA if len(hba) < 5 || hba[0] != "local all all peer" { t.Errorf("shared pg_hba: %q", hba) } if _, local := doc.PostgreSQL["pg_hba"]; local { t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one") } } func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { var patched []string hba := []any{"local all all peer", ClientLine("10.0.0.140")} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}}) case http.MethodPatch: var body struct { PostgreSQL struct { PgHBA []string `json:"pg_hba"` } `json:"postgresql"` } _ = json.NewDecoder(r.Body).Decode(&body) patched = body.PostgreSQL.PgHBA } })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "") if err != nil || !changed { t.Fatalf("%v %v", changed, err) } // Everything that was there is kept, and the new line added once. if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") { t.Errorf("patched %q", patched) } if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed { t.Errorf("one already let in was changed again: %v %v", changed, err) } hba = nil // the rules are in each member's file if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil || !strings.Contains(err.Error(), "each member") { t.Errorf("rules kept per member: %v", err) } } func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) { cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "the-api-pw"}, true) var doc struct { RestAPI struct { Authentication struct{ Username, Password string } `yaml:"authentication"` CertFile string `yaml:"certfile"` CAFile string `yaml:"cafile"` } `yaml:"restapi"` Ctl struct { CACert string `yaml:"cacert"` } `yaml:"ctl"` Etcd3 struct { Protocol, CACert, Cert, Key string } `yaml:"etcd3"` Bootstrap struct { DCS struct { PostgreSQL struct { Parameters map[string]string `yaml:"parameters"` PgHBA []string `yaml:"pg_hba"` } `yaml:"postgresql"` } `yaml:"dcs"` } `yaml:"bootstrap"` } if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { t.Fatalf("%v\n%s", err, cfg) } if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" { t.Errorf("rest api authentication %+v", a) } if doc.RestAPI.CertFile != ServerCert || doc.RestAPI.CAFile != CACert || doc.Ctl.CACert != CACert { t.Errorf("the API is not served over TLS:\n%s", cfg) } if e := doc.Etcd3; e.Protocol != "https" || e.CACert != CACert || e.Cert != ServerCert || e.Key != ServerKey { t.Errorf("etcd is not reached over TLS: %+v", e) } p := doc.Bootstrap.DCS.PostgreSQL.Parameters if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey { t.Errorf("parameters %v", p) } for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA { if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") { t.Errorf("a remote rule without TLS: %q", l) } } } func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) { for in, want := range map[string]string{ "host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256", "host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256", "host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256", "local all all peer": "local all all peer", "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256", } { if got := hostssl(in); got != want { t.Errorf("%q: %q", in, got) } } } func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) { // The second member holds the three PostgreSQL passwords and no API one. a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}} up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}, Status: func(context.Context, []string) (Cluster, error) { return up, nil }} if _, err := s.Run(context.Background()); err != nil { t.Fatal(err) } if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" { t.Errorf("the API password was not carried: %q", s.RestPassword) } for _, m := range []*machine{a, b} { if !m.ranAny(`"username": "patroni"`) { t.Errorf("%s: the API was not given its password", m.name) } } } func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) { var patched map[string]any var user, pass string hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}}) case http.MethodPatch: user, pass, _ = r.BasicAuth() _ = json.NewDecoder(r.Body).Decode(&patched) } })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() ca, err := tlsconf.NewCA("test") if err != nil { t.Fatal(err) } s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca} if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil { t.Fatal(err) } if user != RestUser || pass != "pw" { t.Errorf("auth %q %q", user, pass) } pg := patched["postgresql"].(map[string]any) if pg["parameters"].(map[string]any)["ssl"] != "on" { t.Errorf("parameters %v", pg["parameters"]) } got := pg["pg_hba"].([]any) if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) { t.Errorf("hba %v", got) } // A control plane let in afterwards is let in by TLS. hba = got if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil { t.Fatal(err) } last := patched["postgresql"].(map[string]any)["pg_hba"].([]any) if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) { t.Errorf("let in without TLS: %v", last) } } func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) { ssl := "on" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}}) })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}}, Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem", Status: func(context.Context, []string) (Cluster, error) { return up, nil }} dsn, err := s.Run(context.Background()) if err != nil || !strings.Contains(dsn, "sslmode=verify-full") { t.Fatalf("%s %v", dsn, err) } ssl = "off" if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") { t.Errorf("TLS asked of a cluster without it: %s", dsn) } } func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) { total, ok := MemTotal("MemTotal: 4009848 kB\nMemFree: 976000 kB\n") if !ok || total != 4009848<<10 { t.Fatalf("MemTotal %d %v", total, ok) } own := MemorySettings(total, false) if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" || own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" { t.Errorf("a machine of its own: %v", own) } // Beside a control plane, less of it. if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" { t.Errorf("a shared machine: %v", sh) } // Never below what PostgreSQL needs, never past what it can use. if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" { t.Errorf("a small machine: %v", tiny) } if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" { t.Errorf("a large machine: %v", big) } } func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) { var mu sync.Mutex var restarted []string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPost && r.URL.Path == "/restart" { if _, pass, _ := r.BasicAuth(); pass != "pw" { w.WriteHeader(http.StatusUnauthorized) return } host, _, _ := strings.Cut(r.Host, ":") mu.Lock() restarted = append(restarted, host) mu.Unlock() } })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"} leader := &machine{name: "a", answers: answers} replica := &machine{name: "b", answers: answers} up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true}, {Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}} s := Setup{ DB: []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}}, Status: func(context.Context, []string) (Cluster, error) { return up, nil }, Wait: time.Second, } if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil { t.Fatal(err) } if strings.Join(restarted, " ") != "localhost 127.0.0.1" { t.Errorf("restarted %v; the replica first, then the leader", restarted) } } func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) { // A run that stopped after writing the configuration, or restarted before // Patroni had taken it: nothing to change, but PostgreSQL still on the old. var restarted []string srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { if r.URL.Path == "/restart" { host, _, _ := strings.Cut(r.Host, ":") restarted = append(restarted, host) } })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"} up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"}, {Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}} s := Setup{ DB: []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}}, Status: func(context.Context, []string) (Cluster, error) { return up, nil }, Wait: time.Second, } if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil { t.Fatal(err) } if strings.Join(restarted, " ") != "localhost" { t.Errorf("restarted %v; only the member pending", restarted) } }