package pgha import ( "context" "encoding/json" "net/http" "net/http/httptest" "net/url" "strconv" "strings" "sync" "testing" "time" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/expfmt" "go.yaml.in/yaml/v3" ) var ( pg01 = Member{"llmgw-pg01", "10.0.0.149"} ctl02 = Member{"llmgw-ctl02", "10.0.0.150"} ctl = Member{"llmgw-ctl", "10.0.0.140"} ) func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) { sec := Secrets{"su", "rep", "app"} cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec) for _, want := range []string{ "name: llmgw-pg01", "listen: 10.0.0.149:8008", "hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379", "data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`, "- host replication replicator 10.0.0.150/32 scram-sha-256", "- host llmgw llmgw 10.0.0.140/32 scram-sha-256", // A member can run a control plane too. "- host llmgw llmgw 10.0.0.150/32 scram-sha-256", } { if !strings.Contains(cfg, want) { t.Errorf("patroni config lacks %q", want) } } if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 { t.Error("a member's address is allowed twice") } e := Etcd(ctl, []Member{pg01, ctl02, ctl}) if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") || !strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") { t.Errorf("etcd defaults:\n%s", e) } if got := DSN([]Member{pg01, ctl02}, "pw"); got != "postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" { t.Errorf("dsn %s", got) } } func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) { var c Cluster if err := json.Unmarshal([]byte(`{"members":[ {"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0}, {"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil { t.Fatal(err) } if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) { t.Errorf("%s: leader %q", c, c.Leader()) } c.Members[0].State = "starting" if c.Complete(2) { t.Error("a replica still starting counted as streaming") } if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 { t.Error("lag not read") } } // machine is a fake host: it records what it was asked to run and answers // from a table of script fragments. type machine struct { name string mu sync.Mutex ran []string answers map[string]string } func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) { m.mu.Lock() defer m.mu.Unlock() m.ran = append(m.ran, script) for frag, out := range m.answers { if strings.Contains(script, frag) { return []byte(out), nil } } return nil, nil } func (m *machine) ranAny(frag string) bool { for _, s := range m.ran { if strings.Contains(s, frag) { return true } } return false } const pws = "superuser=a\nreplicator=b\nllmgw=c\n" func healthy(n int) string { return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n) } func TestARunningClusterIsLeftAsItIs(t *testing.T) { // What init does when run again, or against the reference cluster: find // it up, change nothing, and give the address. a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}} b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} w := &machine{name: "ctl"} up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}} s := Setup{ DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, w}, Status: func(context.Context, []string) (Cluster, error) { return up, nil }, } dsn, err := s.Run(context.Background()) if err != nil { t.Fatal(err) } if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") { t.Errorf("dsn %s", dsn) } for _, m := range []*machine{a, b, w} { for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} { if m.ranAny(frag) { t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag) } } } if !a.ranAny("CREATE ROLE") { t.Error("the role was not made sure of on the leader") } } func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) { s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}} if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") { t.Errorf("got %v", err) } } func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) { a := &machine{answers: map[string]string{"openssl rand": pws}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}} s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}} _, err := s.Run(context.Background()) if err == nil || !strings.Contains(err.Error(), "differ") { t.Fatalf("got %v", err) } if b.ranAny("cat > /root/llmgw-pg") { t.Error("the other cluster's passwords were replaced") } } func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) { a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3), "pg_dropcluster": "holds-data\n"}} b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}} s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}, Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil }, Wait: time.Second} _, err := s.Run(context.Background()) if err == nil || !strings.Contains(err.Error(), "databases in it") { t.Fatalf("got %v", err) } if a.ranAny("systemctl restart patroni") { t.Error("patroni was started over somebody's data") } } func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) { c := NewCollector([]string{"10.0.0.149"}) c.status = func(context.Context, []string) (Cluster, error) { return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"}, {Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil } got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes") if err != nil { t.Fatal(err) } for _, want := range []string{ "llmgw_store_patroni_up 1", `llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`, `llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`, } { if !strings.Contains(string(got), want) { t.Errorf("missing %s in\n%s", want, got) } } if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) { t.Error("the leader was given a lag") } } func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) { c := NewCollector([]string{"10.0.0.153", "10.0.0.154"}) c.status = func(context.Context, []string) (Cluster, error) { return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil } cl, err := c.Read() if err != nil { t.Fatal(err) } if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" { t.Errorf("%s", cl) } } func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) { // Where `server add` can change them for every member at once. cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"}) var doc struct { Bootstrap struct { DCS struct { PostgreSQL struct { PgHBA []string `yaml:"pg_hba"` } `yaml:"postgresql"` } `yaml:"dcs"` } `yaml:"bootstrap"` PostgreSQL map[string]any `yaml:"postgresql"` } if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { t.Fatalf("not YAML: %v\n%s", err, cfg) } hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA if len(hba) < 5 || hba[0] != "local all all peer" { t.Errorf("shared pg_hba: %q", hba) } if _, local := doc.PostgreSQL["pg_hba"]; local { t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one") } } func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { var patched []string hba := []any{"local all all peer", ClientLine("10.0.0.140")} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}}) case http.MethodPatch: var body struct { PostgreSQL struct { PgHBA []string `json:"pg_hba"` } `json:"postgresql"` } _ = json.NewDecoder(r.Body).Decode(&body) patched = body.PostgreSQL.PgHBA } })) defer srv.Close() u, _ := url.Parse(srv.URL) old := restPort restPort, _ = strconv.Atoi(u.Port()) defer func() { restPort = old }() changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150") if err != nil || !changed { t.Fatalf("%v %v", changed, err) } // Everything that was there is kept, and the new line added once. if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") { t.Errorf("patched %q", patched) } if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed { t.Errorf("one already let in was changed again: %v %v", changed, err) } hba = nil // the rules are in each member's file if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil || !strings.Contains(err.Error(), "each member") { t.Errorf("rules kept per member: %v", err) } }