init --patroni user@host ... installs PostgreSQL, Patroni and etcd on the given machines over ssh, with this one as the third etcd vote, makes the role and database, and moves the store in use into the cluster. Each step checks first, so it changes nothing on a cluster that is up. For a store of several hosts the control plane publishes each member's role, state and lag from Patroni, a stopped member as missing, prints it in ctl status, and alerts on it. Making an administrator on PostgreSQL sent a bool to an integer column; it sends an integer now. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
211 lines
7.3 KiB
Go
211 lines
7.3 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/prometheus/common/expfmt"
|
|
)
|
|
|
|
var (
|
|
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
|
|
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
|
|
ctl = Member{"llmgw-ctl", "10.0.0.140"}
|
|
)
|
|
|
|
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
|
|
sec := Secrets{"su", "rep", "app"}
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec)
|
|
for _, want := range []string{
|
|
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
|
|
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
|
|
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
|
|
"- host replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
// A member can run a control plane too.
|
|
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
|
|
} {
|
|
if !strings.Contains(cfg, want) {
|
|
t.Errorf("patroni config lacks %q", want)
|
|
}
|
|
}
|
|
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
|
|
t.Error("a member's address is allowed twice")
|
|
}
|
|
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
|
|
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
|
|
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
|
|
t.Errorf("etcd defaults:\n%s", e)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw"); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
|
|
t.Errorf("dsn %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
|
|
var c Cluster
|
|
if err := json.Unmarshal([]byte(`{"members":[
|
|
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
|
|
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
|
|
t.Errorf("%s: leader %q", c, c.Leader())
|
|
}
|
|
c.Members[0].State = "starting"
|
|
if c.Complete(2) {
|
|
t.Error("a replica still starting counted as streaming")
|
|
}
|
|
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
|
|
t.Error("lag not read")
|
|
}
|
|
}
|
|
|
|
// machine is a fake host: it records what it was asked to run and answers
|
|
// from a table of script fragments.
|
|
type machine struct {
|
|
name string
|
|
mu sync.Mutex
|
|
ran []string
|
|
answers map[string]string
|
|
}
|
|
|
|
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.ran = append(m.ran, script)
|
|
for frag, out := range m.answers {
|
|
if strings.Contains(script, frag) {
|
|
return []byte(out), nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *machine) ranAny(frag string) bool {
|
|
for _, s := range m.ran {
|
|
if strings.Contains(s, frag) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const pws = "superuser=a\nreplicator=b\nllmgw=c\n"
|
|
|
|
func healthy(n int) string {
|
|
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
|
|
}
|
|
|
|
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
|
|
// What init does when run again, or against the reference cluster: find
|
|
// it up, change nothing, and give the address.
|
|
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
w := &machine{name: "ctl"}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{
|
|
DB: []Node{{pg01, a}, {ctl02, b}},
|
|
Witness: &Node{ctl, w},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
|
|
t.Errorf("dsn %s", dsn)
|
|
}
|
|
for _, m := range []*machine{a, b, w} {
|
|
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} {
|
|
if m.ranAny(frag) {
|
|
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
|
|
}
|
|
}
|
|
}
|
|
if !a.ranAny("CREATE ROLE") {
|
|
t.Error("the role was not made sure of on the leader")
|
|
}
|
|
}
|
|
|
|
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
|
|
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
|
|
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "differ") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if b.ranAny("cat > /root/llmgw-pg") {
|
|
t.Error("the other cluster's passwords were replaced")
|
|
}
|
|
}
|
|
|
|
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
|
|
"pg_dropcluster": "holds-data\n"}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
|
|
Wait: time.Second}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "databases in it") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if a.ranAny("systemctl restart patroni") {
|
|
t.Error("patroni was started over somebody's data")
|
|
}
|
|
}
|
|
|
|
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.149"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
|
|
}
|
|
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"llmgw_store_patroni_up 1",
|
|
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
|
|
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
|
|
} {
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("missing %s in\n%s", want, got)
|
|
}
|
|
}
|
|
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
|
|
t.Error("the leader was given a lag")
|
|
}
|
|
}
|
|
|
|
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
|
|
}
|
|
cl, err := c.Read()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
|
|
t.Errorf("%s", cl)
|
|
}
|
|
}
|