Files
llmgw/internal/pgha/pgha_test.go
T
Ilker ManapandClaude Opus 5.5 975c93994f Set up a replicated store with server init --patroni, and watch it
init --patroni user@host ... installs PostgreSQL, Patroni and etcd on the
given machines over ssh, with this one as the third etcd vote, makes the role
and database, and moves the store in use into the cluster. Each step checks
first, so it changes nothing on a cluster that is up. For a store of several
hosts the control plane publishes each member's role, state and lag from
Patroni, a stopped member as missing, prints it in ctl status, and alerts on
it. Making an administrator on PostgreSQL sent a bool to an integer column;
it sends an integer now.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 20:57:41 +02:00

211 lines
7.3 KiB
Go

package pgha
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/expfmt"
)
var (
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
ctl = Member{"llmgw-ctl", "10.0.0.140"}
)
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
sec := Secrets{"su", "rep", "app"}
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec)
for _, want := range []string{
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
"- host replication replicator 10.0.0.150/32 scram-sha-256",
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
// A member can run a control plane too.
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
} {
if !strings.Contains(cfg, want) {
t.Errorf("patroni config lacks %q", want)
}
}
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
t.Error("a member's address is allowed twice")
}
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
t.Errorf("etcd defaults:\n%s", e)
}
if got := DSN([]Member{pg01, ctl02}, "pw"); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
t.Errorf("dsn %s", got)
}
}
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
var c Cluster
if err := json.Unmarshal([]byte(`{"members":[
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
t.Fatal(err)
}
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
t.Errorf("%s: leader %q", c, c.Leader())
}
c.Members[0].State = "starting"
if c.Complete(2) {
t.Error("a replica still starting counted as streaming")
}
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
t.Error("lag not read")
}
}
// machine is a fake host: it records what it was asked to run and answers
// from a table of script fragments.
type machine struct {
name string
mu sync.Mutex
ran []string
answers map[string]string
}
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.ran = append(m.ran, script)
for frag, out := range m.answers {
if strings.Contains(script, frag) {
return []byte(out), nil
}
}
return nil, nil
}
func (m *machine) ranAny(frag string) bool {
for _, s := range m.ran {
if strings.Contains(s, frag) {
return true
}
}
return false
}
const pws = "superuser=a\nreplicator=b\nllmgw=c\n"
func healthy(n int) string {
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
}
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
// What init does when run again, or against the reference cluster: find
// it up, change nothing, and give the address.
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
w := &machine{name: "ctl"}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{
DB: []Node{{pg01, a}, {ctl02, b}},
Witness: &Node{ctl, w},
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
}
dsn, err := s.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
t.Errorf("dsn %s", dsn)
}
for _, m := range []*machine{a, b, w} {
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} {
if m.ranAny(frag) {
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
}
}
}
if !a.ranAny("CREATE ROLE") {
t.Error("the role was not made sure of on the leader")
}
}
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
t.Errorf("got %v", err)
}
}
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "differ") {
t.Fatalf("got %v", err)
}
if b.ranAny("cat > /root/llmgw-pg") {
t.Error("the other cluster's passwords were replaced")
}
}
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
"pg_dropcluster": "holds-data\n"}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
Wait: time.Second}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "databases in it") {
t.Fatalf("got %v", err)
}
if a.ranAny("systemctl restart patroni") {
t.Error("patroni was started over somebody's data")
}
}
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
c := NewCollector([]string{"10.0.0.149"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
}
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
"llmgw_store_patroni_up 1",
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
} {
if !strings.Contains(string(got), want) {
t.Errorf("missing %s in\n%s", want, got)
}
}
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
t.Error("the leader was given a lag")
}
}
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
}
cl, err := c.Read()
if err != nil {
t.Fatal(err)
}
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
t.Errorf("%s", cl)
}
}