CI / test (push) Failing after 30m6s
Measured on three VMs: cutting the store primary's power cost about 45 s of hanging requests. New connections tried the dead member first with no connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of both cost a further 17 s of 503s while the new leader knew no judge. The store dials each member for 3 s and closes connections to a machine that is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing clusters through its API. Judges that all fail at once are asked again every 250 ms while the policy's budget lasts, and reported once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
601 lines
23 KiB
Go
601 lines
23 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/prometheus/common/expfmt"
|
|
"go.yaml.in/yaml/v3"
|
|
)
|
|
|
|
var (
|
|
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
|
|
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
|
|
ctl = Member{"llmgw-ctl", "10.0.0.140"}
|
|
)
|
|
|
|
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
|
|
sec := Secrets{"su", "rep", "app", "api"}
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false)
|
|
for _, want := range []string{
|
|
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
|
|
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
|
|
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
|
|
"- host replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
// A member can run a control plane too.
|
|
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
|
|
} {
|
|
if !strings.Contains(cfg, want) {
|
|
t.Errorf("patroni config lacks %q", want)
|
|
}
|
|
}
|
|
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
|
|
t.Error("a member's address is allowed twice")
|
|
}
|
|
e := Etcd(ctl, []Member{pg01, ctl02, ctl}, false)
|
|
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
|
|
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
|
|
t.Errorf("etcd defaults:\n%s", e)
|
|
}
|
|
e = Etcd(ctl, []Member{pg01, ctl02, ctl}, true)
|
|
for _, want := range []string{
|
|
"llmgw-pg01=https://10.0.0.149:2380",
|
|
"ETCD_LISTEN_CLIENT_URLS=https://10.0.0.140:2379,https://127.0.0.1:2379\n",
|
|
"ETCD_ADVERTISE_CLIENT_URLS=https://10.0.0.140:2379\n",
|
|
"ETCD_CLIENT_CERT_AUTH=true\n", "ETCD_PEER_CLIENT_CERT_AUTH=true\n",
|
|
"ETCD_TRUSTED_CA_FILE=/etc/etcd/tls/ca.crt\n",
|
|
} {
|
|
if !strings.Contains(e, want) {
|
|
t.Errorf("etcd over TLS lacks %q:\n%s", want, e)
|
|
}
|
|
}
|
|
if strings.Contains(e, "http://") {
|
|
t.Errorf("etcd over TLS still listens in the clear:\n%s", e)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw", ""); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
|
|
t.Errorf("dsn %s", got)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" {
|
|
t.Errorf("dsn with tls %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
|
|
var c Cluster
|
|
if err := json.Unmarshal([]byte(`{"members":[
|
|
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
|
|
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
|
|
t.Errorf("%s: leader %q", c, c.Leader())
|
|
}
|
|
c.Members[0].State = "starting"
|
|
if c.Complete(2) {
|
|
t.Error("a replica still starting counted as streaming")
|
|
}
|
|
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
|
|
t.Error("lag not read")
|
|
}
|
|
}
|
|
|
|
// machine is a fake host: it records what it was asked to run and answers
|
|
// from a table of script fragments.
|
|
type machine struct {
|
|
name string
|
|
mu sync.Mutex
|
|
ran []string
|
|
answers map[string]string
|
|
}
|
|
|
|
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.ran = append(m.ran, script)
|
|
for frag, out := range m.answers {
|
|
if strings.Contains(script, frag) {
|
|
return []byte(out), nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *machine) ranAny(frag string) bool {
|
|
for _, s := range m.ran {
|
|
if strings.Contains(s, frag) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n"
|
|
|
|
func healthy(n int) string {
|
|
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
|
|
}
|
|
|
|
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
|
|
// What init does when run again, or against the reference cluster: find
|
|
// it up, change nothing, and give the address.
|
|
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
w := &machine{name: "ctl"}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{
|
|
DB: []Node{{pg01, a}, {ctl02, b}},
|
|
Witness: &Node{ctl, w},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
|
|
t.Errorf("dsn %s", dsn)
|
|
}
|
|
for _, m := range []*machine{a, b, w} {
|
|
// Looked at, for the API's password, but not rewritten or restarted.
|
|
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} {
|
|
if m.ranAny(frag) {
|
|
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
|
|
}
|
|
}
|
|
}
|
|
if !a.ranAny("CREATE ROLE") {
|
|
t.Error("the role was not made sure of on the leader")
|
|
}
|
|
// Enabled where the package is there, which on a running cluster is
|
|
// all a rerun does about it: it installs nothing.
|
|
if !a.ranAny("CREATE EXTENSION IF NOT EXISTS vector") {
|
|
t.Error("pgvector was not enabled in llmgw's database on the leader")
|
|
}
|
|
}
|
|
|
|
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
|
|
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
|
|
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "differ") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if b.ranAny("cat > /root/llmgw-pg") {
|
|
t.Error("the other cluster's passwords were replaced")
|
|
}
|
|
}
|
|
|
|
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
|
|
"pg_dropcluster": "holds-data\n"}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
|
|
Wait: time.Second}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "databases in it") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if a.ranAny("systemctl restart patroni") {
|
|
t.Error("patroni was started over somebody's data")
|
|
}
|
|
}
|
|
|
|
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.149"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
|
|
}
|
|
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"llmgw_store_patroni_up 1",
|
|
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
|
|
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
|
|
} {
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("missing %s in\n%s", want, got)
|
|
}
|
|
}
|
|
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
|
|
t.Error("the leader was given a lag")
|
|
}
|
|
}
|
|
|
|
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
|
|
}
|
|
cl, err := c.Read()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
|
|
t.Errorf("%s", cl)
|
|
}
|
|
}
|
|
|
|
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
|
|
// Where `server add` can change them for every member at once.
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false)
|
|
var doc struct {
|
|
Bootstrap struct {
|
|
DCS struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `yaml:"pg_hba"`
|
|
} `yaml:"postgresql"`
|
|
} `yaml:"dcs"`
|
|
} `yaml:"bootstrap"`
|
|
PostgreSQL map[string]any `yaml:"postgresql"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
|
|
t.Fatalf("not YAML: %v\n%s", err, cfg)
|
|
}
|
|
hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
|
|
if len(hba) < 5 || hba[0] != "local all all peer" {
|
|
t.Errorf("shared pg_hba: %q", hba)
|
|
}
|
|
if _, local := doc.PostgreSQL["pg_hba"]; local {
|
|
t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
|
|
}
|
|
}
|
|
|
|
func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
|
|
var patched []string
|
|
hba := []any{"local all all peer", ClientLine("10.0.0.140")}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
|
|
case http.MethodPatch:
|
|
var body struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `json:"pg_hba"`
|
|
} `json:"postgresql"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
patched = body.PostgreSQL.PgHBA
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "")
|
|
if err != nil || !changed {
|
|
t.Fatalf("%v %v", changed, err)
|
|
}
|
|
// Everything that was there is kept, and the new line added once.
|
|
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
|
|
t.Errorf("patched %q", patched)
|
|
}
|
|
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed {
|
|
t.Errorf("one already let in was changed again: %v %v", changed, err)
|
|
}
|
|
|
|
hba = nil // the rules are in each member's file
|
|
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil ||
|
|
!strings.Contains(err.Error(), "each member") {
|
|
t.Errorf("rules kept per member: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) {
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16",
|
|
Secrets{"a", "b", "c", "the-api-pw"}, true)
|
|
var doc struct {
|
|
RestAPI struct {
|
|
Authentication struct{ Username, Password string } `yaml:"authentication"`
|
|
CertFile string `yaml:"certfile"`
|
|
CAFile string `yaml:"cafile"`
|
|
} `yaml:"restapi"`
|
|
Ctl struct {
|
|
CACert string `yaml:"cacert"`
|
|
} `yaml:"ctl"`
|
|
Etcd3 struct {
|
|
Protocol, CACert, Cert, Key string
|
|
} `yaml:"etcd3"`
|
|
Bootstrap struct {
|
|
DCS struct {
|
|
PostgreSQL struct {
|
|
Parameters map[string]string `yaml:"parameters"`
|
|
PgHBA []string `yaml:"pg_hba"`
|
|
} `yaml:"postgresql"`
|
|
} `yaml:"dcs"`
|
|
} `yaml:"bootstrap"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
|
|
t.Fatalf("%v\n%s", err, cfg)
|
|
}
|
|
if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" {
|
|
t.Errorf("rest api authentication %+v", a)
|
|
}
|
|
if doc.RestAPI.CertFile != ServerCert || doc.RestAPI.CAFile != CACert || doc.Ctl.CACert != CACert {
|
|
t.Errorf("the API is not served over TLS:\n%s", cfg)
|
|
}
|
|
if e := doc.Etcd3; e.Protocol != "https" || e.CACert != CACert || e.Cert != ServerCert || e.Key != ServerKey {
|
|
t.Errorf("etcd is not reached over TLS: %+v", e)
|
|
}
|
|
p := doc.Bootstrap.DCS.PostgreSQL.Parameters
|
|
if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey {
|
|
t.Errorf("parameters %v", p)
|
|
}
|
|
for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA {
|
|
if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") {
|
|
t.Errorf("a remote rule without TLS: %q", l)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
"host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256",
|
|
"local all all peer": "local all all peer",
|
|
"hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256",
|
|
} {
|
|
if got := hostssl(in); got != want {
|
|
t.Errorf("%q: %q", in, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) {
|
|
// The second member holds the three PostgreSQL passwords and no API one.
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
|
|
if _, err := s.Run(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" {
|
|
t.Errorf("the API password was not carried: %q", s.RestPassword)
|
|
}
|
|
for _, m := range []*machine{a, b} {
|
|
if !m.ranAny(`"username": "patroni"`) {
|
|
t.Errorf("%s: the API was not given its password", m.name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) {
|
|
var patched map[string]any
|
|
var user, pass string
|
|
hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}})
|
|
case http.MethodPatch:
|
|
user, pass, _ = r.BasicAuth()
|
|
_ = json.NewDecoder(r.Body).Decode(&patched)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
ca, err := tlsconf.NewCA("test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
|
|
if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if user != RestUser || pass != "pw" {
|
|
t.Errorf("auth %q %q", user, pass)
|
|
}
|
|
pg := patched["postgresql"].(map[string]any)
|
|
if pg["parameters"].(map[string]any)["ssl"] != "on" {
|
|
t.Errorf("parameters %v", pg["parameters"])
|
|
}
|
|
got := pg["pg_hba"].([]any)
|
|
if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) {
|
|
t.Errorf("hba %v", got)
|
|
}
|
|
// A control plane let in afterwards is let in by TLS.
|
|
hba = got
|
|
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
last := patched["postgresql"].(map[string]any)["pg_hba"].([]any)
|
|
if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) {
|
|
t.Errorf("let in without TLS: %v", last)
|
|
}
|
|
}
|
|
|
|
func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) {
|
|
ssl := "on"
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}})
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}},
|
|
Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem",
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil || !strings.Contains(dsn, "sslmode=verify-full") {
|
|
t.Fatalf("%s %v", dsn, err)
|
|
}
|
|
ssl = "off"
|
|
if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") {
|
|
t.Errorf("TLS asked of a cluster without it: %s", dsn)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) {
|
|
total, ok := MemTotal("MemTotal: 4009848 kB\nMemFree: 976000 kB\n")
|
|
if !ok || total != 4009848<<10 {
|
|
t.Fatalf("MemTotal %d %v", total, ok)
|
|
}
|
|
own := MemorySettings(total, false)
|
|
if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" ||
|
|
own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" {
|
|
t.Errorf("a machine of its own: %v", own)
|
|
}
|
|
// Beside a control plane, less of it.
|
|
if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" {
|
|
t.Errorf("a shared machine: %v", sh)
|
|
}
|
|
// Never below what PostgreSQL needs, never past what it can use.
|
|
if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" {
|
|
t.Errorf("a small machine: %v", tiny)
|
|
}
|
|
if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" {
|
|
t.Errorf("a large machine: %v", big)
|
|
}
|
|
}
|
|
|
|
func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var restarted []string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodPost && r.URL.Path == "/restart" {
|
|
if _, pass, _ := r.BasicAuth(); pass != "pw" {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
host, _, _ := strings.Cut(r.Host, ":")
|
|
mu.Lock()
|
|
restarted = append(restarted, host)
|
|
mu.Unlock()
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"}
|
|
leader := &machine{name: "a", answers: answers}
|
|
replica := &machine{name: "b", answers: answers}
|
|
up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true},
|
|
{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
|
|
s := Setup{
|
|
DB: []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
Wait: time.Second,
|
|
}
|
|
if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Join(restarted, " ") != "localhost 127.0.0.1" {
|
|
t.Errorf("restarted %v; the replica first, then the leader", restarted)
|
|
}
|
|
}
|
|
|
|
func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) {
|
|
// A run that stopped after writing the configuration, or restarted before
|
|
// Patroni had taken it: nothing to change, but PostgreSQL still on the old.
|
|
var restarted []string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/restart" {
|
|
host, _, _ := strings.Cut(r.Host, ":")
|
|
restarted = append(restarted, host)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"}
|
|
up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"},
|
|
{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
|
|
s := Setup{
|
|
DB: []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
Wait: time.Second,
|
|
}
|
|
if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Join(restarted, " ") != "localhost" {
|
|
t.Errorf("restarted %v; only the member pending", restarted)
|
|
}
|
|
}
|
|
|
|
func TestALostPrimaryIsReplacedWithinTwentySeconds(t *testing.T) {
|
|
cfg := map[string]any{"ttl": 30, "loop_wait": 10, "retry_timeout": 10}
|
|
patches := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(cfg)
|
|
case http.MethodPatch:
|
|
patches++
|
|
_ = json.NewDecoder(r.Body).Decode(&cfg)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
ca, err := tlsconf.NewCA("test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
|
|
for range 2 {
|
|
if err := s.timings(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if patches != 1 || cfg["ttl"].(float64) != 20 || cfg["loop_wait"].(float64) != 5 || cfg["retry_timeout"].(float64) != 5 {
|
|
t.Errorf("patched %d times to %v", patches, cfg)
|
|
}
|
|
// Patroni's own rule, or it refuses them.
|
|
if loopWait+2*retryTimeout > leaderTTL {
|
|
t.Error("loop_wait + 2*retry_timeout is more than ttl")
|
|
}
|
|
if c := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, nil, "16", Secrets{}, false); !strings.Contains(c, "ttl: 20\n loop_wait: 5\n retry_timeout: 5\n") {
|
|
t.Errorf("a new cluster bootstraps with others:\n%s", c)
|
|
}
|
|
}
|