server add gives another machine this llmgw, a certificate, the configuration and store address, lets it into the store through Patroni, and starts it; --vip keeps a floating address on the leader with keepalived, following a new /leader endpoint. Nodes learn every control plane from the leader and notice a dead one by HTTP/2 ping instead of after the kernel's fifteen minutes. The setup, console, node and init screens are drawn full screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
283 lines
9.8 KiB
Go
283 lines
9.8 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/prometheus/common/expfmt"
|
|
"go.yaml.in/yaml/v3"
|
|
)
|
|
|
|
var (
|
|
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
|
|
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
|
|
ctl = Member{"llmgw-ctl", "10.0.0.140"}
|
|
)
|
|
|
|
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
|
|
sec := Secrets{"su", "rep", "app"}
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec)
|
|
for _, want := range []string{
|
|
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
|
|
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
|
|
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
|
|
"- host replication replicator 10.0.0.150/32 scram-sha-256",
|
|
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
|
|
// A member can run a control plane too.
|
|
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
|
|
} {
|
|
if !strings.Contains(cfg, want) {
|
|
t.Errorf("patroni config lacks %q", want)
|
|
}
|
|
}
|
|
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
|
|
t.Error("a member's address is allowed twice")
|
|
}
|
|
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
|
|
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
|
|
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
|
|
t.Errorf("etcd defaults:\n%s", e)
|
|
}
|
|
if got := DSN([]Member{pg01, ctl02}, "pw"); got !=
|
|
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
|
|
t.Errorf("dsn %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
|
|
var c Cluster
|
|
if err := json.Unmarshal([]byte(`{"members":[
|
|
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
|
|
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
|
|
t.Errorf("%s: leader %q", c, c.Leader())
|
|
}
|
|
c.Members[0].State = "starting"
|
|
if c.Complete(2) {
|
|
t.Error("a replica still starting counted as streaming")
|
|
}
|
|
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
|
|
t.Error("lag not read")
|
|
}
|
|
}
|
|
|
|
// machine is a fake host: it records what it was asked to run and answers
|
|
// from a table of script fragments.
|
|
type machine struct {
|
|
name string
|
|
mu sync.Mutex
|
|
ran []string
|
|
answers map[string]string
|
|
}
|
|
|
|
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.ran = append(m.ran, script)
|
|
for frag, out := range m.answers {
|
|
if strings.Contains(script, frag) {
|
|
return []byte(out), nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *machine) ranAny(frag string) bool {
|
|
for _, s := range m.ran {
|
|
if strings.Contains(s, frag) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
const pws = "superuser=a\nreplicator=b\nllmgw=c\n"
|
|
|
|
func healthy(n int) string {
|
|
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
|
|
}
|
|
|
|
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
|
|
// What init does when run again, or against the reference cluster: find
|
|
// it up, change nothing, and give the address.
|
|
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
|
|
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
w := &machine{name: "ctl"}
|
|
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
|
|
s := Setup{
|
|
DB: []Node{{pg01, a}, {ctl02, b}},
|
|
Witness: &Node{ctl, w},
|
|
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
|
|
}
|
|
dsn, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
|
|
t.Errorf("dsn %s", dsn)
|
|
}
|
|
for _, m := range []*machine{a, b, w} {
|
|
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} {
|
|
if m.ranAny(frag) {
|
|
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
|
|
}
|
|
}
|
|
}
|
|
if !a.ranAny("CREATE ROLE") {
|
|
t.Error("the role was not made sure of on the leader")
|
|
}
|
|
}
|
|
|
|
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
|
|
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
|
|
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
|
|
t.Errorf("got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "differ") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if b.ranAny("cat > /root/llmgw-pg") {
|
|
t.Error("the other cluster's passwords were replaced")
|
|
}
|
|
}
|
|
|
|
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
|
|
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
|
|
"pg_dropcluster": "holds-data\n"}}
|
|
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
|
|
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
|
|
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
|
|
Wait: time.Second}
|
|
_, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "databases in it") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
if a.ranAny("systemctl restart patroni") {
|
|
t.Error("patroni was started over somebody's data")
|
|
}
|
|
}
|
|
|
|
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.149"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
|
|
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
|
|
}
|
|
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"llmgw_store_patroni_up 1",
|
|
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
|
|
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
|
|
} {
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("missing %s in\n%s", want, got)
|
|
}
|
|
}
|
|
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
|
|
t.Error("the leader was given a lag")
|
|
}
|
|
}
|
|
|
|
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
|
|
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
|
|
c.status = func(context.Context, []string) (Cluster, error) {
|
|
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
|
|
}
|
|
cl, err := c.Read()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
|
|
t.Errorf("%s", cl)
|
|
}
|
|
}
|
|
|
|
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
|
|
// Where `server add` can change them for every member at once.
|
|
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"})
|
|
var doc struct {
|
|
Bootstrap struct {
|
|
DCS struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `yaml:"pg_hba"`
|
|
} `yaml:"postgresql"`
|
|
} `yaml:"dcs"`
|
|
} `yaml:"bootstrap"`
|
|
PostgreSQL map[string]any `yaml:"postgresql"`
|
|
}
|
|
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
|
|
t.Fatalf("not YAML: %v\n%s", err, cfg)
|
|
}
|
|
hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
|
|
if len(hba) < 5 || hba[0] != "local all all peer" {
|
|
t.Errorf("shared pg_hba: %q", hba)
|
|
}
|
|
if _, local := doc.PostgreSQL["pg_hba"]; local {
|
|
t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
|
|
}
|
|
}
|
|
|
|
func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
|
|
var patched []string
|
|
hba := []any{"local all all peer", ClientLine("10.0.0.140")}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
|
|
case http.MethodPatch:
|
|
var body struct {
|
|
PostgreSQL struct {
|
|
PgHBA []string `json:"pg_hba"`
|
|
} `json:"postgresql"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
patched = body.PostgreSQL.PgHBA
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
u, _ := url.Parse(srv.URL)
|
|
old := restPort
|
|
restPort, _ = strconv.Atoi(u.Port())
|
|
defer func() { restPort = old }()
|
|
|
|
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150")
|
|
if err != nil || !changed {
|
|
t.Fatalf("%v %v", changed, err)
|
|
}
|
|
// Everything that was there is kept, and the new line added once.
|
|
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
|
|
t.Errorf("patched %q", patched)
|
|
}
|
|
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed {
|
|
t.Errorf("one already let in was changed again: %v %v", changed, err)
|
|
}
|
|
|
|
hba = nil // the rules are in each member's file
|
|
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil ||
|
|
!strings.Contains(err.Error(), "each member") {
|
|
t.Errorf("rules kept per member: %v", err)
|
|
}
|
|
}
|