Files
llmgw/internal/pgha/pgha_test.go
T
Ilker ManapandClaude Opus 5.5 2b29016d5b Ask for a password on Patroni's API and give the store TLS, new clusters and existing ones in place
Patroni's REST API accepted changes from anyone on port 8008, and
PostgreSQL ran with sslmode=disable between machines. Setup now keeps a
fourth secret for the API (carried to members of older clusters), issues
each member a certificate from the cluster's authority, writes
restapi.authentication into each member's config and reloads it, and
turns ssl on with hostssl remote rules through an authenticated PATCH of
the shared configuration. The DSN becomes sslmode=verify-full against the
authority's certificate whenever the cluster has TLS, so a control plane
without the authority's key reconnects too. server add authenticates its
PATCH and lets the new control plane in by hostssl. Tried on the e2e
cluster: unauthenticated PATCH 401, plaintext refused, replication
streaming, the control plane on verify-full.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 10:11:28 +02:00

434 lines
16 KiB
Go

package pgha
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/expfmt"
"go.yaml.in/yaml/v3"
)
var (
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
ctl = Member{"llmgw-ctl", "10.0.0.140"}
)
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
sec := Secrets{"su", "rep", "app", "api"}
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false)
for _, want := range []string{
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
"- host replication replicator 10.0.0.150/32 scram-sha-256",
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
// A member can run a control plane too.
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
} {
if !strings.Contains(cfg, want) {
t.Errorf("patroni config lacks %q", want)
}
}
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
t.Error("a member's address is allowed twice")
}
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
t.Errorf("etcd defaults:\n%s", e)
}
if got := DSN([]Member{pg01, ctl02}, "pw", ""); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
t.Errorf("dsn %s", got)
}
if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" {
t.Errorf("dsn with tls %s", got)
}
}
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
var c Cluster
if err := json.Unmarshal([]byte(`{"members":[
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
t.Fatal(err)
}
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
t.Errorf("%s: leader %q", c, c.Leader())
}
c.Members[0].State = "starting"
if c.Complete(2) {
t.Error("a replica still starting counted as streaming")
}
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
t.Error("lag not read")
}
}
// machine is a fake host: it records what it was asked to run and answers
// from a table of script fragments.
type machine struct {
name string
mu sync.Mutex
ran []string
answers map[string]string
}
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.ran = append(m.ran, script)
for frag, out := range m.answers {
if strings.Contains(script, frag) {
return []byte(out), nil
}
}
return nil, nil
}
func (m *machine) ranAny(frag string) bool {
for _, s := range m.ran {
if strings.Contains(s, frag) {
return true
}
}
return false
}
const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n"
func healthy(n int) string {
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
}
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
// What init does when run again, or against the reference cluster: find
// it up, change nothing, and give the address.
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
w := &machine{name: "ctl"}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{
DB: []Node{{pg01, a}, {ctl02, b}},
Witness: &Node{ctl, w},
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
}
dsn, err := s.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
t.Errorf("dsn %s", dsn)
}
for _, m := range []*machine{a, b, w} {
// Looked at, for the API's password, but not rewritten or restarted.
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} {
if m.ranAny(frag) {
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
}
}
}
if !a.ranAny("CREATE ROLE") {
t.Error("the role was not made sure of on the leader")
}
}
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
t.Errorf("got %v", err)
}
}
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "differ") {
t.Fatalf("got %v", err)
}
if b.ranAny("cat > /root/llmgw-pg") {
t.Error("the other cluster's passwords were replaced")
}
}
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
"pg_dropcluster": "holds-data\n"}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
Wait: time.Second}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "databases in it") {
t.Fatalf("got %v", err)
}
if a.ranAny("systemctl restart patroni") {
t.Error("patroni was started over somebody's data")
}
}
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
c := NewCollector([]string{"10.0.0.149"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
}
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
"llmgw_store_patroni_up 1",
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
} {
if !strings.Contains(string(got), want) {
t.Errorf("missing %s in\n%s", want, got)
}
}
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
t.Error("the leader was given a lag")
}
}
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
}
cl, err := c.Read()
if err != nil {
t.Fatal(err)
}
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
t.Errorf("%s", cl)
}
}
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
// Where `server add` can change them for every member at once.
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false)
var doc struct {
Bootstrap struct {
DCS struct {
PostgreSQL struct {
PgHBA []string `yaml:"pg_hba"`
} `yaml:"postgresql"`
} `yaml:"dcs"`
} `yaml:"bootstrap"`
PostgreSQL map[string]any `yaml:"postgresql"`
}
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
t.Fatalf("not YAML: %v\n%s", err, cfg)
}
hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
if len(hba) < 5 || hba[0] != "local all all peer" {
t.Errorf("shared pg_hba: %q", hba)
}
if _, local := doc.PostgreSQL["pg_hba"]; local {
t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
}
}
func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
var patched []string
hba := []any{"local all all peer", ClientLine("10.0.0.140")}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
case http.MethodPatch:
var body struct {
PostgreSQL struct {
PgHBA []string `json:"pg_hba"`
} `json:"postgresql"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
patched = body.PostgreSQL.PgHBA
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "")
if err != nil || !changed {
t.Fatalf("%v %v", changed, err)
}
// Everything that was there is kept, and the new line added once.
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
t.Errorf("patched %q", patched)
}
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed {
t.Errorf("one already let in was changed again: %v %v", changed, err)
}
hba = nil // the rules are in each member's file
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil ||
!strings.Contains(err.Error(), "each member") {
t.Errorf("rules kept per member: %v", err)
}
}
func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) {
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16",
Secrets{"a", "b", "c", "the-api-pw"}, true)
var doc struct {
RestAPI struct {
Authentication struct{ Username, Password string } `yaml:"authentication"`
} `yaml:"restapi"`
Bootstrap struct {
DCS struct {
PostgreSQL struct {
Parameters map[string]string `yaml:"parameters"`
PgHBA []string `yaml:"pg_hba"`
} `yaml:"postgresql"`
} `yaml:"dcs"`
} `yaml:"bootstrap"`
}
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
t.Fatalf("%v\n%s", err, cfg)
}
if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" {
t.Errorf("rest api authentication %+v", a)
}
p := doc.Bootstrap.DCS.PostgreSQL.Parameters
if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey {
t.Errorf("parameters %v", p)
}
for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA {
if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") {
t.Errorf("a remote rule without TLS: %q", l)
}
}
}
func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) {
for in, want := range map[string]string{
"host llmgw llmgw 10.0.0.140/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256",
"host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256",
"host all all 127.0.0.1/32 scram-sha-256": "host all all 127.0.0.1/32 scram-sha-256",
"local all all peer": "local all all peer",
"hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256": "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256",
} {
if got := hostssl(in); got != want {
t.Errorf("%q: %q", in, got)
}
}
}
func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) {
// The second member holds the three PostgreSQL passwords and no API one.
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
if _, err := s.Run(context.Background()); err != nil {
t.Fatal(err)
}
if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" {
t.Errorf("the API password was not carried: %q", s.RestPassword)
}
for _, m := range []*machine{a, b} {
if !m.ranAny(`"username": "patroni"`) {
t.Errorf("%s: the API was not given its password", m.name)
}
}
}
func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) {
var patched map[string]any
var user, pass string
hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}})
case http.MethodPatch:
user, pass, _ = r.BasicAuth()
_ = json.NewDecoder(r.Body).Decode(&patched)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
ca, err := tlsconf.NewCA("test")
if err != nil {
t.Fatal(err)
}
s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
t.Fatal(err)
}
if user != RestUser || pass != "pw" {
t.Errorf("auth %q %q", user, pass)
}
pg := patched["postgresql"].(map[string]any)
if pg["parameters"].(map[string]any)["ssl"] != "on" {
t.Errorf("parameters %v", pg["parameters"])
}
got := pg["pg_hba"].([]any)
if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) {
t.Errorf("hba %v", got)
}
// A control plane let in afterwards is let in by TLS.
hba = got
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil {
t.Fatal(err)
}
last := patched["postgresql"].(map[string]any)["pg_hba"].([]any)
if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) {
t.Errorf("let in without TLS: %v", last)
}
}
func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) {
ssl := "on"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}})
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}},
Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem",
Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
dsn, err := s.Run(context.Background())
if err != nil || !strings.Contains(dsn, "sslmode=verify-full") {
t.Fatalf("%s %v", dsn, err)
}
ssl = "off"
if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") {
t.Errorf("TLS asked of a cluster without it: %s", dsn)
}
}