llmgw/internal/pgha/pgha_test.go

package pgha

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/prometheus/common/expfmt"
	"go.yaml.in/yaml/v3"
)

var (
	pg01  = Member{"llmgw-pg01", "10.0.0.149"}
	ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
	ctl   = Member{"llmgw-ctl", "10.0.0.140"}
)

func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
	sec := Secrets{"su", "rep", "app", "api"}
	cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec, false)
	for _, want := range []string{
		"name: llmgw-pg01", "listen: 10.0.0.149:8008",
		"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
		"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
		"- host replication replicator 10.0.0.150/32 scram-sha-256",
		"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
		// A member can run a control plane too.
		"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
	} {
		if !strings.Contains(cfg, want) {
			t.Errorf("patroni config lacks %q", want)
		}
	}
	if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
		t.Error("a member's address is allowed twice")
	}
	e := Etcd(ctl, []Member{pg01, ctl02, ctl}, false)
	if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
		!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
		t.Errorf("etcd defaults:\n%s", e)
	}
	e = Etcd(ctl, []Member{pg01, ctl02, ctl}, true)
	for _, want := range []string{
		"llmgw-pg01=https://10.0.0.149:2380",
		"ETCD_LISTEN_CLIENT_URLS=https://10.0.0.140:2379,https://127.0.0.1:2379\n",
		"ETCD_ADVERTISE_CLIENT_URLS=https://10.0.0.140:2379\n",
		"ETCD_CLIENT_CERT_AUTH=true\n", "ETCD_PEER_CLIENT_CERT_AUTH=true\n",
		"ETCD_TRUSTED_CA_FILE=/etc/etcd/tls/ca.crt\n",
	} {
		if !strings.Contains(e, want) {
			t.Errorf("etcd over TLS lacks %q:\n%s", want, e)
		}
	}
	if strings.Contains(e, "http://") {
		t.Errorf("etcd over TLS still listens in the clear:\n%s", e)
	}
	if got := DSN([]Member{pg01, ctl02}, "pw", ""); got !=
		"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
		t.Errorf("dsn %s", got)
	}
	if got := DSN([]Member{pg01, ctl02}, "pw", "/etc/llmgw/ca.pem"); got !=
		"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fca.pem&target_session_attrs=read-write" {
		t.Errorf("dsn with tls %s", got)
	}
}

func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
	var c Cluster
	if err := json.Unmarshal([]byte(`{"members":[
		{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
		{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
		t.Fatal(err)
	}
	if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
		t.Errorf("%s: leader %q", c, c.Leader())
	}
	c.Members[0].State = "starting"
	if c.Complete(2) {
		t.Error("a replica still starting counted as streaming")
	}
	if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
		t.Error("lag not read")
	}
}

// machine is a fake host: it records what it was asked to run and answers
// from a table of script fragments.
type machine struct {
	name    string
	mu      sync.Mutex
	ran     []string
	answers map[string]string
}

func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.ran = append(m.ran, script)
	for frag, out := range m.answers {
		if strings.Contains(script, frag) {
			return []byte(out), nil
		}
	}
	return nil, nil
}

func (m *machine) ranAny(frag string) bool {
	for _, s := range m.ran {
		if strings.Contains(s, frag) {
			return true
		}
	}
	return false
}

const pws = "superuser=a\nreplicator=b\nllmgw=c\nrestapi=d\n"

func healthy(n int) string {
	return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
}

func TestARunningClusterIsLeftAsItIs(t *testing.T) {
	// What init does when run again, or against the reference cluster: find
	// it up, change nothing, and give the address.
	a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
	b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
	w := &machine{name: "ctl"}
	up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
		{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
	s := Setup{
		DB:      []Node{{pg01, a}, {ctl02, b}},
		Witness: &Node{ctl, w},
		Status:  func(context.Context, []string) (Cluster, error) { return up, nil },
	}
	dsn, err := s.Run(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
		t.Errorf("dsn %s", dsn)
	}
	for _, m := range []*machine{a, b, w} {
		// Looked at, for the API's password, but not rewritten or restarted.
		for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "config.yml.new\nsudo systemctl"} {
			if m.ranAny(frag) {
				t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
			}
		}
	}
	if !a.ranAny("CREATE ROLE") {
		t.Error("the role was not made sure of on the leader")
	}
	// Enabled where the package is there, which on a running cluster is
	// all a rerun does about it: it installs nothing.
	if !a.ranAny("CREATE EXTENSION IF NOT EXISTS vector") {
		t.Error("pgvector was not enabled in llmgw's database on the leader")
	}
}

func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
	s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
	if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
		t.Errorf("got %v", err)
	}
}

func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
	a := &machine{answers: map[string]string{"openssl rand": pws}}
	b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
	s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
	_, err := s.Run(context.Background())
	if err == nil || !strings.Contains(err.Error(), "differ") {
		t.Fatalf("got %v", err)
	}
	if b.ranAny("cat > /root/llmgw-pg") {
		t.Error("the other cluster's passwords were replaced")
	}
}

func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
	a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
		"pg_dropcluster": "holds-data\n"}}
	b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
	s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
		Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
		Wait:   time.Second}
	_, err := s.Run(context.Background())
	if err == nil || !strings.Contains(err.Error(), "databases in it") {
		t.Fatalf("got %v", err)
	}
	if a.ranAny("systemctl restart patroni") {
		t.Error("patroni was started over somebody's data")
	}
}

func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
	c := NewCollector([]string{"10.0.0.149"})
	c.status = func(context.Context, []string) (Cluster, error) {
		return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
			{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
	}
	got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
	if err != nil {
		t.Fatal(err)
	}
	for _, want := range []string{
		"llmgw_store_patroni_up 1",
		`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
		`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
	} {
		if !strings.Contains(string(got), want) {
			t.Errorf("missing %s in\n%s", want, got)
		}
	}
	if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
		t.Error("the leader was given a lag")
	}
}

func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
	c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
	c.status = func(context.Context, []string) (Cluster, error) {
		return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
	}
	cl, err := c.Read()
	if err != nil {
		t.Fatal(err)
	}
	if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
		t.Errorf("%s", cl)
	}
}

func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
	// Where `server add` can change them for every member at once.
	cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c", "d"}, false)
	var doc struct {
		Bootstrap struct {
			DCS struct {
				PostgreSQL struct {
					PgHBA []string `yaml:"pg_hba"`
				} `yaml:"postgresql"`
			} `yaml:"dcs"`
		} `yaml:"bootstrap"`
		PostgreSQL map[string]any `yaml:"postgresql"`
	}
	if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
		t.Fatalf("not YAML: %v\n%s", err, cfg)
	}
	hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
	if len(hba) < 5 || hba[0] != "local all all peer" {
		t.Errorf("shared pg_hba: %q", hba)
	}
	if _, local := doc.PostgreSQL["pg_hba"]; local {
		t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
	}
}

func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
	var patched []string
	hba := []any{"local all all peer", ClientLine("10.0.0.140")}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case http.MethodGet:
			_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
		case http.MethodPatch:
			var body struct {
				PostgreSQL struct {
					PgHBA []string `json:"pg_hba"`
				} `json:"postgresql"`
			}
			_ = json.NewDecoder(r.Body).Decode(&body)
			patched = body.PostgreSQL.PgHBA
		}
	}))
	defer srv.Close()
	u, _ := url.Parse(srv.URL)
	old := restPort
	restPort, _ = strconv.Atoi(u.Port())
	defer func() { restPort = old }()

	changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "")
	if err != nil || !changed {
		t.Fatalf("%v %v", changed, err)
	}
	// Everything that was there is kept, and the new line added once.
	if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
		t.Errorf("patched %q", patched)
	}
	if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140", ""); err != nil || changed {
		t.Errorf("one already let in was changed again: %v %v", changed, err)
	}

	hba = nil // the rules are in each member's file
	if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151", ""); err == nil ||
		!strings.Contains(err.Error(), "each member") {
		t.Errorf("rules kept per member: %v", err)
	}
}

func TestANewClusterAsksForAPasswordAndSpeaksTLS(t *testing.T) {
	cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16",
		Secrets{"a", "b", "c", "the-api-pw"}, true)
	var doc struct {
		RestAPI struct {
			Authentication struct{ Username, Password string } `yaml:"authentication"`
			CertFile       string                              `yaml:"certfile"`
			CAFile         string                              `yaml:"cafile"`
		} `yaml:"restapi"`
		Ctl struct {
			CACert string `yaml:"cacert"`
		} `yaml:"ctl"`
		Etcd3 struct {
			Protocol, CACert, Cert, Key string
		} `yaml:"etcd3"`
		Bootstrap struct {
			DCS struct {
				PostgreSQL struct {
					Parameters map[string]string `yaml:"parameters"`
					PgHBA      []string          `yaml:"pg_hba"`
				} `yaml:"postgresql"`
			} `yaml:"dcs"`
		} `yaml:"bootstrap"`
	}
	if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
		t.Fatalf("%v\n%s", err, cfg)
	}
	if a := doc.RestAPI.Authentication; a.Username != RestUser || a.Password != "the-api-pw" {
		t.Errorf("rest api authentication %+v", a)
	}
	if doc.RestAPI.CertFile != ServerCert || doc.RestAPI.CAFile != CACert || doc.Ctl.CACert != CACert {
		t.Errorf("the API is not served over TLS:\n%s", cfg)
	}
	if e := doc.Etcd3; e.Protocol != "https" || e.CACert != CACert || e.Cert != ServerCert || e.Key != ServerKey {
		t.Errorf("etcd is not reached over TLS: %+v", e)
	}
	p := doc.Bootstrap.DCS.PostgreSQL.Parameters
	if p["ssl"] != "on" || p["ssl_cert_file"] != ServerCert || p["ssl_key_file"] != ServerKey {
		t.Errorf("parameters %v", p)
	}
	for _, l := range doc.Bootstrap.DCS.PostgreSQL.PgHBA {
		if strings.HasPrefix(l, "host ") && !strings.Contains(l, " 127.0.0.1/32 ") {
			t.Errorf("a remote rule without TLS: %q", l)
		}
	}
}

func TestOnlyRemoteRulesAreMadeHostssl(t *testing.T) {
	for in, want := range map[string]string{
		"host llmgw llmgw 10.0.0.140/32 scram-sha-256":            "hostssl llmgw llmgw 10.0.0.140/32 scram-sha-256",
		"host replication replicator 10.0.0.150/32 scram-sha-256": "hostssl replication replicator 10.0.0.150/32 scram-sha-256",
		"host all all 127.0.0.1/32 scram-sha-256":                 "host all all 127.0.0.1/32 scram-sha-256",
		"local all all peer":                                      "local all all peer",
		"hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256":           "hostssl llmgw llmgw 10.0.0.1/32 scram-sha-256",
	} {
		if got := hostssl(in); got != want {
			t.Errorf("%q: %q", in, got)
		}
	}
}

func TestAClusterFromBeforeTheAPIPasswordIsGivenOne(t *testing.T) {
	// The second member holds the three PostgreSQL passwords and no API one.
	a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
	b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=a\nreplicator=b\nllmgw=c\n"}}
	up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
		{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
	s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
		Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
	if _, err := s.Run(context.Background()); err != nil {
		t.Fatal(err)
	}
	if !b.ranAny(`"restapi $api"`) || s.RestPassword != "d" {
		t.Errorf("the API password was not carried: %q", s.RestPassword)
	}
	for _, m := range []*machine{a, b} {
		if !m.ranAny(`"username": "patroni"`) {
			t.Errorf("%s: the API was not given its password", m.name)
		}
	}
}

func TestTLSIsTurnedOnInTheSharedConfigurationWithThePassword(t *testing.T) {
	var patched map[string]any
	var user, pass string
	hba := []any{"local all all peer", "host all all 127.0.0.1/32 scram-sha-256", ClientLine("10.0.0.140")}
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case http.MethodGet:
			_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba, "parameters": map[string]any{}}})
		case http.MethodPatch:
			user, pass, _ = r.BasicAuth()
			_ = json.NewDecoder(r.Body).Decode(&patched)
		}
	}))
	defer srv.Close()
	u, _ := url.Parse(srv.URL)
	old := restPort
	restPort, _ = strconv.Atoi(u.Port())
	defer func() { restPort = old }()

	ca, err := tlsconf.NewCA("test")
	if err != nil {
		t.Fatal(err)
	}
	s := Setup{DB: []Node{{Member{"m", "127.0.0.1"}, &machine{}}}, CA: ca}
	if err := s.tlsInShared(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
		t.Fatal(err)
	}
	if user != RestUser || pass != "pw" {
		t.Errorf("auth %q %q", user, pass)
	}
	pg := patched["postgresql"].(map[string]any)
	if pg["parameters"].(map[string]any)["ssl"] != "on" {
		t.Errorf("parameters %v", pg["parameters"])
	}
	got := pg["pg_hba"].([]any)
	if got[1] != "host all all 127.0.0.1/32 scram-sha-256" || got[2] != hostssl(ClientLine("10.0.0.140")) {
		t.Errorf("hba %v", got)
	}
	// A control plane let in afterwards is let in by TLS.
	hba = got
	if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150", "pw"); err != nil {
		t.Fatal(err)
	}
	last := patched["postgresql"].(map[string]any)["pg_hba"].([]any)
	if last[len(last)-1] != hostssl(ClientLine("10.0.0.150")) {
		t.Errorf("let in without TLS: %v", last)
	}
}

func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *testing.T) {
	ssl := "on"
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"parameters": map[string]any{"ssl": ssl}}})
	}))
	defer srv.Close()
	u, _ := url.Parse(srv.URL)
	old := restPort
	restPort, _ = strconv.Atoi(u.Port())
	defer func() { restPort = old }()

	a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
	b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
	up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
		{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
	s := Setup{DB: []Node{{Member{"llmgw-pg01", "127.0.0.1"}, a}, {Member{"llmgw-ctl02", "127.0.0.1"}, b}},
		Witness: &Node{ctl, &machine{}}, CAFile: "/etc/llmgw/ca.pem",
		Status: func(context.Context, []string) (Cluster, error) { return up, nil }}
	dsn, err := s.Run(context.Background())
	if err != nil || !strings.Contains(dsn, "sslmode=verify-full") {
		t.Fatalf("%s %v", dsn, err)
	}
	ssl = "off"
	if dsn, _ = s.Run(context.Background()); !strings.Contains(dsn, "sslmode=disable") {
		t.Errorf("TLS asked of a cluster without it: %s", dsn)
	}
}

func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) {
	total, ok := MemTotal("MemTotal:        4009848 kB\nMemFree:          976000 kB\n")
	if !ok || total != 4009848<<10 {
		t.Fatalf("MemTotal %d %v", total, ok)
	}
	own := MemorySettings(total, false)
	if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" ||
		own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" {
		t.Errorf("a machine of its own: %v", own)
	}
	// Beside a control plane, less of it.
	if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" {
		t.Errorf("a shared machine: %v", sh)
	}
	// Never below what PostgreSQL needs, never past what it can use.
	if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" {
		t.Errorf("a small machine: %v", tiny)
	}
	if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" {
		t.Errorf("a large machine: %v", big)
	}
}

func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) {
	var mu sync.Mutex
	var restarted []string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodPost && r.URL.Path == "/restart" {
			if _, pass, _ := r.BasicAuth(); pass != "pw" {
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			host, _, _ := strings.Cut(r.Host, ":")
			mu.Lock()
			restarted = append(restarted, host)
			mu.Unlock()
		}
	}))
	defer srv.Close()
	u, _ := url.Parse(srv.URL)
	old := restPort
	restPort, _ = strconv.Atoi(u.Port())
	defer func() { restPort = old }()

	answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"}
	leader := &machine{name: "a", answers: answers}
	replica := &machine{name: "b", answers: answers}
	up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true},
		{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
	s := Setup{
		DB:     []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}},
		Status: func(context.Context, []string) (Cluster, error) { return up, nil },
		Wait:   time.Second,
	}
	if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
		t.Fatal(err)
	}
	if strings.Join(restarted, " ") != "localhost 127.0.0.1" {
		t.Errorf("restarted %v; the replica first, then the leader", restarted)
	}
}

func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) {
	// A run that stopped after writing the configuration, or restarted before
	// Patroni had taken it: nothing to change, but PostgreSQL still on the old.
	var restarted []string
	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/restart" {
			host, _, _ := strings.Cut(r.Host, ":")
			restarted = append(restarted, host)
		}
	}))
	defer srv.Close()
	u, _ := url.Parse(srv.URL)
	old := restPort
	restPort, _ = strconv.Atoi(u.Port())
	defer func() { restPort = old }()

	answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"}
	up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"},
		{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
	s := Setup{
		DB:     []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}},
		Status: func(context.Context, []string) (Cluster, error) { return up, nil },
		Wait:   time.Second,
	}
	if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
		t.Fatal(err)
	}
	if strings.Join(restarted, " ") != "localhost" {
		t.Errorf("restarted %v; only the member pending", restarted)
	}
}