Set up a replicated store with server init --patroni, and watch it

init --patroni user@host ... installs PostgreSQL, Patroni and etcd on the
given machines over ssh, with this one as the third etcd vote, makes the role
and database, and moves the store in use into the cluster. Each step checks
first, so it changes nothing on a cluster that is up. For a store of several
hosts the control plane publishes each member's role, state and lag from
Patroni, a stopped member as missing, prints it in ctl status, and alerts on
it. Making an administrator on PostgreSQL sent a bool to an integer column;
it sends an integer now.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 20:57:41 +02:00
1 parent 7a562ac1e7
commit 975c93994f
22 files changed
+1495 -43

No files matched your search

+171
View File
@@ -0,0 +1,171 @@
// Package pgha sets up the store as PostgreSQL under Patroni: two or more
// machines, one primary streaming to the others, and a standby promoted when
// the primary is lost. Patroni keeps its leader lock in etcd, which decides
// only with a majority, so etcd runs on three machines — the database
// machines and, when they are two, the control plane as the third vote.
//
// Everything is done over the same ssh the rest of onboarding uses, and every
// step looks before it acts: run against a cluster that is already up, it
// changes nothing.
package pgha
import (
"fmt"
"net/url"
"strings"
)
// Scope is the Patroni cluster's name, and the etcd cluster token.
const Scope = "llmgw"
// PatroniPort is where Patroni's REST API listens on every member: what the
// control plane reads the cluster's state from.
const PatroniPort = 8008
// Role and Database are what the control plane connects as.
const (
Role = "llmgw"
Database = "llmgw"
)
// Member is one machine of the cluster, by the address the others reach it on.
type Member struct {
Name string
IP string
}
// Secrets are the cluster's three passwords.
type Secrets struct {
Superuser, Replicator, App string
}
// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
func Etcd(m Member, all []Member) string {
var peers []string
for _, p := range all {
peers = append(peers, fmt.Sprintf("%s=http://%s:2380", p.Name, p.IP))
}
return fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock.
ETCD_NAME=%[1]s
ETCD_DATA_DIR=/var/lib/etcd/default
ETCD_LISTEN_PEER_URLS=http://%[2]s:2380
ETCD_LISTEN_CLIENT_URLS=http://%[2]s:2379,http://127.0.0.1:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%[2]s:2380
ETCD_ADVERTISE_CLIENT_URLS=http://%[2]s:2379
ETCD_INITIAL_CLUSTER=%[3]s
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
`, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni")
}
// Patroni is a database member's /etc/patroni/config.yml.
//
// clients are the addresses the control planes connect from; the members
// themselves are always allowed, since a second control plane usually runs on
// one of them.
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string {
var hosts []string
for _, e := range etcd {
hosts = append(hosts, e.IP+":2379")
}
var hba []string
hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256")
for _, d := range db {
hba = append(hba,
"host replication replicator "+d.IP+"/32 scram-sha-256",
"host all postgres "+d.IP+"/32 scram-sha-256")
}
seen := map[string]bool{}
for _, c := range append(append([]string{}, clients...), ips(db)...) {
if c == "" || seen[c] {
continue
}
seen[c] = true
hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
}
var b strings.Builder
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
scope: %[1]s
namespace: /service/
name: %[2]s
restapi:
listen: %[3]s:%[4]d
connect_address: %[3]s:%[4]d
etcd3:
hosts: %[5]s
bootstrap:
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
postgresql:
use_pg_rewind: true
use_slots: true
parameters:
wal_level: replica
hot_standby: "on"
max_wal_senders: 10
max_replication_slots: 10
wal_keep_size: 1GB
wal_log_hints: "on"
password_encryption: scram-sha-256
initdb:
- encoding: UTF8
- data-checksums
postgresql:
listen: 127.0.0.1,%[3]s:5432
connect_address: %[3]s:5432
data_dir: /var/lib/postgresql/%[6]s/llmgw
bin_dir: /usr/lib/postgresql/%[6]s/bin
pgpass: /var/lib/postgresql/.pgpass_patroni
authentication:
superuser:
username: postgres
password: "%[7]s"
replication:
username: replicator
password: "%[8]s"
rewind:
username: postgres
password: "%[7]s"
parameters:
unix_socket_directories: /var/run/postgresql
pg_hba:
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator)
for _, h := range hba {
fmt.Fprintf(&b, " - %s\n", h)
}
b.WriteString(`
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
`)
return b.String()
}
// DSN is the store address the control planes are given: every member, and
// the driver told to use whichever accepts writes, so a failover is followed
// without anything in front of the database.
func DSN(db []Member, password string) string {
var hosts []string
for _, d := range db {
hosts = append(hosts, d.IP+":5432")
}
return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write",
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database)
}
func ips(ms []Member) []string {
out := make([]string, len(ms))
for i, m := range ms {
out[i] = m.IP
}
return out
}
+100
View File
@@ -0,0 +1,100 @@
package pgha
import (
"context"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
)
// Collector publishes the store cluster's state as the control plane sees it
// through Patroni's API: who leads, who streams, how far behind. Read at most
// every fifteen seconds, however often it is scraped.
type Collector struct {
hosts []string
status func(ctx context.Context, hosts []string) (Cluster, error)
mu sync.Mutex
at time.Time
last Cluster
err error
up, member, lag *prometheus.Desc
}
// NewCollector reads the cluster from the Patroni of each host in turn.
func NewCollector(hosts []string) *Collector {
return &Collector{
hosts: hosts,
status: HTTPStatus,
up: prometheus.NewDesc("llmgw_store_patroni_up",
"1 when a member's Patroni answered for the store's cluster.", nil, nil),
member: prometheus.NewDesc("llmgw_store_member",
"Each member of the store's cluster, by role (leader, replica) and state (running, streaming, …); always 1.",
[]string{"member", "host", "role", "state"}, nil),
lag: prometheus.NewDesc("llmgw_store_replica_lag_bytes",
"How far each replica is behind the leader; -1 when Patroni does not know.",
[]string{"member"}, nil),
}
}
// Describe implements prometheus.Collector.
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.up
ch <- c.member
ch <- c.lag
}
// Collect implements prometheus.Collector.
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
cl, err := c.read()
if err != nil {
ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 0)
return
}
ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 1)
for _, m := range cl.Members {
ch <- prometheus.MustNewConstMetric(c.member, prometheus.GaugeValue, 1, m.Name, m.Host, m.Role, m.State)
if m.Role != "leader" {
ch <- prometheus.MustNewConstMetric(c.lag, prometheus.GaugeValue, float64(m.Lag), m.Name)
}
}
}
// Read is the cluster's state, at most fifteen seconds old.
func (c *Collector) Read() (Cluster, error) { return c.read() }
func (c *Collector) read() (Cluster, error) {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.at) < 15*time.Second {
return c.last, c.err
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
c.last, c.err = c.status(ctx, c.hosts)
if c.err == nil {
c.last = withMissing(c.last, c.hosts)
}
c.at = time.Now()
return c.last, c.err
}
// withMissing adds every host of the store address that the cluster does not
// list, as a member in state "missing". A member that stops is dropped from
// Patroni's list once its key expires — so without this a cluster down to
// one machine reads as a healthy cluster of one, and nothing says a standby
// is gone.
func withMissing(c Cluster, hosts []string) Cluster {
seen := map[string]bool{}
for _, m := range c.Members {
seen[m.Host] = true
}
for _, h := range hosts {
if !seen[h] {
c.Members = append(c.Members, MemberState{Name: h, Host: h, Role: "replica", State: "missing", Lag: -1})
}
}
return c
}
+210
View File
@@ -0,0 +1,210 @@
package pgha
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/expfmt"
)
var (
pg01 = Member{"llmgw-pg01", "10.0.0.149"}
ctl02 = Member{"llmgw-ctl02", "10.0.0.150"}
ctl = Member{"llmgw-ctl", "10.0.0.140"}
)
func TestTheConfigurationsNameEveryMemberAndOnlyTheClientsGiven(t *testing.T) {
sec := Secrets{"su", "rep", "app"}
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", sec)
for _, want := range []string{
"name: llmgw-pg01", "listen: 10.0.0.149:8008",
"hosts: 10.0.0.149:2379,10.0.0.150:2379,10.0.0.140:2379",
"data_dir: /var/lib/postgresql/16/llmgw", `password: "rep"`,
"- host replication replicator 10.0.0.150/32 scram-sha-256",
"- host llmgw llmgw 10.0.0.140/32 scram-sha-256",
// A member can run a control plane too.
"- host llmgw llmgw 10.0.0.150/32 scram-sha-256",
} {
if !strings.Contains(cfg, want) {
t.Errorf("patroni config lacks %q", want)
}
}
if strings.Count(cfg, "host llmgw llmgw 10.0.0.149/32") != 1 {
t.Error("a member's address is allowed twice")
}
e := Etcd(ctl, []Member{pg01, ctl02, ctl})
if !strings.Contains(e, "ETCD_NAME=llmgw-ctl\n") ||
!strings.Contains(e, "llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380,llmgw-ctl=http://10.0.0.140:2380") {
t.Errorf("etcd defaults:\n%s", e)
}
if got := DSN([]Member{pg01, ctl02}, "pw"); got !=
"postgres://llmgw:pw@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write" {
t.Errorf("dsn %s", got)
}
}
func TestAClusterIsCompleteWithALeaderAndEveryOtherMemberStreaming(t *testing.T) {
var c Cluster
if err := json.Unmarshal([]byte(`{"members":[
{"name":"llmgw-ctl02","role":"replica","state":"streaming","host":"10.0.0.150","lag":0},
{"name":"llmgw-pg01","role":"leader","state":"running","host":"10.0.0.149"}]}`), &c); err != nil {
t.Fatal(err)
}
if c.Leader() != "llmgw-pg01" || !c.Complete(2) || c.Complete(3) {
t.Errorf("%s: leader %q", c, c.Leader())
}
c.Members[0].State = "starting"
if c.Complete(2) {
t.Error("a replica still starting counted as streaming")
}
if lagOf(json.RawMessage(`"unknown"`)) != -1 || lagOf(json.RawMessage(`1024`)) != 1024 {
t.Error("lag not read")
}
}
// machine is a fake host: it records what it was asked to run and answers
// from a table of script fragments.
type machine struct {
name string
mu sync.Mutex
ran []string
answers map[string]string
}
func (m *machine) Run(_ context.Context, script string, _ []byte) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.ran = append(m.ran, script)
for frag, out := range m.answers {
if strings.Contains(script, frag) {
return []byte(out), nil
}
}
return nil, nil
}
func (m *machine) ranAny(frag string) bool {
for _, s := range m.ran {
if strings.Contains(s, frag) {
return true
}
}
return false
}
const pws = "superuser=a\nreplicator=b\nllmgw=c\n"
func healthy(n int) string {
return strings.Repeat("http://x:2379 is healthy: successfully committed proposal\n", n)
}
func TestARunningClusterIsLeftAsItIs(t *testing.T) {
// What init does when run again, or against the reference cluster: find
// it up, change nothing, and give the address.
a := &machine{name: "pg01", answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3)}}
b := &machine{name: "ctl02", answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
w := &machine{name: "ctl"}
up := Cluster{Members: []MemberState{{Name: "llmgw-pg01", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Role: "replica", State: "streaming"}}}
s := Setup{
DB: []Node{{pg01, a}, {ctl02, b}},
Witness: &Node{ctl, w},
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
}
dsn, err := s.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if !strings.Contains(dsn, "llmgw:c@10.0.0.149:5432,10.0.0.150:5432") {
t.Errorf("dsn %s", dsn)
}
for _, m := range []*machine{a, b, w} {
for _, frag := range []string{"apt-get", "systemctl restart", "/etc/default/etcd", "/etc/patroni/config.yml"} {
if m.ranAny(frag) {
t.Errorf("%s was changed (%q) though the cluster is up", m.name, frag)
}
}
}
if !a.ranAny("CREATE ROLE") {
t.Error("the role was not made sure of on the leader")
}
}
func TestTwoDatabaseMachinesWithoutAThirdVoteAreRefused(t *testing.T) {
s := Setup{DB: []Node{{pg01, &machine{}}, {ctl02, &machine{}}}}
if _, err := s.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "third etcd vote") {
t.Errorf("got %v", err)
}
}
func TestAMachineHoldingOtherPasswordsIsNotOverwritten(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": "superuser=x\nreplicator=y\nllmgw=z\n"}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}}}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "differ") {
t.Fatalf("got %v", err)
}
if b.ranAny("cat > /root/llmgw-pg") {
t.Error("the other cluster's passwords were replaced")
}
}
func TestAPostgreSQLWithDatabasesInItIsNotReplaced(t *testing.T) {
a := &machine{answers: map[string]string{"openssl rand": pws, "endpoint health": healthy(3),
"pg_dropcluster": "holds-data\n"}}
b := &machine{answers: map[string]string{"sudo cat /root/llmgw-pg": pws}}
s := Setup{DB: []Node{{pg01, a}, {ctl02, b}}, Witness: &Node{ctl, &machine{}},
Status: func(context.Context, []string) (Cluster, error) { return Cluster{}, nil },
Wait: time.Second}
_, err := s.Run(context.Background())
if err == nil || !strings.Contains(err.Error(), "databases in it") {
t.Fatalf("got %v", err)
}
if a.ranAny("systemctl restart patroni") {
t.Error("patroni was started over somebody's data")
}
}
func TestTheCollectorSaysWhoLeadsAndHowFarBehindTheRestAre(t *testing.T) {
c := NewCollector([]string{"10.0.0.149"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "llmgw-pg01", Host: "10.0.0.149", Role: "leader", State: "running"},
{Name: "llmgw-ctl02", Host: "10.0.0.150", Role: "replica", State: "streaming", Lag: 2048}}}, nil
}
got, err := testutil.CollectAndFormat(c, expfmt.TypeTextPlain, "llmgw_store_patroni_up", "llmgw_store_member", "llmgw_store_replica_lag_bytes")
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
"llmgw_store_patroni_up 1",
`llmgw_store_member{host="10.0.0.149",member="llmgw-pg01",role="leader",state="running"} 1`,
`llmgw_store_replica_lag_bytes{member="llmgw-ctl02"} 2048`,
} {
if !strings.Contains(string(got), want) {
t.Errorf("missing %s in\n%s", want, got)
}
}
if strings.Contains(string(got), `lag_bytes{member="llmgw-pg01"}`) {
t.Error("the leader was given a lag")
}
}
func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
c := NewCollector([]string{"10.0.0.153", "10.0.0.154"})
c.status = func(context.Context, []string) (Cluster, error) {
return Cluster{Members: []MemberState{{Name: "hatest-pg2", Host: "10.0.0.154", Role: "leader", State: "running"}}}, nil
}
cl, err := c.Read()
if err != nil {
t.Fatal(err)
}
if len(cl.Members) != 2 || cl.Members[1].Host != "10.0.0.153" || cl.Members[1].State != "missing" {
t.Errorf("%s", cl)
}
}
+417
View File
@@ -0,0 +1,417 @@
package pgha
import (
"bytes"
"context"
"fmt"
"os/exec"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
)
// Node is a machine the setup reaches, and how.
type Node struct {
Member
Remote onboard.Remote
}
// Setup builds, or finds, a Patroni cluster.
type Setup struct {
// DB are the database machines; the first is where the cluster starts.
DB []Node
// Witness is the third etcd vote when there are only two database
// machines: the control plane, run locally. Ignored with three or more.
Witness *Node
// Clients are the addresses control planes connect from.
Clients []string
// Status reads the cluster from Patroni's API; nil asks over HTTP.
Status func(ctx context.Context, hosts []string) (Cluster, error)
// Emit reports progress: a step, what happened, and whether anything
// was changed or found as it should be.
Emit func(step, detail string, changed bool)
// Wait bounds each wait for the cluster to come up.
Wait time.Duration
}
// Local is a Remote for the machine this runs on, which init runs as root.
type Local struct{}
// Run runs script with sh here.
func (Local) Run(ctx context.Context, script string, stdin []byte) ([]byte, error) {
cmd := exec.CommandContext(ctx, "sh", "-c", script)
if stdin != nil {
cmd.Stdin = bytes.NewReader(stdin)
}
var out, errb bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &errb
if err := cmd.Run(); err != nil {
return append(out.Bytes(), errb.Bytes()...), err
}
return out.Bytes(), nil
}
func (s *Setup) emit(step string, changed bool, format string, a ...any) {
if s.Emit != nil {
s.Emit(step, fmt.Sprintf(format, a...), changed)
}
}
// etcdMembers are the machines that vote: the database machines, and the
// witness when they are only two.
func (s *Setup) etcdMembers() []Node {
out := append([]Node{}, s.DB...)
if len(s.DB) == 2 && s.Witness != nil {
out = append(out, *s.Witness)
}
return out
}
// Run sets the cluster up, or confirms it is, and returns the store address.
func (s *Setup) Run(ctx context.Context) (string, error) {
if len(s.DB) < 2 {
return "", fmt.Errorf("a replicated store needs at least two database machines, not %d", len(s.DB))
}
if len(s.DB) == 2 && s.Witness == nil {
return "", fmt.Errorf("two database machines need a third etcd vote, or losing either stops failover")
}
if s.Wait == 0 {
s.Wait = 3 * time.Minute
}
if s.Status == nil {
s.Status = HTTPStatus
}
sec, err := s.secrets(ctx)
if err != nil {
return "", fmt.Errorf("passwords: %w", err)
}
if err := s.etcd(ctx); err != nil {
return "", fmt.Errorf("etcd: %w", err)
}
if err := s.patroni(ctx, sec); err != nil {
return "", fmt.Errorf("patroni: %w", err)
}
if err := s.database(ctx); err != nil {
return "", fmt.Errorf("database: %w", err)
}
members := make([]Member, len(s.DB))
for i, d := range s.DB {
members[i] = d.Member
}
return DSN(members, sec.App), nil
}
// secretDir holds the passwords on every database machine, readable by root
// alone. Generated on the first, carried to the others.
const secretDir = "/root/llmgw-pg"
var secretNames = []string{"superuser", "replicator", "llmgw"}
func (s *Setup) secrets(ctx context.Context) (Secrets, error) {
script := `set -e
sudo install -d -m 700 ` + secretDir + `
for f in superuser replicator llmgw; do
sudo test -s ` + secretDir + `/$f.pw || sudo sh -c "umask 077; openssl rand -hex 24 > ` + secretDir + `/$f.pw"
done
for f in superuser replicator llmgw; do printf '%s=' $f; sudo cat ` + secretDir + `/$f.pw; done`
out, err := s.DB[0].Remote.Run(ctx, script, nil)
if err != nil {
return Secrets{}, fmt.Errorf("on %s: %v: %s", s.DB[0].Name, err, strings.TrimSpace(string(out)))
}
got := parseSecrets(out)
if got.Superuser == "" || got.Replicator == "" || got.App == "" {
return Secrets{}, fmt.Errorf("could not read them back from %s", s.DB[0].Name)
}
for _, d := range s.DB[1:] {
have, err := d.Remote.Run(ctx, `for f in superuser replicator llmgw; do
if sudo test -s `+secretDir+`/$f.pw; then printf '%s=' $f; sudo cat `+secretDir+`/$f.pw; fi
done`, nil)
if err != nil {
return Secrets{}, fmt.Errorf("on %s: %v", d.Name, err)
}
if h := parseSecrets(have); h != (Secrets{}) {
if h != got {
// Not overwritten: a member holding other passwords belongs
// to another cluster, or to an earlier one somebody meant to
// keep, and replacing them would lock it out of itself.
return Secrets{}, fmt.Errorf("%s holds passwords in %s that differ from %s's; "+
"remove them there if that machine is meant to join this cluster", d.Name, secretDir, s.DB[0].Name)
}
continue
}
body := fmt.Sprintf("%s\n%s\n%s\n", got.Superuser, got.Replicator, got.App)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -m 700 `+secretDir+`
read su; read rep; read app
for kv in "superuser $su" "replicator $rep" "llmgw $app"; do
set -- $kv
printf '%s\n' "$2" | sudo sh -c "umask 077; cat > `+secretDir+`/$1.pw"
done`, []byte(body)); err != nil {
return Secrets{}, fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
s.emit("passwords", true, "carried to %s", d.Name)
}
s.emit("passwords", false, "kept in %s on every database machine", secretDir)
return got, nil
}
func parseSecrets(out []byte) Secrets {
var s Secrets
for _, line := range strings.Split(string(out), "\n") {
k, v, ok := strings.Cut(strings.TrimSpace(line), "=")
if !ok {
continue
}
switch k {
case "superuser":
s.Superuser = v
case "replicator":
s.Replicator = v
case "llmgw":
s.App = v
}
}
return s
}
// etcd brings up the voting members, unless every one of them is already
// healthy in one cluster — in which case nothing is touched, since rewriting
// a member's configuration means emptying its data.
func (s *Setup) etcd(ctx context.Context) error {
nodes := s.etcdMembers()
var members []Member
var endpoints []string
for _, n := range nodes {
members = append(members, n.Member)
endpoints = append(endpoints, "http://"+n.IP+":2379")
}
if s.etcdHealthy(ctx, nodes[0], endpoints, len(nodes)) {
s.emit("etcd", false, "%d members healthy; left as they are", len(nodes))
return nil
}
changed := make([]bool, len(nodes))
for i, n := range nodes {
out, err := n.Remote.Run(ctx, `set -e
if ! command -v etcd >/dev/null 2>&1; then
sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null
sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y etcd-server etcd-client >/dev/null
fi
new=$(cat)
if [ "$(sudo cat /etc/default/etcd 2>/dev/null)" != "$new" ]; then
sudo systemctl stop etcd >/dev/null 2>&1 || true
sudo rm -rf /var/lib/etcd/default
printf '%s\n' "$new" | sudo tee /etc/default/etcd >/dev/null
echo changed
fi
sudo systemctl enable etcd >/dev/null 2>&1`, []byte(Etcd(n.Member, members)))
if err != nil {
return fmt.Errorf("on %s: %v: %s", n.Name, err, strings.TrimSpace(string(out)))
}
changed[i] = strings.Contains(string(out), "changed")
}
// Started together: a new member waits for the others before it says it
// is up, so starting them one after another would wait for ever on the
// first.
var wg sync.WaitGroup
errs := make([]error, len(nodes))
for i, n := range nodes {
wg.Add(1)
go func(i int, n Node) {
defer wg.Done()
verb := "start"
if changed[i] {
verb = "restart"
}
if out, err := n.Remote.Run(ctx, "sudo systemctl "+verb+" --no-block etcd", nil); err != nil {
errs[i] = fmt.Errorf("on %s: %v: %s", n.Name, err, strings.TrimSpace(string(out)))
}
}(i, n)
}
wg.Wait()
for _, err := range errs {
if err != nil {
return err
}
}
deadline := time.Now().Add(s.Wait)
for !s.etcdHealthy(ctx, nodes[0], endpoints, len(nodes)) {
if time.Now().After(deadline) {
return fmt.Errorf("the %d members did not become healthy within %s; `journalctl -u etcd` on each says why", len(nodes), s.Wait)
}
if err := sleep(ctx, 2*time.Second); err != nil {
return err
}
}
s.emit("etcd", true, "%d members up", len(nodes))
return nil
}
func (s *Setup) etcdHealthy(ctx context.Context, n Node, endpoints []string, want int) bool {
out, err := n.Remote.Run(ctx, "ETCDCTL_API=3 etcdctl --dial-timeout=3s --endpoints="+
strings.Join(endpoints, ",")+" endpoint health 2>&1", nil)
return err == nil && strings.Count(string(out), "is healthy") == want
}
// patroni installs PostgreSQL and Patroni on every database machine, unless
// the cluster already has a leader and every member streaming.
func (s *Setup) patroni(ctx context.Context, sec Secrets) error {
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
if c, err := s.Status(ctx, hosts); err == nil && c.Complete(len(s.DB)) {
s.emit("patroni", false, "%s leads, %d streaming; left as it is", c.Leader(), len(s.DB)-1)
return nil
}
var version string
for _, d := range s.DB {
v, err := s.preparePostgres(ctx, d)
if err != nil {
return err
}
if version != "" && v != version {
// Streaming replication is between one major version.
return fmt.Errorf("%s has PostgreSQL %s and %s has %s; they must match", s.DB[0].Name, version, d.Name, v)
}
version = v
}
var etcd, db []Member
for _, n := range s.etcdMembers() {
etcd = append(etcd, n.Member)
}
for _, d := range s.DB {
db = append(db, d.Member)
}
for i, d := range s.DB {
cfg := Patroni(d.Member, db, etcd, s.Clients, version, sec)
if out, err := d.Remote.Run(ctx, `set -e
sudo install -d -m 755 /etc/patroni
sudo install -d -o postgres -g postgres -m 700 /var/lib/postgresql/`+version+`/llmgw
sudo sh -c 'umask 027; cat > /etc/patroni/config.yml.new'
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl enable patroni >/dev/null 2>&1
sudo systemctl restart patroni`, []byte(cfg)); err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
// The first starts the cluster; the rest clone it, which they can
// only do once it has a leader.
want := func(c Cluster) bool { return c.Leader() != "" }
if i > 0 {
want = func(c Cluster) bool { return c.Complete(i + 1) }
}
if err := s.waitFor(ctx, hosts, want); err != nil {
return fmt.Errorf("after starting %s: %w", d.Name, err)
}
s.emit("patroni", true, "%s up", d.Name)
}
return nil
}
// preparePostgres installs the packages and clears the distribution's own
// cluster, which would hold the port Patroni's needs — unless it holds a
// database somebody made, which is not this setup's to delete.
func (s *Setup) preparePostgres(ctx context.Context, d Node) (string, error) {
out, err := d.Remote.Run(ctx, `set -e
need=""
command -v patroni >/dev/null 2>&1 || need="$need patroni"
ls /usr/lib/postgresql/*/bin/postgres >/dev/null 2>&1 || need="$need postgresql"
if [ -n "$need" ]; then
sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null
sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y $need >/dev/null
fi
v=$(ls /usr/lib/postgresql | sort -n | tail -1)
if [ -d /var/lib/postgresql/$v/main ]; then
if sudo systemctl is-active --quiet postgresql@$v-main; then
n=$(sudo -u postgres psql -Atc "select count(*) from pg_database where datname not in ('postgres','template0','template1')")
if [ "$n" != "0" ]; then echo "holds-data"; exit 0; fi
fi
sudo systemctl disable --now postgresql@$v-main postgresql >/dev/null 2>&1 || true
sudo pg_dropcluster --stop $v main >/dev/null 2>&1 || sudo rm -rf /var/lib/postgresql/$v/main
fi
sudo systemctl disable --now postgresql >/dev/null 2>&1 || true
echo "version=$v"`, nil)
if err != nil {
return "", fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
if strings.Contains(string(out), "holds-data") {
return "", fmt.Errorf("%s runs a PostgreSQL with databases in it; move or drop them first — "+
"this setup replaces that cluster", d.Name)
}
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "version="); ok && v != "" {
return v, nil
}
}
return "", fmt.Errorf("no PostgreSQL found on %s after installing it", d.Name)
}
// database makes the role and database on whichever member leads.
func (s *Setup) database(ctx context.Context) error {
hosts := make([]string, len(s.DB))
for i, d := range s.DB {
hosts[i] = d.IP
}
c, err := s.Status(ctx, hosts)
if err != nil {
return err
}
var leader *Node
for i := range s.DB {
if s.DB[i].Name == c.Leader() {
leader = &s.DB[i]
}
}
if leader == nil {
return fmt.Errorf("no member leads")
}
out, err := leader.Remote.Run(ctx, `set -e
pw=$(sudo cat `+secretDir+`/llmgw.pw)
sudo -u postgres psql -q -v ON_ERROR_STOP=1 -v pw="$pw" <<'SQL'
SELECT format('CREATE ROLE %I LOGIN', 'llmgw') WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'llmgw') \gexec
ALTER ROLE llmgw PASSWORD :'pw';
SELECT format('CREATE DATABASE %I OWNER %I', 'llmgw', 'llmgw') WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'llmgw') \gexec
SQL`, nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", leader.Name, err, strings.TrimSpace(string(out)))
}
s.emit("database", true, "role and database %s on %s", Database, leader.Name)
return nil
}
func (s *Setup) waitFor(ctx context.Context, hosts []string, ok func(Cluster) bool) error {
deadline := time.Now().Add(s.Wait)
var last error
for {
c, err := s.Status(ctx, hosts)
if err == nil && ok(c) {
return nil
}
if err != nil {
last = err
} else {
last = fmt.Errorf("the cluster is %s", c)
}
if time.Now().After(deadline) {
return fmt.Errorf("not up within %s: %v; `journalctl -u patroni` says why", s.Wait, last)
}
if err := sleep(ctx, 3*time.Second); err != nil {
return err
}
}
}
func sleep(ctx context.Context, d time.Duration) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(d):
return nil
}
}
+114
View File
@@ -0,0 +1,114 @@
package pgha
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
)
// MemberState is one member as Patroni reports it.
type MemberState struct {
Name string `json:"name"`
// Role is "leader", "replica" or "sync_standby".
Role string `json:"role"`
// State is "running" for a leader, "streaming" for a replica keeping up,
// and anything else for one that is not.
State string `json:"state"`
Host string `json:"host"`
Timeline int `json:"timeline"`
// Lag is how far behind the leader a replica is, in bytes; -1 when
// Patroni does not know, which is itself worth an alert.
Lag int64 `json:"-"`
RawLag json.RawMessage `json:"lag"`
}
// Cluster is the cluster's state, as any member's API reports it.
type Cluster struct {
Members []MemberState `json:"members"`
}
// Leader is the leading member's name, or "".
func (c Cluster) Leader() string {
for _, m := range c.Members {
if m.Role == "leader" || m.Role == "standby_leader" {
return m.Name
}
}
return ""
}
// Complete reports whether n members are up: one leading, the rest streaming.
func (c Cluster) Complete(n int) bool {
if c.Leader() == "" || len(c.Members) < n {
return false
}
streaming := 0
for _, m := range c.Members {
if m.Role != "leader" && m.State == "streaming" {
streaming++
}
}
return streaming >= n-1
}
func (c Cluster) String() string {
if len(c.Members) == 0 {
return "empty"
}
var parts []string
for _, m := range c.Members {
parts = append(parts, m.Name+" "+m.Role+"/"+m.State)
}
return strings.Join(parts, ", ")
}
// HTTPStatus asks each host's Patroni in turn and returns the first answer:
// any member describes the whole cluster.
func HTTPStatus(ctx context.Context, hosts []string) (Cluster, error) {
client := &http.Client{Timeout: 3 * time.Second}
var last error
for _, h := range hosts {
u := "http://" + net.JoinHostPort(h, strconv.Itoa(PatroniPort)) + "/cluster"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return Cluster{}, err
}
resp, err := client.Do(req)
if err != nil {
last = err
continue
}
var c Cluster
err = json.NewDecoder(resp.Body).Decode(&c)
resp.Body.Close()
if err != nil {
last = fmt.Errorf("%s: %w", u, err)
continue
}
for i := range c.Members {
c.Members[i].Lag = lagOf(c.Members[i].RawLag)
}
return c, nil
}
if last == nil {
last = fmt.Errorf("no hosts to ask")
}
return Cluster{}, fmt.Errorf("no member's Patroni answered: %w", last)
}
// lagOf reads Patroni's lag: a number of bytes, or "unknown".
func lagOf(raw json.RawMessage) int64 {
if len(raw) == 0 {
return 0
}
var n int64
if json.Unmarshal(raw, &n) == nil {
return n
}
return -1
}