server add gives another machine this llmgw, a certificate, the configuration and store address, lets it into the store through Patroni, and starts it; --vip keeps a floating address on the leader with keepalived, following a new /leader endpoint. Nodes learn every control plane from the leader and notice a dead one by HTTP/2 ping instead of after the kernel's fifteen minutes. The setup, console, node and init screens are drawn full screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
179 lines
5.2 KiB
Go
179 lines
5.2 KiB
Go
// Package pgha sets up the store as PostgreSQL under Patroni: two or more
|
|
// machines, one primary streaming to the others, and a standby promoted when
|
|
// the primary is lost. Patroni keeps its leader lock in etcd, which decides
|
|
// only with a majority, so etcd runs on three machines — the database
|
|
// machines and, when they are two, the control plane as the third vote.
|
|
//
|
|
// Everything is done over the same ssh the rest of onboarding uses, and every
|
|
// step looks before it acts: run against a cluster that is already up, it
|
|
// changes nothing.
|
|
package pgha
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
)
|
|
|
|
// Scope is the Patroni cluster's name, and the etcd cluster token.
|
|
const Scope = "llmgw"
|
|
|
|
// PatroniPort is where Patroni's REST API listens on every member: what the
|
|
// control plane reads the cluster's state from.
|
|
const PatroniPort = 8008
|
|
|
|
// restPort is where this package asks Patroni; a test moves it.
|
|
var restPort = PatroniPort
|
|
|
|
// Role and Database are what the control plane connects as.
|
|
const (
|
|
Role = "llmgw"
|
|
Database = "llmgw"
|
|
)
|
|
|
|
// Member is one machine of the cluster, by the address the others reach it on.
|
|
type Member struct {
|
|
Name string
|
|
IP string
|
|
}
|
|
|
|
// Secrets are the cluster's three passwords.
|
|
type Secrets struct {
|
|
Superuser, Replicator, App string
|
|
}
|
|
|
|
// Etcd is a member's /etc/default/etcd, for Ubuntu's etcd-server.
|
|
func Etcd(m Member, all []Member) string {
|
|
var peers []string
|
|
for _, p := range all {
|
|
peers = append(peers, fmt.Sprintf("%s=http://%s:2380", p.Name, p.IP))
|
|
}
|
|
return fmt.Sprintf(`# llmgw: one member of the etcd that holds Patroni's leader lock.
|
|
ETCD_NAME=%[1]s
|
|
ETCD_DATA_DIR=/var/lib/etcd/default
|
|
ETCD_LISTEN_PEER_URLS=http://%[2]s:2380
|
|
ETCD_LISTEN_CLIENT_URLS=http://%[2]s:2379,http://127.0.0.1:2379
|
|
ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%[2]s:2380
|
|
ETCD_ADVERTISE_CLIENT_URLS=http://%[2]s:2379
|
|
ETCD_INITIAL_CLUSTER=%[3]s
|
|
ETCD_INITIAL_CLUSTER_STATE=new
|
|
ETCD_INITIAL_CLUSTER_TOKEN=%[4]s
|
|
`, m.Name, m.IP, strings.Join(peers, ","), Scope+"-patroni")
|
|
}
|
|
|
|
// Patroni is a database member's /etc/patroni/config.yml.
|
|
//
|
|
// clients are the addresses the control planes connect from; the members
|
|
// themselves are always allowed, since a second control plane usually runs on
|
|
// one of them.
|
|
func Patroni(m Member, db, etcd []Member, clients []string, version string, s Secrets) string {
|
|
var hosts []string
|
|
for _, e := range etcd {
|
|
hosts = append(hosts, e.IP+":2379")
|
|
}
|
|
var hba []string
|
|
hba = append(hba, "local all all peer", "host all all 127.0.0.1/32 scram-sha-256")
|
|
for _, d := range db {
|
|
hba = append(hba,
|
|
"host replication replicator "+d.IP+"/32 scram-sha-256",
|
|
"host all postgres "+d.IP+"/32 scram-sha-256")
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, c := range append(append([]string{}, clients...), ips(db)...) {
|
|
if c == "" || seen[c] {
|
|
continue
|
|
}
|
|
seen[c] = true
|
|
hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
|
|
}
|
|
var hbaLines string
|
|
for _, h := range hba {
|
|
hbaLines += " - " + h + "\n"
|
|
}
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
|
|
scope: %[1]s
|
|
namespace: /service/
|
|
name: %[2]s
|
|
|
|
restapi:
|
|
listen: %[3]s:%[4]d
|
|
connect_address: %[3]s:%[4]d
|
|
|
|
etcd3:
|
|
hosts: %[5]s
|
|
|
|
bootstrap:
|
|
dcs:
|
|
ttl: 30
|
|
loop_wait: 10
|
|
retry_timeout: 10
|
|
maximum_lag_on_failover: 1048576
|
|
postgresql:
|
|
use_pg_rewind: true
|
|
use_slots: true
|
|
parameters:
|
|
wal_level: replica
|
|
hot_standby: "on"
|
|
max_wal_senders: 10
|
|
max_replication_slots: 10
|
|
wal_keep_size: 1GB
|
|
wal_log_hints: "on"
|
|
password_encryption: scram-sha-256
|
|
# In the cluster's shared configuration rather than each member's, so a
|
|
# control plane added later is let in by one change through Patroni's
|
|
# API (llmgw server add) instead of an edit on every member.
|
|
pg_hba:
|
|
%[9]s initdb:
|
|
- encoding: UTF8
|
|
- data-checksums
|
|
|
|
postgresql:
|
|
listen: 127.0.0.1,%[3]s:5432
|
|
connect_address: %[3]s:5432
|
|
data_dir: /var/lib/postgresql/%[6]s/llmgw
|
|
bin_dir: /usr/lib/postgresql/%[6]s/bin
|
|
pgpass: /var/lib/postgresql/.pgpass_patroni
|
|
authentication:
|
|
superuser:
|
|
username: postgres
|
|
password: "%[7]s"
|
|
replication:
|
|
username: replicator
|
|
password: "%[8]s"
|
|
rewind:
|
|
username: postgres
|
|
password: "%[7]s"
|
|
parameters:
|
|
unix_socket_directories: /var/run/postgresql
|
|
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines)
|
|
b.WriteString(`
|
|
tags:
|
|
nofailover: false
|
|
noloadbalance: false
|
|
clonefrom: false
|
|
nosync: false
|
|
`)
|
|
return b.String()
|
|
}
|
|
|
|
// DSN is the store address the control planes are given: every member, and
|
|
// the driver told to use whichever accepts writes, so a failover is followed
|
|
// without anything in front of the database.
|
|
func DSN(db []Member, password string) string {
|
|
var hosts []string
|
|
for _, d := range db {
|
|
hosts = append(hosts, d.IP+":5432")
|
|
}
|
|
return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable&target_session_attrs=read-write",
|
|
Role, url.QueryEscape(password), strings.Join(hosts, ","), Database)
|
|
}
|
|
|
|
func ips(ms []Member) []string {
|
|
out := make([]string, len(ms))
|
|
for i, m := range ms {
|
|
out[i] = m.IP
|
|
}
|
|
return out
|
|
}
|