Keep the store under Patroni, and back up from a store of several hosts
deploy/ha documents PostgreSQL under Patroni with a three-member etcd, as running on the reference cluster, and the address that follows the primary. Backups read such an address as libpq host and port lists and prefer a standby, rather than resolving the list as one name. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
febcf251e4
commit
7a562ac1e7
6 files changed
+223
-10
No files matched your search
+57
-10
@@ -178,30 +178,77 @@ func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
|
||||
// command line: a DSN carries the password, and every user on the machine
|
||||
// can read another process's arguments.
|
||||
func pgDump(ctx context.Context, dsn, out string) error {
|
||||
u, err := url.Parse(dsn)
|
||||
vars, err := pgEnv(dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the store address is not a URL")
|
||||
return err
|
||||
}
|
||||
env := os.Environ()
|
||||
cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
|
||||
cmd.Env = append(os.Environ(), vars...)
|
||||
if b, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pgEnv turns a store address into libpq's environment.
|
||||
//
|
||||
// Several hosts are allowed, as pgx allows them — "h1:5432,h2:5432", the
|
||||
// members of a replicated cluster — and become libpq's own lists in PGHOST and
|
||||
// PGPORT. url.Parse does not read such a host: it took everything before the
|
||||
// last colon as one name, which libpq could not resolve, and a backup worked
|
||||
// only because it fell through to the second host.
|
||||
func pgEnv(dsn string) ([]string, error) {
|
||||
scheme, rest, ok := strings.Cut(dsn, "://")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the store address is not a URL")
|
||||
}
|
||||
authority, tail := rest, ""
|
||||
if i := strings.IndexAny(rest, "/?"); i >= 0 {
|
||||
authority, tail = rest[:i], rest[i:]
|
||||
}
|
||||
userinfo, hostlist := "", authority
|
||||
if i := strings.LastIndexByte(authority, '@'); i >= 0 {
|
||||
userinfo, hostlist = authority[:i+1], authority[i+1:]
|
||||
}
|
||||
var hosts, ports []string
|
||||
for _, hp := range strings.Split(hostlist, ",") {
|
||||
u, err := url.Parse("x://" + hp)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return nil, fmt.Errorf("the store address has a host that is not one: %q", hp)
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "5432"
|
||||
}
|
||||
hosts, ports = append(hosts, u.Hostname()), append(ports, port)
|
||||
}
|
||||
// The rest of the address with one placeholder host, which url.Parse reads.
|
||||
u, err := url.Parse(scheme + "://" + userinfo + "host" + tail)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the store address is not a URL")
|
||||
}
|
||||
var env []string
|
||||
set := func(k, v string) {
|
||||
if v != "" {
|
||||
env = append(env, k+"="+v)
|
||||
}
|
||||
}
|
||||
set("PGHOST", u.Hostname())
|
||||
set("PGPORT", u.Port())
|
||||
set("PGHOST", strings.Join(hosts, ","))
|
||||
set("PGPORT", strings.Join(ports, ","))
|
||||
set("PGUSER", u.User.Username())
|
||||
if pw, ok := u.User.Password(); ok {
|
||||
set("PGPASSWORD", pw)
|
||||
}
|
||||
set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
|
||||
set("PGSSLMODE", u.Query().Get("sslmode"))
|
||||
cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
|
||||
cmd.Env = env
|
||||
if b, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
|
||||
// A standby if there is one, else whichever answers — not the address's
|
||||
// read-write: a dump only reads, it need not load the primary, and one
|
||||
// that insists on the primary fails exactly while the cluster is failing
|
||||
// over.
|
||||
if len(hosts) > 1 {
|
||||
set("PGTARGETSESSIONATTRS", "prefer-standby")
|
||||
}
|
||||
return nil
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func addTree(tw *tar.Writer, root, as string) error {
|
||||
|
||||
@@ -139,3 +139,26 @@ func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreOfSeveralHostsIsGivenToPgDumpAsLibpqLists(t *testing.T) {
|
||||
// The address of a Patroni cluster: url.Parse read its host as one name,
|
||||
// and pg_dump reached the database only by falling through to the second.
|
||||
env, err := pgEnv("postgres://llmgw:s3cret@10.0.0.149:5432,10.0.0.150/llmgw?sslmode=disable&target_session_attrs=read-write")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(env, " ")
|
||||
for _, want := range []string{"PGHOST=10.0.0.149,10.0.0.150", "PGPORT=5432,5432", "PGUSER=llmgw",
|
||||
"PGPASSWORD=s3cret", "PGDATABASE=llmgw", "PGSSLMODE=disable", "PGTARGETSESSIONATTRS=prefer-standby"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("%s is not in %s", want, got)
|
||||
}
|
||||
}
|
||||
one, err := pgEnv("postgres://u:p@db.internal:6432/llmgw")
|
||||
if err != nil || !strings.Contains(strings.Join(one, " "), "PGHOST=db.internal PGPORT=6432") {
|
||||
t.Errorf("one host: %v, %v", one, err)
|
||||
}
|
||||
if _, err := pgEnv("postgres://u:p@a:5432,:5432/x"); err == nil {
|
||||
t.Error("an empty host in the list was accepted")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user