The store's PostgreSQL sized to its machine's memory

Each Patroni member gets a share of its own RAM in its own config.yml:
shared_buffers a quarter (15% beside a control plane or agent),
effective_cache_size, maintenance_work_mem and work_mem scaled to it, all
clamped. It is read again on every server init --patroni, so a machine
given more memory gets the use of it. A single-machine PostgreSQL init
installs is sized the same way, as a shared machine, instead of a fixed
96MB.

The step itself went in with 85025df by accident, in an early form that
restarted PostgreSQL before Patroni had taken the reload, leaving the old
shared_buffers with a restart pending. It now waits for Patroni to report
the restart pending, restarts with restart_pending (replicas first, the
leader last), and restarts a member it finds pending from an earlier run.

Applied to the PoC: pg01 978MB, ctl02 (control plane beside it) 587MB,
both from 128MB; streaming, lag 0, gateways unaffected.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -56,6 +56,21 @@ APIs change between minors without a deprecation period.
   not the organization, so two teams of one organization no longer share
   cached passages.
 
+### Changed
+
+- **The store's PostgreSQL is sized to its machine's memory.** Before, it ran
+  on PostgreSQL's defaults (128MB cache) under Patroni, or on 96MB on one
+  machine. Now each member gets a share of its own RAM, a smaller one where
+  a control plane or agent shares the machine, and the share is read again
+  every time `server init --patroni` runs. The new cache is applied by
+  restarting one member at a time, the leader last.
+
+### Fixed
+
+- `llmgw server backup` and `restore` against a replicated store over TLS
+  failed looking for `~/.postgresql/root.crt`: pg_dump was given
+  `sslmode=verify-full` without the cluster's authority.
+
 ## v0.14.0 — 2026-10-08
 
 Knowledge bases: a company's documents, put in by people from their own

docs/install.md

@@ -201,6 +201,16 @@ this by running the same command again, on the control plane that holds the
 authority: etcd is rebuilt on TLS with Patroni in maintenance mode, so
 PostgreSQL serves throughout and no failover happens.
 
+Each member's PostgreSQL is sized to the machine's memory, read each time
+the command runs: a quarter of it as cache (`shared_buffers`), three quarters
+counted on for the planner, and room to sort and build indexes, all capped. A
+member that also runs a control plane or an agent gets less: about 15% as
+cache, so the gateway is not pushed into swap. A knowledge base's vector
+index is searched well only from memory, so give a store machine more RAM and
+run the command again. The new cache takes a PostgreSQL restart, which Patroni
+does one member at a time, the leader last. The leader's restart costs a few
+seconds without writes, and there is no failover.
+
 Every step looks before it acts. Running it again finishes an install that
 stopped halfway and changes nothing on one that did not; it never overwrites
 `server.yaml`, the authority, or an existing account's password.
@@ -1736,9 +1746,11 @@ the file again.
 
 On a machine that also runs the control plane, PostgreSQL's defaults assume it
 owns the box. A gateway swapped out to give the database cache is a worse
-outcome than a smaller cache, so cap it: `shared_buffers = 96MB` and
-`max_connections = 40` are enough for a store whose working set is measured in
-megabytes.
+outcome than a smaller cache, so size it to the machine. `server init` does this
+for a PostgreSQL it installs: `shared_buffers` about 15% of memory (at least
+96MB), `effective_cache_size` half, `work_mem` and `maintenance_work_mem`
+scaled to it, and `max_connections = 40`. A store with a large knowledge base
+wants a machine of its own, where its vector index fits in memory.
 
 ## Tagging requests
 

internal/cli/serverinit_postgres.go

@@ -16,6 +16,7 @@ import (
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 )
 
@@ -348,15 +349,23 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
 	}
 
 	if installed {
-		// Its defaults assume it owns the machine. A gateway swapped out to
-		// give the database cache is worse than a smaller cache, and a store
-		// whose working set is megabytes needs little. Only a PostgreSQL init
-		// installed itself: one that was already here may serve something
-		// else that needs its connections.
-		for _, q := range []string{
-			"ALTER SYSTEM SET shared_buffers = '96MB'",
-			"ALTER SYSTEM SET max_connections = 40",
-		} {
+		// Sized to the machine's memory, as one it shares with the gateway:
+		// a gateway swapped out to give the database cache is worse than a
+		// smaller cache, but a knowledge base's vector index is searched
+		// well only from memory. Only a PostgreSQL init installed itself:
+		// one that was already here may serve something else that needs its
+		// connections.
+		meminfo, _ := os.ReadFile("/proc/meminfo")
+		total, ok := pgha.MemTotal(string(meminfo))
+		if !ok {
+			total = 1 << 30
+		}
+		mem := pgha.MemorySettings(total, true)
+		qs := []string{"ALTER SYSTEM SET max_connections = 40"}
+		for _, k := range []string{"shared_buffers", "effective_cache_size", "maintenance_work_mem", "work_mem"} {
+			qs = append(qs, "ALTER SYSTEM SET "+k+" = '"+mem[k]+"'")
+		}
+		for _, q := range qs {
 			if _, err := in.psql(ctx, q); err != nil {
 				return "", err
 			}
@@ -367,7 +376,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
 		if err := in.waitForPostgres(ctx); err != nil {
 			return "", err
 		}
-		in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
+		in.done("PostgreSQL sized for a machine it shares, with %d MB of memory (shared_buffers %s, work_mem %s, 40 connections)",
+			total>>20, mem["shared_buffers"], mem["work_mem"])
 	}
 
 	in.setupPgvector(ctx, discovery.DiscoverHost())

internal/pgha/memory.go

@@ -113,13 +113,39 @@ echo "$r"`, body)
 			s.emit("memory", true, "%s", what)
 		}
 	}
+	// Patroni takes a reload in its own time: restarted before it has written
+	// the new configuration, PostgreSQL comes back on the old one. So each is
+	// waited on until Patroni says a restart is pending — and a member found
+	// pending, by a run that stopped before restarting it, is restarted too.
+	pending := func(c Cluster, name string) bool {
+		for _, m := range c.Members {
+			if m.Name == name {
+				return m.PendingRestart
+			}
+		}
+		return false
+	}
+	var c Cluster
+	err := s.waitFor(ctx, s.dbHosts(), func(got Cluster) bool {
+		c = got
+		for _, d := range restart {
+			if !pending(got, d.Name) {
+				return false
+			}
+		}
+		return true
+	})
+	if err != nil {
+		return fmt.Errorf("waiting for Patroni to take the new configuration: %w", err)
+	}
+	for _, d := range s.DB {
+		if pending(c, d.Name) && !slices.ContainsFunc(restart, func(n Node) bool { return n.Name == d.Name }) {
+			restart = append(restart, d)
+		}
+	}
 	if len(restart) == 0 {
 		return nil
 	}
-	c, err := s.Status(ctx, s.dbHosts())
-	if err != nil {
-		return err
-	}
 	// The leader last: its restart is the only one the control planes see,
 	// a few seconds without writes, and by then the rest are back.
 	slices.SortStableFunc(restart, func(a, b Node) int {
@@ -133,7 +159,7 @@ echo "$r"`, body)
 		return 0
 	})
 	for _, d := range restart {
-		if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{}`), sec.RestAPI, 2*time.Minute); err != nil {
+		if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{"restart_pending": true}`), sec.RestAPI, 2*time.Minute); err != nil {
 			return fmt.Errorf("restarting PostgreSQL on %s for its new shared_buffers: %w", d.Name, err)
 		}
 		if err := s.waitFor(ctx, s.dbHosts(), func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil {

internal/pgha/pgha_test.go

@@ -465,3 +465,97 @@ func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *tes
 		t.Errorf("TLS asked of a cluster without it: %s", dsn)
 	}
 }
+
+func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) {
+	total, ok := MemTotal("MemTotal:        4009848 kB\nMemFree:          976000 kB\n")
+	if !ok || total != 4009848<<10 {
+		t.Fatalf("MemTotal %d %v", total, ok)
+	}
+	own := MemorySettings(total, false)
+	if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" ||
+		own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" {
+		t.Errorf("a machine of its own: %v", own)
+	}
+	// Beside a control plane, less of it.
+	if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" {
+		t.Errorf("a shared machine: %v", sh)
+	}
+	// Never below what PostgreSQL needs, never past what it can use.
+	if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" {
+		t.Errorf("a small machine: %v", tiny)
+	}
+	if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" {
+		t.Errorf("a large machine: %v", big)
+	}
+}
+
+func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) {
+	var mu sync.Mutex
+	var restarted []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodPost && r.URL.Path == "/restart" {
+			if _, pass, _ := r.BasicAuth(); pass != "pw" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			host, _, _ := strings.Cut(r.Host, ":")
+			mu.Lock()
+			restarted = append(restarted, host)
+			mu.Unlock()
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+	old := restPort
+	restPort, _ = strconv.Atoi(u.Port())
+	defer func() { restPort = old }()
+
+	answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"}
+	leader := &machine{name: "a", answers: answers}
+	replica := &machine{name: "b", answers: answers}
+	up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true},
+		{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
+	s := Setup{
+		DB:     []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}},
+		Status: func(context.Context, []string) (Cluster, error) { return up, nil },
+		Wait:   time.Second,
+	}
+	if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(restarted, " ") != "localhost 127.0.0.1" {
+		t.Errorf("restarted %v; the replica first, then the leader", restarted)
+	}
+}
+
+func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) {
+	// A run that stopped after writing the configuration, or restarted before
+	// Patroni had taken it: nothing to change, but PostgreSQL still on the old.
+	var restarted []string
+	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/restart" {
+			host, _, _ := strings.Cut(r.Host, ":")
+			restarted = append(restarted, host)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+	old := restPort
+	restPort, _ = strconv.Atoi(u.Port())
+	defer func() { restPort = old }()
+
+	answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"}
+	up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"},
+		{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
+	s := Setup{
+		DB:     []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}},
+		Status: func(context.Context, []string) (Cluster, error) { return up, nil },
+		Wait:   time.Second,
+	}
+	if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(restarted, " ") != "localhost" {
+		t.Errorf("restarted %v; only the member pending", restarted)
+	}
+}

internal/pgha/status.go

@@ -19,6 +19,9 @@ type MemberState struct {
 	State    string `json:"state"`
 	Host     string `json:"host"`
 	Timeline int    `json:"timeline"`
+	// PendingRestart is a parameter changed that PostgreSQL takes only on
+	// restart.
+	PendingRestart bool `json:"pending_restart"`
 	// Lag is how far behind the leader a replica is, in bytes; -1 when
 	// Patroni does not know, which is itself worth an alert.
 	Lag    int64           `json:"-"`