From 42d3b7ea2256281e15991eda89bae1f2e836b14b Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 8 Oct 2026 21:48:53 +0200 Subject: [PATCH] The store's PostgreSQL sized to its machine's memory Each Patroni member gets a share of its own RAM in its own config.yml: shared_buffers a quarter (15% beside a control plane or agent), effective_cache_size, maintenance_work_mem and work_mem scaled to it, all clamped. It is read again on every server init --patroni, so a machine given more memory gets the use of it. A single-machine PostgreSQL init installs is sized the same way, as a shared machine, instead of a fixed 96MB. The step itself went in with 85025df by accident, in an early form that restarted PostgreSQL before Patroni had taken the reload, leaving the old shared_buffers with a restart pending. It now waits for Patroni to report the restart pending, restarts with restart_pending (replicas first, the leader last), and restarts a member it finds pending from an earlier run. Applied to the PoC: pg01 978MB, ctl02 (control plane beside it) 587MB, both from 128MB; streaming, lag 0, gateways unaffected. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +++++ docs/install.md | 18 +++++- internal/cli/serverinit_postgres.go | 30 ++++++--- internal/pgha/memory.go | 36 +++++++++-- internal/pgha/pgha_test.go | 94 +++++++++++++++++++++++++++++ internal/pgha/status.go | 3 + 6 files changed, 178 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef8d307..2cc3f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,21 @@ APIs change between minors without a deprecation period. not the organization, so two teams of one organization no longer share cached passages. +### Changed + +- **The store's PostgreSQL is sized to its machine's memory.** Before, it ran + on PostgreSQL's defaults (128MB cache) under Patroni, or on 96MB on one + machine. Now each member gets a share of its own RAM, a smaller one where + a control plane or agent shares the machine, and the share is read again + every time `server init --patroni` runs. The new cache is applied by + restarting one member at a time, the leader last. + +### Fixed + +- `llmgw server backup` and `restore` against a replicated store over TLS + failed looking for `~/.postgresql/root.crt`: pg_dump was given + `sslmode=verify-full` without the cluster's authority. + ## v0.14.0 — 2026-10-08 Knowledge bases: a company's documents, put in by people from their own diff --git a/docs/install.md b/docs/install.md index 2028ffd..2a00268 100644 --- a/docs/install.md +++ b/docs/install.md @@ -201,6 +201,16 @@ this by running the same command again, on the control plane that holds the authority: etcd is rebuilt on TLS with Patroni in maintenance mode, so PostgreSQL serves throughout and no failover happens. +Each member's PostgreSQL is sized to the machine's memory, read each time +the command runs: a quarter of it as cache (`shared_buffers`), three quarters +counted on for the planner, and room to sort and build indexes, all capped. A +member that also runs a control plane or an agent gets less: about 15% as +cache, so the gateway is not pushed into swap. A knowledge base's vector +index is searched well only from memory, so give a store machine more RAM and +run the command again. The new cache takes a PostgreSQL restart, which Patroni +does one member at a time, the leader last. The leader's restart costs a few +seconds without writes, and there is no failover. + Every step looks before it acts. Running it again finishes an install that stopped halfway and changes nothing on one that did not; it never overwrites `server.yaml`, the authority, or an existing account's password. @@ -1736,9 +1746,11 @@ the file again. On a machine that also runs the control plane, PostgreSQL's defaults assume it owns the box. A gateway swapped out to give the database cache is a worse -outcome than a smaller cache, so cap it: `shared_buffers = 96MB` and -`max_connections = 40` are enough for a store whose working set is measured in -megabytes. +outcome than a smaller cache, so size it to the machine. `server init` does this +for a PostgreSQL it installs: `shared_buffers` about 15% of memory (at least +96MB), `effective_cache_size` half, `work_mem` and `maintenance_work_mem` +scaled to it, and `max_connections = 40`. A store with a large knowledge base +wants a machine of its own, where its vector index fits in memory. ## Tagging requests diff --git a/internal/cli/serverinit_postgres.go b/internal/cli/serverinit_postgres.go index 4141719..c2dfd4a 100644 --- a/internal/cli/serverinit_postgres.go +++ b/internal/cli/serverinit_postgres.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -348,15 +349,23 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) { } if installed { - // Its defaults assume it owns the machine. A gateway swapped out to - // give the database cache is worse than a smaller cache, and a store - // whose working set is megabytes needs little. Only a PostgreSQL init - // installed itself: one that was already here may serve something - // else that needs its connections. - for _, q := range []string{ - "ALTER SYSTEM SET shared_buffers = '96MB'", - "ALTER SYSTEM SET max_connections = 40", - } { + // Sized to the machine's memory, as one it shares with the gateway: + // a gateway swapped out to give the database cache is worse than a + // smaller cache, but a knowledge base's vector index is searched + // well only from memory. Only a PostgreSQL init installed itself: + // one that was already here may serve something else that needs its + // connections. + meminfo, _ := os.ReadFile("/proc/meminfo") + total, ok := pgha.MemTotal(string(meminfo)) + if !ok { + total = 1 << 30 + } + mem := pgha.MemorySettings(total, true) + qs := []string{"ALTER SYSTEM SET max_connections = 40"} + for _, k := range []string{"shared_buffers", "effective_cache_size", "maintenance_work_mem", "work_mem"} { + qs = append(qs, "ALTER SYSTEM SET "+k+" = '"+mem[k]+"'") + } + for _, q := range qs { if _, err := in.psql(ctx, q); err != nil { return "", err } @@ -367,7 +376,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) { if err := in.waitForPostgres(ctx); err != nil { return "", err } - in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)") + in.done("PostgreSQL sized for a machine it shares, with %d MB of memory (shared_buffers %s, work_mem %s, 40 connections)", + total>>20, mem["shared_buffers"], mem["work_mem"]) } in.setupPgvector(ctx, discovery.DiscoverHost()) diff --git a/internal/pgha/memory.go b/internal/pgha/memory.go index 4655ff2..f22cc9a 100644 --- a/internal/pgha/memory.go +++ b/internal/pgha/memory.go @@ -113,13 +113,39 @@ echo "$r"`, body) s.emit("memory", true, "%s", what) } } + // Patroni takes a reload in its own time: restarted before it has written + // the new configuration, PostgreSQL comes back on the old one. So each is + // waited on until Patroni says a restart is pending — and a member found + // pending, by a run that stopped before restarting it, is restarted too. + pending := func(c Cluster, name string) bool { + for _, m := range c.Members { + if m.Name == name { + return m.PendingRestart + } + } + return false + } + var c Cluster + err := s.waitFor(ctx, s.dbHosts(), func(got Cluster) bool { + c = got + for _, d := range restart { + if !pending(got, d.Name) { + return false + } + } + return true + }) + if err != nil { + return fmt.Errorf("waiting for Patroni to take the new configuration: %w", err) + } + for _, d := range s.DB { + if pending(c, d.Name) && !slices.ContainsFunc(restart, func(n Node) bool { return n.Name == d.Name }) { + restart = append(restart, d) + } + } if len(restart) == 0 { return nil } - c, err := s.Status(ctx, s.dbHosts()) - if err != nil { - return err - } // The leader last: its restart is the only one the control planes see, // a few seconds without writes, and by then the rest are back. slices.SortStableFunc(restart, func(a, b Node) int { @@ -133,7 +159,7 @@ echo "$r"`, body) return 0 }) for _, d := range restart { - if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{}`), sec.RestAPI, 2*time.Minute); err != nil { + if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{"restart_pending": true}`), sec.RestAPI, 2*time.Minute); err != nil { return fmt.Errorf("restarting PostgreSQL on %s for its new shared_buffers: %w", d.Name, err) } if err := s.waitFor(ctx, s.dbHosts(), func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil { diff --git a/internal/pgha/pgha_test.go b/internal/pgha/pgha_test.go index dc7eb79..e63cfef 100644 --- a/internal/pgha/pgha_test.go +++ b/internal/pgha/pgha_test.go @@ -465,3 +465,97 @@ func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *tes t.Errorf("TLS asked of a cluster without it: %s", dsn) } } + +func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) { + total, ok := MemTotal("MemTotal: 4009848 kB\nMemFree: 976000 kB\n") + if !ok || total != 4009848<<10 { + t.Fatalf("MemTotal %d %v", total, ok) + } + own := MemorySettings(total, false) + if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" || + own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" { + t.Errorf("a machine of its own: %v", own) + } + // Beside a control plane, less of it. + if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" { + t.Errorf("a shared machine: %v", sh) + } + // Never below what PostgreSQL needs, never past what it can use. + if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" { + t.Errorf("a small machine: %v", tiny) + } + if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" { + t.Errorf("a large machine: %v", big) + } +} + +func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) { + var mu sync.Mutex + var restarted []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/restart" { + if _, pass, _ := r.BasicAuth(); pass != "pw" { + w.WriteHeader(http.StatusUnauthorized) + return + } + host, _, _ := strings.Cut(r.Host, ":") + mu.Lock() + restarted = append(restarted, host) + mu.Unlock() + } + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"} + leader := &machine{name: "a", answers: answers} + replica := &machine{name: "b", answers: answers} + up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true}, + {Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}} + s := Setup{ + DB: []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}}, + Status: func(context.Context, []string) (Cluster, error) { return up, nil }, + Wait: time.Second, + } + if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil { + t.Fatal(err) + } + if strings.Join(restarted, " ") != "localhost 127.0.0.1" { + t.Errorf("restarted %v; the replica first, then the leader", restarted) + } +} + +func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) { + // A run that stopped after writing the configuration, or restarted before + // Patroni had taken it: nothing to change, but PostgreSQL still on the old. + var restarted []string + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/restart" { + host, _, _ := strings.Cut(r.Host, ":") + restarted = append(restarted, host) + } + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"} + up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"}, + {Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}} + s := Setup{ + DB: []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}}, + Status: func(context.Context, []string) (Cluster, error) { return up, nil }, + Wait: time.Second, + } + if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil { + t.Fatal(err) + } + if strings.Join(restarted, " ") != "localhost" { + t.Errorf("restarted %v; only the member pending", restarted) + } +} diff --git a/internal/pgha/status.go b/internal/pgha/status.go index e217ecc..c363687 100644 --- a/internal/pgha/status.go +++ b/internal/pgha/status.go @@ -19,6 +19,9 @@ type MemberState struct { State string `json:"state"` Host string `json:"host"` Timeline int `json:"timeline"` + // PendingRestart is a parameter changed that PostgreSQL takes only on + // restart. + PendingRestart bool `json:"pending_restart"` // Lag is how far behind the leader a replica is, in bytes; -1 when // Patroni does not know, which is itself worth an alert. Lag int64 `json:"-"`