The store's PostgreSQL sized to its machine's memory

Each Patroni member gets a share of its own RAM in its own config.yml:
shared_buffers a quarter (15% beside a control plane or agent),
effective_cache_size, maintenance_work_mem and work_mem scaled to it, all
clamped. It is read again on every server init --patroni, so a machine
given more memory gets the use of it. A single-machine PostgreSQL init
installs is sized the same way, as a shared machine, instead of a fixed
96MB.

The step itself went in with 85025df by accident, in an early form that
restarted PostgreSQL before Patroni had taken the reload, leaving the old
shared_buffers with a restart pending. It now waits for Patroni to report
the restart pending, restarts with restart_pending (replicas first, the
leader last), and restarts a member it finds pending from an earlier run.

Applied to the PoC: pg01 978MB, ctl02 (control plane beside it) 587MB,
both from 128MB; streaming, lag 0, gateways unaffected.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 21:48:53 +02:00
1 parent 85025dfb03
commit 42d3b7ea22
6 files changed
+178 -18

No files matched your search

+20 -10
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -348,15 +349,23 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
}
if installed {
// Its defaults assume it owns the machine. A gateway swapped out to
// give the database cache is worse than a smaller cache, and a store
// whose working set is megabytes needs little. Only a PostgreSQL init
// installed itself: one that was already here may serve something
// else that needs its connections.
for _, q := range []string{
"ALTER SYSTEM SET shared_buffers = '96MB'",
"ALTER SYSTEM SET max_connections = 40",
} {
// Sized to the machine's memory, as one it shares with the gateway:
// a gateway swapped out to give the database cache is worse than a
// smaller cache, but a knowledge base's vector index is searched
// well only from memory. Only a PostgreSQL init installed itself:
// one that was already here may serve something else that needs its
// connections.
meminfo, _ := os.ReadFile("/proc/meminfo")
total, ok := pgha.MemTotal(string(meminfo))
if !ok {
total = 1 << 30
}
mem := pgha.MemorySettings(total, true)
qs := []string{"ALTER SYSTEM SET max_connections = 40"}
for _, k := range []string{"shared_buffers", "effective_cache_size", "maintenance_work_mem", "work_mem"} {
qs = append(qs, "ALTER SYSTEM SET "+k+" = '"+mem[k]+"'")
}
for _, q := range qs {
if _, err := in.psql(ctx, q); err != nil {
return "", err
}
@@ -367,7 +376,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
if err := in.waitForPostgres(ctx); err != nil {
return "", err
}
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
in.done("PostgreSQL sized for a machine it shares, with %d MB of memory (shared_buffers %s, work_mem %s, 40 connections)",
total>>20, mem["shared_buffers"], mem["work_mem"])
}
in.setupPgvector(ctx, discovery.DiscoverHost())
+31 -5
View File
@@ -113,13 +113,39 @@ echo "$r"`, body)
s.emit("memory", true, "%s", what)
}
}
// Patroni takes a reload in its own time: restarted before it has written
// the new configuration, PostgreSQL comes back on the old one. So each is
// waited on until Patroni says a restart is pending — and a member found
// pending, by a run that stopped before restarting it, is restarted too.
pending := func(c Cluster, name string) bool {
for _, m := range c.Members {
if m.Name == name {
return m.PendingRestart
}
}
return false
}
var c Cluster
err := s.waitFor(ctx, s.dbHosts(), func(got Cluster) bool {
c = got
for _, d := range restart {
if !pending(got, d.Name) {
return false
}
}
return true
})
if err != nil {
return fmt.Errorf("waiting for Patroni to take the new configuration: %w", err)
}
for _, d := range s.DB {
if pending(c, d.Name) && !slices.ContainsFunc(restart, func(n Node) bool { return n.Name == d.Name }) {
restart = append(restart, d)
}
}
if len(restart) == 0 {
return nil
}
c, err := s.Status(ctx, s.dbHosts())
if err != nil {
return err
}
// The leader last: its restart is the only one the control planes see,
// a few seconds without writes, and by then the rest are back.
slices.SortStableFunc(restart, func(a, b Node) int {
@@ -133,7 +159,7 @@ echo "$r"`, body)
return 0
})
for _, d := range restart {
if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{}`), sec.RestAPI, 2*time.Minute); err != nil {
if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{"restart_pending": true}`), sec.RestAPI, 2*time.Minute); err != nil {
return fmt.Errorf("restarting PostgreSQL on %s for its new shared_buffers: %w", d.Name, err)
}
if err := s.waitFor(ctx, s.dbHosts(), func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil {
+94
View File
@@ -465,3 +465,97 @@ func TestAControlPlaneWithoutTheAuthorityConnectsByTLSOnceTheClusterHasIt(t *tes
t.Errorf("TLS asked of a cluster without it: %s", dsn)
}
}
func TestPostgreSQLIsGivenAShareOfTheMachinesMemory(t *testing.T) {
total, ok := MemTotal("MemTotal: 4009848 kB\nMemFree: 976000 kB\n")
if !ok || total != 4009848<<10 {
t.Fatalf("MemTotal %d %v", total, ok)
}
own := MemorySettings(total, false)
if own["shared_buffers"] != "978MB" || own["effective_cache_size"] != "2936MB" ||
own["maintenance_work_mem"] != "244MB" || own["work_mem"] != "15MB" {
t.Errorf("a machine of its own: %v", own)
}
// Beside a control plane, less of it.
if sh := MemorySettings(total, true); sh["shared_buffers"] != "587MB" || sh["work_mem"] != "7MB" {
t.Errorf("a shared machine: %v", sh)
}
// Never below what PostgreSQL needs, never past what it can use.
if tiny := MemorySettings(512<<20, true); tiny["shared_buffers"] != "96MB" || tiny["work_mem"] != "4MB" {
t.Errorf("a small machine: %v", tiny)
}
if big := MemorySettings(1<<40, false); big["shared_buffers"] != "32768MB" || big["maintenance_work_mem"] != "2048MB" {
t.Errorf("a large machine: %v", big)
}
}
func TestNewSharedBuffersRestartTheReplicasBeforeTheLeader(t *testing.T) {
var mu sync.Mutex
var restarted []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == "/restart" {
if _, pass, _ := r.BasicAuth(); pass != "pw" {
w.WriteHeader(http.StatusUnauthorized)
return
}
host, _, _ := strings.Cut(r.Host, ":")
mu.Lock()
restarted = append(restarted, host)
mu.Unlock()
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "restart\n"}
leader := &machine{name: "a", answers: answers}
replica := &machine{name: "b", answers: answers}
up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running", PendingRestart: true},
{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
s := Setup{
DB: []Node{{Member{"a", "127.0.0.1"}, leader}, {Member{"b", "localhost"}, replica}},
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
Wait: time.Second,
}
if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
t.Fatal(err)
}
if strings.Join(restarted, " ") != "localhost 127.0.0.1" {
t.Errorf("restarted %v; the replica first, then the leader", restarted)
}
}
func TestAMemberLeftPendingARestartIsRestartedOnTheNextRun(t *testing.T) {
// A run that stopped after writing the configuration, or restarted before
// Patroni had taken it: nothing to change, but PostgreSQL still on the old.
var restarted []string
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/restart" {
host, _, _ := strings.Cut(r.Host, ":")
restarted = append(restarted, host)
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
answers := map[string]string{"cat /proc/meminfo": "MemTotal: 4009848 kB\n", "WANT": "same\n"}
up := Cluster{Members: []MemberState{{Name: "a", Role: "leader", State: "running"},
{Name: "b", Role: "replica", State: "streaming", PendingRestart: true}}}
s := Setup{
DB: []Node{{Member{"a", "127.0.0.1"}, &machine{answers: answers}}, {Member{"b", "localhost"}, &machine{answers: answers}}},
Status: func(context.Context, []string) (Cluster, error) { return up, nil },
Wait: time.Second,
}
if err := s.memory(context.Background(), Secrets{RestAPI: "pw"}); err != nil {
t.Fatal(err)
}
if strings.Join(restarted, " ") != "localhost" {
t.Errorf("restarted %v; only the member pending", restarted)
}
}
+3
View File
@@ -19,6 +19,9 @@ type MemberState struct {
State string `json:"state"`
Host string `json:"host"`
Timeline int `json:"timeline"`
// PendingRestart is a parameter changed that PostgreSQL takes only on
// restart.
PendingRestart bool `json:"pending_restart"`
// Lag is how far behind the leader a replica is, in bytes; -1 when
// Patroni does not know, which is itself worth an alert.
Lag int64 `json:"-"`