Files
llmgw/internal/pgha/memory.go
T
Ilker ManapandClaude Opus 5.5 85025dfb03 Trained adapters published, private to their owners like the knowledge bases they come from
An adapter now has an owner (org, team, project, user or key): only callers
there see it, and to anyone else it is a model that does not exist, absent
from /v1/models too. A training job's owner defaults to the one collection
its pairs come from, must be able to read every collection they come from,
and pairs from a knowledge base are never made public. ctl train publish
keeps the GGUF or PEFT files on the mirror, which accepts uploads now, and
sets the owned adapter after the usual architecture check.

The knowledge bases' cache_salt follows the collections searched rather
than the organization, so two teams of one organization no longer share
cached passages.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-08 21:30:50 +02:00

146 lines
5.4 KiB
Go

package pgha
import (
"context"
"encoding/json"
"fmt"
"net/http"
"slices"
"strconv"
"strings"
"time"
)
// PostgreSQL's own defaults — 128MB of cache, 4MB to sort in — are for a
// machine it barely uses. A knowledge base's vector index is searched well
// only from memory: one that does not fit is read from disk on every search.
// So each member is sized to the memory it has, read when setup runs, and
// again on every later run, so a machine given more memory gets the use of it.
// MemorySettings are PostgreSQL's memory parameters for a machine with total
// bytes of memory. shared is a machine that also runs a control plane or an
// agent: the database is given less of it, since a gateway swapped out to
// give the database cache is worse than a smaller cache.
func MemorySettings(total int64, shared bool) map[string]string {
const mb = 1 << 20
ram := total / mb
// Shares of memory, in MB: the cache PostgreSQL keeps itself, what the
// planner may count on the kernel caching besides, what building or
// vacuuming an index may take, and what one sort may.
buffers, cache, maint, sort := ram/4, ram*3/4, ram/16, ram/256
if shared {
buffers, cache, maint, sort = ram*3/20, ram/2, ram/32, ram/512
}
clamp := func(v, lo, hi int64) int64 { return min(max(v, lo), hi) }
return map[string]string{
"shared_buffers": strconv.FormatInt(clamp(buffers, 96, 32<<10), 10) + "MB",
"effective_cache_size": strconv.FormatInt(clamp(cache, 256, 1<<20), 10) + "MB",
"maintenance_work_mem": strconv.FormatInt(clamp(maint, 64, 2<<10), 10) + "MB",
"work_mem": strconv.FormatInt(clamp(sort, 4, 64), 10) + "MB",
}
}
// MemTotal reads the machine's memory, in bytes, from /proc/meminfo.
func MemTotal(meminfo string) (int64, bool) {
for _, line := range strings.Split(meminfo, "\n") {
f := strings.Fields(line)
if len(f) >= 2 && f[0] == "MemTotal:" {
kb, err := strconv.ParseInt(f[1], 10, 64)
return kb << 10, err == nil && kb > 0
}
}
return 0, false
}
// memory sizes each member to its machine, in its own configuration — the
// machines need not be alike. What takes effect on reload is applied so;
// shared_buffers needs a restart, given one member at a time, the leader
// last, each waited on before the next.
func (s *Setup) memory(ctx context.Context, sec Secrets) error {
var restart []Node
for _, d := range s.DB {
out, err := d.Remote.Run(ctx, `cat /proc/meminfo
if [ -f /etc/llmgw/server.yaml ] || [ -f /etc/llmgw/agent.yaml ]; then echo "llmgw-shared"; fi`, nil)
total, ok := MemTotal(string(out))
if err != nil || !ok {
s.emit("memory", false, "%s: could not read its memory; PostgreSQL left as it is", d.Name)
continue
}
shared := strings.Contains(string(out), "llmgw-shared")
params := MemorySettings(total, shared)
body, _ := json.Marshal(params)
out, err = d.Remote.Run(ctx, `set -e
test -f /etc/patroni/config.yml || { echo missing; exit 0; }
want=$(cat)
r=$(sudo env WANT="$want" python3 - <<'PY'
import json, os, yaml
p = "/etc/patroni/config.yml"
want = json.loads(os.environ["WANT"])
c = yaml.safe_load(open(p))
pg = c.setdefault("postgresql", {})
params = pg.get("parameters") or {}
pg["parameters"] = params
if all(str(params.get(k)) == v for k, v in want.items()):
print("same")
else:
restart = str(params.get("shared_buffers")) != want["shared_buffers"]
params.update(want)
open(p + ".new", "w").write("# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.\n" +
yaml.safe_dump(c, default_flow_style=False, sort_keys=False))
print("restart" if restart else "changed")
PY
)
if [ "$r" != same ]; then
sudo chown root:postgres /etc/patroni/config.yml.new
sudo chmod 640 /etc/patroni/config.yml.new
sudo mv -f /etc/patroni/config.yml.new /etc/patroni/config.yml
sudo systemctl reload patroni
fi
echo "$r"`, body)
if err != nil {
return fmt.Errorf("on %s: %v: %s", d.Name, err, strings.TrimSpace(string(out)))
}
what := fmt.Sprintf("%s: %d MB of memory%s; shared_buffers %s, effective_cache_size %s, maintenance_work_mem %s, work_mem %s",
d.Name, total>>20, map[bool]string{true: ", shared with llmgw"}[shared],
params["shared_buffers"], params["effective_cache_size"], params["maintenance_work_mem"], params["work_mem"])
switch r := strings.TrimSpace(lastLine(out)); r {
case "same", "missing":
s.emit("memory", false, "%s", what)
case "restart":
s.emit("memory", true, "%s", what)
restart = append(restart, d)
default:
s.emit("memory", true, "%s", what)
}
}
if len(restart) == 0 {
return nil
}
c, err := s.Status(ctx, s.dbHosts())
if err != nil {
return err
}
// The leader last: its restart is the only one the control planes see,
// a few seconds without writes, and by then the rest are back.
slices.SortStableFunc(restart, func(a, b Node) int {
al, bl := a.Name == c.Leader(), b.Name == c.Leader()
switch {
case al && !bl:
return 1
case bl && !al:
return -1
}
return 0
})
for _, d := range restart {
if _, err := restJSON(ctx, d.IP, http.MethodPost, "/restart", []byte(`{}`), sec.RestAPI, 2*time.Minute); err != nil {
return fmt.Errorf("restarting PostgreSQL on %s for its new shared_buffers: %w", d.Name, err)
}
if err := s.waitFor(ctx, s.dbHosts(), func(c Cluster) bool { return c.Complete(len(s.DB)) }); err != nil {
return fmt.Errorf("after restarting %s: %w", d.Name, err)
}
s.emit("memory", true, "%s: PostgreSQL restarted with the new shared_buffers", d.Name)
}
return nil
}