Describe v0.8.8

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitea/workflows/release.yml

@@ -55,6 +55,13 @@ jobs:
             (cd dist && sha256sum "${name}" >> checksums.txt)
           done
 
+          # The reverse proxy configurations, so a release page carries
+          # everything install.md asks for and the repository is not needed.
+          for f in deploy/proxy/Caddyfile deploy/proxy/nginx.conf; do
+            cp "$f" dist/
+            (cd dist && sha256sum "$(basename "$f")" >> checksums.txt)
+          done
+
           # An NVML build for nodes with NVLink, which the portable binary
           # cannot serve.
           #

CHANGELOG.md

@@ -18,6 +18,67 @@ is only released once its **exit criterion** is met, not when its code lands.
 Until v1.0.0 the minor version is the breaking one: configuration formats and Go
 APIs change between minors without a deprecation period.
 
+## v0.8.8 — 2026-09-28
+
+### Fixes
+
+- **`server init --postgres` failed on Rocky Linux** with "postgresql-setup:
+  executable file not found". The PostgreSQL project's own packages (PGDG)
+  provide `postgresql-server` too, so dnf can answer that name with
+  `postgresql16-server`, which names its unit `postgresql-16.service` and keeps
+  its tools in `/usr/pgsql-16/bin`. init now finds the server unit systemd has,
+  reads the data directory from it, and runs `initdb` itself when the directory
+  is empty. A PostgreSQL that was already installed is used as it is.
+
+### Known gaps
+
+Not yet run on a real machine. Three arrangements are not handled: a
+PostgreSQL initialised with `ident` for TCP logins (the connection check fails
+and says so), one listening on a port other than 5432, and several PGDG
+versions where the highest is not the one on 5432.
+
+## v0.8.7 — 2026-09-28
+
+### PostgreSQL in one command
+
+```
+sudo llmgw server init --postgres
+```
+
+Installs PostgreSQL with the machine's own package manager when there is none,
+creates the `llmgw` role and database with a generated password kept in
+`/etc/llmgw/secrets/store-dsn`, and checks that it connects. On a control plane
+already running on SQLite it stops the gateway, copies the store across so
+every account and key survives, points `auth.store` at PostgreSQL and starts
+the gateway again; if the move fails, the gateway is started again on SQLite.
+Given on the first run, there is never a SQLite store.
+
+`server.yaml` is edited in place with its comments intact, and the previous
+file is kept beside it. A PostgreSQL that init did not install is left sized as
+it was, since something else may depend on it.
+
+The PostgreSQL installation itself has not yet run on a real machine; the move,
+the configuration edit and the rerun behaviour are tested.
+
+### Fixes
+
+- **A new control plane with no models did not start.** A configuration with
+  no static endpoints and no models was refused as having nothing to serve, so
+  under systemd the unit restarted every two seconds and nothing listened on
+  8080. It now runs as a control plane with an empty catalog.
+
+## v0.8.6 — 2026-09-28
+
+### Reverse proxy configurations
+
+`deploy/proxy/` has a Caddyfile and an nginx configuration for the client
+port, and both are attached to every release beside the binaries. They do not
+buffer streamed replies, their read timeout outlasts a model's wait before its
+first token, their body limit matches the router's, and they set
+`X-Forwarded-Proto` and `X-Forwarded-Host` themselves — llmgw marks session
+cookies `Secure` from the first and shows the second as the base URL on its
+landing page.
+
 ## v0.8.5 — 2026-09-28
 
 ### A control plane in one command

README.md

@@ -66,6 +66,8 @@ on a machine that has them.
   does not exist or miss one that does
 - [deploy/mirror/](deploy/mirror/) — a model mirror, which is the cheapest
   and most useful thing to build around llmgw
+- [deploy/proxy/](deploy/proxy/) — Caddy and nginx in front of the client
+  port, with streaming and the forwarded headers right
 - [deploy/ha/](deploy/ha/) — running two control planes, and the one health
   check that has to be right
 - [docs/node-onboarding.md](docs/node-onboarding.md),

deploy/proxy/Caddyfile

@@ -0,0 +1,41 @@
+# llmgw behind Caddy: the client-facing port (8080) on a public name, with a
+# certificate Caddy obtains and renews by itself.
+#
+# Replace llm.example.com, point its DNS at this machine, open 80 and 443, and
+# run: caddy run --config /etc/caddy/Caddyfile
+#
+# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's
+# own authority and must reach llmgw directly; 9090 and 9091 stay private.
+
+llm.example.com {
+	# The router reads at most 32 MiB of body (router.max_body_bytes). Refusing
+	# more here gives the client a 413 before the upload, not after it.
+	request_body {
+		max_size 32MB
+	}
+
+	reverse_proxy 127.0.0.1:8080 {
+		# Tokens are written the moment they are generated. Without this a
+		# streamed reply can arrive in bursts, and a client measuring time to
+		# first token measures the proxy instead.
+		flush_interval -1
+
+		# Caddy sets X-Forwarded-Proto and X-Forwarded-Host itself and, with
+		# no trusted_proxies, discards whatever the client sent. llmgw relies
+		# on that: the proto decides whether session cookies are Secure, and
+		# the host is the base URL the landing page tells people to use.
+		# Behind a load balancer of your own, name it in trusted_proxies.
+
+		transport http {
+			# A long generation can be silent for a while before the first
+			# token. llmgw has its own limits (timeouts.*); this must not be
+			# the shorter one.
+			response_header_timeout 15m
+		}
+	}
+
+	# No `encode` on purpose: compressing a stream buffers it.
+
+	# Access log to stderr, which under systemd is the journal.
+	log
+}

deploy/proxy/README.md

@@ -0,0 +1,36 @@
+# A reverse proxy in front of llmgw
+
+The client-facing port (8080) is plain HTTP. On anything but a trusted network,
+put it behind a proxy that holds a publicly trusted certificate.
+
+| File | For |
+| --- | --- |
+| [Caddyfile](Caddyfile) | Caddy — obtains and renews its own certificate; the least work |
+| [nginx.conf](nginx.conf) | nginx, with a certificate from certbot |
+
+Both are also attached to every release.
+
+Four things both files get right, and any other proxy has to as well:
+
+- **Streaming is not buffered.** Replies are server-sent events; a buffering
+  proxy delivers them in one burst at the end.
+- **The read timeout is longer than a model's silence.** nginx's default of 60
+  seconds turns a slow first token into a 504.
+- **`X-Forwarded-Proto` and `X-Forwarded-Host` are set by the proxy**, replacing
+  whatever the client sent. llmgw uses the first to mark session cookies
+  `Secure` and the second for the base URL on its landing page.
+- **The body limit matches llmgw's** (32 MiB, `router.max_body_bytes`), so an
+  oversized request is refused before it is uploaded.
+
+Only port 8080 is proxied. Agents reach 7443 directly — it is mutual TLS with
+the cluster's own authority, and a proxy terminating it would take away the
+node identity it carries. 9090 (admin, pprof) stays on loopback; 9091
+(metrics) goes to your monitoring host only.
+
+With a proxy on the same machine, bind llmgw to loopback so nothing reaches it
+around the proxy:
+
+```yaml
+# /etc/llmgw/server.yaml
+listen: "127.0.0.1:8080"
+```

deploy/proxy/nginx.conf

@@ -0,0 +1,63 @@
+# llmgw behind nginx: the client-facing port (8080) on a public name.
+#
+# Put this in /etc/nginx/sites-available/llmgw (or conf.d/llmgw.conf), replace
+# llm.example.com, and get a certificate — certbot --nginx -d llm.example.com
+# fills in the ssl_ lines below. Then: nginx -t && systemctl reload nginx
+#
+# Only 8080 goes through here. 7443 (agents) is mutual TLS with the cluster's
+# own authority and must reach llmgw directly; 9090 and 9091 stay private.
+
+upstream llmgw {
+    server 127.0.0.1:8080;
+    keepalive 32;
+}
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name llm.example.com;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    # `listen ... http2` rather than `http2 on;`: the latter needs nginx
+    # 1.25.1, and Ubuntu 22.04 and Debian 12 ship older ones. Newer nginx
+    # accepts this form with a deprecation warning.
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+    server_name llm.example.com;
+
+    ssl_certificate     /etc/letsencrypt/live/llm.example.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/llm.example.com/privkey.pem;
+
+    # The router reads at most 32 MiB of body (router.max_body_bytes).
+    client_max_body_size 32m;
+
+    location / {
+        proxy_pass http://llmgw;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+
+        # Set, never appended: llmgw decides from X-Forwarded-Proto whether
+        # session cookies are Secure, and builds the base URL the landing page
+        # shows from X-Forwarded-Host. A value passed through from the client
+        # would be the client's choice.
+        proxy_set_header Host              $host;
+        proxy_set_header X-Forwarded-Host  $host;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+
+        # Streamed replies: tokens go out as they are generated. Buffering
+        # turns a stream into one late burst, and gzip buffers too.
+        proxy_buffering off;
+        proxy_cache off;
+        gzip off;
+
+        # nginx's default is 60 seconds between two reads from upstream. A
+        # large model can think longer than that before its first token, and
+        # the client then sees a 504 for a request that was about to succeed.
+        # llmgw has its own limits (timeouts.*); this must not be the shorter.
+        proxy_read_timeout 15m;
+        proxy_send_timeout 15m;
+    }
+}

docs/install.md

@@ -105,6 +105,10 @@ That is the whole step. In order, it:
    once**
 6. installs the systemd unit, starts it, and waits until `/health` answers
 
+Add `--postgres` to keep accounts and keys in PostgreSQL rather than SQLite:
+init installs it and sets it up, and on a machine already running on SQLite it
+moves everything across. See [PostgreSQL](#postgresql-when-sqlite-is-no-longer-the-right-shape).
+
 Every step looks before it acts. Running it again finishes an install that
 stopped halfway and changes nothing on one that did not; it never overwrites
 `server.yaml`, the authority, or an existing account's password.
@@ -128,8 +132,10 @@ If this machine is exposed, move the directory elsewhere once the nodes are in
 and pass `--pki` to onboarding.
 
 This covers the fleet-facing port only. The client-facing API is public and
-belongs behind an ordinary publicly trusted certificate, usually terminated by a
-reverse proxy in front of port 8080.
+belongs behind an ordinary publicly trusted certificate, terminated by a
+reverse proxy in front of port 8080. [deploy/proxy/](../deploy/proxy/) has a
+Caddyfile and an nginx configuration that get streaming, timeouts and the
+forwarded headers right; both are attached to every release.
 
 ### Check it
 
@@ -741,6 +747,20 @@ turn, which is the first per-request load this store has had; and SQLite takes
 one writer at a time, so those turns serialise. And a second control plane is
 impossible while the store is a file on one machine's disk.
 
+On the control plane, one command does all of it — installs PostgreSQL if the
+machine has none, creates its role and database with a generated password kept
+in `/etc/llmgw/secrets/store-dsn`, stops the gateway, copies the SQLite store
+into it so every account and key survives, points `server.yaml` at it, and
+starts the gateway again:
+
+```
+sudo llmgw server init --postgres
+```
+
+It can be given on the first run too, and then there is never a SQLite store.
+The rest of this section is what that command does, for a PostgreSQL somewhere
+else.
+
 The setting is the same one, and the address decides:
 
 ```yaml

internal/agent/provision/install.go

@@ -290,6 +290,14 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
 	return nil
 }
 
+// InstallPackages installs whichever of these packages the machine lacks,
+// from its own package sources, waiting out another package manager's lock.
+// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
+func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
+	var st Status
+	return in.packages(ctx, byFamily, &st)
+}
+
 // lockWait bounds how long a package command waits for another to finish.
 const lockWait = 10 * time.Minute
 

internal/cli/serverinit.go

@@ -66,6 +66,13 @@ type serverInit struct {
 	user  string
 	hosts []string
 	admin string
+	// postgres keeps the store in a PostgreSQL on this machine, installed if
+	// need be, moving whatever SQLite already holds.
+	postgres bool
+	// pgSetup prepares PostgreSQL and returns the store address; a seam for
+	// tests. Nil does it for real.
+	pgSetup func(context.Context) (string, error)
+	pg      pgServer
 
 	out io.Writer
 
@@ -88,6 +95,10 @@ func newServerInitCmd() *cobra.Command {
 			"control plane's certificate for every address this machine has, a\n" +
 			"configuration, an administrator account, and the systemd unit — then\n" +
 			"starts it and waits until it answers.\n\n" +
+			"With --postgres the store is a PostgreSQL database on this machine:\n" +
+			"installed if there is none, with its own role and a generated password\n" +
+			"kept in /etc/llmgw/secrets. An existing SQLite store is copied into it,\n" +
+			"so accounts and keys survive, and server.yaml is pointed at it.\n\n" +
 			"Each step is skipped when its result already exists, so it is safe to run\n" +
 			"again: to finish an install that stopped, or with another --host to\n" +
 			"reissue the certificate. It never overwrites server.yaml or the authority.\n\n" +
@@ -110,6 +121,8 @@ func newServerInitCmd() *cobra.Command {
 		"an address agents dial besides the ones found on this machine; repeatable")
 	f.StringVar(&in.admin, "admin", "", "email of the first administrator; a password is generated and printed once")
 	f.StringVar(&in.pkiDir, "pki", in.pkiDir, "directory holding the certificate authority")
+	f.BoolVar(&in.postgres, "postgres", false,
+		"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
 	return cmd
 }
 
@@ -122,6 +135,7 @@ func (in *serverInit) run(ctx context.Context) error {
 		{"directories", in.ensureDirs},
 		{"certificates", in.ensureCerts},
 		{"configuration", in.ensureConfig},
+		{"database", in.ensureDatabase},
 		{"store", in.ensureStore},
 	}
 	for _, s := range steps {
@@ -351,7 +365,7 @@ func (in *serverInit) ensureConfig(context.Context) error {
 			contact = "platform@example.com"
 		}
 		body := fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
-			filepath.Join(in.stateDir, "llmgw.db"), contact)
+			in.storeAddress(), contact)
 		if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
 			return err
 		}
@@ -393,7 +407,7 @@ control:
 
 auth:
   enabled: true
-  store: %s
+  store: "%s"   # a SQLite path, or @file holding a PostgreSQL address
   signup_open: true
   # Applied to any key no scope gives a limit to. Without them, a project
   # created without limits is metered by nothing at all.

internal/cli/serverinit_postgres.go

@@ -0,0 +1,525 @@
+package cli
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// pgRole and pgDatabase are what init creates in a PostgreSQL it manages.
+const (
+	pgRole     = "llmgw"
+	pgDatabase = "llmgw"
+)
+
+// dsnPath is where the PostgreSQL address is kept, out of server.yaml, so the
+// password in it is not in a file that gets copied, diffed and pasted.
+func (in *serverInit) dsnPath() string { return filepath.Join(in.etcDir, "secrets", "store-dsn") }
+
+// storeAddress is what auth.store says for the store init manages.
+func (in *serverInit) storeAddress() string {
+	if in.postgres {
+		return "@" + in.dsnPath()
+	}
+	return filepath.Join(in.stateDir, "llmgw.db")
+}
+
+// ensureDatabase puts the store in PostgreSQL when asked to, moving whatever
+// the SQLite store already holds.
+//
+// The move is the point. Keys are stored hashed, so starting PostgreSQL empty
+// means reissuing every key and being unable to say which was which; and the
+// administrator init created a minute ago would be gone too.
+func (in *serverInit) ensureDatabase(ctx context.Context) (err error) {
+	if !in.postgres {
+		return nil
+	}
+	setup := in.pgSetup
+	if setup == nil {
+		setup = in.setupPostgres
+	}
+	target, err := setup(ctx)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(in.configPath())
+	if err != nil {
+		return err
+	}
+	current := cfg.Auth.Store
+	if current == target {
+		in.kept("store in PostgreSQL (%s)", in.dsnPath())
+		return nil
+	}
+	if isPostgresAddress(current) {
+		// Two PostgreSQL stores, and a guess about which is meant would lose
+		// one of them.
+		return fmt.Errorf("server.yaml already names a PostgreSQL store (%s); "+
+			"init moves SQLite into PostgreSQL, not one PostgreSQL into another", current)
+	}
+
+	// Nothing may write the store while it is copied: a row landing after its
+	// table has been passed would be absent, and nothing would say so.
+	stopped, err := in.stopService(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		// A move that failed leaves the configuration naming SQLite, which
+		// still holds everything: start the service on it again rather than
+		// leave the gateway down over a database that did not work out.
+		if err != nil && stopped {
+			_ = exec.Command("systemctl", "start", "llmgw-server").Run()
+			fmt.Fprintf(in.out, "  ..    llmgw-server started again on its SQLite store\n")
+		}
+	}()
+
+	if current != "" {
+		if _, err := os.Stat(current); err == nil {
+			if err := in.copyStore(ctx, current, target); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := setAuthStore(in.configPath(), target); err != nil {
+		return err
+	}
+	in.done("server.yaml now keeps the store in PostgreSQL")
+	return nil
+}
+
+func (in *serverInit) copyStore(ctx context.Context, from, to string) error {
+	src, err := store.Open(ctx, from)
+	if err != nil {
+		return fmt.Errorf("opening %s: %w", from, err)
+	}
+	defer src.Close()
+	dst, err := store.Open(ctx, to)
+	if err != nil {
+		return fmt.Errorf("opening PostgreSQL: %w", err)
+	}
+	defer dst.Close()
+
+	empty, err := dst.Empty(ctx)
+	if err != nil {
+		return err
+	}
+	if !empty {
+		// A previous run that copied and then stopped before changing the
+		// configuration. The service was stopped for that copy too, so what
+		// is there is what SQLite held.
+		in.kept("PostgreSQL already holds a store; %s is left as it is", from)
+		return nil
+	}
+	res, err := src.CopyTo(ctx, dst)
+	if err != nil {
+		return fmt.Errorf("copying %s into PostgreSQL: %w", from, err)
+	}
+	var rows int64
+	for _, r := range res {
+		rows += r.Rows
+	}
+	in.done("copied %d rows from %s into PostgreSQL (the SQLite file is kept)", rows, from)
+	return nil
+}
+
+func isPostgresAddress(a string) bool {
+	return strings.HasPrefix(a, "@") || strings.HasPrefix(a, "postgres://") ||
+		strings.HasPrefix(a, "postgresql://")
+}
+
+// setAuthStore changes auth.store in server.yaml and nothing else.
+//
+// Edited as text rather than unmarshalled and written back, because writing
+// it back would drop every comment in a file people annotate. The old file is
+// kept beside it.
+func setAuthStore(path, address string) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(string(b), "\n")
+	value := fmt.Sprintf("%q", address)
+
+	authAt, storeAt, indent := -1, -1, "  "
+	for i, l := range lines {
+		trimmed := strings.TrimSpace(l)
+		top := l != "" && l[0] != ' ' && l[0] != '\t' && !strings.HasPrefix(trimmed, "#")
+		if top {
+			if authAt >= 0 && i > authAt {
+				break // past the auth block
+			}
+			if strings.HasPrefix(trimmed, "auth:") {
+				authAt = i
+			}
+			continue
+		}
+		if authAt >= 0 && strings.HasPrefix(trimmed, "store:") {
+			storeAt = i
+			indent = l[:len(l)-len(strings.TrimLeft(l, " \t"))]
+			break
+		}
+	}
+	switch {
+	case storeAt >= 0:
+		lines[storeAt] = indent + "store: " + value
+	case authAt >= 0:
+		lines = append(lines[:authAt+1], append([]string{indent + "store: " + value}, lines[authAt+1:]...)...)
+	default:
+		lines = append(lines, "auth:", "  enabled: true", "  store: "+value)
+	}
+
+	backup := fmt.Sprintf("%s.bak-%s", path, time.Now().Format("20060102-150405"))
+	if err := os.WriteFile(backup, b, 0o640); err != nil {
+		return err
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, []byte(strings.Join(lines, "\n")), info.Mode().Perm())
+}
+
+// pgServer is the PostgreSQL installation on this machine, as systemd and the
+// file system describe it.
+//
+// Found rather than assumed. A distribution's own package calls its unit
+// postgresql.service and puts initdb and psql on PATH; the PostgreSQL
+// project's packages (PGDG) call it postgresql-16.service and keep them in
+// /usr/pgsql-16/bin — and they provide "postgresql-server" too, so asking dnf
+// for that name can install or find one of them. Assuming the first found a
+// Rocky machine with the second unable to initialise the database it had.
+type pgServer struct {
+	unit   string // postgresql.service, postgresql-16.service
+	pgdata string // "" when the unit does not say, which is Debian's case
+	bin    string // directory holding initdb and psql; "" means PATH
+}
+
+func (p pgServer) tool(name string) string {
+	if p.bin != "" {
+		return filepath.Join(p.bin, name)
+	}
+	return name
+}
+
+// pickPostgresUnit chooses the server unit from systemctl's list of
+// postgresql* unit files: the plain one when there is one, else the highest
+// version. Templates (postgresql@.service) are not a server by themselves.
+func pickPostgresUnit(list string) string {
+	best, bestVersion := "", -1
+	for _, line := range strings.Split(list, "\n") {
+		f := strings.Fields(line)
+		if len(f) == 0 {
+			continue
+		}
+		unit := f[0]
+		if !strings.HasSuffix(unit, ".service") || strings.Contains(unit, "@") {
+			continue
+		}
+		name := strings.TrimSuffix(unit, ".service")
+		if name == "postgresql" {
+			return unit
+		}
+		v := strings.TrimPrefix(name, "postgresql-")
+		if n, err := strconv.Atoi(strings.SplitN(v, ".", 2)[0]); err == nil && n > bestVersion {
+			best, bestVersion = unit, n
+		}
+	}
+	return best
+}
+
+// pgdataFrom reads PGDATA out of `systemctl show -p Environment --value`.
+func pgdataFrom(env string) string {
+	for _, kv := range strings.Fields(env) {
+		if v, ok := strings.CutPrefix(kv, "PGDATA="); ok {
+			return v
+		}
+	}
+	return ""
+}
+
+func findPostgres(ctx context.Context) (pgServer, bool) {
+	out, _ := exec.CommandContext(ctx, "systemctl", "list-unit-files", "--type=service",
+		"--no-legend", "--plain", "postgresql*").Output()
+	unit := pickPostgresUnit(string(out))
+	if unit == "" {
+		return pgServer{}, false
+	}
+	p := pgServer{unit: unit}
+	env, _ := exec.CommandContext(ctx, "systemctl", "show", "-p", "Environment", "--value", unit).Output()
+	p.pgdata = pgdataFrom(string(env))
+
+	if v, ok := strings.CutPrefix(strings.TrimSuffix(unit, ".service"), "postgresql-"); ok {
+		if dir := "/usr/pgsql-" + v + "/bin"; isDir(dir) {
+			p.bin = dir
+		}
+		if p.pgdata == "" && isDir("/var/lib/pgsql/"+v) {
+			p.pgdata = "/var/lib/pgsql/" + v + "/data"
+		}
+	}
+	if p.bin == "" {
+		if _, err := exec.LookPath("psql"); err != nil {
+			// Debian keeps the server's tools per version and puts only
+			// wrappers on PATH; without even those, look where it keeps them.
+			if dirs, _ := filepath.Glob("/usr/lib/postgresql/*/bin"); len(dirs) > 0 {
+				p.bin = dirs[len(dirs)-1]
+			}
+		}
+	}
+	return p, true
+}
+
+func isDir(p string) bool {
+	st, err := os.Stat(p)
+	return err == nil && st.IsDir()
+}
+
+// setupPostgres makes sure a PostgreSQL on this machine holds a database for
+// llmgw, installing the server if there is none, and returns the store
+// address.
+func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
+	installed, err := in.installPostgres(ctx)
+	if err != nil {
+		return "", err
+	}
+	if err := in.waitForPostgres(ctx); err != nil {
+		return "", err
+	}
+
+	exists, err := in.psqlBool(ctx, "SELECT 1 FROM pg_roles WHERE rolname = '"+pgRole+"'")
+	if err != nil {
+		return "", err
+	}
+	_, statErr := os.Stat(in.dsnPath())
+	if exists && statErr == nil {
+		in.kept("PostgreSQL role %s", pgRole)
+	} else {
+		// A role with no address written down is one whose password nobody
+		// knows; giving it a new one is the only way to use it.
+		password := make([]byte, 24)
+		_, _ = rand.Read(password)
+		pass := hex.EncodeToString(password)
+		verb := "CREATE ROLE"
+		if exists {
+			verb = "ALTER ROLE"
+		}
+		if _, err := in.psql(ctx, verb+" "+pgRole+" WITH LOGIN PASSWORD '"+pass+"'"); err != nil {
+			return "", err
+		}
+		dsn := fmt.Sprintf("postgres://%s:%s@127.0.0.1:5432/%s?sslmode=disable", pgRole, pass, pgDatabase)
+		if err := in.writeDSN(dsn); err != nil {
+			return "", err
+		}
+		in.done("PostgreSQL role %s, its address in %s", pgRole, in.dsnPath())
+	}
+
+	exists, err = in.psqlBool(ctx, "SELECT 1 FROM pg_database WHERE datname = '"+pgDatabase+"'")
+	if err != nil {
+		return "", err
+	}
+	if exists {
+		in.kept("PostgreSQL database %s", pgDatabase)
+	} else {
+		if _, err := in.psql(ctx, "CREATE DATABASE "+pgDatabase+" OWNER "+pgRole); err != nil {
+			return "", err
+		}
+		in.done("PostgreSQL database %s", pgDatabase)
+	}
+
+	if installed {
+		// Its defaults assume it owns the machine. A gateway swapped out to
+		// give the database cache is worse than a smaller cache, and a store
+		// whose working set is megabytes needs little. Only a PostgreSQL init
+		// installed itself: one that was already here may serve something
+		// else that needs its connections.
+		for _, q := range []string{
+			"ALTER SYSTEM SET shared_buffers = '96MB'",
+			"ALTER SYSTEM SET max_connections = 40",
+		} {
+			if _, err := in.psql(ctx, q); err != nil {
+				return "", err
+			}
+		}
+		if out, err := exec.CommandContext(ctx, "systemctl", "restart", in.pg.unit).CombinedOutput(); err != nil {
+			return "", fmt.Errorf("restarting PostgreSQL: %v: %s", err, strings.TrimSpace(string(out)))
+		}
+		if err := in.waitForPostgres(ctx); err != nil {
+			return "", err
+		}
+		in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
+	}
+
+	// Proof the address works, before anything is moved into it.
+	address := "@" + in.dsnPath()
+	s, err := store.Open(ctx, address)
+	if err != nil {
+		return "", fmt.Errorf("connecting with the address in %s: %w (PostgreSQL's pg_hba.conf "+
+			"must allow password logins from 127.0.0.1: scram-sha-256 or md5, not ident)", in.dsnPath(), err)
+	}
+	_ = s.Close()
+	return address, nil
+}
+
+func (in *serverInit) writeDSN(dsn string) error {
+	dir := filepath.Dir(in.dsnPath())
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return err
+	}
+	if err := in.chown(dir, false); err != nil {
+		return err
+	}
+	if err := os.WriteFile(in.dsnPath(), []byte(dsn+"\n"), 0o600); err != nil {
+		return err
+	}
+	// Read by the service, which runs as llmgw; root reads it anyway.
+	return in.chown(in.dsnPath(), true)
+}
+
+// installPostgres installs the server when this machine has none, makes sure
+// its database is initialised and running, and says whether it installed it.
+func (in *serverInit) installPostgres(ctx context.Context) (bool, error) {
+	host := discovery.DiscoverHost()
+	pg, found := findPostgres(ctx)
+	installed := false
+	if found {
+		in.kept("PostgreSQL server (%s)", pg.unit)
+	} else {
+		if host.Family() == "" {
+			return false, fmt.Errorf("no PostgreSQL here and no package manager init knows (%s); "+
+				"install PostgreSQL and run init again", host.OSID)
+		}
+		fmt.Fprintf(in.out, "  ..    installing PostgreSQL (a minute or two)\n")
+		pi := &provision.Installer{Host: host, Log: slog.Default()}
+		if err := pi.InstallPackages(ctx, map[string][]string{
+			"deb": {"postgresql"},
+			"rpm": {"postgresql-server", "postgresql"},
+		}); err != nil {
+			return false, err
+		}
+		if pg, found = findPostgres(ctx); !found {
+			q := "rpm -qa 'postgresql*'"
+			if host.Family() == "deb" {
+				q = "dpkg -l 'postgresql*'"
+			}
+			have, _ := exec.CommandContext(ctx, "sh", "-c", q).CombinedOutput()
+			return false, fmt.Errorf("PostgreSQL was installed but no postgresql service unit "+
+				"appeared; what is installed:\n%s", strings.TrimSpace(string(have)))
+		}
+		installed = true
+		in.done("PostgreSQL installed (%s)", pg.unit)
+	}
+	in.pg = pg
+
+	// Debian's package creates and starts a cluster itself. The RHEL family's,
+	// and PGDG's, leave the data directory empty until someone runs initdb.
+	if pg.pgdata != "" {
+		if _, err := os.Stat(filepath.Join(pg.pgdata, "PG_VERSION")); err != nil {
+			// Directly rather than through postgresql-setup, which is named
+			// differently by every packager. Password logins over TCP are
+			// asked for: RHEL's default for them is ident, which no service
+			// account passes.
+			if err := os.MkdirAll(pg.pgdata, 0o700); err != nil {
+				return false, err
+			}
+			if out, err := exec.CommandContext(ctx, "chown", "postgres:postgres", pg.pgdata).CombinedOutput(); err != nil {
+				return false, fmt.Errorf("chown %s: %v: %s", pg.pgdata, err, strings.TrimSpace(string(out)))
+			}
+			cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--", pg.tool("initdb"),
+				"-D", pg.pgdata, "--auth-host=scram-sha-256", "--auth-local=peer")
+			cmd.Dir = "/"
+			if out, err := cmd.CombinedOutput(); err != nil {
+				return false, fmt.Errorf("initdb %s: %v: %s", pg.pgdata, err, lastLines(out, 5))
+			}
+			in.done("PostgreSQL database cluster initialised in %s", pg.pgdata)
+		}
+	}
+	if exec.CommandContext(ctx, "systemctl", "is-active", "--quiet", pg.unit).Run() != nil {
+		if out, err := exec.CommandContext(ctx, "systemctl", "enable", "--now", pg.unit).CombinedOutput(); err != nil {
+			return false, fmt.Errorf("starting %s: %v: %s", pg.unit, err, strings.TrimSpace(string(out)))
+		}
+		in.done("%s started", pg.unit)
+	}
+	return installed, nil
+}
+
+func lastLines(b []byte, n int) string {
+	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}
+
+// psql runs one statement as the postgres superuser, over the local socket.
+func (in *serverInit) psql(ctx context.Context, query string) (string, error) {
+	cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
+		in.pg.tool("psql"), "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
+	// postgres cannot enter root's home, and psql says so on every call.
+	cmd.Dir = "/"
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		// The statement can hold a password; the error names only its verb.
+		verb := strings.Join(strings.Fields(query)[:2], " ")
+		return "", fmt.Errorf("psql (%s): %v: %s", verb, err, strings.TrimSpace(string(out)))
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func (in *serverInit) psqlBool(ctx context.Context, query string) (bool, error) {
+	out, err := in.psql(ctx, query)
+	return out == "1", err
+}
+
+// waitForPostgres waits for a server that was just started or restarted to
+// take connections.
+func (in *serverInit) waitForPostgres(ctx context.Context) error {
+	deadline := time.Now().Add(60 * time.Second)
+	for {
+		_, err := in.psql(ctx, "SELECT 1")
+		if err == nil {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("PostgreSQL is not taking connections after 60s: %w", err)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}
+
+// stopService stops the control plane if it runs, so the store is still.
+// It says whether it stopped it, so a failed move can start it again.
+func (in *serverInit) stopService(ctx context.Context) (bool, error) {
+	if in.unitPath == "" {
+		return false, nil
+	}
+	if _, err := exec.LookPath("systemctl"); err != nil {
+		return false, nil
+	}
+	if exec.CommandContext(ctx, "systemctl", "is-active", "--quiet", "llmgw-server").Run() != nil {
+		return false, nil
+	}
+	if out, err := exec.CommandContext(ctx, "systemctl", "stop", "llmgw-server").CombinedOutput(); err != nil {
+		return false, fmt.Errorf("stopping llmgw-server to move its store: %v: %s", err, strings.TrimSpace(string(out)))
+	}
+	in.done("llmgw-server stopped while its store moves")
+	return true, nil
+}

internal/cli/serverinit_postgres_test.go

@@ -0,0 +1,203 @@
+package cli
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func TestSetAuthStoreChangesOnlyAuthStore(t *testing.T) {
+	p := filepath.Join(t.TempDir(), "server.yaml")
+	body := `# my notes
+threads:
+  store: /var/lib/llmgw/threads.db   # not this one
+  retain_hours: 24
+auth:
+  enabled: true
+  # the store
+  store: /var/lib/llmgw/llmgw.db
+  signup_open: true
+landing:
+  contact: "x"
+`
+	if err := os.WriteFile(p, []byte(body), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	if err := setAuthStore(p, "@/etc/llmgw/secrets/store-dsn"); err != nil {
+		t.Fatal(err)
+	}
+	got, _ := os.ReadFile(p)
+	want := strings.Replace(body, "  store: /var/lib/llmgw/llmgw.db",
+		`  store: "@/etc/llmgw/secrets/store-dsn"`, 1)
+	if string(got) != want {
+		t.Errorf("got:\n%s\nwant:\n%s", got, want)
+	}
+	cfg, err := config.Load(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.Auth.Store != "@/etc/llmgw/secrets/store-dsn" || cfg.Threads.Store != "/var/lib/llmgw/threads.db" {
+		t.Errorf("auth.store %q, threads.store %q", cfg.Auth.Store, cfg.Threads.Store)
+	}
+	backups, _ := filepath.Glob(p + ".bak-*")
+	if len(backups) != 1 {
+		t.Errorf("the old file should be kept beside it, found %v", backups)
+	}
+}
+
+func TestSetAuthStoreAddsTheLineWhenAuthHasNone(t *testing.T) {
+	p := filepath.Join(t.TempDir(), "server.yaml")
+	if err := os.WriteFile(p, []byte("auth:\n  enabled: true\nlanding:\n  contact: x\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	if err := setAuthStore(p, "@/x"); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := config.Load(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.Auth.Store != "@/x" || !cfg.Auth.Enabled {
+		t.Errorf("auth = %+v", cfg.Auth)
+	}
+}
+
+// fakePostgres stands in for setupPostgres the way it really answers: an
+// address file under /etc/llmgw/secrets, returned as "@file". What the file
+// holds is a SQLite path, so everything after the setup runs for real.
+func fakePostgres(t *testing.T, in *serverInit) (address, db string) {
+	t.Helper()
+	db = filepath.Join(t.TempDir(), "pg-standin.db")
+	in.pgSetup = func(context.Context) (string, error) {
+		if err := os.MkdirAll(filepath.Dir(in.dsnPath()), 0o750); err != nil {
+			return "", err
+		}
+		return "@" + in.dsnPath(), os.WriteFile(in.dsnPath(), []byte(db+"\n"), 0o600)
+	}
+	return "@" + in.dsnPath(), db
+}
+
+// Moving an install that began on SQLite: the administrator it created must
+// be in the new store, the configuration must name it, and the SQLite file
+// must still be there. The new store is SQLite too here — what PostgreSQL
+// adds is the setup, which needs a machine to run on.
+func TestInitMovesAnExistingStore(t *testing.T) {
+	in, out := newTestInit(t)
+	if err := in.run(context.Background()); err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	old := filepath.Join(in.stateDir, "llmgw.db")
+
+	in.postgres = true
+	target, db := fakePostgres(t, in)
+	out.Reset()
+	if err := in.run(context.Background()); err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if !strings.Contains(out.String(), "copied") {
+		t.Errorf("nothing was copied:\n%s", out)
+	}
+
+	cfg, err := config.Load(in.configPath())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.Auth.Store != target {
+		t.Fatalf("server.yaml still names %q", cfg.Auth.Store)
+	}
+	s, err := store.Open(context.Background(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	u, err := s.UserByEmail(context.Background(), "admin@example.com")
+	s.Close()
+	if err != nil || !u.IsAdmin {
+		t.Fatalf("the administrator did not move: %v %+v", err, u)
+	}
+	if _, err := os.Stat(old); err != nil {
+		t.Error("the SQLite store should be kept")
+	}
+
+	// And once moved, again is nothing.
+	in.changed = false
+	out.Reset()
+	if err := in.run(context.Background()); err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if in.changed {
+		t.Errorf("a run after the move changed something:\n%s", out)
+	}
+}
+
+func TestInitWritesAPostgresConfigurationFromTheStart(t *testing.T) {
+	in, out := newTestInit(t)
+	in.postgres = true
+	target, _ := fakePostgres(t, in)
+	if err := in.run(context.Background()); err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	cfg, _ := config.Load(in.configPath())
+	if cfg.Auth.Store != target {
+		t.Errorf("auth.store = %q", cfg.Auth.Store)
+	}
+	if _, err := os.Stat(filepath.Join(in.stateDir, "llmgw.db")); err == nil {
+		t.Error("a PostgreSQL install should not leave a SQLite store behind")
+	}
+}
+
+func TestInitWillNotMoveOnePostgresIntoAnother(t *testing.T) {
+	in, out := newTestInit(t)
+	if err := os.MkdirAll(in.etcDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	cfg := "control:\n  enabled: true\nauth:\n  enabled: true\n  store: postgres://elsewhere/llmgw\n"
+	if err := os.WriteFile(in.configPath(), []byte(cfg), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	in.postgres = true
+	in.pgSetup = func(context.Context) (string, error) { return "@/somewhere/else", nil }
+	err := in.run(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "already names a PostgreSQL store") {
+		t.Fatalf("expected a refusal, got %v\n%s", err, out)
+	}
+}
+
+// The lists below are what `systemctl list-unit-files --type=service
+// --no-legend --plain 'postgresql*'` prints on each kind of machine.
+func TestPickPostgresUnit(t *testing.T) {
+	for _, c := range []struct{ name, list, want string }{
+		{"rocky, distribution package",
+			"postgresql.service disabled disabled\npostgresql@.service disabled disabled\n", "postgresql.service"},
+		{"rocky, PGDG package",
+			"postgresql-16.service disabled disabled\n", "postgresql-16.service"},
+		{"two PGDG versions",
+			"postgresql-13.service enabled disabled\npostgresql-16.service disabled disabled\n", "postgresql-16.service"},
+		{"debian",
+			"postgresql.service enabled enabled\npostgresql@.service indirect enabled\n", "postgresql.service"},
+		{"only the template", "postgresql@.service indirect enabled\n", ""},
+		{"none", "", ""},
+	} {
+		if got := pickPostgresUnit(c.list); got != c.want {
+			t.Errorf("%s: got %q, want %q", c.name, got, c.want)
+		}
+	}
+}
+
+// What `systemctl show -p Environment --value` prints for each unit, from
+// the unit files the packages ship.
+func TestPGDATAFromTheUnit(t *testing.T) {
+	for env, want := range map[string]string{
+		"PG_OOM_ADJUST_FILE=/proc/self/oom_score_adj PG_OOM_ADJUST_VALUE=0 PGDATA=/var/lib/pgsql/data": "/var/lib/pgsql/data",
+		"PGDATA=/var/lib/pgsql/16/data/ PG_OOM_ADJUST_FILE=/proc/self/oom_score_adj":                   "/var/lib/pgsql/16/data/",
+		"": "",
+	} {
+		if got := pgdataFrom(env); got != want {
+			t.Errorf("%q: got %q, want %q", env, got, want)
+		}
+	}
+}

internal/config/config.go

@@ -540,8 +540,14 @@ func (s Server) Catalog() (*catalog.Catalog, error) {
 // A model with no profiles cannot be placed, so a configuration of bare names
 // plus a static endpoint list is the router-only mode used for testing against
 // sim-engines.
+//
+// With no static endpoints there is nothing a router-only process could serve,
+// so a configuration naming neither endpoints nor models is a control plane
+// with an empty catalog — which is what every new cluster is. It used to be
+// refused as having nothing to serve, and under systemd that is a unit
+// restarting every two seconds with nothing on 8080.
 func (s Server) ControlPlaneMode() bool {
-	if s.Control.Enabled {
+	if s.Control.Enabled || len(s.Endpoints) == 0 {
 		return true
 	}
 	for _, m := range s.Models {

internal/config/config_test.go

@@ -155,19 +155,26 @@ func TestValidateAcceptsDistinctAliases(t *testing.T) {
 	}
 }
 
-func TestAControlPlaneCanBeAskedForWithNoModelsInTheFile(t *testing.T) {
-	// Without this, a deployment whose catalog lives entirely in the database
-	// cannot start: the mode is inferred from the file, an empty models block
-	// reads as "router only", and the process refuses to start for want of
-	// anything to serve — so the stored models it would have loaded are never
-	// read at all.
+func TestAnEmptyConfigurationIsAControlPlane(t *testing.T) {
+	// A new cluster has no models and no endpoints. Refusing it as having
+	// nothing to serve left a systemd unit restarting every two seconds and
+	// nothing listening on 8080 — on the first start anyone ever made.
 	var s Server
+	if !s.ControlPlaneMode() {
+		t.Fatal("a configuration with no endpoints and no models should run a control plane")
+	}
+}
+
+func TestStaticEndpointsAloneAreRouterOnly(t *testing.T) {
+	var s Server
+	s.Endpoints = []Endpoint{{ID: "e", URL: "http://127.0.0.1:1"}}
+	s.Models = []catalog.Model{{Name: "m"}}
 	if s.ControlPlaneMode() {
-		t.Fatal("an empty configuration is not a control plane")
+		t.Error("static endpoints and bare model names are the router-only mode")
 	}
 	s.Control.Enabled = true
 	if !s.ControlPlaneMode() {
-		t.Error("control.enabled should run the control plane with no models declared")
+		t.Error("control.enabled should run the control plane alongside static endpoints")
 	}
 }
 

internal/store/copy.go

@@ -96,6 +96,13 @@ func (s *DB) CopyTo(ctx context.Context, dst *DB) ([]CopyResult, error) {
 }
 
 // isEmpty reports whether any table has rows, and which one first does.
+// Empty reports whether the store holds no rows at all, which is what CopyTo
+// needs of its destination.
+func (s *DB) Empty(ctx context.Context) (bool, error) {
+	empty, _, err := s.isEmpty(ctx)
+	return empty, err
+}
+
 func (s *DB) isEmpty(ctx context.Context) (bool, string, error) {
 	for _, table := range copyOrder {
 		var n int64