From 1da4a2e1ef3b7ae3df0f1cd8def3548196e12300 Mon Sep 17 00:00:00 2001
From: Ilker Manap <ilker@Ilkers-MacBook-Pro.local>
Date: Wed, 30 Sep 2026 01:44:32 +0200
Subject: [PATCH] Back up the store and the certificate authority, and alert
 when it stops
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There were no backups: docs/install.md gave a pg_dump command, and nothing
ran it. The store holds every key, person, quota and usage record, the models
added at runtime, and now the secret each engine's key is derived from; the
certificate authority's key sits in /etc/llmgw/pki beside it. Losing the disk
lost all of it.

`llmgw server backup` writes both to one archive: pg_dump's custom format, or
a VACUUM INTO snapshot for SQLite, plus /etc/llmgw. The archive holds the CA's
key, so it is 0600; the directory is listable, so the control plane — not
root — can see when the newest was written. pg_dump is given the connection
through its environment, since a DSN carries the password and a process's
arguments are readable by every user. Written beside its name and renamed,
so an interrupted backup is never the newest; the newest 14 are kept, and
nothing the command did not write is ever removed.

llmgw-backup.timer runs it nightly and catches up after downtime. The control
plane publishes the newest archive's time, and BackupTooOld fires past 26
hours — or when there has never been one. The install guide now covers
restoring, and says to copy archives off the machine, since they share its
disk.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md                        |  14 ++
 deploy/monitoring/llmgw-rules.yml   |  11 ++
 deploy/systemd/llmgw-backup.service |  11 ++
 deploy/systemd/llmgw-backup.timer   |  12 ++
 docs/install.md                     |  74 +++++--
 docs/reference.md                   |   9 +
 internal/cli/backup.go              | 289 ++++++++++++++++++++++++++++
 internal/cli/backup_test.go         | 106 ++++++++++
 internal/cli/server.go              |  19 ++
 internal/config/config.go           |   5 +
 internal/store/open.go              |   6 +
 internal/store/sqlite.go            |  11 ++
 12 files changed, 549 insertions(+), 18 deletions(-)
 create mode 100644 deploy/systemd/llmgw-backup.service
 create mode 100644 deploy/systemd/llmgw-backup.timer
 create mode 100644 internal/cli/backup.go
 create mode 100644 internal/cli/backup_test.go

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d81d80e..0da36a1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -66,8 +66,22 @@ ControlPlaneLeadership, CertificateExpiresSoon and CertificateExpiresVerySoon.
 than its minimum, so an on-demand model that was asked for and fitted nowhere
 no longer reads as satisfied.
 
+### Backups
+
+`llmgw server backup` writes the store and `/etc/llmgw` — configuration,
+secrets, and the certificate authority's key — to one archive readable by root
+only, keeping the newest 14. `deploy/systemd/llmgw-backup.timer` runs it
+nightly. The control plane publishes the newest archive's time as
+`llmgw_backup_last_success_timestamp_seconds`, and BackupTooOld fires when it is
+more than 26 hours old or there has never been one. docs/install.md now says how
+to restore, including why running engines survive it: the engine-key secret
+comes back with the store.
+
 ### Fixes
 
+- Tests that slept a fixed time and then asserted now wait for their
+  condition. They are the likeliest cause of CI's intermittent failures on a
+  loaded runner.
 - `ctl` reported a change the API had queued (202 Accepted) as an error, as
   `ctl restart` did on its first real use.
 
diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml
index 128f11e..fc022b5 100644
--- a/deploy/monitoring/llmgw-rules.yml
+++ b/deploy/monitoring/llmgw-rules.yml
@@ -123,6 +123,17 @@ groups:
           summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
           description: See CertificateExpiresSoon; a week is not long.
 
+      - alert: BackupTooOld
+        expr: time() - llmgw_backup_last_success_timestamp_seconds > 26 * 3600
+        labels: { severity: warning }
+        annotations:
+          summary: "The newest llmgw backup is {{ $value | humanizeDuration }} old"
+          description: >-
+            llmgw-backup.timer runs nightly. Also fires when there has never
+            been a backup (the series is 0 then). `journalctl -u
+            llmgw-backup.service` on the control plane says why the last one
+            failed; docs/install.md, "Backing up", says how to set it up.
+
       - alert: PoolSaturated
         expr: llmgw_pool_saturation > 1
         for: 5m
diff --git a/deploy/systemd/llmgw-backup.service b/deploy/systemd/llmgw-backup.service
new file mode 100644
index 0000000..268fbe5
--- /dev/null
+++ b/deploy/systemd/llmgw-backup.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=llmgw backup: the store and /etc/llmgw, to /var/backups/llmgw
+After=llmgw-server.service postgresql.service
+
+[Service]
+Type=oneshot
+# Root, because the archive includes /etc/llmgw/secrets and the certificate
+# authority's key, which the llmgw user deliberately cannot read all of.
+ExecStart=/usr/local/bin/llmgw server backup --config /etc/llmgw/server.yaml --keep 14
+Nice=10
+IOSchedulingClass=idle
diff --git a/deploy/systemd/llmgw-backup.timer b/deploy/systemd/llmgw-backup.timer
new file mode 100644
index 0000000..7db9a3a
--- /dev/null
+++ b/deploy/systemd/llmgw-backup.timer
@@ -0,0 +1,12 @@
+[Unit]
+Description=Back up llmgw every night
+
+[Timer]
+OnCalendar=*-*-* 03:15:00
+# A machine that was off at 03:15 backs up when it next starts, rather than
+# waiting another day with a gap nobody notices until the alert.
+Persistent=true
+RandomizedDelaySec=10m
+
+[Install]
+WantedBy=timers.target
diff --git a/docs/install.md b/docs/install.md
index c6d7bc9..b282b0a 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -961,36 +961,74 @@ the choice is yours rather than silent.
 
 ## Backing up
 
-Three things, in descending order of how badly you want them.
+`llmgw server backup` writes one archive holding everything a control plane
+cannot be rebuilt without:
 
-**The certificate authority.** `ca-key.pem`, `/etc/llmgw/pki` unless you kept it elsewhere (step 2).
-Losing it does not stop the cluster, and it means every node identity has to be
-reissued the next time you touch one. Nothing in llmgw can reconstruct it.
+- **the store** — users, API keys, the tenancy tree, limits and budgets, usage
+  history, models added at runtime, measured capacity, and the secret every
+  engine key is derived from (`pg_dump` in its custom format for PostgreSQL, a
+  consistent `VACUUM INTO` snapshot for SQLite; both are safe against a running
+  gateway);
+- **`/etc/llmgw`** — the configuration, the store's address, and the
+  certificate authority's key (`pki/`). Nothing in llmgw can reconstruct that
+  key; losing it means reissuing every node's identity.
 
-**The store.** Users, API keys, organisations, teams, projects, every limit and
-budget, all usage history, models added at runtime, and measured capacity.
+Archives go to `/var/backups/llmgw`, readable by root only because of the CA
+key, and the newest 14 are kept. Run it nightly with the timer:
 
 ```
-# SQLite
-sqlite3 /var/lib/llmgw/llmgw.db ".backup '/backup/llmgw-$(date +%F).db'"
-
-# PostgreSQL
-pg_dump --format=custom llmgw > /backup/llmgw-$(date +%F).dump
+sudo install -m 0644 deploy/systemd/llmgw-backup.service deploy/systemd/llmgw-backup.timer /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now llmgw-backup.timer
+sudo systemctl start llmgw-backup.service     # one now, to see it work
+ls -l /var/backups/llmgw
 ```
 
-`.backup` and `pg_dump` are both safe against a running gateway. Copying the
-SQLite file with `cp` while it is being written is not.
+The control plane publishes when the newest archive was written as
+`llmgw_backup_last_success_timestamp_seconds`, and **BackupTooOld** fires when
+it is more than 26 hours old — including when there has never been one.
+
+**Copy the archives off the machine.** They sit on the disk they back up and do
+not survive losing it. Anything that copies files will do — rsync from the
+mirror, a NAS mount, an offsite bucket — as long as it keeps the archives
+readable by root only.
 
 **Keys are stored hashed and cannot be recovered from a backup or from
 anywhere else.** Restoring an old store restores which keys exist and who owns
 them, not their secrets. A key created after the backup is gone and has to be
-reissued — which is why the store is worth backing up on a schedule rather than
-before upgrades.
+reissued.
+
+### Restoring
+
+On a machine with llmgw installed (the same version or newer) and, for
+PostgreSQL, an empty database and role as `server init --postgres` would create
+them:
+
+```
+sudo systemctl stop llmgw-server
+mkdir /tmp/restore && sudo tar -xzf /var/backups/llmgw/llmgw-backup-<host>-<time>.tar.gz -C /tmp/restore
+cat /tmp/restore/MANIFEST                     # version and time it was taken
+
+# /etc/llmgw: configuration, secrets, certificates and the CA
+sudo cp -a /tmp/restore/etc/llmgw/. /etc/llmgw/
+
+# the store — PostgreSQL
+sudo -u postgres pg_restore --clean --if-exists --no-owner --role=llmgw -d llmgw /tmp/restore/store.pgdump
+# the store — SQLite
+sudo install -o llmgw -g llmgw -m 0600 /tmp/restore/store.sqlite /var/lib/llmgw/llmgw.db
+
+sudo systemctl start llmgw-server
+sudo rm -rf /tmp/restore
+```
+
+Running engines keep serving throughout: agents reconnect to the restored
+control plane and their replicas are adopted, and because the engine-key secret
+came back with the store, the router can still reach them.
 
 **The mirror's directory** is the one thing you can rebuild, by downloading it
-all again. Worth remembering that a pinned version may no longer be upstream to
-download: that is exactly what pinning it protected you from, and a backup is
-what makes the protection outlive the disk.
+all again. A pinned version may no longer be upstream to download: that is
+exactly what pinning it protected you from, and a copy of the directory is what
+makes the protection outlive the disk.
 
 ## Reclaiming disk
 
diff --git a/docs/reference.md b/docs/reference.md
index d3ac0c1..670905e 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -169,6 +169,7 @@ rather than repeated.
 | `monitoring` | `Monitoring` | [Server](#server) |
 | `mirror` | `Mirror` | [Server](#server) |
 | `engines` | `map[string]string` | [Server](#server) |
+| `backup_dir` | `string` | [Server](#server) |
 | `router` | `Router` | [Server](#server) |
 | `control` | `Control` | [Server](#server) |
 | `models` | `[]catalog.Model` | [Server](#server) |
@@ -1265,6 +1266,14 @@ never trusted to already have what somebody once put there.
 Unset keeps DefaultEngines. A version replaces the default; an empty
 one leaves that engine to be installed by hand.
 
+### `backup_dir`
+
+`string`
+
+BackupDir is where `llmgw server backup` writes, and where the newest
+archive's age is read for llmgw_backup_last_success_timestamp_seconds.
+Empty means /var/backups/llmgw.
+
 ### `router`
 
 `Router`
diff --git a/internal/cli/backup.go b/internal/cli/backup.go
new file mode 100644
index 0000000..906ee30
--- /dev/null
+++ b/internal/cli/backup.go
@@ -0,0 +1,289 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+// DefaultBackupDir is where backups are written and where the control plane
+// looks for the newest one.
+const DefaultBackupDir = "/var/backups/llmgw"
+
+func newBackupCmd() *cobra.Command {
+	var (
+		configPath string
+		etcDir     string
+		dir        string
+		keep       int
+	)
+	c := &cobra.Command{
+		Use:   "backup",
+		Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
+		Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
+			"people, quotas, usage, the models added at runtime, the secret engine\n" +
+			"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
+			"store's address and the certificate authority's key.\n\n" +
+			"The archive holds that key, so it is written readable by root only. It\n" +
+			"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
+			"not survive the loss of this machine.\n\n" +
+			"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
+			"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
+			"BackupTooOld fires when it is more than a day old.",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, err := config.Load(configPath)
+			if err != nil {
+				return err
+			}
+			path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
+			if err != nil {
+				return err
+			}
+			removed, err := pruneBackups(dir, keep)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
+			if removed > 0 {
+				fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
+			}
+			fmt.Fprintln(cmd.OutOrStdout())
+			return nil
+		},
+	}
+	c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
+	c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
+	c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
+	c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
+	return c
+}
+
+const backupPrefix = "llmgw-backup-"
+
+// writeBackup writes one archive and returns its path. Written beside its
+// final name and renamed, so a backup interrupted halfway is never the newest
+// one the control plane reports.
+func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
+	// Listable by anyone, so the control plane — not root — can see when the
+	// newest archive was written. The archives themselves are root's alone.
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	host, _ := os.Hostname()
+	name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
+	final := filepath.Join(dir, name)
+	part := final + ".part"
+
+	f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return "", err
+	}
+	ok := false
+	defer func() {
+		if !ok {
+			_ = os.Remove(part)
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	tw := tar.NewWriter(gz)
+
+	manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
+		time.Now().UTC().Format(time.RFC3339))
+	if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
+		return "", err
+	}
+	if storeAddr != "" {
+		if err := addStore(ctx, tw, storeAddr, dir); err != nil {
+			return "", fmt.Errorf("the store: %w", err)
+		}
+	}
+	if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
+		return "", fmt.Errorf("%s: %w", etcDir, err)
+	}
+	if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
+		return "", err
+	}
+	if err := os.Rename(part, final); err != nil {
+		return "", err
+	}
+	ok = true
+	return final, nil
+}
+
+// addStore dumps the store into the archive.
+func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
+	addr, err := store.ResolveAddress(addr)
+	if err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(scratch, ".store-*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	_ = tmp.Close()
+	defer os.Remove(tmpPath)
+
+	name := "store.sqlite"
+	if store.IsPostgres(addr) {
+		name = "store.pgdump"
+		if err := pgDump(ctx, addr, tmpPath); err != nil {
+			return err
+		}
+	} else {
+		// VACUUM INTO refuses a file that exists.
+		_ = os.Remove(tmpPath)
+		db, err := store.Open(ctx, addr)
+		if err != nil {
+			return err
+		}
+		err = db.SnapshotTo(ctx, tmpPath)
+		_ = db.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return addFile(tw, tmpPath, name)
+}
+
+// pgDump runs pg_dump in its custom format, which pg_restore can restore
+// selectively. The connection is given through the environment, not the
+// command line: a DSN carries the password, and every user on the machine
+// can read another process's arguments.
+func pgDump(ctx context.Context, dsn, out string) error {
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return fmt.Errorf("the store address is not a URL")
+	}
+	env := os.Environ()
+	set := func(k, v string) {
+		if v != "" {
+			env = append(env, k+"="+v)
+		}
+	}
+	set("PGHOST", u.Hostname())
+	set("PGPORT", u.Port())
+	set("PGUSER", u.User.Username())
+	if pw, ok := u.User.Password(); ok {
+		set("PGPASSWORD", pw)
+	}
+	set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
+	set("PGSSLMODE", u.Query().Get("sslmode"))
+	cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
+	cmd.Env = env
+	if b, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
+	}
+	return nil
+}
+
+func addTree(tw *tar.Writer, root, as string) error {
+	return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, _ := filepath.Rel(root, p)
+		name := filepath.ToSlash(filepath.Join(as, rel))
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		switch {
+		case d.IsDir():
+			return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
+				Typeflag: tar.TypeDir, ModTime: info.ModTime()})
+		case info.Mode().IsRegular():
+			return addFile(tw, p, name)
+		}
+		return nil // sockets, links: not configuration
+	})
+}
+
+func addFile(tw *tar.Writer, path, name string) error {
+	f, err := os.Open(path) //nolint:gosec // files we chose to back up
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
+		Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err = io.Copy(tw, f)
+	return err
+}
+
+func addBytes(tw *tar.Writer, name string, b []byte) error {
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
+		ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err := tw.Write(b)
+	return err
+}
+
+// pruneBackups keeps the newest keep archives. Only files this command
+// wrote are considered, so nothing else in the directory is ever removed.
+func pruneBackups(dir string, keep int) (int, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return 0, err
+	}
+	var names []string
+	for _, e := range entries {
+		if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
+			names = append(names, n)
+		}
+	}
+	// The name carries a UTC timestamp, so name order is age order.
+	slices.Sort(names)
+	removed := 0
+	for len(names) > max(keep, 1) {
+		if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
+			return removed, err
+		}
+		names = names[1:]
+		removed++
+	}
+	return removed, nil
+}
+
+// NewestBackup is the modification time of the newest complete archive in
+// dir, or zero when there is none.
+func NewestBackup(dir string) time.Time {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return time.Time{}
+	}
+	var newest time.Time
+	for _, e := range entries {
+		if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
+			continue
+		}
+		if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	return newest
+}
diff --git a/internal/cli/backup_test.go b/internal/cli/backup_test.go
new file mode 100644
index 0000000..0e92b6a
--- /dev/null
+++ b/internal/cli/backup_test.go
@@ -0,0 +1,106 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
+	tmp := t.TempDir()
+	dbPath := filepath.Join(tmp, "store.db")
+	db, err := store.Open(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secret, err := db.ClusterSecret(t.Context(), "engine-key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = db.Close()
+
+	etc := filepath.Join(tmp, "etc")
+	_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
+	_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
+	_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
+
+	out := filepath.Join(tmp, "backups")
+	path, err := writeBackup(t.Context(), dbPath, etc, out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, _ := os.Stat(path)
+	if info.Mode().Perm() != 0o600 {
+		t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
+	}
+
+	files := map[string][]byte{}
+	f, _ := os.Open(path)
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		b, _ := io.ReadAll(tr)
+		files[h.Name] = b
+	}
+	for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
+		if _, ok := files[want]; !ok {
+			t.Errorf("%s is not in the archive", want)
+		}
+	}
+
+	// And the store in it is a working store, with what was in it.
+	restored := filepath.Join(tmp, "restored.db")
+	_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
+	rdb, err := store.Open(t.Context(), restored)
+	if err != nil {
+		t.Fatalf("the backed-up store does not open: %v", err)
+	}
+	defer rdb.Close()
+	if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
+		t.Error("the restored store lost the engine key secret; every engine would be locked out")
+	}
+}
+
+func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
+	dir := t.TempDir()
+	for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
+		p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
+		_ = os.WriteFile(p, []byte("x"), 0o600)
+		mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
+		_ = os.Chtimes(p, mt, mt)
+	}
+	_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
+	_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
+
+	removed, err := pruneBackups(dir, 2)
+	if err != nil || removed != 1 {
+		t.Fatalf("removed %d, %v", removed, err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
+		t.Error("a file the backup did not write was removed")
+	}
+	newest := NewestBackup(dir)
+	if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("newest %v; an interrupted .part must not count", newest)
+	}
+	if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
+		t.Error("a missing directory reported a backup")
+	}
+}
diff --git a/internal/cli/server.go b/internal/cli/server.go
index 2c6884d..cfe7d9a 100644
--- a/internal/cli/server.go
+++ b/internal/cli/server.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
+	"github.com/prometheus/client_golang/prometheus"
 	"log/slog"
 	"net"
 	"net/http"
@@ -132,6 +133,23 @@ func newServerCmd() *cobra.Command {
 				// On the same /metrics as the traffic figures: correlating
 				// "the cluster was full" with "requests were queueing" should
 				// not need two systems.
+				// When the newest backup was written, so its absence alerts
+				// rather than being found on the day it is needed.
+				backupDir := cfg.BackupDir
+				if backupDir == "" {
+					backupDir = DefaultBackupDir
+				}
+				if err := rt.RegisterMetrics(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+					Name: "llmgw_backup_last_success_timestamp_seconds",
+					Help: "When the newest backup archive was written, as a Unix time; 0 when there is none.",
+				}, func() float64 {
+					if t := NewestBackup(backupDir); !t.IsZero() {
+						return float64(t.Unix())
+					}
+					return 0
+				})); err != nil {
+					return err
+				}
 				if err := rt.RegisterMetrics(tlsconf.ExpiryCollector(map[string]string{
 					"agent-server": cfg.AgentTLS.Cert, "agent-ca": cfg.AgentTLS.CA,
 				})); err != nil {
@@ -367,6 +385,7 @@ func newServerCmd() *cobra.Command {
 	f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
 
 	cmd.AddCommand(newServerInitCmd())
+	cmd.AddCommand(newBackupCmd())
 	return cmd
 }
 
diff --git a/internal/config/config.go b/internal/config/config.go
index e0f5a17..8b1ff1e 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -81,6 +81,11 @@ type Server struct {
 	// one leaves that engine to be installed by hand.
 	Engines map[string]string `koanf:"engines"`
 
+	// BackupDir is where `llmgw server backup` writes, and where the newest
+	// archive's age is read for llmgw_backup_last_success_timestamp_seconds.
+	// Empty means /var/backups/llmgw.
+	BackupDir string `koanf:"backup_dir"`
+
 	Router  Router          `koanf:"router"`
 	Control Control         `koanf:"control"`
 	Models  []catalog.Model `koanf:"models"`
diff --git a/internal/store/open.go b/internal/store/open.go
index 2637014..606019f 100644
--- a/internal/store/open.go
+++ b/internal/store/open.go
@@ -59,3 +59,9 @@ func resolveAddress(address string) (string, error) {
 	}
 	return out, nil
 }
+
+// ResolveAddress expands an @file store address, as Open does.
+func ResolveAddress(address string) (string, error) { return resolveAddress(address) }
+
+// IsPostgres reports whether a resolved address names a PostgreSQL server.
+func IsPostgres(address string) bool { return isPostgres(address) }
diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go
index bb00cdf..a9ed2ec 100644
--- a/internal/store/sqlite.go
+++ b/internal/store/sqlite.go
@@ -817,3 +817,14 @@ func (s *DB) UsageByModel(ctx context.Context, since time.Time) ([]ModelUsage, e
 	}
 	return out, rows.Err()
 }
+
+// SnapshotTo writes a consistent copy of a SQLite store to path, while it is
+// in use. VACUUM INTO takes a read transaction, so the copy is of one moment
+// even with writes going on, which copying the file is not.
+func (s *DB) SnapshotTo(ctx context.Context, path string) error {
+	if s.dialect != SQLiteDialect {
+		return fmt.Errorf("a snapshot is for SQLite; back up PostgreSQL with pg_dump")
+	}
+	_, err := s.raw.ExecContext(ctx, `VACUUM INTO ?`, path)
+	return err
+}