Ilker ManapandClaude Opus 5.5
1da4a2e1ef
There were no backups: docs/install.md gave a pg_dump command, and nothing ran it. The store holds every key, person, quota and usage record, the models added at runtime, and now the secret each engine's key is derived from; the certificate authority's key sits in /etc/llmgw/pki beside it. Losing the disk lost all of it. `llmgw server backup` writes both to one archive: pg_dump's custom format, or a VACUUM INTO snapshot for SQLite, plus /etc/llmgw. The archive holds the CA's key, so it is 0600; the directory is listable, so the control plane — not root — can see when the newest was written. pg_dump is given the connection through its environment, since a DSN carries the password and a process's arguments are readable by every user. Written beside its name and renamed, so an interrupted backup is never the newest; the newest 14 are kept, and nothing the command did not write is ever removed. llmgw-backup.timer runs it nightly and catches up after downtime. The control plane publishes the newest archive's time, and BackupTooOld fires past 26 hours — or when there has never been one. The install guide now covers restoring, and says to copy archives off the machine, since they share its disk. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
107 lines
3.0 KiB
Go
107 lines
3.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
|
|
tmp := t.TempDir()
|
|
dbPath := filepath.Join(tmp, "store.db")
|
|
db, err := store.Open(t.Context(), dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret, err := db.ClusterSecret(t.Context(), "engine-key")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = db.Close()
|
|
|
|
etc := filepath.Join(tmp, "etc")
|
|
_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
|
|
_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
|
|
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
|
|
|
|
out := filepath.Join(tmp, "backups")
|
|
path, err := writeBackup(t.Context(), dbPath, etc, out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
info, _ := os.Stat(path)
|
|
if info.Mode().Perm() != 0o600 {
|
|
t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
|
|
}
|
|
|
|
files := map[string][]byte{}
|
|
f, _ := os.Open(path)
|
|
defer f.Close()
|
|
gz, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tr := tar.NewReader(gz)
|
|
for {
|
|
h, err := tr.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, _ := io.ReadAll(tr)
|
|
files[h.Name] = b
|
|
}
|
|
for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
|
|
if _, ok := files[want]; !ok {
|
|
t.Errorf("%s is not in the archive", want)
|
|
}
|
|
}
|
|
|
|
// And the store in it is a working store, with what was in it.
|
|
restored := filepath.Join(tmp, "restored.db")
|
|
_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
|
|
rdb, err := store.Open(t.Context(), restored)
|
|
if err != nil {
|
|
t.Fatalf("the backed-up store does not open: %v", err)
|
|
}
|
|
defer rdb.Close()
|
|
if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
|
|
t.Error("the restored store lost the engine key secret; every engine would be locked out")
|
|
}
|
|
}
|
|
|
|
func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
|
|
p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
|
|
_ = os.WriteFile(p, []byte("x"), 0o600)
|
|
mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
|
|
_ = os.Chtimes(p, mt, mt)
|
|
}
|
|
_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
|
|
_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
|
|
|
|
removed, err := pruneBackups(dir, 2)
|
|
if err != nil || removed != 1 {
|
|
t.Fatalf("removed %d, %v", removed, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
|
|
t.Error("a file the backup did not write was removed")
|
|
}
|
|
newest := NewestBackup(dir)
|
|
if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("newest %v; an interrupted .part must not count", newest)
|
|
}
|
|
if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
|
|
t.Error("a missing directory reported a backup")
|
|
}
|
|
}
|