The first backup on the live control plane was written, and the control plane reported none: /var/backups/llmgw already existed, made by an earlier move to PostgreSQL for its own dumps, owned by postgres and 0700. MkdirAll leaves an existing directory as it is, so the archive landed where the llmgw user could not list it, and llmgw_backup_last_success_timestamp_seconds said 0. Opening that directory up would have exposed the dumps already in it, so backups get their own, /var/backups/llmgw-server. And `llmgw server backup` now warns when the directory it wrote to is one the control plane cannot list, rather than leaving that to be found from an alert. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
119 lines
3.3 KiB
Go
119 lines
3.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
|
|
tmp := t.TempDir()
|
|
dbPath := filepath.Join(tmp, "store.db")
|
|
db, err := store.Open(t.Context(), dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret, err := db.ClusterSecret(t.Context(), "engine-key")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = db.Close()
|
|
|
|
etc := filepath.Join(tmp, "etc")
|
|
_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
|
|
_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
|
|
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
|
|
|
|
out := filepath.Join(tmp, "backups")
|
|
path, err := writeBackup(t.Context(), dbPath, etc, out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
info, _ := os.Stat(path)
|
|
if info.Mode().Perm() != 0o600 {
|
|
t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
|
|
}
|
|
|
|
files := map[string][]byte{}
|
|
f, _ := os.Open(path)
|
|
defer f.Close()
|
|
gz, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tr := tar.NewReader(gz)
|
|
for {
|
|
h, err := tr.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, _ := io.ReadAll(tr)
|
|
files[h.Name] = b
|
|
}
|
|
for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
|
|
if _, ok := files[want]; !ok {
|
|
t.Errorf("%s is not in the archive", want)
|
|
}
|
|
}
|
|
|
|
// And the store in it is a working store, with what was in it.
|
|
restored := filepath.Join(tmp, "restored.db")
|
|
_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
|
|
rdb, err := store.Open(t.Context(), restored)
|
|
if err != nil {
|
|
t.Fatalf("the backed-up store does not open: %v", err)
|
|
}
|
|
defer rdb.Close()
|
|
if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
|
|
t.Error("the restored store lost the engine key secret; every engine would be locked out")
|
|
}
|
|
}
|
|
|
|
func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
|
|
p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
|
|
_ = os.WriteFile(p, []byte("x"), 0o600)
|
|
mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
|
|
_ = os.Chtimes(p, mt, mt)
|
|
}
|
|
_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
|
|
_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
|
|
|
|
removed, err := pruneBackups(dir, 2)
|
|
if err != nil || removed != 1 {
|
|
t.Fatalf("removed %d, %v", removed, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
|
|
t.Error("a file the backup did not write was removed")
|
|
}
|
|
newest := NewestBackup(dir)
|
|
if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("newest %v; an interrupted .part must not count", newest)
|
|
}
|
|
if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
|
|
t.Error("a missing directory reported a backup")
|
|
}
|
|
}
|
|
|
|
func TestABackupDirectoryTheControlPlaneCannotListIsSaid(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_ = os.Chmod(dir, 0o700)
|
|
if unlistable(dir) == "" {
|
|
t.Error("a directory only its owner can list was taken as visible")
|
|
}
|
|
_ = os.Chmod(dir, 0o755)
|
|
if why := unlistable(dir); why != "" {
|
|
t.Errorf("a listable directory: %s", why)
|
|
}
|
|
}
|