Encrypt backups to backup_recipients with age

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:38:47 +02:00
1 parent 3b05ec0538
commit 12efb7d085
10 files changed
+204 -12

No files matched your search

+37 -9
View File
@@ -15,6 +15,7 @@ import (
"strings"
"time"
agecrypt "filippo.io/age"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
@@ -44,9 +45,11 @@ func newBackupCmd() *cobra.Command {
"people, quotas, usage, the models added at runtime, the secret engine\n" +
"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
"store's address and the certificate authority's key.\n\n" +
"The archive holds that key, so it is written readable by root only. It\n" +
"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
"not survive the loss of this machine.\n\n" +
"The archive holds that key, so it is written readable by root only, and\n" +
"encrypted to backup_recipients in server.yaml when they are set (age\n" +
"recipients or SSH public keys; open it with `age -d -i <key>`). It is on\n" +
"the same disk as what it backs up: copy it elsewhere, or it does not\n" +
"survive the loss of this machine.\n\n" +
"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
"BackupTooOld fires when it is more than a day old.",
@@ -55,7 +58,11 @@ func newBackupCmd() *cobra.Command {
if err != nil {
return err
}
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
to, err := config.ParseBackupRecipients(cfg.BackupRecipients)
if err != nil {
return err
}
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir, to)
if err != nil {
return err
}
@@ -87,7 +94,9 @@ const backupPrefix = "llmgw-backup-"
// writeBackup writes one archive and returns its path. Written beside its
// final name and renamed, so a backup interrupted halfway is never the newest
// one the control plane reports.
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
//
// With recipients, the archive is encrypted to them, and named .tar.gz.age.
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string, recipients []agecrypt.Recipient) (string, error) {
// Listable by anyone, so the control plane — not root — can see when the
// newest archive was written. The archives themselves are root's alone.
if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -95,6 +104,9 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
}
host, _ := os.Hostname()
name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
if len(recipients) > 0 {
name += ".age"
}
final := filepath.Join(dir, name)
part := final + ".part"
@@ -108,7 +120,13 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
_ = os.Remove(part)
}
}()
gz := gzip.NewWriter(f)
var sealed io.WriteCloser = nopCloser{f}
if len(recipients) > 0 {
if sealed, err = agecrypt.Encrypt(f, recipients...); err != nil {
return "", err
}
}
gz := gzip.NewWriter(sealed)
tw := tar.NewWriter(gz)
manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
@@ -124,7 +142,7 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
return "", fmt.Errorf("%s: %w", etcDir, err)
}
if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
if err := errors.Join(tw.Close(), gz.Close(), sealed.Close(), f.Sync(), f.Close()); err != nil {
return "", err
}
if err := os.Rename(part, final); err != nil {
@@ -134,6 +152,16 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
return final, nil
}
type nopCloser struct{ io.Writer }
func (nopCloser) Close() error { return nil }
// isBackup reports whether a file name is an archive this command wrote.
func isBackup(name string) bool {
return strings.HasPrefix(name, backupPrefix) &&
(strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tar.gz.age"))
}
// addStore dumps the store into the archive.
func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
addr, err := store.ResolveAddress(addr)
@@ -280,7 +308,7 @@ func pruneBackups(dir string, keep int) (int, error) {
}
var names []string
for _, e := range entries {
if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
if n := e.Name(); isBackup(n) {
names = append(names, n)
}
}
@@ -306,7 +334,7 @@ func NewestBackup(dir string) time.Time {
}
var newest time.Time
for _, e := range entries {
if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
if n := e.Name(); !isBackup(n) {
continue
}
if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
+78 -2
View File
@@ -3,6 +3,7 @@ package cli
import (
"archive/tar"
"compress/gzip"
"crypto/ed25519"
"io"
"os"
"path/filepath"
@@ -10,9 +11,84 @@ import (
"testing"
"time"
agecrypt "filippo.io/age"
"golang.org/x/crypto/ssh"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestABackupIsEncryptedToItsRecipients(t *testing.T) {
tmp := t.TempDir()
etc := filepath.Join(tmp, "etc")
_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("THE-CA-KEY"), 0o600)
id, err := agecrypt.GenerateX25519Identity()
if err != nil {
t.Fatal(err)
}
to, err := config.ParseBackupRecipients([]string{id.Recipient().String()})
if err != nil {
t.Fatal(err)
}
out := filepath.Join(tmp, "backups")
path, err := writeBackup(t.Context(), "", etc, out, to)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(path, ".tar.gz.age") {
t.Errorf("named %s", filepath.Base(path))
}
raw, _ := os.ReadFile(path)
if strings.Contains(string(raw), "THE-CA-KEY") {
t.Fatal("the CA key is in the archive in the clear")
}
f, _ := os.Open(path)
defer f.Close()
plain, err := agecrypt.Decrypt(f, id)
if err != nil {
t.Fatalf("the recipient cannot open it: %v", err)
}
gz, err := gzip.NewReader(plain)
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(gz)
found := false
for {
h, err := tr.Next()
if err != nil {
break
}
found = found || h.Name == "etc/llmgw/pki/ca-key.pem"
}
if !found {
t.Error("the opened archive lacks the CA key")
}
// Seen and pruned like any other.
if NewestBackup(out).IsZero() {
t.Error("the control plane would not see an encrypted archive")
}
}
func TestBackupRecipientsMustBeKeys(t *testing.T) {
if _, err := config.ParseBackupRecipients([]string{"not a key"}); err == nil {
t.Error("nonsense was taken as a recipient")
}
pub, _, _ := ed25519.GenerateKey(nil)
sp, err := ssh.NewPublicKey(pub)
if err != nil {
t.Fatal(err)
}
if _, err := config.ParseBackupRecipients([]string{
strings.TrimSpace(string(ssh.MarshalAuthorizedKey(sp))) + " ops@example",
}); err != nil {
t.Errorf("an SSH key was refused: %v", err)
}
}
func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
tmp := t.TempDir()
dbPath := filepath.Join(tmp, "store.db")
@@ -32,7 +108,7 @@ func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
out := filepath.Join(tmp, "backups")
path, err := writeBackup(t.Context(), dbPath, etc, out)
path, err := writeBackup(t.Context(), dbPath, etc, out, nil)
if err != nil {
t.Fatal(err)
}
@@ -129,7 +205,7 @@ func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
}
_ = db.Close()
out := filepath.Join(tmp, "backups")
if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil {
if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out, nil); err != nil {
t.Fatal(err)
}
entries, _ := os.ReadDir(out)