Encrypt backups to backup_recipients with age
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3b05ec0538
commit
12efb7d085
10 files changed
+204
-12
No files matched your search
+37
-9
@@ -15,6 +15,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
agecrypt "filippo.io/age"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
@@ -44,9 +45,11 @@ func newBackupCmd() *cobra.Command {
|
||||
"people, quotas, usage, the models added at runtime, the secret engine\n" +
|
||||
"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
|
||||
"store's address and the certificate authority's key.\n\n" +
|
||||
"The archive holds that key, so it is written readable by root only. It\n" +
|
||||
"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
|
||||
"not survive the loss of this machine.\n\n" +
|
||||
"The archive holds that key, so it is written readable by root only, and\n" +
|
||||
"encrypted to backup_recipients in server.yaml when they are set (age\n" +
|
||||
"recipients or SSH public keys; open it with `age -d -i <key>`). It is on\n" +
|
||||
"the same disk as what it backs up: copy it elsewhere, or it does not\n" +
|
||||
"survive the loss of this machine.\n\n" +
|
||||
"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
|
||||
"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
|
||||
"BackupTooOld fires when it is more than a day old.",
|
||||
@@ -55,7 +58,11 @@ func newBackupCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
|
||||
to, err := config.ParseBackupRecipients(cfg.BackupRecipients)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir, to)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -87,7 +94,9 @@ const backupPrefix = "llmgw-backup-"
|
||||
// writeBackup writes one archive and returns its path. Written beside its
|
||||
// final name and renamed, so a backup interrupted halfway is never the newest
|
||||
// one the control plane reports.
|
||||
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
|
||||
//
|
||||
// With recipients, the archive is encrypted to them, and named .tar.gz.age.
|
||||
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string, recipients []agecrypt.Recipient) (string, error) {
|
||||
// Listable by anyone, so the control plane — not root — can see when the
|
||||
// newest archive was written. The archives themselves are root's alone.
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
@@ -95,6 +104,9 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
|
||||
if len(recipients) > 0 {
|
||||
name += ".age"
|
||||
}
|
||||
final := filepath.Join(dir, name)
|
||||
part := final + ".part"
|
||||
|
||||
@@ -108,7 +120,13 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
|
||||
_ = os.Remove(part)
|
||||
}
|
||||
}()
|
||||
gz := gzip.NewWriter(f)
|
||||
var sealed io.WriteCloser = nopCloser{f}
|
||||
if len(recipients) > 0 {
|
||||
if sealed, err = agecrypt.Encrypt(f, recipients...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
gz := gzip.NewWriter(sealed)
|
||||
tw := tar.NewWriter(gz)
|
||||
|
||||
manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
|
||||
@@ -124,7 +142,7 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
|
||||
if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
|
||||
return "", fmt.Errorf("%s: %w", etcDir, err)
|
||||
}
|
||||
if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
|
||||
if err := errors.Join(tw.Close(), gz.Close(), sealed.Close(), f.Sync(), f.Close()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(part, final); err != nil {
|
||||
@@ -134,6 +152,16 @@ func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, er
|
||||
return final, nil
|
||||
}
|
||||
|
||||
type nopCloser struct{ io.Writer }
|
||||
|
||||
func (nopCloser) Close() error { return nil }
|
||||
|
||||
// isBackup reports whether a file name is an archive this command wrote.
|
||||
func isBackup(name string) bool {
|
||||
return strings.HasPrefix(name, backupPrefix) &&
|
||||
(strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tar.gz.age"))
|
||||
}
|
||||
|
||||
// addStore dumps the store into the archive.
|
||||
func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
|
||||
addr, err := store.ResolveAddress(addr)
|
||||
@@ -280,7 +308,7 @@ func pruneBackups(dir string, keep int) (int, error) {
|
||||
}
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
|
||||
if n := e.Name(); isBackup(n) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
@@ -306,7 +334,7 @@ func NewestBackup(dir string) time.Time {
|
||||
}
|
||||
var newest time.Time
|
||||
for _, e := range entries {
|
||||
if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
|
||||
if n := e.Name(); !isBackup(n) {
|
||||
continue
|
||||
}
|
||||
if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"crypto/ed25519"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,9 +11,84 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agecrypt "filippo.io/age"
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestABackupIsEncryptedToItsRecipients(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
etc := filepath.Join(tmp, "etc")
|
||||
_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
|
||||
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("THE-CA-KEY"), 0o600)
|
||||
|
||||
id, err := agecrypt.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
to, err := config.ParseBackupRecipients([]string{id.Recipient().String()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(tmp, "backups")
|
||||
path, err := writeBackup(t.Context(), "", etc, out, to)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(path, ".tar.gz.age") {
|
||||
t.Errorf("named %s", filepath.Base(path))
|
||||
}
|
||||
raw, _ := os.ReadFile(path)
|
||||
if strings.Contains(string(raw), "THE-CA-KEY") {
|
||||
t.Fatal("the CA key is in the archive in the clear")
|
||||
}
|
||||
|
||||
f, _ := os.Open(path)
|
||||
defer f.Close()
|
||||
plain, err := agecrypt.Decrypt(f, id)
|
||||
if err != nil {
|
||||
t.Fatalf("the recipient cannot open it: %v", err)
|
||||
}
|
||||
gz, err := gzip.NewReader(plain)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
found := false
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
found = found || h.Name == "etc/llmgw/pki/ca-key.pem"
|
||||
}
|
||||
if !found {
|
||||
t.Error("the opened archive lacks the CA key")
|
||||
}
|
||||
// Seen and pruned like any other.
|
||||
if NewestBackup(out).IsZero() {
|
||||
t.Error("the control plane would not see an encrypted archive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRecipientsMustBeKeys(t *testing.T) {
|
||||
if _, err := config.ParseBackupRecipients([]string{"not a key"}); err == nil {
|
||||
t.Error("nonsense was taken as a recipient")
|
||||
}
|
||||
pub, _, _ := ed25519.GenerateKey(nil)
|
||||
sp, err := ssh.NewPublicKey(pub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := config.ParseBackupRecipients([]string{
|
||||
strings.TrimSpace(string(ssh.MarshalAuthorizedKey(sp))) + " ops@example",
|
||||
}); err != nil {
|
||||
t.Errorf("an SSH key was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
dbPath := filepath.Join(tmp, "store.db")
|
||||
@@ -32,7 +108,7 @@ func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
|
||||
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
|
||||
|
||||
out := filepath.Join(tmp, "backups")
|
||||
path, err := writeBackup(t.Context(), dbPath, etc, out)
|
||||
path, err := writeBackup(t.Context(), dbPath, etc, out, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -129,7 +205,7 @@ func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
|
||||
}
|
||||
_ = db.Close()
|
||||
out := filepath.Join(tmp, "backups")
|
||||
if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil {
|
||||
if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, _ := os.ReadDir(out)
|
||||
|
||||
Reference in new issue
Block a user