Files
llmgw/internal/cli/backup.go
T
Ilker ManapandClaude Opus 5.5 9bf8256577
Release / build (push) Successful in 1m1s
Release / notes (push) Successful in 2s
CI / test (push) Successful in 3m35s
Keep backups where the control plane can see them
The first backup on the live control plane was written, and the control
plane reported none: /var/backups/llmgw already existed, made by an earlier
move to PostgreSQL for its own dumps, owned by postgres and 0700. MkdirAll
leaves an existing directory as it is, so the archive landed where the
llmgw user could not list it, and llmgw_backup_last_success_timestamp_seconds
said 0.

Opening that directory up would have exposed the dumps already in it, so
backups get their own, /var/backups/llmgw-server. And `llmgw server backup`
now warns when the directory it wrote to is one the control plane cannot
list, rather than leaving that to be found from an alert.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 01:50:33 +02:00

311 lines
9.0 KiB
Go

package cli
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// DefaultBackupDir is where backups are written and where the control plane
// looks for the newest one.
//
// Its own directory rather than /var/backups/llmgw, which a PostgreSQL move
// had already made for its own dumps, owned by postgres and closed to
// everyone else — so the control plane could not see a backup landing in it.
const DefaultBackupDir = "/var/backups/llmgw-server"
func newBackupCmd() *cobra.Command {
var (
configPath string
etcDir string
dir string
keep int
)
c := &cobra.Command{
Use: "backup",
Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
"people, quotas, usage, the models added at runtime, the secret engine\n" +
"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
"store's address and the certificate authority's key.\n\n" +
"The archive holds that key, so it is written readable by root only. It\n" +
"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
"not survive the loss of this machine.\n\n" +
"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
"BackupTooOld fires when it is more than a day old.",
RunE: func(cmd *cobra.Command, _ []string) error {
cfg, err := config.Load(configPath)
if err != nil {
return err
}
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
if err != nil {
return err
}
if why := unlistable(dir); why != "" {
fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s; the control plane will report no backup "+
"and BackupTooOld will fire\n", why)
}
removed, err := pruneBackups(dir, keep)
if err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
if removed > 0 {
fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
}
fmt.Fprintln(cmd.OutOrStdout())
return nil
},
}
c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
return c
}
const backupPrefix = "llmgw-backup-"
// writeBackup writes one archive and returns its path. Written beside its
// final name and renamed, so a backup interrupted halfway is never the newest
// one the control plane reports.
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
// Listable by anyone, so the control plane — not root — can see when the
// newest archive was written. The archives themselves are root's alone.
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
host, _ := os.Hostname()
name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
final := filepath.Join(dir, name)
part := final + ".part"
f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
if err != nil {
return "", err
}
ok := false
defer func() {
if !ok {
_ = os.Remove(part)
}
}()
gz := gzip.NewWriter(f)
tw := tar.NewWriter(gz)
manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
time.Now().UTC().Format(time.RFC3339))
if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
return "", err
}
if storeAddr != "" {
if err := addStore(ctx, tw, storeAddr, dir); err != nil {
return "", fmt.Errorf("the store: %w", err)
}
}
if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
return "", fmt.Errorf("%s: %w", etcDir, err)
}
if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
return "", err
}
if err := os.Rename(part, final); err != nil {
return "", err
}
ok = true
return final, nil
}
// addStore dumps the store into the archive.
func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
addr, err := store.ResolveAddress(addr)
if err != nil {
return err
}
tmp, err := os.CreateTemp(scratch, ".store-*")
if err != nil {
return err
}
tmpPath := tmp.Name()
_ = tmp.Close()
defer os.Remove(tmpPath)
name := "store.sqlite"
if store.IsPostgres(addr) {
name = "store.pgdump"
if err := pgDump(ctx, addr, tmpPath); err != nil {
return err
}
} else {
// VACUUM INTO refuses a file that exists.
_ = os.Remove(tmpPath)
db, err := store.Open(ctx, addr)
if err != nil {
return err
}
err = db.SnapshotTo(ctx, tmpPath)
_ = db.Close()
if err != nil {
return err
}
}
return addFile(tw, tmpPath, name)
}
// pgDump runs pg_dump in its custom format, which pg_restore can restore
// selectively. The connection is given through the environment, not the
// command line: a DSN carries the password, and every user on the machine
// can read another process's arguments.
func pgDump(ctx context.Context, dsn, out string) error {
u, err := url.Parse(dsn)
if err != nil {
return fmt.Errorf("the store address is not a URL")
}
env := os.Environ()
set := func(k, v string) {
if v != "" {
env = append(env, k+"="+v)
}
}
set("PGHOST", u.Hostname())
set("PGPORT", u.Port())
set("PGUSER", u.User.Username())
if pw, ok := u.User.Password(); ok {
set("PGPASSWORD", pw)
}
set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
set("PGSSLMODE", u.Query().Get("sslmode"))
cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
cmd.Env = env
if b, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
}
return nil
}
func addTree(tw *tar.Writer, root, as string) error {
return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, _ := filepath.Rel(root, p)
name := filepath.ToSlash(filepath.Join(as, rel))
info, err := d.Info()
if err != nil {
return err
}
switch {
case d.IsDir():
return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
Typeflag: tar.TypeDir, ModTime: info.ModTime()})
case info.Mode().IsRegular():
return addFile(tw, p, name)
}
return nil // sockets, links: not configuration
})
}
func addFile(tw *tar.Writer, path, name string) error {
f, err := os.Open(path) //nolint:gosec // files we chose to back up
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
return err
}
_, err = io.Copy(tw, f)
return err
}
func addBytes(tw *tar.Writer, name string, b []byte) error {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
return err
}
_, err := tw.Write(b)
return err
}
// pruneBackups keeps the newest keep archives. Only files this command
// wrote are considered, so nothing else in the directory is ever removed.
func pruneBackups(dir string, keep int) (int, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return 0, err
}
var names []string
for _, e := range entries {
if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
names = append(names, n)
}
}
// The name carries a UTC timestamp, so name order is age order.
slices.Sort(names)
removed := 0
for len(names) > max(keep, 1) {
if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
return removed, err
}
names = names[1:]
removed++
}
return removed, nil
}
// NewestBackup is the modification time of the newest complete archive in
// dir, or zero when there is none.
func NewestBackup(dir string) time.Time {
entries, err := os.ReadDir(dir)
if err != nil {
return time.Time{}
}
var newest time.Time
for _, e := range entries {
if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
continue
}
if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
newest = info.ModTime()
}
}
return newest
}
// unlistable says why the control plane, which does not run as root, could
// not see the archives in dir, or "" when it can.
func unlistable(dir string) string {
info, err := os.Stat(dir)
if err != nil {
return err.Error()
}
if info.Mode().Perm()&0o005 != 0o005 {
return fmt.Sprintf("%s is %v, so only its owner can list it", dir, info.Mode().Perm())
}
return ""
}