llmgw/internal/cli/backup.go

package cli

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"time"

	agecrypt "filippo.io/age"
	"github.com/spf13/cobra"

	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)

// DefaultBackupDir is where backups are written and where the control plane
// looks for the newest one.
//
// Its own directory rather than /var/backups/llmgw, which a PostgreSQL move
// had already made for its own dumps, owned by postgres and closed to
// everyone else — so the control plane could not see a backup landing in it.
const DefaultBackupDir = "/var/backups/llmgw-server"

func newBackupCmd() *cobra.Command {
	var (
		configPath string
		etcDir     string
		dir        string
		keep       int
	)
	c := &cobra.Command{
		Use:   "backup",
		Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
		Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
			"people, quotas, usage, the models added at runtime, the secret engine\n" +
			"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
			"store's address and the certificate authority's key.\n\n" +
			"The archive holds that key, so it is written readable by root only, and\n" +
			"encrypted to backup_recipients in server.yaml when they are set (age\n" +
			"recipients or SSH public keys; open it with `age -d -i <key>`). It is on\n" +
			"the same disk as what it backs up: copy it elsewhere, or it does not\n" +
			"survive the loss of this machine.\n\n" +
			"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
			"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
			"BackupTooOld fires when it is more than a day old.",
		RunE: func(cmd *cobra.Command, _ []string) error {
			cfg, err := config.Load(configPath)
			if err != nil {
				return err
			}
			to, err := config.ParseBackupRecipients(cfg.BackupRecipients)
			if err != nil {
				return err
			}
			path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir, to)
			if err != nil {
				return err
			}
			if why := unlistable(dir); why != "" {
				fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s; the control plane will report no backup "+
					"and BackupTooOld will fire\n", why)
			}
			removed, err := pruneBackups(dir, keep)
			if err != nil {
				return err
			}
			if len(cfg.BackupCopyTo) > 0 {
				if err := copyBackup(cmd.Context(), path, cfg.BackupCopyTo, cfg.BackupCopyKey,
					filepath.Join(etcDir, "backup_known_hosts"), keep,
					func(s string) { fmt.Fprintln(cmd.OutOrStdout(), s) }); err != nil {
					// The archive is here; that it did not travel is said, and
					// the exit says it failed, so the timer's unit shows it.
					fmt.Fprintf(cmd.OutOrStdout(), "wrote %s\n", path)
					return fmt.Errorf("the archive was not copied off this machine: %w", err)
				}
			}
			fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
			if removed > 0 {
				fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
			}
			fmt.Fprintln(cmd.OutOrStdout())
			return nil
		},
	}
	c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
	c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
	c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
	c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
	return c
}

const backupPrefix = "llmgw-backup-"

// writeBackup writes one archive and returns its path. Written beside its
// final name and renamed, so a backup interrupted halfway is never the newest
// one the control plane reports.
//
// With recipients, the archive is encrypted to them, and named .tar.gz.age.
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string, recipients []agecrypt.Recipient) (string, error) {
	// Listable by anyone, so the control plane — not root — can see when the
	// newest archive was written. The archives themselves are root's alone.
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return "", err
	}
	host, _ := os.Hostname()
	name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
	if len(recipients) > 0 {
		name += ".age"
	}
	final := filepath.Join(dir, name)
	part := final + ".part"

	f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
	if err != nil {
		return "", err
	}
	ok := false
	defer func() {
		if !ok {
			_ = os.Remove(part)
		}
	}()
	var sealed io.WriteCloser = nopCloser{f}
	if len(recipients) > 0 {
		if sealed, err = agecrypt.Encrypt(f, recipients...); err != nil {
			return "", err
		}
	}
	gz := gzip.NewWriter(sealed)
	tw := tar.NewWriter(gz)

	manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
		time.Now().UTC().Format(time.RFC3339))
	if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
		return "", err
	}
	if storeAddr != "" {
		if err := addStore(ctx, tw, storeAddr, dir); err != nil {
			return "", fmt.Errorf("the store: %w", err)
		}
	}
	if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
		return "", fmt.Errorf("%s: %w", etcDir, err)
	}
	if err := errors.Join(tw.Close(), gz.Close(), sealed.Close(), f.Sync(), f.Close()); err != nil {
		return "", err
	}
	if err := os.Rename(part, final); err != nil {
		return "", err
	}
	ok = true
	return final, nil
}

type nopCloser struct{ io.Writer }

func (nopCloser) Close() error { return nil }

// isBackup reports whether a file name is an archive this command wrote.
func isBackup(name string) bool {
	return strings.HasPrefix(name, backupPrefix) &&
		(strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tar.gz.age"))
}

// addStore dumps the store into the archive.
func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
	addr, err := store.ResolveAddress(addr)
	if err != nil {
		return err
	}
	// Inside a directory only its owner can enter. The archive directory is
	// listable on purpose, and VACUUM INTO creates its file with the default
	// mode: made there directly, a full copy of the store — password hashes,
	// the engine-key secret — sat readable by any local user for as long as
	// the backup took.
	tmpDir, err := os.MkdirTemp(scratch, ".store-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	tmpPath := filepath.Join(tmpDir, "store")

	name := "store.sqlite"
	if store.IsPostgres(addr) {
		name = "store.pgdump"
		if err := pgDump(ctx, addr, tmpPath); err != nil {
			return err
		}
	} else {
		db, err := store.Open(ctx, addr)
		if err != nil {
			return err
		}
		err = db.SnapshotTo(ctx, tmpPath)
		_ = db.Close()
		if err != nil {
			return err
		}
	}
	return addFile(tw, tmpPath, name)
}

// pgDump runs pg_dump in its custom format, which pg_restore can restore
// selectively. The connection is given through the environment, not the
// command line: a DSN carries the password, and every user on the machine
// can read another process's arguments.
func pgDump(ctx context.Context, dsn, out string) error {
	vars, err := pgEnv(dsn)
	if err != nil {
		return err
	}
	cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
	cmd.Env = append(os.Environ(), vars...)
	if b, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
	}
	return nil
}

// pgEnv turns a store address into libpq's environment. Several hosts — the
// members of a replicated cluster — become libpq's own lists in PGHOST and
// PGPORT; read as one URL host, the list was one name libpq could not
// resolve, and a backup worked only by falling through to the second.
func pgEnv(dsn string) ([]string, error) {
	hosts, ports, u, err := store.SplitPostgres(dsn)
	if err != nil {
		return nil, err
	}
	var env []string
	set := func(k, v string) {
		if v != "" {
			env = append(env, k+"="+v)
		}
	}
	set("PGHOST", strings.Join(hosts, ","))
	set("PGPORT", strings.Join(ports, ","))
	set("PGUSER", u.User.Username())
	if pw, ok := u.User.Password(); ok {
		set("PGPASSWORD", pw)
	}
	set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
	set("PGSSLMODE", u.Query().Get("sslmode"))
	// A standby if there is one, else whichever answers — not the address's
	// read-write: a dump only reads, it need not load the primary, and one
	// that insists on the primary fails exactly while the cluster is failing
	// over.
	if len(hosts) > 1 {
		set("PGTARGETSESSIONATTRS", "prefer-standby")
	}
	return env, nil
}

func addTree(tw *tar.Writer, root, as string) error {
	return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, _ := filepath.Rel(root, p)
		name := filepath.ToSlash(filepath.Join(as, rel))
		info, err := d.Info()
		if err != nil {
			return err
		}
		switch {
		case d.IsDir():
			h := ownedHeader(info)
			h.Name, h.Mode, h.Typeflag, h.ModTime = name+"/", int64(info.Mode().Perm()), tar.TypeDir, info.ModTime()
			return tw.WriteHeader(h)
		case info.Mode().IsRegular():
			return addFile(tw, p, name)
		}
		return nil // sockets, links: not configuration
	})
}

func addFile(tw *tar.Writer, path, name string) error {
	f, err := os.Open(path) //nolint:gosec // files we chose to back up
	if err != nil {
		return err
	}
	defer f.Close()
	info, err := f.Stat()
	if err != nil {
		return err
	}
	h := ownedHeader(info)
	h.Name, h.Mode, h.Size, h.ModTime, h.Typeflag = name, int64(info.Mode().Perm()), info.Size(), info.ModTime(), tar.TypeReg
	if err := tw.WriteHeader(h); err != nil {
		return err
	}
	_, err = io.Copy(tw, f)
	return err
}

// ownedHeader is a header with the file's owner and group by name, which is
// what a restore on another machine can give back: the llmgw user there has
// another number.
func ownedHeader(info fs.FileInfo) *tar.Header {
	h, err := tar.FileInfoHeader(info, "")
	if err != nil {
		return &tar.Header{}
	}
	return &tar.Header{Uid: h.Uid, Gid: h.Gid, Uname: h.Uname, Gname: h.Gname}
}

func addBytes(tw *tar.Writer, name string, b []byte) error {
	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
		ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
		return err
	}
	_, err := tw.Write(b)
	return err
}

// pruneBackups keeps the newest keep archives. Only files this command
// wrote are considered, so nothing else in the directory is ever removed.
func pruneBackups(dir string, keep int) (int, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return 0, err
	}
	var names []string
	for _, e := range entries {
		if n := e.Name(); isBackup(n) {
			names = append(names, n)
		}
	}
	// The name carries a UTC timestamp, so name order is age order.
	slices.Sort(names)
	removed := 0
	for len(names) > max(keep, 1) {
		if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
			return removed, err
		}
		names = names[1:]
		removed++
	}
	return removed, nil
}

// NewestBackup is the modification time of the newest complete archive in
// dir, or zero when there is none.
func NewestBackup(dir string) time.Time {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return time.Time{}
	}
	var newest time.Time
	for _, e := range entries {
		if n := e.Name(); !isBackup(n) {
			continue
		}
		if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
			newest = info.ModTime()
		}
	}
	return newest
}

// unlistable says why the control plane, which does not run as root, could
// not see the archives in dir, or "" when it can.
func unlistable(dir string) string {
	info, err := os.Stat(dir)
	if err != nil {
		return err.Error()
	}
	if info.Mode().Perm()&0o005 != 0o005 {
		return fmt.Sprintf("%s is %v, so only its owner can list it", dir, info.Mode().Perm())
	}
	return ""
}