server restore: an archive checked, dry-run, and put back — configuration with modes and owners, the store in SQLite or PostgreSQL; backup_copy_to copies archives off the machine
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d29f694aaf
commit
10466416df
11 files changed
+889
-25
No files matched your search
+27
-4
@@ -74,6 +74,16 @@ func newBackupCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cfg.BackupCopyTo) > 0 {
|
||||
if err := copyBackup(cmd.Context(), path, cfg.BackupCopyTo, cfg.BackupCopyKey,
|
||||
filepath.Join(etcDir, "backup_known_hosts"), keep,
|
||||
func(s string) { fmt.Fprintln(cmd.OutOrStdout(), s) }); err != nil {
|
||||
// The archive is here; that it did not travel is said, and
|
||||
// the exit says it failed, so the timer's unit shows it.
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "wrote %s\n", path)
|
||||
return fmt.Errorf("the archive was not copied off this machine: %w", err)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
|
||||
if removed > 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
|
||||
@@ -263,8 +273,9 @@ func addTree(tw *tar.Writer, root, as string) error {
|
||||
}
|
||||
switch {
|
||||
case d.IsDir():
|
||||
return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
|
||||
Typeflag: tar.TypeDir, ModTime: info.ModTime()})
|
||||
h := ownedHeader(info)
|
||||
h.Name, h.Mode, h.Typeflag, h.ModTime = name+"/", int64(info.Mode().Perm()), tar.TypeDir, info.ModTime()
|
||||
return tw.WriteHeader(h)
|
||||
case info.Mode().IsRegular():
|
||||
return addFile(tw, p, name)
|
||||
}
|
||||
@@ -282,14 +293,26 @@ func addFile(tw *tar.Writer, path, name string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
|
||||
Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
|
||||
h := ownedHeader(info)
|
||||
h.Name, h.Mode, h.Size, h.ModTime, h.Typeflag = name, int64(info.Mode().Perm()), info.Size(), info.ModTime(), tar.TypeReg
|
||||
if err := tw.WriteHeader(h); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(tw, f)
|
||||
return err
|
||||
}
|
||||
|
||||
// ownedHeader is a header with the file's owner and group by name, which is
|
||||
// what a restore on another machine can give back: the llmgw user there has
|
||||
// another number.
|
||||
func ownedHeader(info fs.FileInfo) *tar.Header {
|
||||
h, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return &tar.Header{}
|
||||
}
|
||||
return &tar.Header{Uid: h.Uid, Gid: h.Gid, Uname: h.Uname, Gname: h.Gname}
|
||||
}
|
||||
|
||||
func addBytes(tw *tar.Writer, name string, b []byte) error {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
|
||||
ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
)
|
||||
|
||||
// copyBackup puts an archive where server.yaml's backup_copy_to says, and
|
||||
// keeps as many there as here. It says each copy, and fails on the first
|
||||
// that cannot be made.
|
||||
func copyBackup(ctx context.Context, archive string, targets []string, key, knownHosts string, keep int, say func(string)) error {
|
||||
for _, t := range targets {
|
||||
if strings.HasPrefix(t, "ssh://") {
|
||||
if err := copyOverSSH(ctx, archive, t, key, knownHosts, keep); err != nil {
|
||||
return fmt.Errorf("%s: %w", t, err)
|
||||
}
|
||||
} else {
|
||||
if err := copyToDir(archive, t, keep); err != nil {
|
||||
return fmt.Errorf("%s: %w", t, err)
|
||||
}
|
||||
}
|
||||
say("copied to " + t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyToDir(archive, dir string, keep int) error {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := os.ReadFile(archive) //nolint:gosec // the archive just written
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
final := filepath.Join(dir, filepath.Base(archive))
|
||||
if err := os.WriteFile(final+".part", b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(final+".part", final); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = pruneBackups(dir, keep)
|
||||
return err
|
||||
}
|
||||
|
||||
func copyOverSSH(ctx context.Context, archive, target, key, knownHosts string, keep int) error {
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.User == nil || u.Host == "" || u.Path == "" {
|
||||
return fmt.Errorf("not ssh://user@host/path")
|
||||
}
|
||||
port := 22
|
||||
if p := u.Port(); p != "" {
|
||||
port, _ = strconv.Atoi(p)
|
||||
}
|
||||
b, err := os.ReadFile(archive) //nolint:gosec // the archive just written
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := &onboard.GoSSH{Addr: u.Hostname(), Port: port, User: u.User.Username(), KeyFile: key, KnownHosts: knownHosts}
|
||||
defer c.Close()
|
||||
dir, name := shellQuote(u.Path), shellQuote(filepath.Base(archive))
|
||||
// Written beside its name and renamed, then the oldest beyond keep
|
||||
// removed: only archives this command wrote, by their names.
|
||||
script := fmt.Sprintf(`set -e
|
||||
umask 077
|
||||
mkdir -p %[1]s
|
||||
cat > %[1]s/%[2]s.part
|
||||
mv %[1]s/%[2]s.part %[1]s/%[2]s
|
||||
ls -1t %[1]s | grep -E '^llmgw-backup-.*\.tar\.gz(\.age)?$' | tail -n +%[3]d | while read -r f; do rm -f %[1]s/"$f"; done
|
||||
`, dir, name, keep+1)
|
||||
if out, err := c.Run(ctx, script, b); err != nil {
|
||||
return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
|
||||
@@ -0,0 +1,546 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
agecrypt "filippo.io/age"
|
||||
"filippo.io/age/agessh"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
// restore is `llmgw server restore`: an archive `server backup` wrote, put
|
||||
// back on this machine.
|
||||
type restore struct {
|
||||
archive string
|
||||
identities []string
|
||||
etcDir string
|
||||
storeAddr string // overrides the restored server.yaml's
|
||||
dryRun bool
|
||||
force bool
|
||||
// service is the unit stopped before and started after; "" touches no
|
||||
// service, for tests and machines without systemd.
|
||||
service string
|
||||
out io.Writer
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newRestoreCmd() *cobra.Command {
|
||||
rs := restore{etcDir: "/etc/llmgw", service: "llmgw-server", now: time.Now}
|
||||
c := &cobra.Command{
|
||||
Use: "restore ARCHIVE",
|
||||
Short: "Put a backup back: /etc/llmgw and the store, on this machine",
|
||||
Long: "Reads an archive `llmgw server backup` wrote — opening an encrypted one\n" +
|
||||
"with --identity, an age identity file or an SSH private key — and checks\n" +
|
||||
"it before anything is touched: that it is whole, that this llmgw is not\n" +
|
||||
"older than the one that wrote it, and where the store goes. --dry-run\n" +
|
||||
"stops there and says what it would do.\n\n" +
|
||||
"Then: the server is stopped, /etc/llmgw is moved aside to\n" +
|
||||
"/etc/llmgw.before-restore-<time> and the archive's put in its place, the\n" +
|
||||
"store is restored — a SQLite file replaced, the old one kept beside it;\n" +
|
||||
"PostgreSQL by pg_restore --clean into the database the address names,\n" +
|
||||
"which must exist — and the server is started and asked whether it\n" +
|
||||
"answers.\n\n" +
|
||||
"Running engines keep serving throughout: their agents reconnect, and the\n" +
|
||||
"engine-key secret came back with the store. For a whole drill, see\n" +
|
||||
"\"Restoring\" in docs/install.md.",
|
||||
Example: " sudo llmgw server restore /var/backups/llmgw-server/llmgw-backup-ctl-20261004T020000Z.tar.gz --dry-run\n" +
|
||||
" sudo llmgw server restore backup.tar.gz.age --identity ~/llmgw-backup-key.txt",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
rs.archive, rs.out = args[0], cmd.OutOrStdout()
|
||||
return rs.run(cmd.Context())
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringArrayVar(&rs.identities, "identity", nil, "a key that opens an encrypted archive: an age identity file or an SSH private key; repeatable")
|
||||
f.StringVar(&rs.etcDir, "etc", rs.etcDir, "the configuration directory to restore into")
|
||||
f.StringVar(&rs.storeAddr, "store", "", "restore the store here rather than where the archive's server.yaml says: a SQLite path or a postgres:// address")
|
||||
f.BoolVar(&rs.dryRun, "dry-run", false, "check the archive and say what would be done; change nothing")
|
||||
f.BoolVar(&rs.force, "force", false, "restore an archive written by a newer llmgw")
|
||||
f.StringVar(&rs.service, "service", rs.service, "the unit stopped before and started after; empty touches none")
|
||||
return c
|
||||
}
|
||||
|
||||
func (rs *restore) say(format string, a ...any) { fmt.Fprintf(rs.out, format+"\n", a...) }
|
||||
|
||||
func (rs *restore) run(ctx context.Context) error {
|
||||
tmp, err := os.MkdirTemp("", "llmgw-restore-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
// 1. The archive, opened and unpacked where only this user can read it.
|
||||
man, err := rs.unpack(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rs.say("archive written by llmgw %s on %s at %s", man.version, man.host, man.time)
|
||||
if newer(man.version, version.Version) && !rs.force {
|
||||
return fmt.Errorf("the archive was written by llmgw %s, newer than this one (%s): its store may have "+
|
||||
"changes this one does not know. Install %s or newer first, or pass --force", man.version, version.Version, man.version)
|
||||
}
|
||||
restoredEtc := filepath.Join(tmp, "etc", "llmgw")
|
||||
if _, err := os.Stat(filepath.Join(restoredEtc, "server.yaml")); err != nil {
|
||||
return errors.New("the archive has no etc/llmgw/server.yaml: not a control plane's backup")
|
||||
}
|
||||
n := 0
|
||||
_ = filepath.WalkDir(restoredEtc, func(_ string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
n++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
rs.say("config %d files for %s", n, rs.etcDir)
|
||||
|
||||
// 2. Where the store goes, and what it is.
|
||||
dump, kind := filepath.Join(tmp, "store.sqlite"), "sqlite"
|
||||
if _, err := os.Stat(dump); err != nil {
|
||||
dump, kind = filepath.Join(tmp, "store.pgdump"), "postgres"
|
||||
if _, err := os.Stat(dump); err != nil {
|
||||
dump, kind = "", ""
|
||||
}
|
||||
}
|
||||
target, err := rs.storeTarget(restoredEtc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case kind == "":
|
||||
rs.say("store none in the archive; nothing to restore")
|
||||
case kind == "postgres" && !store.IsPostgres(target):
|
||||
return fmt.Errorf("the archive holds a PostgreSQL dump and the store goes to %s: "+
|
||||
"give --store a postgres:// address", target)
|
||||
case kind == "sqlite" && store.IsPostgres(target):
|
||||
return fmt.Errorf("the archive holds a SQLite store and the store goes to PostgreSQL: give --store a file, " +
|
||||
"then `llmgw keys move-store` copies it into PostgreSQL")
|
||||
case kind == "postgres":
|
||||
if _, err := exec.LookPath("pg_restore"); err != nil {
|
||||
return errors.New("the store is a PostgreSQL dump and pg_restore is not installed here (apt-get install postgresql-client)")
|
||||
}
|
||||
rs.say("store PostgreSQL dump into %s, with pg_restore --clean", redactDSN(target))
|
||||
default:
|
||||
rs.say("store SQLite file to %s%s", target, aside(target, rs.now()))
|
||||
}
|
||||
if rs.service != "" {
|
||||
if exec.Command("systemctl", "is-active", "--quiet", rs.service).Run() == nil { //nolint:gosec // our unit
|
||||
rs.say("service %s is running: stopped first, started after", rs.service)
|
||||
} else {
|
||||
rs.say("service %s is not running: started after", rs.service)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(rs.etcDir); err == nil {
|
||||
rs.say("current %s moved aside to %s", rs.etcDir, rs.etcDir+".before-restore-"+stamp(rs.now()))
|
||||
}
|
||||
if rs.dryRun {
|
||||
rs.say("\ndry run: nothing was changed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// 3. Done, in an order that can be undone by hand at every step.
|
||||
if rs.service != "" {
|
||||
_ = exec.CommandContext(ctx, "systemctl", "stop", rs.service).Run() //nolint:gosec // our unit
|
||||
}
|
||||
if _, err := os.Stat(rs.etcDir); err == nil {
|
||||
if err := os.Rename(rs.etcDir, rs.etcDir+".before-restore-"+stamp(rs.now())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := copyTree(restoredEtc, rs.etcDir); err != nil {
|
||||
return fmt.Errorf("putting back %s: %w", rs.etcDir, err)
|
||||
}
|
||||
rs.say("ok %s restored", rs.etcDir)
|
||||
switch kind {
|
||||
case "sqlite":
|
||||
if err := restoreSQLite(dump, target, rs.now()); err != nil {
|
||||
return fmt.Errorf("the store: %w", err)
|
||||
}
|
||||
rs.say("ok store restored to %s", target)
|
||||
case "postgres":
|
||||
if err := pgRestore(ctx, target, dump); err != nil {
|
||||
return fmt.Errorf("the store: %w", err)
|
||||
}
|
||||
rs.say("ok store restored into %s", redactDSN(target))
|
||||
}
|
||||
if rs.service != "" {
|
||||
if err := exec.CommandContext(ctx, "systemctl", "start", rs.service).Run(); err != nil { //nolint:gosec // our unit
|
||||
return fmt.Errorf("starting %s: %w; `journalctl -u %s` says why", rs.service, err, rs.service)
|
||||
}
|
||||
if err := rs.waitHealthy(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
rs.say("ok %s answers", rs.service)
|
||||
}
|
||||
rs.say("\nRestored. `llmgw ctl status` shows the cluster as it comes back; agents reconnect by themselves.")
|
||||
return nil
|
||||
}
|
||||
|
||||
type manifest struct{ version, host, time string }
|
||||
|
||||
// unpack opens the archive — decrypting it when it is sealed — and writes
|
||||
// its files under dir, keeping their modes and, by name, their owners.
|
||||
func (rs *restore) unpack(dir string) (manifest, error) {
|
||||
f, err := os.Open(rs.archive)
|
||||
if err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
var r io.Reader = bufio.NewReader(f)
|
||||
if strings.HasSuffix(rs.archive, ".age") {
|
||||
if len(rs.identities) == 0 {
|
||||
return manifest{}, errors.New("the archive is encrypted: give the key that opens it with --identity")
|
||||
}
|
||||
ids, err := readIdentities(rs.identities)
|
||||
if err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
if r, err = agecrypt.Decrypt(r, ids...); err != nil {
|
||||
return manifest{}, fmt.Errorf("opening the archive: %w", err)
|
||||
}
|
||||
}
|
||||
gz, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return manifest{}, fmt.Errorf("not an archive server backup wrote: %w", err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
var man manifest
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return manifest{}, fmt.Errorf("the archive is damaged: %w", err)
|
||||
}
|
||||
name := filepath.Clean(h.Name)
|
||||
if filepath.IsAbs(name) || strings.HasPrefix(name, "..") {
|
||||
return manifest{}, fmt.Errorf("the archive names %q, outside itself", h.Name)
|
||||
}
|
||||
p := filepath.Join(dir, name)
|
||||
switch h.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(p, 0o700); err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
out, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
_, err = io.Copy(out, tr)
|
||||
if cerr := out.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
if name == "MANIFEST" {
|
||||
b, _ := os.ReadFile(p)
|
||||
man = parseManifest(b)
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
// The mode and owner, applied when the tree is copied into place.
|
||||
if err := writeMeta(p, h); err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
}
|
||||
if man.version == "" {
|
||||
return manifest{}, errors.New("the archive has no MANIFEST: not one server backup wrote, or cut short")
|
||||
}
|
||||
return man, nil
|
||||
}
|
||||
|
||||
func parseManifest(b []byte) manifest {
|
||||
var m manifest
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
k, v, _ := strings.Cut(l, " ")
|
||||
switch k {
|
||||
case "llmgw":
|
||||
m.version = v
|
||||
case "host":
|
||||
m.host = v
|
||||
case "time":
|
||||
m.time = v
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// meta is a restored file's mode and owner, kept beside it until it is
|
||||
// copied into place.
|
||||
const metaSuffix = ".llmgw-restore-meta"
|
||||
|
||||
func writeMeta(p string, h *tar.Header) error {
|
||||
return os.WriteFile(p+metaSuffix, fmt.Appendf(nil, "%o %s %s %d %d", h.Mode&0o7777, h.Uname, h.Gname, h.Uid, h.Gid), 0o600)
|
||||
}
|
||||
|
||||
// copyTree puts the unpacked tree in place with each file's mode and owner.
|
||||
func copyTree(from, to string) error {
|
||||
return filepath.WalkDir(from, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil || strings.HasSuffix(p, metaSuffix) {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(from, p)
|
||||
dst := filepath.Join(to, rel)
|
||||
mode, uid, gid := readMeta(p, d.IsDir())
|
||||
if d.IsDir() {
|
||||
if err := os.MkdirAll(dst, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
b, err := os.ReadFile(p) //nolint:gosec // our own temporary copy
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dst, b, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Chmod(dst, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
if os.Geteuid() == 0 && uid >= 0 {
|
||||
return os.Lchown(dst, uid, gid)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// readMeta is a file's mode, and its owner resolved by name on this machine,
|
||||
// falling back to the number; -1 leaves it as it is.
|
||||
func readMeta(p string, dir bool) (os.FileMode, int, int) {
|
||||
mode := os.FileMode(0o640)
|
||||
if dir {
|
||||
mode = 0o750
|
||||
}
|
||||
b, err := os.ReadFile(p + metaSuffix)
|
||||
if err != nil {
|
||||
return mode, -1, -1
|
||||
}
|
||||
var m uint32
|
||||
var un, gn string
|
||||
var uid, gid int
|
||||
if _, err := fmt.Sscanf(string(b), "%o %s %s %d %d", &m, &un, &gn, &uid, &gid); err != nil {
|
||||
// Names may be empty: read what is there.
|
||||
f := strings.Fields(string(b))
|
||||
if len(f) == 0 {
|
||||
return mode, -1, -1
|
||||
}
|
||||
v, _ := strconv.ParseUint(f[0], 8, 32)
|
||||
return os.FileMode(v), -1, -1
|
||||
}
|
||||
if u, err := user.Lookup(un); err == nil {
|
||||
uid, _ = strconv.Atoi(u.Uid)
|
||||
}
|
||||
if g, err := user.LookupGroup(gn); err == nil {
|
||||
gid, _ = strconv.Atoi(g.Gid)
|
||||
}
|
||||
return os.FileMode(m), uid, gid
|
||||
}
|
||||
|
||||
func readIdentities(paths []string) ([]agecrypt.Identity, error) {
|
||||
var out []agecrypt.Identity
|
||||
for _, p := range paths {
|
||||
b, err := os.ReadFile(p) //nolint:gosec // the operator's key file
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if bytes.Contains(b, []byte("PRIVATE KEY")) {
|
||||
id, err := agessh.ParseIdentity(b)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
out = append(out, id)
|
||||
continue
|
||||
}
|
||||
ids, err := agecrypt.ParseIdentities(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s is neither an age identity file nor an SSH private key: %w", p, err)
|
||||
}
|
||||
out = append(out, ids...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// storeTarget is where the store is restored: --store, or the restored
|
||||
// server.yaml's auth.store, read from where it was unpacked.
|
||||
func (rs *restore) storeTarget(restoredEtc string) (string, error) {
|
||||
if rs.storeAddr != "" {
|
||||
return store.ResolveAddress(rs.storeAddr)
|
||||
}
|
||||
cfg, err := config.Load(filepath.Join(restoredEtc, "server.yaml"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the archive's server.yaml: %w", err)
|
||||
}
|
||||
addr := cfg.Auth.Store
|
||||
if addr == "" {
|
||||
return "/var/lib/llmgw/llmgw.db", nil
|
||||
}
|
||||
// "@/etc/llmgw/secrets/…" names a file the archive brought back.
|
||||
if f, ok := strings.CutPrefix(addr, "@"); ok && strings.HasPrefix(f, rs.etcDir+"/") {
|
||||
addr = "@" + filepath.Join(restoredEtc, strings.TrimPrefix(f, rs.etcDir+"/"))
|
||||
}
|
||||
return store.ResolveAddress(addr)
|
||||
}
|
||||
|
||||
func restoreSQLite(dump, target string, now time.Time) error {
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
if err := os.Rename(target, target+".before-restore-"+stamp(now)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, side := range []string{"-wal", "-shm"} {
|
||||
_ = os.Remove(target + side)
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(dump) //nolint:gosec // our own temporary copy
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(target, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if os.Geteuid() == 0 {
|
||||
if u, err := user.Lookup("llmgw"); err == nil {
|
||||
uid, _ := strconv.Atoi(u.Uid)
|
||||
gid, _ := strconv.Atoi(u.Gid)
|
||||
return os.Chown(target, uid, gid)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pgRestore loads a dump into the database the address names, as its
|
||||
// owner, on the primary.
|
||||
func pgRestore(ctx context.Context, dsn, dump string) error {
|
||||
env, err := pgEnv(dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, e := range env {
|
||||
if strings.HasPrefix(e, "PGTARGETSESSIONATTRS=") {
|
||||
env[i] = "PGTARGETSESSIONATTRS=read-write"
|
||||
}
|
||||
}
|
||||
env = append(env, "PGTARGETSESSIONATTRS=read-write")
|
||||
db := ""
|
||||
for _, e := range env {
|
||||
if v, ok := strings.CutPrefix(e, "PGDATABASE="); ok {
|
||||
db = v
|
||||
}
|
||||
}
|
||||
if db == "" {
|
||||
return errors.New("the store's address names no database")
|
||||
}
|
||||
// Without --dbname pg_restore writes SQL out rather than restoring; the
|
||||
// rest of the connection comes from the environment, the password with it.
|
||||
cmd := exec.CommandContext(ctx, "pg_restore", "--clean", "--if-exists", "--no-owner", "--single-transaction", //nolint:gosec // fixed binary, our own file
|
||||
"--exit-on-error", "--dbname="+db, dump)
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
if b, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("pg_restore: %v: %s", err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitHealthy waits for the restored server to answer, for a minute.
|
||||
func (rs *restore) waitHealthy(ctx context.Context) error {
|
||||
cfg, err := config.Load(filepath.Join(rs.etcDir, "server.yaml"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := newCtlClient(cfg.AdminListen, adminTokenPath(filepath.Join(rs.etcDir, "server.yaml"), cfg))
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for {
|
||||
var v map[string]any
|
||||
if err := c.get(ctx, "/api/v1/summary", &v); err == nil {
|
||||
return nil
|
||||
} else if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s started but does not answer: %w", rs.service, err)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func stamp(t time.Time) string { return t.UTC().Format("20060102T150405Z") }
|
||||
|
||||
func aside(target string, now time.Time) string {
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return "; the current one kept as " + filepath.Base(target) + ".before-restore-" + stamp(now)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// redactDSN is an address without its password, for saying where.
|
||||
func redactDSN(dsn string) string {
|
||||
if i := strings.Index(dsn, "://"); i >= 0 {
|
||||
if j := strings.Index(dsn[i+3:], "@"); j >= 0 {
|
||||
cred := dsn[i+3 : i+3+j]
|
||||
if k := strings.IndexByte(cred, ':'); k >= 0 {
|
||||
return dsn[:i+3] + cred[:k] + ":…" + dsn[i+3+j:]
|
||||
}
|
||||
}
|
||||
}
|
||||
return dsn
|
||||
}
|
||||
|
||||
// newer says whether version a is newer than b, by their leading
|
||||
// major.minor.patch; anything unreadable is not newer.
|
||||
func newer(a, b string) bool {
|
||||
pa, oka := semverOf(a)
|
||||
pb, okb := semverOf(b)
|
||||
if !oka || !okb {
|
||||
return false
|
||||
}
|
||||
for i := range pa {
|
||||
if pa[i] != pb[i] {
|
||||
return pa[i] > pb[i]
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func semverOf(v string) ([3]int, bool) {
|
||||
v = strings.TrimPrefix(v, "v")
|
||||
if i := strings.IndexAny(v, "-+"); i >= 0 {
|
||||
v = v[:i]
|
||||
}
|
||||
parts := strings.Split(v, ".")
|
||||
var out [3]int
|
||||
if len(parts) != 3 {
|
||||
return out, false
|
||||
}
|
||||
for i, p := range parts {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil {
|
||||
return out, false
|
||||
}
|
||||
out[i] = n
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agecrypt "filippo.io/age"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestABackupIsRestoredWhereItSays(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
etc := filepath.Join(tmp, "etc")
|
||||
_ = os.MkdirAll(filepath.Join(etc, "secrets"), 0o750)
|
||||
dbPath := filepath.Join(tmp, "llmgw.db")
|
||||
_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("auth:\n enabled: true\n store: "+dbPath+"\n"), 0o640)
|
||||
_ = os.WriteFile(filepath.Join(etc, "secrets", "admin-token"), []byte(strings.Repeat("t", 40)), 0o600)
|
||||
|
||||
db, err := store.Open(t.Context(), dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.CreateOrg(t.Context(), "acme"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = db.Close()
|
||||
|
||||
id, _ := agecrypt.GenerateX25519Identity()
|
||||
to, _ := config.ParseBackupRecipients([]string{id.Recipient().String()})
|
||||
archive, err := writeBackup(t.Context(), dbPath, etc, filepath.Join(tmp, "backups"), to)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := filepath.Join(tmp, "key.txt")
|
||||
_ = os.WriteFile(key, []byte(id.String()+"\n"), 0o600)
|
||||
|
||||
// Onto another machine, as it were: an empty place for both.
|
||||
newEtc, newDB := filepath.Join(tmp, "new-etc"), filepath.Join(tmp, "new", "llmgw.db")
|
||||
var out bytes.Buffer
|
||||
rs := restore{archive: archive, identities: []string{key}, etcDir: newEtc, storeAddr: newDB,
|
||||
dryRun: true, out: &out, now: time.Now}
|
||||
if err := rs.run(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(newEtc); err == nil || !strings.Contains(out.String(), "dry run") {
|
||||
t.Fatalf("a dry run changed something:\n%s", out.String())
|
||||
}
|
||||
out.Reset()
|
||||
rs.dryRun = false
|
||||
if err := rs.run(t.Context()); err != nil {
|
||||
t.Fatalf("%v\n%s", err, out.String())
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(newEtc, "secrets", "admin-token")); len(b) != 40 {
|
||||
t.Error("the configuration did not come back")
|
||||
}
|
||||
if fi, err := os.Stat(filepath.Join(newEtc, "secrets", "admin-token")); err != nil || fi.Mode().Perm() != 0o600 {
|
||||
t.Errorf("modes: %v %v", fi.Mode(), err)
|
||||
}
|
||||
db, err = store.Open(t.Context(), newDB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if orgs, _ := db.AllOrgs(t.Context()); len(orgs) != 1 || orgs[0].Name != "acme" {
|
||||
t.Errorf("the store did not come back: %+v", orgs)
|
||||
}
|
||||
|
||||
// Again over what is there: the current one is kept aside, not lost.
|
||||
out.Reset()
|
||||
if err := rs.run(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matches, _ := filepath.Glob(newEtc + ".before-restore-*")
|
||||
if len(matches) != 1 {
|
||||
t.Errorf("the configuration replaced was not kept: %v", matches)
|
||||
}
|
||||
|
||||
// Without the key it is said, not guessed.
|
||||
rs.identities = nil
|
||||
if err := rs.run(t.Context()); err == nil || !strings.Contains(err.Error(), "--identity") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewerArchiveIsRefused(t *testing.T) {
|
||||
if !newer("v0.12.0", "v0.11.1-106-gabc") || newer("v0.11.1", "v0.11.1-5-g1") || newer("dev", "v1.0.0") {
|
||||
t.Error("version comparison")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveIsCopiedAndTheOldestPruned(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
dest := filepath.Join(tmp, "share")
|
||||
for i := range 4 {
|
||||
p := filepath.Join(tmp, "llmgw-backup-h-2026100"+string(rune('1'+i))+"T000000Z.tar.gz.age")
|
||||
_ = os.WriteFile(p, []byte("x"), 0o600)
|
||||
_ = os.Chtimes(p, time.Now().Add(time.Duration(i)*time.Hour), time.Now().Add(time.Duration(i)*time.Hour))
|
||||
if err := copyToDir(p, dest, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = os.Chtimes(filepath.Join(dest, filepath.Base(p)), time.Now().Add(time.Duration(i)*time.Hour), time.Now().Add(time.Duration(i)*time.Hour))
|
||||
}
|
||||
left, _ := filepath.Glob(filepath.Join(dest, "llmgw-backup-*"))
|
||||
if len(left) != 2 {
|
||||
t.Errorf("kept %v", left)
|
||||
}
|
||||
c := config.Default()
|
||||
c.BackupCopyTo = []string{"/mnt/x"}
|
||||
if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "backup_recipients") {
|
||||
t.Errorf("an unencrypted copy off the machine was allowed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -576,7 +576,7 @@ func newServerCmd() *cobra.Command {
|
||||
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
||||
|
||||
cmd.AddCommand(newServerInitCmd(), newServerAddCmd())
|
||||
cmd.AddCommand(newBackupCmd())
|
||||
cmd.AddCommand(newBackupCmd(), newRestoreCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user