Move a control plane to PostgreSQL with server init --postgres

Installs PostgreSQL with the machine's own package manager when it has
none (initdb with password auth on the RHEL family, whose default is
ident), creates the llmgw role and database with a generated password
kept in /etc/llmgw/secrets/store-dsn, and checks the address connects.
On a machine already on SQLite it stops the gateway, copies the store
so accounts and keys survive, points auth.store at the new address and
starts it again. A move that fails starts the gateway on SQLite again.

server.yaml is edited as text so its comments survive, and kept beside
it. A PostgreSQL init did not install is not resized: it may serve
something else. It refuses to move one PostgreSQL store into another.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 14:33:03 +02:00
1 parent 6dba6bd93a
commit cba65f5be7
6 files changed
+612 -2

No files matched your search

+18
View File
@@ -105,6 +105,10 @@ That is the whole step. In order, it:
once**
6. installs the systemd unit, starts it, and waits until `/health` answers
Add `--postgres` to keep accounts and keys in PostgreSQL rather than SQLite:
init installs it and sets it up, and on a machine already running on SQLite it
moves everything across. See [PostgreSQL](#postgresql-when-sqlite-is-no-longer-the-right-shape).
Every step looks before it acts. Running it again finishes an install that
stopped halfway and changes nothing on one that did not; it never overwrites
`server.yaml`, the authority, or an existing account's password.
@@ -743,6 +747,20 @@ turn, which is the first per-request load this store has had; and SQLite takes
one writer at a time, so those turns serialise. And a second control plane is
impossible while the store is a file on one machine's disk.
On the control plane, one command does all of it — installs PostgreSQL if the
machine has none, creates its role and database with a generated password kept
in `/etc/llmgw/secrets/store-dsn`, stops the gateway, copies the SQLite store
into it so every account and key survives, points `server.yaml` at it, and
starts the gateway again:
```
sudo llmgw server init --postgres
```
It can be given on the first run too, and then there is never a SQLite store.
The rest of this section is what that command does, for a PostgreSQL somewhere
else.
The setting is the same one, and the address decides:
```yaml
+8
View File
@@ -290,6 +290,14 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
return nil
}
// InstallPackages installs whichever of these packages the machine lacks,
// from its own package sources, waiting out another package manager's lock.
// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
var st Status
return in.packages(ctx, byFamily, &st)
}
// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute
+15 -2
View File
@@ -66,6 +66,12 @@ type serverInit struct {
user string
hosts []string
admin string
// postgres keeps the store in a PostgreSQL on this machine, installed if
// need be, moving whatever SQLite already holds.
postgres bool
// pgSetup prepares PostgreSQL and returns the store address; a seam for
// tests. Nil does it for real.
pgSetup func(context.Context) (string, error)
out io.Writer
@@ -88,6 +94,10 @@ func newServerInitCmd() *cobra.Command {
"control plane's certificate for every address this machine has, a\n" +
"configuration, an administrator account, and the systemd unit — then\n" +
"starts it and waits until it answers.\n\n" +
"With --postgres the store is a PostgreSQL database on this machine:\n" +
"installed if there is none, with its own role and a generated password\n" +
"kept in /etc/llmgw/secrets. An existing SQLite store is copied into it,\n" +
"so accounts and keys survive, and server.yaml is pointed at it.\n\n" +
"Each step is skipped when its result already exists, so it is safe to run\n" +
"again: to finish an install that stopped, or with another --host to\n" +
"reissue the certificate. It never overwrites server.yaml or the authority.\n\n" +
@@ -110,6 +120,8 @@ func newServerInitCmd() *cobra.Command {
"an address agents dial besides the ones found on this machine; repeatable")
f.StringVar(&in.admin, "admin", "", "email of the first administrator; a password is generated and printed once")
f.StringVar(&in.pkiDir, "pki", in.pkiDir, "directory holding the certificate authority")
f.BoolVar(&in.postgres, "postgres", false,
"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
return cmd
}
@@ -122,6 +134,7 @@ func (in *serverInit) run(ctx context.Context) error {
{"directories", in.ensureDirs},
{"certificates", in.ensureCerts},
{"configuration", in.ensureConfig},
{"database", in.ensureDatabase},
{"store", in.ensureStore},
}
for _, s := range steps {
@@ -351,7 +364,7 @@ func (in *serverInit) ensureConfig(context.Context) error {
contact = "platform@example.com"
}
body := fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
filepath.Join(in.stateDir, "llmgw.db"), contact)
in.storeAddress(), contact)
if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
return err
}
@@ -393,7 +406,7 @@ control:
auth:
enabled: true
store: %s
store: "%s" # a SQLite path, or @file holding a PostgreSQL address
signup_open: true
# Applied to any key no scope gives a limit to. Without them, a project
# created without limits is metered by nothing at all.
+396
View File
@@ -0,0 +1,396 @@
package cli
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"log/slog"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// pgRole and pgDatabase are what init creates in a PostgreSQL it manages.
const (
pgRole = "llmgw"
pgDatabase = "llmgw"
)
// dsnPath is where the PostgreSQL address is kept, out of server.yaml, so the
// password in it is not in a file that gets copied, diffed and pasted.
func (in *serverInit) dsnPath() string { return filepath.Join(in.etcDir, "secrets", "store-dsn") }
// storeAddress is what auth.store says for the store init manages.
func (in *serverInit) storeAddress() string {
if in.postgres {
return "@" + in.dsnPath()
}
return filepath.Join(in.stateDir, "llmgw.db")
}
// ensureDatabase puts the store in PostgreSQL when asked to, moving whatever
// the SQLite store already holds.
//
// The move is the point. Keys are stored hashed, so starting PostgreSQL empty
// means reissuing every key and being unable to say which was which; and the
// administrator init created a minute ago would be gone too.
func (in *serverInit) ensureDatabase(ctx context.Context) (err error) {
if !in.postgres {
return nil
}
setup := in.pgSetup
if setup == nil {
setup = in.setupPostgres
}
target, err := setup(ctx)
if err != nil {
return err
}
cfg, err := config.Load(in.configPath())
if err != nil {
return err
}
current := cfg.Auth.Store
if current == target {
in.kept("store in PostgreSQL (%s)", in.dsnPath())
return nil
}
if isPostgresAddress(current) {
// Two PostgreSQL stores, and a guess about which is meant would lose
// one of them.
return fmt.Errorf("server.yaml already names a PostgreSQL store (%s); "+
"init moves SQLite into PostgreSQL, not one PostgreSQL into another", current)
}
// Nothing may write the store while it is copied: a row landing after its
// table has been passed would be absent, and nothing would say so.
stopped, err := in.stopService(ctx)
if err != nil {
return err
}
defer func() {
// A move that failed leaves the configuration naming SQLite, which
// still holds everything: start the service on it again rather than
// leave the gateway down over a database that did not work out.
if err != nil && stopped {
_ = exec.Command("systemctl", "start", "llmgw-server").Run()
fmt.Fprintf(in.out, " .. llmgw-server started again on its SQLite store\n")
}
}()
if current != "" {
if _, err := os.Stat(current); err == nil {
if err := in.copyStore(ctx, current, target); err != nil {
return err
}
}
}
if err := setAuthStore(in.configPath(), target); err != nil {
return err
}
in.done("server.yaml now keeps the store in PostgreSQL")
return nil
}
func (in *serverInit) copyStore(ctx context.Context, from, to string) error {
src, err := store.Open(ctx, from)
if err != nil {
return fmt.Errorf("opening %s: %w", from, err)
}
defer src.Close()
dst, err := store.Open(ctx, to)
if err != nil {
return fmt.Errorf("opening PostgreSQL: %w", err)
}
defer dst.Close()
empty, err := dst.Empty(ctx)
if err != nil {
return err
}
if !empty {
// A previous run that copied and then stopped before changing the
// configuration. The service was stopped for that copy too, so what
// is there is what SQLite held.
in.kept("PostgreSQL already holds a store; %s is left as it is", from)
return nil
}
res, err := src.CopyTo(ctx, dst)
if err != nil {
return fmt.Errorf("copying %s into PostgreSQL: %w", from, err)
}
var rows int64
for _, r := range res {
rows += r.Rows
}
in.done("copied %d rows from %s into PostgreSQL (the SQLite file is kept)", rows, from)
return nil
}
func isPostgresAddress(a string) bool {
return strings.HasPrefix(a, "@") || strings.HasPrefix(a, "postgres://") ||
strings.HasPrefix(a, "postgresql://")
}
// setAuthStore changes auth.store in server.yaml and nothing else.
//
// Edited as text rather than unmarshalled and written back, because writing
// it back would drop every comment in a file people annotate. The old file is
// kept beside it.
func setAuthStore(path, address string) error {
b, err := os.ReadFile(path)
if err != nil {
return err
}
lines := strings.Split(string(b), "\n")
value := fmt.Sprintf("%q", address)
authAt, storeAt, indent := -1, -1, " "
for i, l := range lines {
trimmed := strings.TrimSpace(l)
top := l != "" && l[0] != ' ' && l[0] != '\t' && !strings.HasPrefix(trimmed, "#")
if top {
if authAt >= 0 && i > authAt {
break // past the auth block
}
if strings.HasPrefix(trimmed, "auth:") {
authAt = i
}
continue
}
if authAt >= 0 && strings.HasPrefix(trimmed, "store:") {
storeAt = i
indent = l[:len(l)-len(strings.TrimLeft(l, " \t"))]
break
}
}
switch {
case storeAt >= 0:
lines[storeAt] = indent + "store: " + value
case authAt >= 0:
lines = append(lines[:authAt+1], append([]string{indent + "store: " + value}, lines[authAt+1:]...)...)
default:
lines = append(lines, "auth:", " enabled: true", " store: "+value)
}
backup := fmt.Sprintf("%s.bak-%s", path, time.Now().Format("20060102-150405"))
if err := os.WriteFile(backup, b, 0o640); err != nil {
return err
}
info, err := os.Stat(path)
if err != nil {
return err
}
return os.WriteFile(path, []byte(strings.Join(lines, "\n")), info.Mode().Perm())
}
// setupPostgres makes sure a PostgreSQL on this machine holds a database for
// llmgw, installing the server if there is none, and returns the store
// address.
func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
installed, err := in.installPostgres(ctx)
if err != nil {
return "", err
}
if err := waitForPostgres(ctx); err != nil {
return "", err
}
exists, err := psqlBool(ctx, "SELECT 1 FROM pg_roles WHERE rolname = '"+pgRole+"'")
if err != nil {
return "", err
}
_, statErr := os.Stat(in.dsnPath())
if exists && statErr == nil {
in.kept("PostgreSQL role %s", pgRole)
} else {
// A role with no address written down is one whose password nobody
// knows; giving it a new one is the only way to use it.
password := make([]byte, 24)
_, _ = rand.Read(password)
pass := hex.EncodeToString(password)
verb := "CREATE ROLE"
if exists {
verb = "ALTER ROLE"
}
if _, err := psql(ctx, verb+" "+pgRole+" WITH LOGIN PASSWORD '"+pass+"'"); err != nil {
return "", err
}
dsn := fmt.Sprintf("postgres://%s:%s@127.0.0.1:5432/%s?sslmode=disable", pgRole, pass, pgDatabase)
if err := in.writeDSN(dsn); err != nil {
return "", err
}
in.done("PostgreSQL role %s, its address in %s", pgRole, in.dsnPath())
}
exists, err = psqlBool(ctx, "SELECT 1 FROM pg_database WHERE datname = '"+pgDatabase+"'")
if err != nil {
return "", err
}
if exists {
in.kept("PostgreSQL database %s", pgDatabase)
} else {
if _, err := psql(ctx, "CREATE DATABASE "+pgDatabase+" OWNER "+pgRole); err != nil {
return "", err
}
in.done("PostgreSQL database %s", pgDatabase)
}
if installed {
// Its defaults assume it owns the machine. A gateway swapped out to
// give the database cache is worse than a smaller cache, and a store
// whose working set is megabytes needs little. Only a PostgreSQL init
// installed itself: one that was already here may serve something
// else that needs its connections.
for _, q := range []string{
"ALTER SYSTEM SET shared_buffers = '96MB'",
"ALTER SYSTEM SET max_connections = 40",
} {
if _, err := psql(ctx, q); err != nil {
return "", err
}
}
if out, err := exec.CommandContext(ctx, "systemctl", "restart", "postgresql").CombinedOutput(); err != nil {
return "", fmt.Errorf("restarting PostgreSQL: %v: %s", err, strings.TrimSpace(string(out)))
}
if err := waitForPostgres(ctx); err != nil {
return "", err
}
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
}
// Proof the address works, before anything is moved into it.
address := "@" + in.dsnPath()
s, err := store.Open(ctx, address)
if err != nil {
return "", fmt.Errorf("connecting with %s: %w", in.dsnPath(), err)
}
_ = s.Close()
return address, nil
}
func (in *serverInit) writeDSN(dsn string) error {
dir := filepath.Dir(in.dsnPath())
if err := os.MkdirAll(dir, 0o750); err != nil {
return err
}
if err := in.chown(dir, false); err != nil {
return err
}
if err := os.WriteFile(in.dsnPath(), []byte(dsn+"\n"), 0o600); err != nil {
return err
}
// Read by the service, which runs as llmgw; root reads it anyway.
return in.chown(in.dsnPath(), true)
}
// installPostgres installs the server when this machine has none, and says
// whether it did.
func (in *serverInit) installPostgres(ctx context.Context) (bool, error) {
if exec.CommandContext(ctx, "systemctl", "cat", "postgresql.service").Run() == nil {
in.kept("PostgreSQL server")
return false, nil
}
host := discovery.DiscoverHost()
family := host.Family()
if family == "" {
return false, fmt.Errorf("no PostgreSQL here and no package manager init knows (%s); "+
"install PostgreSQL and run init again", host.OSID)
}
fmt.Fprintf(in.out, " .. installing PostgreSQL (a minute or two)\n")
pi := &provision.Installer{Host: host, Log: slog.Default()}
if err := pi.InstallPackages(ctx, map[string][]string{
"deb": {"postgresql"},
"rpm": {"postgresql-server", "postgresql"},
}); err != nil {
return false, err
}
if family == "rpm" {
// Debian's package creates and starts a cluster; RHEL's leaves an
// empty data directory. Password authentication over TCP is asked
// for, because RHEL's default for it is ident, which no service
// account passes.
if _, err := os.Stat("/var/lib/pgsql/data/PG_VERSION"); err != nil {
cmd := exec.CommandContext(ctx, "postgresql-setup", "--initdb")
cmd.Env = append(os.Environ(), "PGSETUP_INITDB_OPTIONS=--auth-host=scram-sha-256 --auth-local=peer")
if out, err := cmd.CombinedOutput(); err != nil {
return false, fmt.Errorf("postgresql-setup --initdb: %v: %s", err, strings.TrimSpace(string(out)))
}
}
}
if out, err := exec.CommandContext(ctx, "systemctl", "enable", "--now", "postgresql").CombinedOutput(); err != nil {
return false, fmt.Errorf("starting PostgreSQL: %v: %s", err, strings.TrimSpace(string(out)))
}
in.done("PostgreSQL installed and started")
return true, nil
}
// psql runs one statement as the postgres superuser, over the local socket.
func psql(ctx context.Context, query string) (string, error) {
cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
"psql", "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
// postgres cannot enter root's home, and psql says so on every call.
cmd.Dir = "/"
out, err := cmd.CombinedOutput()
if err != nil {
// The statement can hold a password; the error names only its verb.
verb := strings.Join(strings.Fields(query)[:2], " ")
return "", fmt.Errorf("psql (%s): %v: %s", verb, err, strings.TrimSpace(string(out)))
}
return strings.TrimSpace(string(out)), nil
}
func psqlBool(ctx context.Context, query string) (bool, error) {
out, err := psql(ctx, query)
return out == "1", err
}
// waitForPostgres waits for a server that was just started or restarted to
// take connections.
func waitForPostgres(ctx context.Context) error {
deadline := time.Now().Add(60 * time.Second)
for {
_, err := psql(ctx, "SELECT 1")
if err == nil {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("PostgreSQL is not taking connections after 60s: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
}
// stopService stops the control plane if it runs, so the store is still.
// It says whether it stopped it, so a failed move can start it again.
func (in *serverInit) stopService(ctx context.Context) (bool, error) {
if in.unitPath == "" {
return false, nil
}
if _, err := exec.LookPath("systemctl"); err != nil {
return false, nil
}
if exec.CommandContext(ctx, "systemctl", "is-active", "--quiet", "llmgw-server").Run() != nil {
return false, nil
}
if out, err := exec.CommandContext(ctx, "systemctl", "stop", "llmgw-server").CombinedOutput(); err != nil {
return false, fmt.Errorf("stopping llmgw-server to move its store: %v: %s", err, strings.TrimSpace(string(out)))
}
in.done("llmgw-server stopped while its store moves")
return true, nil
}
+168
View File
@@ -0,0 +1,168 @@
package cli
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestSetAuthStoreChangesOnlyAuthStore(t *testing.T) {
p := filepath.Join(t.TempDir(), "server.yaml")
body := `# my notes
threads:
store: /var/lib/llmgw/threads.db # not this one
retain_hours: 24
auth:
enabled: true
# the store
store: /var/lib/llmgw/llmgw.db
signup_open: true
landing:
contact: "x"
`
if err := os.WriteFile(p, []byte(body), 0o640); err != nil {
t.Fatal(err)
}
if err := setAuthStore(p, "@/etc/llmgw/secrets/store-dsn"); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(p)
want := strings.Replace(body, " store: /var/lib/llmgw/llmgw.db",
` store: "@/etc/llmgw/secrets/store-dsn"`, 1)
if string(got) != want {
t.Errorf("got:\n%s\nwant:\n%s", got, want)
}
cfg, err := config.Load(p)
if err != nil {
t.Fatal(err)
}
if cfg.Auth.Store != "@/etc/llmgw/secrets/store-dsn" || cfg.Threads.Store != "/var/lib/llmgw/threads.db" {
t.Errorf("auth.store %q, threads.store %q", cfg.Auth.Store, cfg.Threads.Store)
}
backups, _ := filepath.Glob(p + ".bak-*")
if len(backups) != 1 {
t.Errorf("the old file should be kept beside it, found %v", backups)
}
}
func TestSetAuthStoreAddsTheLineWhenAuthHasNone(t *testing.T) {
p := filepath.Join(t.TempDir(), "server.yaml")
if err := os.WriteFile(p, []byte("auth:\n enabled: true\nlanding:\n contact: x\n"), 0o640); err != nil {
t.Fatal(err)
}
if err := setAuthStore(p, "@/x"); err != nil {
t.Fatal(err)
}
cfg, err := config.Load(p)
if err != nil {
t.Fatal(err)
}
if cfg.Auth.Store != "@/x" || !cfg.Auth.Enabled {
t.Errorf("auth = %+v", cfg.Auth)
}
}
// fakePostgres stands in for setupPostgres the way it really answers: an
// address file under /etc/llmgw/secrets, returned as "@file". What the file
// holds is a SQLite path, so everything after the setup runs for real.
func fakePostgres(t *testing.T, in *serverInit) (address, db string) {
t.Helper()
db = filepath.Join(t.TempDir(), "pg-standin.db")
in.pgSetup = func(context.Context) (string, error) {
if err := os.MkdirAll(filepath.Dir(in.dsnPath()), 0o750); err != nil {
return "", err
}
return "@" + in.dsnPath(), os.WriteFile(in.dsnPath(), []byte(db+"\n"), 0o600)
}
return "@" + in.dsnPath(), db
}
// Moving an install that began on SQLite: the administrator it created must
// be in the new store, the configuration must name it, and the SQLite file
// must still be there. The new store is SQLite too here — what PostgreSQL
// adds is the setup, which needs a machine to run on.
func TestInitMovesAnExistingStore(t *testing.T) {
in, out := newTestInit(t)
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
old := filepath.Join(in.stateDir, "llmgw.db")
in.postgres = true
target, db := fakePostgres(t, in)
out.Reset()
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
if !strings.Contains(out.String(), "copied") {
t.Errorf("nothing was copied:\n%s", out)
}
cfg, err := config.Load(in.configPath())
if err != nil {
t.Fatal(err)
}
if cfg.Auth.Store != target {
t.Fatalf("server.yaml still names %q", cfg.Auth.Store)
}
s, err := store.Open(context.Background(), db)
if err != nil {
t.Fatal(err)
}
u, err := s.UserByEmail(context.Background(), "admin@example.com")
s.Close()
if err != nil || !u.IsAdmin {
t.Fatalf("the administrator did not move: %v %+v", err, u)
}
if _, err := os.Stat(old); err != nil {
t.Error("the SQLite store should be kept")
}
// And once moved, again is nothing.
in.changed = false
out.Reset()
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
if in.changed {
t.Errorf("a run after the move changed something:\n%s", out)
}
}
func TestInitWritesAPostgresConfigurationFromTheStart(t *testing.T) {
in, out := newTestInit(t)
in.postgres = true
target, _ := fakePostgres(t, in)
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
cfg, _ := config.Load(in.configPath())
if cfg.Auth.Store != target {
t.Errorf("auth.store = %q", cfg.Auth.Store)
}
if _, err := os.Stat(filepath.Join(in.stateDir, "llmgw.db")); err == nil {
t.Error("a PostgreSQL install should not leave a SQLite store behind")
}
}
func TestInitWillNotMoveOnePostgresIntoAnother(t *testing.T) {
in, out := newTestInit(t)
if err := os.MkdirAll(in.etcDir, 0o755); err != nil {
t.Fatal(err)
}
cfg := "control:\n enabled: true\nauth:\n enabled: true\n store: postgres://elsewhere/llmgw\n"
if err := os.WriteFile(in.configPath(), []byte(cfg), 0o640); err != nil {
t.Fatal(err)
}
in.postgres = true
in.pgSetup = func(context.Context) (string, error) { return "@/somewhere/else", nil }
err := in.run(context.Background())
if err == nil || !strings.Contains(err.Error(), "already names a PostgreSQL store") {
t.Fatalf("expected a refusal, got %v\n%s", err, out)
}
}
+7
View File
@@ -96,6 +96,13 @@ func (s *DB) CopyTo(ctx context.Context, dst *DB) ([]CopyResult, error) {
}
// isEmpty reports whether any table has rows, and which one first does.
// Empty reports whether the store holds no rows at all, which is what CopyTo
// needs of its destination.
func (s *DB) Empty(ctx context.Context) (bool, error) {
empty, _, err := s.isEmpty(ctx)
return empty, err
}
func (s *DB) isEmpty(ctx context.Context) (bool, string, error) {
for _, table := range copyOrder {
var n int64