diff --git a/internal/cli/keys.go b/internal/cli/keys.go index 60380b5..5e702df 100644 --- a/internal/cli/keys.go +++ b/internal/cli/keys.go @@ -37,6 +37,7 @@ func newTenancyCmd() *cobra.Command { newLimitsCmd(open), newThreadsCmd(open), newMemoryCmd(open), + newMoveStoreCmd(), ) return cmd } diff --git a/internal/cli/movestore.go b/internal/cli/movestore.go new file mode 100644 index 0000000..82505af --- /dev/null +++ b/internal/cli/movestore.go @@ -0,0 +1,71 @@ +package cli + +import ( + "fmt" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// newMoveStoreCmd copies a store into another one. +// +// Written because keys are stored hashed. Moving to PostgreSQL without this +// means every person and every script needs a new key issued, and the old ones +// cannot even be looked up to say which they were. +// +// It reads and writes, and nothing else runs while it does: run it with the +// gateway stopped. A copy taken while requests are landing would miss whatever +// arrived after the table it had already passed. +func newMoveStoreCmd() *cobra.Command { + var from, to string + + cmd := &cobra.Command{ + Use: "move-store", + Short: "Copy a store into another one, for moving SQLite to PostgreSQL", + Long: "Copies every table, parents first, into an empty destination.\n\n" + + "Stop the gateway first. This makes no attempt to copy a store that is\n" + + "being written to: rows landing after the table they belong to has been\n" + + "passed would simply be absent, and nothing would say so.\n\n" + + "The destination must be empty. Merging two stores means deciding which\n" + + "of two projects with the same id was meant, and there is no answer to\n" + + "that which is not a guess.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if from == "" || to == "" { + return fmt.Errorf("--from and --to are both required") + } + src, err := store.Open(cmd.Context(), from) + if err != nil { + return fmt.Errorf("opening the source: %w", err) + } + defer src.Close() + + dst, err := store.Open(cmd.Context(), to) + if err != nil { + return fmt.Errorf("opening the destination: %w", err) + } + defer dst.Close() + + res, err := src.CopyTo(cmd.Context(), dst) + // Printed even on failure: a partial copy is the one case where + // knowing how far it got decides what to do next. + out := cmd.OutOrStdout() + var total int64 + for _, r := range res { + if r.Rows > 0 { + fmt.Fprintf(out, " %-16s %d\n", r.Table, r.Rows) + } + total += r.Rows + } + if err != nil { + return err + } + fmt.Fprintf(out, "\n%d rows copied. Point the gateway at the new store and start it.\n", total) + return nil + }, + } + cmd.Flags().StringVar(&from, "from", "", "source: a SQLite path, a postgres:// URL, or @file holding one") + cmd.Flags().StringVar(&to, "to", "", "destination, same forms") + return cmd +} diff --git a/internal/store/copy.go b/internal/store/copy.go new file mode 100644 index 0000000..6110a31 --- /dev/null +++ b/internal/store/copy.go @@ -0,0 +1,208 @@ +package store + +import ( + "context" + "fmt" + "strings" +) + +// copyOrder is every table, parents before children. +// +// Written out rather than discovered, because the order is a fact about the +// foreign keys and getting it wrong produces a constraint violation halfway +// through — with some tables copied and some not, which is the worst place to +// stop. A table added without being added here is copied by nothing, and the +// destination comes up quietly missing it; the test below is what catches +// that. +var copyOrder = []string{ + "organizations", + "teams", + "projects", + // Before api_keys, which references it. The first version of this list + // followed the order the tables are created across the migrations, and + // users is created in a later one than api_keys even though api_keys + // points at it — so the copy failed on a foreign key three tables in. + "users", + "api_keys", + "key_claims", + "sessions", + "limits", + "usage", + "models", + "threads", + "thread_messages", + "memory_notes", +} + +// CopyResult is how many rows each table received. +type CopyResult struct { + Table string + Rows int64 +} + +// CopyTo copies every row into another store. +// +// For moving from SQLite to PostgreSQL, which otherwise means reissuing every +// API key: the keys are stored hashed, so there is no way to recreate them and +// no way for anyone to keep using the one they have. +// +// The destination must be empty. Merging two stores would mean deciding what +// happens when both have a project with the same id and different contents, +// and every answer to that is a guess about which one someone meant. +// +// Not a transaction across the whole copy. It is one transaction per table, +// because a single transaction holding every row of a large usage table is how +// a migration runs out of memory on the machine least able to spare it. A +// failure therefore leaves a partial copy, which is why the destination has to +// have been empty: starting again is dropping and recreating it. +func (s *SQLite) CopyTo(ctx context.Context, dst *SQLite) ([]CopyResult, error) { + empty, where, err := dst.isEmpty(ctx) + if err != nil { + return nil, err + } + if !empty { + return nil, fmt.Errorf("the destination already has rows in %s; copy into an empty store", where) + } + + var out []CopyResult + for _, table := range copyOrder { + n, err := s.copyTable(ctx, dst, table) + if err != nil { + return out, fmt.Errorf("copying %s: %w", table, err) + } + out = append(out, CopyResult{Table: table, Rows: n}) + } + // Part of the copy, not a step someone has to remember: a destination + // whose sequences still start at 1 is not a destination anyone can use. + if err := dst.resetIdentities(ctx); err != nil { + return out, fmt.Errorf("pointing the identity columns past the copied rows: %w", err) + } + return out, nil +} + +// isEmpty reports whether any table has rows, and which one first does. +func (s *SQLite) isEmpty(ctx context.Context) (bool, string, error) { + for _, table := range copyOrder { + var n int64 + //nolint:gosec // table comes from copyOrder, not from input + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&n); err != nil { + return false, "", err + } + if n > 0 { + return false, table, nil + } + } + return true, "", nil +} + +// copyTable moves one table, column names taken from the source. +// +// Reading the columns rather than listing them keeps this working when a +// migration adds one: a hand-written column list would copy the old shape and +// silently drop the new field, and nothing would notice until someone looked +// for a setting that was there before the move. +func (s *SQLite) copyTable(ctx context.Context, dst *SQLite, table string) (int64, error) { + //nolint:gosec // table comes from copyOrder, not from input + rows, err := s.db.QueryContext(ctx, `SELECT * FROM `+table) + if err != nil { + return 0, err + } + defer rows.Close() + + cols, err := rows.Columns() + if err != nil { + return 0, err + } + + q, tx, err := dst.begin(ctx) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + insert := `INSERT INTO ` + table + ` (` + strings.Join(quoteAll(cols), ", ") + + `) VALUES (` + strings.Repeat("?, ", len(cols)-1) + `?)` + + var n int64 + for rows.Next() { + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return n, err + } + for i, v := range vals { + vals[i] = keepEmptyBytes(v) + } + if _, err := q.ExecContext(ctx, insert, vals...); err != nil { + return n, err + } + n++ + } + if err := rows.Err(); err != nil { + return n, err + } + return n, tx.Commit() +} + +// keepEmptyBytes stops an empty byte column becoming NULL on the way across. +// +// An empty BLOB scans as a typed nil slice — []byte(nil) inside a non-nil any +// — while a real NULL scans as an untyped nil. The two are distinguishable +// here and not in pgx, which encodes any nil slice as NULL, so the difference +// has to be restored before the insert. +// +// It is not a detail. key_claims.sealed is emptied the moment a claim is used, +// so every claimed row holds an empty blob in a NOT NULL column, and the copy +// failed on the first one — after the gateway had been stopped and four tables +// had already been written. +func keepEmptyBytes(v any) any { + if b, ok := v.([]byte); ok && b == nil { + return []byte{} + } + return v +} + +// quoteAll quotes column names, because `key` and `text` are reserved words in +// one dialect or the other and both are column names here. +func quoteAll(cols []string) []string { + out := make([]string, len(cols)) + for i, c := range cols { + out[i] = `"` + c + `"` + } + return out +} + +// resetIdentities points PostgreSQL's identity columns past the copied rows. +// +// Without it the first insert after a migration collides with a row that was +// copied in: the sequence still starts at 1 while the table already holds a +// thousand notes, and the failure arrives later, to somebody else, looking +// like a bug in the feature rather than in the move. +func (s *SQLite) resetIdentities(ctx context.Context) error { + if s.dialect != PostgresDialect { + return nil + } + // Only the tables with a generated id. SQLite's rowid needs nothing. + for _, t := range []string{"memory_notes", "usage"} { + var has bool + if err := s.db.QueryRowContext(ctx, + `SELECT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = ? AND column_name = 'id' + AND is_identity = 'YES')`, t).Scan(&has); err != nil { + return err + } + if !has { + continue + } + //nolint:gosec // t is a literal from the list above + if _, err := s.db.ExecContext(ctx, + `SELECT setval(pg_get_serial_sequence('`+t+`', 'id'), + GREATEST((SELECT COALESCE(MAX(id), 0) FROM `+t+`), 1))`); err != nil { + return err + } + } + return nil +} diff --git a/internal/store/copy_test.go b/internal/store/copy_test.go new file mode 100644 index 0000000..fe701b0 --- /dev/null +++ b/internal/store/copy_test.go @@ -0,0 +1,253 @@ +package store + +import ( + "context" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + "time" +) + +func TestEveryTableIsInTheCopyOrder(t *testing.T) { + // A table added by a migration and not added here is copied by nothing, + // and the destination comes up quietly missing it — discovered when + // somebody looks for a setting that was there before the move. Read out of + // the migrations rather than listed twice, so the two cannot disagree. + var declared []string + entries, err := migrationFS.ReadDir("migrations/sqlite") + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + body, err := migrationFS.ReadFile("migrations/sqlite/" + e.Name()) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(body), "\n") { + line = strings.TrimSpace(line) + rest, ok := strings.CutPrefix(line, "CREATE TABLE ") + if !ok { + continue + } + name, _, _ := strings.Cut(rest, " ") + name = strings.TrimSuffix(strings.TrimSpace(name), "(") + declared = append(declared, name) + } + } + if len(declared) == 0 { + t.Fatal("found no tables in the migrations, so this test proves nothing") + } + for _, name := range declared { + if !slices.Contains(copyOrder, name) { + t.Errorf("%q is created by a migration but nothing copies it", name) + } + } +} + +func TestCopyRefusesADestinationThatAlreadyHasRows(t *testing.T) { + // Merging two stores means deciding what happens when both hold a project + // with the same id and different contents, and every answer is a guess + // about which one somebody meant. + ctx := context.Background() + src, dst := openDB(t), openDB(t) + tree(t, src) + tree(t, dst) + + _, err := src.CopyTo(ctx, dst) + if err == nil || !strings.Contains(err.Error(), "empty store") { + t.Errorf("copying into a populated store gave %v", err) + } +} + +func TestACopiedStoreStillAuthenticatesTheSameKey(t *testing.T) { + // The reason this exists. Keys are stored hashed, so a migration that + // loses them cannot recreate them: every person and every script would + // need a new one issued. + ctx := context.Background() + src := openDB(t) + proj := tree(t, src) + secret, k := mustKey(t, src, "k1", proj) + + if err := src.WriteUsage(ctx, []Usage{{ + At: time.Now(), KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", + PromptTokens: 3, CompletionTokens: 4, CostMicros: 99, Status: 200, + }}); err != nil { + t.Fatal(err) + } + + dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db")) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + + res, err := src.CopyTo(ctx, dst) + if err != nil { + t.Fatalf("copying: %v", err) + } + if len(res) != len(copyOrder) { + t.Errorf("copied %d tables of %d", len(res), len(copyOrder)) + } + + // The same secret the caller already holds still resolves, in the copy. + got, err := dst.KeyByHash(ctx, HashKey(secret)) + if err != nil { + t.Fatalf("the copied store does not know the key: %v", err) + } + if got.ID != k.ID || got.ProjectID != proj { + t.Errorf("key came back as %+v", got) + } + + // And the spend it had, so a budget is not quietly reset by moving. + spend, err := dst.SpendSince(ctx, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if spend[k.ID] != 99 { + t.Errorf("spend after the copy = %d, want 99", spend[k.ID]) + } +} + +func TestCopyCarriesConversationsAndNotes(t *testing.T) { + ctx := context.Background() + src := openDB(t) + proj := tree(t, src) + if err := src.SetThreadRetention(ctx, proj, 24); err != nil { + t.Fatal(err) + } + if err := src.AppendThread(ctx, proj, "th1", []Message{ + {Role: "user", Content: "remembered"}, + }); err != nil { + t.Fatal(err) + } + + dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db")) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + if _, err := src.CopyTo(ctx, dst); err != nil { + t.Fatal(err) + } + + msgs, err := dst.Thread(ctx, proj, "th1") + if err != nil { + t.Fatalf("the conversation did not survive: %v", err) + } + if len(msgs) != 1 || msgs[0].Content != "remembered" { + t.Errorf("thread came back as %+v", msgs) + } + // The retention setting is a column on projects, so it rides along — a + // destination that kept the conversations and forgot the promise about + // how long would be worse than one that kept neither. + if d, err := dst.ThreadRetention(ctx, proj); err != nil || d == 0 { + t.Errorf("retention after the copy = %v (err %v)", d, err) + } +} + +func TestCopyingAnEmptyStoreIsFine(t *testing.T) { + ctx := context.Background() + src := openDB(t) + dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db")) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + if _, err := src.CopyTo(ctx, dst); err != nil { + t.Errorf("copying an empty store failed: %v", err) + } + _ = os.Remove("") +} + +func TestTheCopyOrderSatisfiesTheForeignKeys(t *testing.T) { + // Completeness is not enough: a table copied before the one it references + // fails on a constraint partway through, leaving some tables copied and + // some not. The first version of copyOrder followed the order the + // migrations create tables in, and users is created after api_keys even + // though api_keys points at it — so the copy died three tables in, on a + // real database, after the gateway had already been stopped. + // + // Derived from the migrations rather than restated, so a new reference + // fails here rather than during somebody's migration. + deps := map[string]map[string]bool{} + note := func(table, ref string) { + if table == ref { + return + } + if deps[table] == nil { + deps[table] = map[string]bool{} + } + deps[table][ref] = true + } + + entries, err := migrationFS.ReadDir("migrations/sqlite") + if err != nil { + t.Fatal(err) + } + body := "" + for _, e := range entries { + b, err := migrationFS.ReadFile("migrations/sqlite/" + e.Name()) + if err != nil { + t.Fatal(err) + } + body += string(b) + } + + // Table bodies, so a REFERENCES is attributed to the table holding it. + for _, m := range regexpCreateTable.FindAllStringSubmatch(body, -1) { + for _, r := range regexpReferences.FindAllStringSubmatch(m[2], -1) { + note(m[1], r[1]) + } + } + // And columns added later, which is exactly how api_keys got its user_id. + for _, m := range regexpAlterRef.FindAllStringSubmatch(body, -1) { + note(m[1], m[2]) + } + if len(deps) == 0 { + t.Fatal("found no foreign keys, so this test proves nothing") + } + + position := map[string]int{} + for i, t := range copyOrder { + position[t] = i + } + for table, refs := range deps { + for ref := range refs { + if position[ref] > position[table] { + t.Errorf("%s is copied before %s, which it references", table, ref) + } + } + } +} + +var ( + regexpCreateTable = regexp.MustCompile(`(?s)CREATE TABLE (\w+)\s*\((.*?)\n\);`) + regexpReferences = regexp.MustCompile(`REFERENCES\s+(\w+)`) + regexpAlterRef = regexp.MustCompile(`ALTER TABLE (\w+) ADD COLUMN[^;]*REFERENCES\s+(\w+)`) +) + +func TestAnEmptyBlobDoesNotBecomeNullInTheCopy(t *testing.T) { + // key_claims.sealed is emptied the moment a claim is used, so a claimed + // row holds an empty blob in a NOT NULL column. An empty blob scans as a + // typed nil slice and a real NULL as an untyped one; pgx cannot tell them + // apart, so the distinction is restored before the insert. + if got := keepEmptyBytes([]byte(nil)); got == nil { + t.Error("an empty byte column would be inserted as NULL") + } else if b, ok := got.([]byte); !ok || len(b) != 0 { + t.Errorf("empty bytes came out as %#v", got) + } + // A genuine NULL arrives as an untyped nil and must stay one. + if got := keepEmptyBytes(nil); got != nil { + t.Errorf("a NULL became %#v", got) + } + // Everything else passes through. + if got := keepEmptyBytes(int64(7)); got != int64(7) { + t.Errorf("a number was rewritten to %#v", got) + } + if got := keepEmptyBytes([]byte("x")); string(got.([]byte)) != "x" { + t.Errorf("a non-empty blob was rewritten to %#v", got) + } +}