Files
llmgw/internal/store/copy.go
T
Ilker ManapandClaude Opus 5 b6de469351 Move a store to another one without reissuing every key
Keys are stored hashed, so a migration that drops them cannot recreate them:
every person and every script needs a new one issued, and the old ones cannot
even be looked up to say which they were. `llmgw keys move-store` copies
instead.

Found by running it against the real cluster, twice, after the gateway had
already been stopped.

The first order came from the order the migrations create tables in, and
users is created in a later migration than api_keys even though api_keys
references it — so the copy died three tables in. The order is now checked by
a test that derives the foreign keys from the migrations rather than restating
them, so a new reference fails there instead of during somebody's migration.
Checked by putting the wrong order back and watching it fail.

Then key_claims. Its sealed column is emptied the moment a claim is used, so
every claimed row holds an empty blob in a NOT NULL column. An empty blob
scans as a typed nil slice and a genuine NULL as an untyped one — a
distinction this side can see and pgx cannot, since it encodes any nil slice
as NULL. Restored before the insert, which is the difference between a copy
that completes and one that stops four tables in.

One transaction per table rather than one for the whole copy: a single
transaction holding every row of a large usage table is how a migration runs
out of memory on the machine least able to spare it. A failure therefore
leaves a partial copy, which is why the destination must have been empty —
starting again is dropping and recreating it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-25 16:28:38 +02:00

209 lines
6.7 KiB
Go

package store
import (
"context"
"fmt"
"strings"
)
// copyOrder is every table, parents before children.
//
// Written out rather than discovered, because the order is a fact about the
// foreign keys and getting it wrong produces a constraint violation halfway
// through — with some tables copied and some not, which is the worst place to
// stop. A table added without being added here is copied by nothing, and the
// destination comes up quietly missing it; the test below is what catches
// that.
var copyOrder = []string{
"organizations",
"teams",
"projects",
// Before api_keys, which references it. The first version of this list
// followed the order the tables are created across the migrations, and
// users is created in a later one than api_keys even though api_keys
// points at it — so the copy failed on a foreign key three tables in.
"users",
"api_keys",
"key_claims",
"sessions",
"limits",
"usage",
"models",
"threads",
"thread_messages",
"memory_notes",
}
// CopyResult is how many rows each table received.
type CopyResult struct {
Table string
Rows int64
}
// CopyTo copies every row into another store.
//
// For moving from SQLite to PostgreSQL, which otherwise means reissuing every
// API key: the keys are stored hashed, so there is no way to recreate them and
// no way for anyone to keep using the one they have.
//
// The destination must be empty. Merging two stores would mean deciding what
// happens when both have a project with the same id and different contents,
// and every answer to that is a guess about which one someone meant.
//
// Not a transaction across the whole copy. It is one transaction per table,
// because a single transaction holding every row of a large usage table is how
// a migration runs out of memory on the machine least able to spare it. A
// failure therefore leaves a partial copy, which is why the destination has to
// have been empty: starting again is dropping and recreating it.
func (s *SQLite) CopyTo(ctx context.Context, dst *SQLite) ([]CopyResult, error) {
empty, where, err := dst.isEmpty(ctx)
if err != nil {
return nil, err
}
if !empty {
return nil, fmt.Errorf("the destination already has rows in %s; copy into an empty store", where)
}
var out []CopyResult
for _, table := range copyOrder {
n, err := s.copyTable(ctx, dst, table)
if err != nil {
return out, fmt.Errorf("copying %s: %w", table, err)
}
out = append(out, CopyResult{Table: table, Rows: n})
}
// Part of the copy, not a step someone has to remember: a destination
// whose sequences still start at 1 is not a destination anyone can use.
if err := dst.resetIdentities(ctx); err != nil {
return out, fmt.Errorf("pointing the identity columns past the copied rows: %w", err)
}
return out, nil
}
// isEmpty reports whether any table has rows, and which one first does.
func (s *SQLite) isEmpty(ctx context.Context) (bool, string, error) {
for _, table := range copyOrder {
var n int64
//nolint:gosec // table comes from copyOrder, not from input
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&n); err != nil {
return false, "", err
}
if n > 0 {
return false, table, nil
}
}
return true, "", nil
}
// copyTable moves one table, column names taken from the source.
//
// Reading the columns rather than listing them keeps this working when a
// migration adds one: a hand-written column list would copy the old shape and
// silently drop the new field, and nothing would notice until someone looked
// for a setting that was there before the move.
func (s *SQLite) copyTable(ctx context.Context, dst *SQLite, table string) (int64, error) {
//nolint:gosec // table comes from copyOrder, not from input
rows, err := s.db.QueryContext(ctx, `SELECT * FROM `+table)
if err != nil {
return 0, err
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
return 0, err
}
q, tx, err := dst.begin(ctx)
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback() }()
insert := `INSERT INTO ` + table + ` (` + strings.Join(quoteAll(cols), ", ") +
`) VALUES (` + strings.Repeat("?, ", len(cols)-1) + `?)`
var n int64
for rows.Next() {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
return n, err
}
for i, v := range vals {
vals[i] = keepEmptyBytes(v)
}
if _, err := q.ExecContext(ctx, insert, vals...); err != nil {
return n, err
}
n++
}
if err := rows.Err(); err != nil {
return n, err
}
return n, tx.Commit()
}
// keepEmptyBytes stops an empty byte column becoming NULL on the way across.
//
// An empty BLOB scans as a typed nil slice — []byte(nil) inside a non-nil any
// — while a real NULL scans as an untyped nil. The two are distinguishable
// here and not in pgx, which encodes any nil slice as NULL, so the difference
// has to be restored before the insert.
//
// It is not a detail. key_claims.sealed is emptied the moment a claim is used,
// so every claimed row holds an empty blob in a NOT NULL column, and the copy
// failed on the first one — after the gateway had been stopped and four tables
// had already been written.
func keepEmptyBytes(v any) any {
if b, ok := v.([]byte); ok && b == nil {
return []byte{}
}
return v
}
// quoteAll quotes column names, because `key` and `text` are reserved words in
// one dialect or the other and both are column names here.
func quoteAll(cols []string) []string {
out := make([]string, len(cols))
for i, c := range cols {
out[i] = `"` + c + `"`
}
return out
}
// resetIdentities points PostgreSQL's identity columns past the copied rows.
//
// Without it the first insert after a migration collides with a row that was
// copied in: the sequence still starts at 1 while the table already holds a
// thousand notes, and the failure arrives later, to somebody else, looking
// like a bug in the feature rather than in the move.
func (s *SQLite) resetIdentities(ctx context.Context) error {
if s.dialect != PostgresDialect {
return nil
}
// Only the tables with a generated id. SQLite's rowid needs nothing.
for _, t := range []string{"memory_notes", "usage"} {
var has bool
if err := s.db.QueryRowContext(ctx,
`SELECT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = ? AND column_name = 'id'
AND is_identity = 'YES')`, t).Scan(&has); err != nil {
return err
}
if !has {
continue
}
//nolint:gosec // t is a literal from the list above
if _, err := s.db.ExecContext(ctx,
`SELECT setval(pg_get_serial_sequence('`+t+`', 'id'),
GREATEST((SELECT COALESCE(MAX(id), 0) FROM `+t+`), 1))`); err != nil {
return err
}
}
return nil
}