Keys are stored hashed, so a migration that drops them cannot recreate them: every person and every script needs a new one issued, and the old ones cannot even be looked up to say which they were. `llmgw keys move-store` copies instead. Found by running it against the real cluster, twice, after the gateway had already been stopped. The first order came from the order the migrations create tables in, and users is created in a later migration than api_keys even though api_keys references it — so the copy died three tables in. The order is now checked by a test that derives the foreign keys from the migrations rather than restating them, so a new reference fails there instead of during somebody's migration. Checked by putting the wrong order back and watching it fail. Then key_claims. Its sealed column is emptied the moment a claim is used, so every claimed row holds an empty blob in a NOT NULL column. An empty blob scans as a typed nil slice and a genuine NULL as an untyped one — a distinction this side can see and pgx cannot, since it encodes any nil slice as NULL. Restored before the insert, which is the difference between a copy that completes and one that stops four tables in. One transaction per table rather than one for the whole copy: a single transaction holding every row of a large usage table is how a migration runs out of memory on the machine least able to spare it. A failure therefore leaves a partial copy, which is why the destination must have been empty — starting again is dropping and recreating it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a76ebdb79a
commit
b6de469351
4 files changed
+533
No files matched your search
@@ -37,6 +37,7 @@ func newTenancyCmd() *cobra.Command {
|
||||
newLimitsCmd(open),
|
||||
newThreadsCmd(open),
|
||||
newMemoryCmd(open),
|
||||
newMoveStoreCmd(),
|
||||
)
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// newMoveStoreCmd copies a store into another one.
|
||||
//
|
||||
// Written because keys are stored hashed. Moving to PostgreSQL without this
|
||||
// means every person and every script needs a new key issued, and the old ones
|
||||
// cannot even be looked up to say which they were.
|
||||
//
|
||||
// It reads and writes, and nothing else runs while it does: run it with the
|
||||
// gateway stopped. A copy taken while requests are landing would miss whatever
|
||||
// arrived after the table it had already passed.
|
||||
func newMoveStoreCmd() *cobra.Command {
|
||||
var from, to string
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "move-store",
|
||||
Short: "Copy a store into another one, for moving SQLite to PostgreSQL",
|
||||
Long: "Copies every table, parents first, into an empty destination.\n\n" +
|
||||
"Stop the gateway first. This makes no attempt to copy a store that is\n" +
|
||||
"being written to: rows landing after the table they belong to has been\n" +
|
||||
"passed would simply be absent, and nothing would say so.\n\n" +
|
||||
"The destination must be empty. Merging two stores means deciding which\n" +
|
||||
"of two projects with the same id was meant, and there is no answer to\n" +
|
||||
"that which is not a guess.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if from == "" || to == "" {
|
||||
return fmt.Errorf("--from and --to are both required")
|
||||
}
|
||||
src, err := store.Open(cmd.Context(), from)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening the source: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := store.Open(cmd.Context(), to)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening the destination: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
res, err := src.CopyTo(cmd.Context(), dst)
|
||||
// Printed even on failure: a partial copy is the one case where
|
||||
// knowing how far it got decides what to do next.
|
||||
out := cmd.OutOrStdout()
|
||||
var total int64
|
||||
for _, r := range res {
|
||||
if r.Rows > 0 {
|
||||
fmt.Fprintf(out, " %-16s %d\n", r.Table, r.Rows)
|
||||
}
|
||||
total += r.Rows
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "\n%d rows copied. Point the gateway at the new store and start it.\n", total)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&from, "from", "", "source: a SQLite path, a postgres:// URL, or @file holding one")
|
||||
cmd.Flags().StringVar(&to, "to", "", "destination, same forms")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// copyOrder is every table, parents before children.
|
||||
//
|
||||
// Written out rather than discovered, because the order is a fact about the
|
||||
// foreign keys and getting it wrong produces a constraint violation halfway
|
||||
// through — with some tables copied and some not, which is the worst place to
|
||||
// stop. A table added without being added here is copied by nothing, and the
|
||||
// destination comes up quietly missing it; the test below is what catches
|
||||
// that.
|
||||
var copyOrder = []string{
|
||||
"organizations",
|
||||
"teams",
|
||||
"projects",
|
||||
// Before api_keys, which references it. The first version of this list
|
||||
// followed the order the tables are created across the migrations, and
|
||||
// users is created in a later one than api_keys even though api_keys
|
||||
// points at it — so the copy failed on a foreign key three tables in.
|
||||
"users",
|
||||
"api_keys",
|
||||
"key_claims",
|
||||
"sessions",
|
||||
"limits",
|
||||
"usage",
|
||||
"models",
|
||||
"threads",
|
||||
"thread_messages",
|
||||
"memory_notes",
|
||||
}
|
||||
|
||||
// CopyResult is how many rows each table received.
|
||||
type CopyResult struct {
|
||||
Table string
|
||||
Rows int64
|
||||
}
|
||||
|
||||
// CopyTo copies every row into another store.
|
||||
//
|
||||
// For moving from SQLite to PostgreSQL, which otherwise means reissuing every
|
||||
// API key: the keys are stored hashed, so there is no way to recreate them and
|
||||
// no way for anyone to keep using the one they have.
|
||||
//
|
||||
// The destination must be empty. Merging two stores would mean deciding what
|
||||
// happens when both have a project with the same id and different contents,
|
||||
// and every answer to that is a guess about which one someone meant.
|
||||
//
|
||||
// Not a transaction across the whole copy. It is one transaction per table,
|
||||
// because a single transaction holding every row of a large usage table is how
|
||||
// a migration runs out of memory on the machine least able to spare it. A
|
||||
// failure therefore leaves a partial copy, which is why the destination has to
|
||||
// have been empty: starting again is dropping and recreating it.
|
||||
func (s *SQLite) CopyTo(ctx context.Context, dst *SQLite) ([]CopyResult, error) {
|
||||
empty, where, err := dst.isEmpty(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !empty {
|
||||
return nil, fmt.Errorf("the destination already has rows in %s; copy into an empty store", where)
|
||||
}
|
||||
|
||||
var out []CopyResult
|
||||
for _, table := range copyOrder {
|
||||
n, err := s.copyTable(ctx, dst, table)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("copying %s: %w", table, err)
|
||||
}
|
||||
out = append(out, CopyResult{Table: table, Rows: n})
|
||||
}
|
||||
// Part of the copy, not a step someone has to remember: a destination
|
||||
// whose sequences still start at 1 is not a destination anyone can use.
|
||||
if err := dst.resetIdentities(ctx); err != nil {
|
||||
return out, fmt.Errorf("pointing the identity columns past the copied rows: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isEmpty reports whether any table has rows, and which one first does.
|
||||
func (s *SQLite) isEmpty(ctx context.Context) (bool, string, error) {
|
||||
for _, table := range copyOrder {
|
||||
var n int64
|
||||
//nolint:gosec // table comes from copyOrder, not from input
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&n); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if n > 0 {
|
||||
return false, table, nil
|
||||
}
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
// copyTable moves one table, column names taken from the source.
|
||||
//
|
||||
// Reading the columns rather than listing them keeps this working when a
|
||||
// migration adds one: a hand-written column list would copy the old shape and
|
||||
// silently drop the new field, and nothing would notice until someone looked
|
||||
// for a setting that was there before the move.
|
||||
func (s *SQLite) copyTable(ctx context.Context, dst *SQLite, table string) (int64, error) {
|
||||
//nolint:gosec // table comes from copyOrder, not from input
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT * FROM `+table)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
q, tx, err := dst.begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
insert := `INSERT INTO ` + table + ` (` + strings.Join(quoteAll(cols), ", ") +
|
||||
`) VALUES (` + strings.Repeat("?, ", len(cols)-1) + `?)`
|
||||
|
||||
var n int64
|
||||
for rows.Next() {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return n, err
|
||||
}
|
||||
for i, v := range vals {
|
||||
vals[i] = keepEmptyBytes(v)
|
||||
}
|
||||
if _, err := q.ExecContext(ctx, insert, vals...); err != nil {
|
||||
return n, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return n, err
|
||||
}
|
||||
return n, tx.Commit()
|
||||
}
|
||||
|
||||
// keepEmptyBytes stops an empty byte column becoming NULL on the way across.
|
||||
//
|
||||
// An empty BLOB scans as a typed nil slice — []byte(nil) inside a non-nil any
|
||||
// — while a real NULL scans as an untyped nil. The two are distinguishable
|
||||
// here and not in pgx, which encodes any nil slice as NULL, so the difference
|
||||
// has to be restored before the insert.
|
||||
//
|
||||
// It is not a detail. key_claims.sealed is emptied the moment a claim is used,
|
||||
// so every claimed row holds an empty blob in a NOT NULL column, and the copy
|
||||
// failed on the first one — after the gateway had been stopped and four tables
|
||||
// had already been written.
|
||||
func keepEmptyBytes(v any) any {
|
||||
if b, ok := v.([]byte); ok && b == nil {
|
||||
return []byte{}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// quoteAll quotes column names, because `key` and `text` are reserved words in
|
||||
// one dialect or the other and both are column names here.
|
||||
func quoteAll(cols []string) []string {
|
||||
out := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
out[i] = `"` + c + `"`
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resetIdentities points PostgreSQL's identity columns past the copied rows.
|
||||
//
|
||||
// Without it the first insert after a migration collides with a row that was
|
||||
// copied in: the sequence still starts at 1 while the table already holds a
|
||||
// thousand notes, and the failure arrives later, to somebody else, looking
|
||||
// like a bug in the feature rather than in the move.
|
||||
func (s *SQLite) resetIdentities(ctx context.Context) error {
|
||||
if s.dialect != PostgresDialect {
|
||||
return nil
|
||||
}
|
||||
// Only the tables with a generated id. SQLite's rowid needs nothing.
|
||||
for _, t := range []string{"memory_notes", "usage"} {
|
||||
var has bool
|
||||
if err := s.db.QueryRowContext(ctx,
|
||||
`SELECT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = ? AND column_name = 'id'
|
||||
AND is_identity = 'YES')`, t).Scan(&has); err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
//nolint:gosec // t is a literal from the list above
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`SELECT setval(pg_get_serial_sequence('`+t+`', 'id'),
|
||||
GREATEST((SELECT COALESCE(MAX(id), 0) FROM `+t+`), 1))`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEveryTableIsInTheCopyOrder(t *testing.T) {
|
||||
// A table added by a migration and not added here is copied by nothing,
|
||||
// and the destination comes up quietly missing it — discovered when
|
||||
// somebody looks for a setting that was there before the move. Read out of
|
||||
// the migrations rather than listed twice, so the two cannot disagree.
|
||||
var declared []string
|
||||
entries, err := migrationFS.ReadDir("migrations/sqlite")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
body, err := migrationFS.ReadFile("migrations/sqlite/" + e.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
rest, ok := strings.CutPrefix(line, "CREATE TABLE ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, _, _ := strings.Cut(rest, " ")
|
||||
name = strings.TrimSuffix(strings.TrimSpace(name), "(")
|
||||
declared = append(declared, name)
|
||||
}
|
||||
}
|
||||
if len(declared) == 0 {
|
||||
t.Fatal("found no tables in the migrations, so this test proves nothing")
|
||||
}
|
||||
for _, name := range declared {
|
||||
if !slices.Contains(copyOrder, name) {
|
||||
t.Errorf("%q is created by a migration but nothing copies it", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyRefusesADestinationThatAlreadyHasRows(t *testing.T) {
|
||||
// Merging two stores means deciding what happens when both hold a project
|
||||
// with the same id and different contents, and every answer is a guess
|
||||
// about which one somebody meant.
|
||||
ctx := context.Background()
|
||||
src, dst := openDB(t), openDB(t)
|
||||
tree(t, src)
|
||||
tree(t, dst)
|
||||
|
||||
_, err := src.CopyTo(ctx, dst)
|
||||
if err == nil || !strings.Contains(err.Error(), "empty store") {
|
||||
t.Errorf("copying into a populated store gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACopiedStoreStillAuthenticatesTheSameKey(t *testing.T) {
|
||||
// The reason this exists. Keys are stored hashed, so a migration that
|
||||
// loses them cannot recreate them: every person and every script would
|
||||
// need a new one issued.
|
||||
ctx := context.Background()
|
||||
src := openDB(t)
|
||||
proj := tree(t, src)
|
||||
secret, k := mustKey(t, src, "k1", proj)
|
||||
|
||||
if err := src.WriteUsage(ctx, []Usage{{
|
||||
At: time.Now(), KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m",
|
||||
PromptTokens: 3, CompletionTokens: 4, CostMicros: 99, Status: 200,
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
res, err := src.CopyTo(ctx, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("copying: %v", err)
|
||||
}
|
||||
if len(res) != len(copyOrder) {
|
||||
t.Errorf("copied %d tables of %d", len(res), len(copyOrder))
|
||||
}
|
||||
|
||||
// The same secret the caller already holds still resolves, in the copy.
|
||||
got, err := dst.KeyByHash(ctx, HashKey(secret))
|
||||
if err != nil {
|
||||
t.Fatalf("the copied store does not know the key: %v", err)
|
||||
}
|
||||
if got.ID != k.ID || got.ProjectID != proj {
|
||||
t.Errorf("key came back as %+v", got)
|
||||
}
|
||||
|
||||
// And the spend it had, so a budget is not quietly reset by moving.
|
||||
spend, err := dst.SpendSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if spend[k.ID] != 99 {
|
||||
t.Errorf("spend after the copy = %d, want 99", spend[k.ID])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyCarriesConversationsAndNotes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
src := openDB(t)
|
||||
proj := tree(t, src)
|
||||
if err := src.SetThreadRetention(ctx, proj, 24); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := src.AppendThread(ctx, proj, "th1", []Message{
|
||||
{Role: "user", Content: "remembered"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
if _, err := src.CopyTo(ctx, dst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
msgs, err := dst.Thread(ctx, proj, "th1")
|
||||
if err != nil {
|
||||
t.Fatalf("the conversation did not survive: %v", err)
|
||||
}
|
||||
if len(msgs) != 1 || msgs[0].Content != "remembered" {
|
||||
t.Errorf("thread came back as %+v", msgs)
|
||||
}
|
||||
// The retention setting is a column on projects, so it rides along — a
|
||||
// destination that kept the conversations and forgot the promise about
|
||||
// how long would be worse than one that kept neither.
|
||||
if d, err := dst.ThreadRetention(ctx, proj); err != nil || d == 0 {
|
||||
t.Errorf("retention after the copy = %v (err %v)", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyingAnEmptyStoreIsFine(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
src := openDB(t)
|
||||
dst, err := Open(ctx, filepath.Join(t.TempDir(), "copy.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
if _, err := src.CopyTo(ctx, dst); err != nil {
|
||||
t.Errorf("copying an empty store failed: %v", err)
|
||||
}
|
||||
_ = os.Remove("")
|
||||
}
|
||||
|
||||
func TestTheCopyOrderSatisfiesTheForeignKeys(t *testing.T) {
|
||||
// Completeness is not enough: a table copied before the one it references
|
||||
// fails on a constraint partway through, leaving some tables copied and
|
||||
// some not. The first version of copyOrder followed the order the
|
||||
// migrations create tables in, and users is created after api_keys even
|
||||
// though api_keys points at it — so the copy died three tables in, on a
|
||||
// real database, after the gateway had already been stopped.
|
||||
//
|
||||
// Derived from the migrations rather than restated, so a new reference
|
||||
// fails here rather than during somebody's migration.
|
||||
deps := map[string]map[string]bool{}
|
||||
note := func(table, ref string) {
|
||||
if table == ref {
|
||||
return
|
||||
}
|
||||
if deps[table] == nil {
|
||||
deps[table] = map[string]bool{}
|
||||
}
|
||||
deps[table][ref] = true
|
||||
}
|
||||
|
||||
entries, err := migrationFS.ReadDir("migrations/sqlite")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := ""
|
||||
for _, e := range entries {
|
||||
b, err := migrationFS.ReadFile("migrations/sqlite/" + e.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body += string(b)
|
||||
}
|
||||
|
||||
// Table bodies, so a REFERENCES is attributed to the table holding it.
|
||||
for _, m := range regexpCreateTable.FindAllStringSubmatch(body, -1) {
|
||||
for _, r := range regexpReferences.FindAllStringSubmatch(m[2], -1) {
|
||||
note(m[1], r[1])
|
||||
}
|
||||
}
|
||||
// And columns added later, which is exactly how api_keys got its user_id.
|
||||
for _, m := range regexpAlterRef.FindAllStringSubmatch(body, -1) {
|
||||
note(m[1], m[2])
|
||||
}
|
||||
if len(deps) == 0 {
|
||||
t.Fatal("found no foreign keys, so this test proves nothing")
|
||||
}
|
||||
|
||||
position := map[string]int{}
|
||||
for i, t := range copyOrder {
|
||||
position[t] = i
|
||||
}
|
||||
for table, refs := range deps {
|
||||
for ref := range refs {
|
||||
if position[ref] > position[table] {
|
||||
t.Errorf("%s is copied before %s, which it references", table, ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
regexpCreateTable = regexp.MustCompile(`(?s)CREATE TABLE (\w+)\s*\((.*?)\n\);`)
|
||||
regexpReferences = regexp.MustCompile(`REFERENCES\s+(\w+)`)
|
||||
regexpAlterRef = regexp.MustCompile(`ALTER TABLE (\w+) ADD COLUMN[^;]*REFERENCES\s+(\w+)`)
|
||||
)
|
||||
|
||||
func TestAnEmptyBlobDoesNotBecomeNullInTheCopy(t *testing.T) {
|
||||
// key_claims.sealed is emptied the moment a claim is used, so a claimed
|
||||
// row holds an empty blob in a NOT NULL column. An empty blob scans as a
|
||||
// typed nil slice and a real NULL as an untyped one; pgx cannot tell them
|
||||
// apart, so the distinction is restored before the insert.
|
||||
if got := keepEmptyBytes([]byte(nil)); got == nil {
|
||||
t.Error("an empty byte column would be inserted as NULL")
|
||||
} else if b, ok := got.([]byte); !ok || len(b) != 0 {
|
||||
t.Errorf("empty bytes came out as %#v", got)
|
||||
}
|
||||
// A genuine NULL arrives as an untyped nil and must stay one.
|
||||
if got := keepEmptyBytes(nil); got != nil {
|
||||
t.Errorf("a NULL became %#v", got)
|
||||
}
|
||||
// Everything else passes through.
|
||||
if got := keepEmptyBytes(int64(7)); got != int64(7) {
|
||||
t.Errorf("a number was rewritten to %#v", got)
|
||||
}
|
||||
if got := keepEmptyBytes([]byte("x")); string(got.([]byte)) != "x" {
|
||||
t.Errorf("a non-empty blob was rewritten to %#v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user