// Package kbsync is llmgw-kb's work (BACKLOG F2): it finds a person's // documents, keeps a ledger of what it has done with each, and sends a // knowledge base only what changed — text and vectors, never the files. package kbsync import ( "context" "database/sql" "fmt" "os" "path/filepath" "time" _ "modernc.org/sqlite" // pure Go, so the client builds for every system ) // Ledger is the record, on this machine, of every document sent or skipped: // which file it came from, its size, modification time and digest, and what // happened. A file whose size and time have not changed is not read again to // learn that it has not changed, and --delete removes only what this // machine sent. type Ledger struct { db *sql.DB } // State is what happened to a document. type State string // The states a document is left in. const ( Sent State = "sent" Skipped State = "skipped" Failed State = "failed" ) // Entry is one document in the ledger. type Entry struct { Gateway string Collection string // Name is the document's name in the collection. Name string // File is the path on disk it came from, absolute. File string Size int64 MTime time.Time Digest string State State Reason string Chunks int At time.Time } // Run is one sync, for history. type Run struct { ID int64 Gateway, Collection string Started, Finished time.Time Sent, Unchanged, Skipped, Failed, Delete int } // DefaultLedgerPath is where the ledger is kept: the user's configuration // directory, which, unlike a cache directory, nothing cleans up. func DefaultLedgerPath() (string, error) { dir, err := os.UserConfigDir() if err != nil { return "", err } return filepath.Join(dir, "llmgw-kb", "ledger.db"), nil } const ledgerSchema = ` CREATE TABLE IF NOT EXISTS documents ( gateway TEXT NOT NULL, collection TEXT NOT NULL, name TEXT NOT NULL, file TEXT NOT NULL, size INTEGER NOT NULL, mtime_ns INTEGER NOT NULL, digest TEXT NOT NULL, state TEXT NOT NULL, reason TEXT NOT NULL DEFAULT '', chunks INTEGER NOT NULL DEFAULT 0, at_ms INTEGER NOT NULL, PRIMARY KEY (gateway, collection, name) ); CREATE INDEX IF NOT EXISTS documents_by_file ON documents (file); CREATE TABLE IF NOT EXISTS runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, gateway TEXT NOT NULL, collection TEXT NOT NULL, started_ms INTEGER NOT NULL, ended_ms INTEGER NOT NULL DEFAULT 0, sent INTEGER NOT NULL DEFAULT 0, unchanged INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0, failed INTEGER NOT NULL DEFAULT 0, deleted INTEGER NOT NULL DEFAULT 0 );` // OpenLedger opens (creating) the ledger at a path. func OpenLedger(ctx context.Context, path string) (*Ledger, error) { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return nil, err } // WAL and a busy timeout: two syncs at once, from two terminals, wait // for each other rather than failing. db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_pragma=synchronous(NORMAL)") if err != nil { return nil, err } db.SetMaxOpenConns(1) if _, err := db.ExecContext(ctx, ledgerSchema); err != nil { _ = db.Close() return nil, fmt.Errorf("opening the ledger %s: %w", path, err) } _ = os.Chmod(path, 0o600) return &Ledger{db: db}, nil } // Close closes the ledger. func (l *Ledger) Close() error { return l.db.Close() } const entryCols = `gateway, collection, name, file, size, mtime_ns, digest, state, reason, chunks, at_ms` func scanEntry(sc interface{ Scan(...any) error }) (Entry, error) { var e Entry var mtime, at int64 var state string err := sc.Scan(&e.Gateway, &e.Collection, &e.Name, &e.File, &e.Size, &mtime, &e.Digest, &state, &e.Reason, &e.Chunks, &at) e.MTime, e.At, e.State = time.Unix(0, mtime), time.UnixMilli(at), State(state) return e, err } // Put records what happened to a document. func (l *Ledger) Put(ctx context.Context, e Entry) error { if e.At.IsZero() { e.At = time.Now() } _, err := l.db.ExecContext(ctx, `INSERT INTO documents (`+entryCols+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (gateway, collection, name) DO UPDATE SET file = excluded.file, size = excluded.size, mtime_ns = excluded.mtime_ns, digest = excluded.digest, state = excluded.state, reason = excluded.reason, chunks = excluded.chunks, at_ms = excluded.at_ms`, e.Gateway, e.Collection, e.Name, e.File, e.Size, e.MTime.UnixNano(), e.Digest, string(e.State), e.Reason, e.Chunks, e.At.UnixMilli()) return err } // Remove forgets a document. func (l *Ledger) Remove(ctx context.Context, gateway, collection, name string) error { _, err := l.db.ExecContext(ctx, `DELETE FROM documents WHERE gateway = ? AND collection = ? AND name = ?`, gateway, collection, name) return err } // Entries lists a collection's documents as this machine left them. func (l *Ledger) Entries(ctx context.Context, gateway, collection string) ([]Entry, error) { rows, err := l.db.QueryContext(ctx, `SELECT `+entryCols+` FROM documents WHERE gateway = ? AND collection = ? ORDER BY name`, gateway, collection) if err != nil { return nil, err } defer rows.Close() var out []Entry for rows.Next() { e, err := scanEntry(rows) if err != nil { return nil, err } out = append(out, e) } return out, rows.Err() } // Known is the digest the ledger has for a file of this size and time, from // any collection: a file is hashed once however many collections it goes to. func (l *Ledger) Known(ctx context.Context, file string, size int64, mtime time.Time) (string, bool) { var d string err := l.db.QueryRowContext(ctx, `SELECT digest FROM documents WHERE file = ? AND size = ? AND mtime_ns = ? LIMIT 1`, file, size, mtime.UnixNano()).Scan(&d) return d, err == nil } // StartRun records that a sync began. func (l *Ledger) StartRun(ctx context.Context, gateway, collection string) (int64, error) { res, err := l.db.ExecContext(ctx, `INSERT INTO runs (gateway, collection, started_ms) VALUES (?, ?, ?)`, gateway, collection, time.Now().UnixMilli()) if err != nil { return 0, err } return res.LastInsertId() } // EndRun records how a sync ended. func (l *Ledger) EndRun(ctx context.Context, id int64, r Run) error { _, err := l.db.ExecContext(ctx, `UPDATE runs SET ended_ms = ?, sent = ?, unchanged = ?, skipped = ?, failed = ?, deleted = ? WHERE id = ?`, time.Now().UnixMilli(), r.Sent, r.Unchanged, r.Skipped, r.Failed, r.Delete, id) return err } // Runs lists the latest syncs, newest first. func (l *Ledger) Runs(ctx context.Context, limit int) ([]Run, error) { rows, err := l.db.QueryContext(ctx, `SELECT id, gateway, collection, started_ms, ended_ms, sent, unchanged, skipped, failed, deleted FROM runs ORDER BY id DESC LIMIT ?`, limit) if err != nil { return nil, err } defer rows.Close() var out []Run for rows.Next() { var r Run var s, e int64 if err := rows.Scan(&r.ID, &r.Gateway, &r.Collection, &s, &e, &r.Sent, &r.Unchanged, &r.Skipped, &r.Failed, &r.Delete); err != nil { return nil, err } r.Started = time.UnixMilli(s) if e > 0 { r.Finished = time.UnixMilli(e) } out = append(out, r) } return out, rows.Err() }