llmgw/internal/kb/kbsync/ledger.go

// Package kbsync is llmgw-kb's work (BACKLOG F2): it finds a person's
// documents, keeps a ledger of what it has done with each, and sends a
// knowledge base only what changed — text and vectors, never the files.
package kbsync

import (
	"context"
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"time"

	_ "modernc.org/sqlite" // pure Go, so the client builds for every system
)

// Ledger is the record, on this machine, of every document sent or skipped:
// which file it came from, its size, modification time and digest, and what
// happened. A file whose size and time have not changed is not read again to
// learn that it has not changed, and --delete removes only what this
// machine sent.
type Ledger struct {
	db *sql.DB
}

// State is what happened to a document.
type State string

// The states a document is left in.
const (
	Sent    State = "sent"
	Skipped State = "skipped"
	Failed  State = "failed"
)

// Entry is one document in the ledger.
type Entry struct {
	Gateway    string
	Collection string
	// Name is the document's name in the collection.
	Name string
	// File is the path on disk it came from, absolute.
	File   string
	Size   int64
	MTime  time.Time
	Digest string
	State  State
	Reason string
	Chunks int
	At     time.Time
}

// Run is one sync, for history.
type Run struct {
	ID                                       int64
	Gateway, Collection                      string
	Started, Finished                        time.Time
	Sent, Unchanged, Skipped, Failed, Delete int
}

// DefaultLedgerPath is where the ledger is kept: the user's configuration
// directory, which, unlike a cache directory, nothing cleans up.
func DefaultLedgerPath() (string, error) {
	dir, err := os.UserConfigDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, "llmgw-kb", "ledger.db"), nil
}

const ledgerSchema = `
CREATE TABLE IF NOT EXISTS documents (
  gateway    TEXT NOT NULL,
  collection TEXT NOT NULL,
  name       TEXT NOT NULL,
  file       TEXT NOT NULL,
  size       INTEGER NOT NULL,
  mtime_ns   INTEGER NOT NULL,
  digest     TEXT NOT NULL,
  state      TEXT NOT NULL,
  reason     TEXT NOT NULL DEFAULT '',
  chunks     INTEGER NOT NULL DEFAULT 0,
  at_ms      INTEGER NOT NULL,
  PRIMARY KEY (gateway, collection, name)
);
CREATE INDEX IF NOT EXISTS documents_by_file ON documents (file);
CREATE TABLE IF NOT EXISTS runs (
  id         INTEGER PRIMARY KEY AUTOINCREMENT,
  gateway    TEXT NOT NULL,
  collection TEXT NOT NULL,
  started_ms INTEGER NOT NULL,
  ended_ms   INTEGER NOT NULL DEFAULT 0,
  sent       INTEGER NOT NULL DEFAULT 0,
  unchanged  INTEGER NOT NULL DEFAULT 0,
  skipped    INTEGER NOT NULL DEFAULT 0,
  failed     INTEGER NOT NULL DEFAULT 0,
  deleted    INTEGER NOT NULL DEFAULT 0
);`

// OpenLedger opens (creating) the ledger at a path.
func OpenLedger(ctx context.Context, path string) (*Ledger, error) {
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		return nil, err
	}
	// WAL and a busy timeout: two syncs at once, from two terminals, wait
	// for each other rather than failing.
	db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_pragma=synchronous(NORMAL)")
	if err != nil {
		return nil, err
	}
	db.SetMaxOpenConns(1)
	if _, err := db.ExecContext(ctx, ledgerSchema); err != nil {
		_ = db.Close()
		return nil, fmt.Errorf("opening the ledger %s: %w", path, err)
	}
	_ = os.Chmod(path, 0o600)
	return &Ledger{db: db}, nil
}

// Close closes the ledger.
func (l *Ledger) Close() error { return l.db.Close() }

const entryCols = `gateway, collection, name, file, size, mtime_ns, digest, state, reason, chunks, at_ms`

func scanEntry(sc interface{ Scan(...any) error }) (Entry, error) {
	var e Entry
	var mtime, at int64
	var state string
	err := sc.Scan(&e.Gateway, &e.Collection, &e.Name, &e.File, &e.Size, &mtime, &e.Digest, &state, &e.Reason, &e.Chunks, &at)
	e.MTime, e.At, e.State = time.Unix(0, mtime), time.UnixMilli(at), State(state)
	return e, err
}

// Put records what happened to a document.
func (l *Ledger) Put(ctx context.Context, e Entry) error {
	if e.At.IsZero() {
		e.At = time.Now()
	}
	_, err := l.db.ExecContext(ctx, `INSERT INTO documents (`+entryCols+`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
		ON CONFLICT (gateway, collection, name) DO UPDATE SET file = excluded.file, size = excluded.size,
		  mtime_ns = excluded.mtime_ns, digest = excluded.digest, state = excluded.state,
		  reason = excluded.reason, chunks = excluded.chunks, at_ms = excluded.at_ms`,
		e.Gateway, e.Collection, e.Name, e.File, e.Size, e.MTime.UnixNano(), e.Digest, string(e.State),
		e.Reason, e.Chunks, e.At.UnixMilli())
	return err
}

// Remove forgets a document.
func (l *Ledger) Remove(ctx context.Context, gateway, collection, name string) error {
	_, err := l.db.ExecContext(ctx, `DELETE FROM documents WHERE gateway = ? AND collection = ? AND name = ?`,
		gateway, collection, name)
	return err
}

// Entries lists a collection's documents as this machine left them.
func (l *Ledger) Entries(ctx context.Context, gateway, collection string) ([]Entry, error) {
	rows, err := l.db.QueryContext(ctx, `SELECT `+entryCols+` FROM documents WHERE gateway = ? AND collection = ? ORDER BY name`,
		gateway, collection)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Entry
	for rows.Next() {
		e, err := scanEntry(rows)
		if err != nil {
			return nil, err
		}
		out = append(out, e)
	}
	return out, rows.Err()
}

// Known is the digest the ledger has for a file of this size and time, from
// any collection: a file is hashed once however many collections it goes to.
func (l *Ledger) Known(ctx context.Context, file string, size int64, mtime time.Time) (string, bool) {
	var d string
	err := l.db.QueryRowContext(ctx, `SELECT digest FROM documents WHERE file = ? AND size = ? AND mtime_ns = ? LIMIT 1`,
		file, size, mtime.UnixNano()).Scan(&d)
	return d, err == nil
}

// StartRun records that a sync began.
func (l *Ledger) StartRun(ctx context.Context, gateway, collection string) (int64, error) {
	res, err := l.db.ExecContext(ctx, `INSERT INTO runs (gateway, collection, started_ms) VALUES (?, ?, ?)`,
		gateway, collection, time.Now().UnixMilli())
	if err != nil {
		return 0, err
	}
	return res.LastInsertId()
}

// EndRun records how a sync ended.
func (l *Ledger) EndRun(ctx context.Context, id int64, r Run) error {
	_, err := l.db.ExecContext(ctx, `UPDATE runs SET ended_ms = ?, sent = ?, unchanged = ?, skipped = ?, failed = ?, deleted = ?
		WHERE id = ?`, time.Now().UnixMilli(), r.Sent, r.Unchanged, r.Skipped, r.Failed, r.Delete, id)
	return err
}

// Runs lists the latest syncs, newest first.
func (l *Ledger) Runs(ctx context.Context, limit int) ([]Run, error) {
	rows, err := l.db.QueryContext(ctx, `SELECT id, gateway, collection, started_ms, ended_ms, sent, unchanged, skipped, failed, deleted
		FROM runs ORDER BY id DESC LIMIT ?`, limit)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Run
	for rows.Next() {
		var r Run
		var s, e int64
		if err := rows.Scan(&r.ID, &r.Gateway, &r.Collection, &s, &e, &r.Sent, &r.Unchanged, &r.Skipped, &r.Failed, &r.Delete); err != nil {
			return nil, err
		}
		r.Started = time.UnixMilli(s)
		if e > 0 {
			r.Finished = time.UnixMilli(e)
		}
		out = append(out, r)
	}
	return out, rows.Err()
}