diff --git a/internal/store/migrations/sqlite/0006_threads.sql b/internal/store/migrations/sqlite/0006_threads.sql new file mode 100644 index 00000000..1e0b2bc7 --- /dev/null +++ b/internal/store/migrations/sqlite/0006_threads.sql @@ -0,0 +1,48 @@ +-- Server-side conversations, so a thread can continue on a different model +-- than it started on. +-- +-- Off unless a project turns it on. Until this migration ran, llmgw stored no +-- message content anywhere — usage rows carry token counts and costs, never +-- text — and that stays true for every project that does not ask. An on-prem +-- gateway keeping nothing anyone typed is much easier to put in front of a +-- team than one keeping everything, so storing is a choice a project makes +-- rather than a default it discovers. + +-- retention_hours is the switch and the limit at once. NULL means this project +-- stores nothing, which is every project that existed before this and every +-- one created since without asking. +-- +-- One column rather than a boolean and a duration: a thread store with no +-- expiry is a transcript of everything a team ever typed, kept forever by +-- omission. Turning it on therefore requires saying for how long. +ALTER TABLE projects ADD COLUMN thread_retention_hours INTEGER; + +CREATE TABLE threads ( + id TEXT PRIMARY KEY, + -- Scoped to a project, and checked on every read. Thread ids are opaque, but + -- an unguessable id is a secret and secrets leak into logs, shell history + -- and bug reports; the scope is what makes a leaked id useless to someone + -- outside the project. + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, + -- Touched on every append, so retention measures idleness rather than age: + -- expiring a conversation someone is still having would be a strange way to + -- keep a promise about storage. + updated_at INTEGER NOT NULL +); +CREATE INDEX threads_by_project ON threads(project_id); +CREATE INDEX threads_by_updated ON threads(updated_at); + +CREATE TABLE thread_messages ( + thread_id TEXT NOT NULL REFERENCES threads(id) ON DELETE CASCADE, + -- Position rather than a timestamp, because order is the point and two + -- messages can share a millisecond. + seq INTEGER NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + -- Which model produced an assistant turn. A thread that moved between models + -- otherwise reads as one voice, and when the answers disagree that is the + -- first thing anyone wants to know. + model TEXT, + PRIMARY KEY (thread_id, seq) +); diff --git a/internal/store/threads.go b/internal/store/threads.go new file mode 100644 index 00000000..944aa301 --- /dev/null +++ b/internal/store/threads.go @@ -0,0 +1,216 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +// ErrThreadsNotEnabled means the project has not asked llmgw to keep +// conversations. +// +// Returned rather than silently ignoring the thread. Ignoring it would send +// the model a conversation with no history, and it would answer confidently +// about something it was never told — which reads as the model being wrong +// rather than as a setting being off. +var ErrThreadsNotEnabled = errors.New("store: this project does not keep conversations") + +// ErrNoThread means the id does not exist, or belongs to another project. +// +// One error for both on purpose. Distinguishing them would turn thread ids +// into an oracle: a caller could learn which ids exist in projects they cannot +// read by watching which error comes back. +var ErrNoThread = errors.New("store: no such thread") + +// Message is one turn of a stored conversation. +type Message struct { + Role string + Content string + // Model is which model produced an assistant turn, empty for a user one. A + // thread that moved between models otherwise reads as a single voice. + Model string +} + +// ThreadRetention reports how long a project keeps conversations. +// +// Zero means it keeps none, which is the default and was the only behaviour +// before threads existed. +func (s *SQLite) ThreadRetention(ctx context.Context, projectID string) (time.Duration, error) { + var hours sql.NullInt64 + err := s.db.QueryRowContext(ctx, + `SELECT thread_retention_hours FROM projects WHERE id = ?`, projectID).Scan(&hours) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + if err != nil { + return 0, err + } + if !hours.Valid || hours.Int64 <= 0 { + return 0, nil + } + return time.Duration(hours.Int64) * time.Hour, nil +} + +// SetThreadRetention turns conversation storage on for a project, or off. +// +// Hours rather than a boolean: a thread store with no expiry is a transcript +// of everything a team ever typed, kept forever because nobody chose a +// number. Turning it on means saying for how long. +func (s *SQLite) SetThreadRetention(ctx context.Context, projectID string, hours int) error { + var v any + if hours > 0 { + v = hours + } + res, err := s.db.ExecContext(ctx, + `UPDATE projects SET thread_retention_hours = ? WHERE id = ?`, v, projectID) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n == 0 { + return fmt.Errorf("store: no project %q", projectID) + } + return nil +} + +// Thread returns a conversation's messages in order. +// +// The project is part of the lookup rather than checked afterwards, so there +// is no path that reads another project's conversation even briefly. +func (s *SQLite) Thread(ctx context.Context, projectID, threadID string) ([]Message, error) { + var owner string + err := s.db.QueryRowContext(ctx, + `SELECT project_id FROM threads WHERE id = ?`, threadID).Scan(&owner) + if errors.Is(err, sql.ErrNoRows) || (err == nil && owner != projectID) { + return nil, ErrNoThread + } + if err != nil { + return nil, err + } + + return queryAll(ctx, s.db, + `SELECT role, content, COALESCE(model, '') FROM thread_messages + WHERE thread_id = ? ORDER BY seq`, + []any{threadID}, func(rows *sql.Rows) (Message, error) { + var m Message + return m, rows.Scan(&m.Role, &m.Content, &m.Model) + }) +} + +// AppendThread adds turns to a conversation, creating it if it is new. +// +// One transaction, because a half-written exchange is worse than a lost one: +// a stored question with no answer would be replayed to the next model as +// though the assistant had ignored it. +func (s *SQLite) AppendThread(ctx context.Context, projectID, threadID string, msgs []Message) error { + if len(msgs) == 0 { + return nil + } + retention, err := s.ThreadRetention(ctx, projectID) + if err != nil { + return err + } + if retention == 0 { + return ErrThreadsNotEnabled + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + now := time.Now().UnixMilli() + var owner string + err = tx.QueryRowContext(ctx, `SELECT project_id FROM threads WHERE id = ?`, threadID).Scan(&owner) + switch { + case errors.Is(err, sql.ErrNoRows): + if _, err := tx.ExecContext(ctx, + `INSERT INTO threads (id, project_id, created_at, updated_at) VALUES (?, ?, ?, ?)`, + threadID, projectID, now, now); err != nil { + return err + } + case err != nil: + return err + case owner != projectID: + // Appending to a thread another project owns would let one team write + // into another's conversation, and then read it back through their + // own replies. + return ErrNoThread + } + + var next int64 + if err := tx.QueryRowContext(ctx, + `SELECT COALESCE(MAX(seq), -1) + 1 FROM thread_messages WHERE thread_id = ?`, + threadID).Scan(&next); err != nil { + return err + } + for i, m := range msgs { + var model any + if m.Model != "" { + model = m.Model + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO thread_messages (thread_id, seq, role, content, model) VALUES (?, ?, ?, ?, ?)`, + threadID, next+int64(i), m.Role, m.Content, model); err != nil { + return err + } + } + // Touched on every append, so retention measures idleness rather than age. + if _, err := tx.ExecContext(ctx, + `UPDATE threads SET updated_at = ? WHERE id = ?`, now, threadID); err != nil { + return err + } + return tx.Commit() +} + +// DeleteThread removes a conversation and everything in it. +// +// Somebody has to be able to say "forget that", and a retention window is not +// an answer to a message sent by mistake. +func (s *SQLite) DeleteThread(ctx context.Context, projectID, threadID string) error { + res, err := s.db.ExecContext(ctx, + `DELETE FROM threads WHERE id = ? AND project_id = ?`, threadID, projectID) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrNoThread + } + return nil +} + +// PruneThreads deletes conversations past their project's retention. +// +// Per project, because the window is per project. A single sweep with one +// cutoff would apply the shortest retention to everyone or the longest to +// nobody, and the number a project chose is the promise made to it. +func (s *SQLite) PruneThreads(ctx context.Context, now time.Time) (int, error) { + type window struct { + project string + hours int64 + } + windows, err := queryAll(ctx, s.db, + `SELECT id, thread_retention_hours FROM projects WHERE thread_retention_hours > 0`, + nil, func(rows *sql.Rows) (window, error) { + var w window + return w, rows.Scan(&w.project, &w.hours) + }) + if err != nil { + return 0, err + } + + var removed int + for _, w := range windows { + cutoff := now.Add(-time.Duration(w.hours) * time.Hour).UnixMilli() + res, err := s.db.ExecContext(ctx, + `DELETE FROM threads WHERE project_id = ? AND updated_at < ?`, w.project, cutoff) + if err != nil { + return removed, err + } + n, _ := res.RowsAffected() + removed += int(n) + } + return removed, nil +} diff --git a/internal/store/threads_test.go b/internal/store/threads_test.go new file mode 100644 index 00000000..30b7dee3 --- /dev/null +++ b/internal/store/threads_test.go @@ -0,0 +1,199 @@ +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestAProjectKeepsNoConversationsUnlessItAsks(t *testing.T) { + // The property being given up here is worth protecting in a test: until a + // project turns this on, llmgw stores no message content anywhere. Every + // project that existed before threads did, and every one created since + // without asking, must stay that way — a gateway that quietly starts + // keeping what a team types is a much harder thing to have deployed. + ctx := context.Background() + s := openDB(t) + proj := tree(t, s) + + if d, err := s.ThreadRetention(ctx, proj); err != nil || d != 0 { + t.Fatalf("a new project retains %v (err %v), want none", d, err) + } + + err := s.AppendThread(ctx, proj, "th1", []Message{{Role: "user", Content: "secret"}}) + if !errors.Is(err, ErrThreadsNotEnabled) { + t.Fatalf("storing into a project that never asked returned %v", err) + } + + // And nothing was written on the way to refusing. + if _, err := s.Thread(ctx, proj, "th1"); !errors.Is(err, ErrNoThread) { + t.Errorf("a refused append left a thread behind: %v", err) + } +} + +func TestATurnedOnProjectStoresAndReplaysInOrder(t *testing.T) { + ctx := context.Background() + s := openDB(t) + proj := tree(t, s) + if err := s.SetThreadRetention(ctx, proj, 24); err != nil { + t.Fatal(err) + } + + turns := []Message{ + {Role: "user", Content: "first"}, + {Role: "assistant", Content: "answer one", Model: "small"}, + {Role: "user", Content: "second"}, + {Role: "assistant", Content: "answer two", Model: "large"}, + } + for _, m := range turns { + if err := s.AppendThread(ctx, proj, "th1", []Message{m}); err != nil { + t.Fatal(err) + } + } + + got, err := s.Thread(ctx, proj, "th1") + if err != nil { + t.Fatal(err) + } + if len(got) != len(turns) { + t.Fatalf("stored %d turns, read back %d", len(turns), len(got)) + } + for i := range turns { + if got[i].Role != turns[i].Role || got[i].Content != turns[i].Content { + t.Errorf("turn %d came back as %+v", i, got[i]) + } + } + // Which model answered, because a thread that moved between models + // otherwise reads as one voice — and when the answers disagree that is the + // first thing anyone asks. + if got[1].Model != "small" || got[3].Model != "large" { + t.Errorf("the models that answered were lost: %q and %q", got[1].Model, got[3].Model) + } +} + +func TestAnotherProjectsThreadIsNotReadable(t *testing.T) { + // Thread ids are opaque, but an unguessable id is a secret, and secrets end + // up in logs, shell history and bug reports. The project scope is what + // makes a leaked id useless to anyone outside it. + ctx := context.Background() + s := openDB(t) + mine := tree(t, s) + if err := s.PutProject(ctx, Project{ID: "proj2", TeamID: "team1", Name: "Other"}); err != nil { + t.Fatal(err) + } + for _, p := range []string{mine, "proj2"} { + if err := s.SetThreadRetention(ctx, p, 24); err != nil { + t.Fatal(err) + } + } + if err := s.AppendThread(ctx, mine, "th1", []Message{{Role: "user", Content: "private"}}); err != nil { + t.Fatal(err) + } + + // The same error as a thread that does not exist. Two errors would let a + // caller learn which ids are real in projects they cannot read. + if _, err := s.Thread(ctx, "proj2", "th1"); !errors.Is(err, ErrNoThread) { + t.Errorf("reading across projects returned %v", err) + } + if _, err := s.Thread(ctx, "proj2", "never-existed"); !errors.Is(err, ErrNoThread) { + t.Errorf("a missing thread returned a different error: %v", err) + } + // Writing across projects is the same hole in the other direction: append, + // then read your own reply back. + if err := s.AppendThread(ctx, "proj2", "th1", []Message{{Role: "user", Content: "x"}}); !errors.Is(err, ErrNoThread) { + t.Errorf("writing into another project's thread returned %v", err) + } +} + +func TestDeletingAThreadTakesItsMessagesWithIt(t *testing.T) { + // Somebody has to be able to say "forget that", and a retention window is + // no answer to a message sent by mistake. + ctx := context.Background() + s := openDB(t) + proj := tree(t, s) + if err := s.SetThreadRetention(ctx, proj, 24); err != nil { + t.Fatal(err) + } + if err := s.AppendThread(ctx, proj, "th1", []Message{{Role: "user", Content: "oops"}}); err != nil { + t.Fatal(err) + } + + if err := s.DeleteThread(ctx, proj, "th1"); err != nil { + t.Fatal(err) + } + if _, err := s.Thread(ctx, proj, "th1"); !errors.Is(err, ErrNoThread) { + t.Errorf("the thread survived deletion: %v", err) + } + + // The rows, not just the row. Without the cascade the content would still + // be in the database after someone was told it was gone. + var left int + if err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM thread_messages WHERE thread_id = ?`, "th1").Scan(&left); err != nil { + t.Fatal(err) + } + if left != 0 { + t.Errorf("%d messages survived the thread being deleted", left) + } +} + +func TestRetentionIsMeasuredFromTheLastTurnNotTheFirst(t *testing.T) { + // Expiring a conversation someone is still having would be a strange way + // to keep a promise about how long things are kept. + ctx := context.Background() + s := openDB(t) + proj := tree(t, s) + if err := s.SetThreadRetention(ctx, proj, 1); err != nil { + t.Fatal(err) + } + if err := s.AppendThread(ctx, proj, "live", []Message{{Role: "user", Content: "hello"}}); err != nil { + t.Fatal(err) + } + + // Two hours on, a thread touched now is still inside a one-hour window. + if n, err := s.PruneThreads(ctx, time.Now().Add(2*time.Hour)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("an idle thread past its window was not pruned: %d", n) + } + + if err := s.AppendThread(ctx, proj, "live2", []Message{{Role: "user", Content: "hello"}}); err != nil { + t.Fatal(err) + } + if n, err := s.PruneThreads(ctx, time.Now()); err != nil { + t.Fatal(err) + } else if n != 0 { + t.Errorf("a thread inside its window was pruned: %d", n) + } +} + +func TestPruningLeavesProjectsThatStoreNothingAlone(t *testing.T) { + // They have nothing to prune, and a sweep that treated "no retention" as + // "retain for zero hours" would be indistinguishable from one that worked. + ctx := context.Background() + s := openDB(t) + tree(t, s) + + if n, err := s.PruneThreads(ctx, time.Now()); err != nil || n != 0 { + t.Errorf("pruning removed %d rows in a store with no threads (err %v)", n, err) + } +} + +func TestTurningRetentionOffAgain(t *testing.T) { + ctx := context.Background() + s := openDB(t) + proj := tree(t, s) + if err := s.SetThreadRetention(ctx, proj, 24); err != nil { + t.Fatal(err) + } + if err := s.SetThreadRetention(ctx, proj, 0); err != nil { + t.Fatal(err) + } + if d, err := s.ThreadRetention(ctx, proj); err != nil || d != 0 { + t.Fatalf("retention is %v after being turned off (err %v)", d, err) + } + if err := s.AppendThread(ctx, proj, "th1", []Message{{Role: "user", Content: "x"}}); !errors.Is(err, ErrThreadsNotEnabled) { + t.Errorf("storing continued after being turned off: %v", err) + } +}