From eec0d92d1801958a90fe8d4199f924ab2496e24f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 8 Oct 2026 13:58:12 +0200 Subject: [PATCH] The console's Knowledge screen, and the manual's page on knowledge bases Collections listed with owner, model, documents and grants; A makes one; on a row its documents, grant, revoke, a search to try, delete. docs/knowledge.md says what administrators do and how people use llmgw-kb on Windows, macOS and Linux: what it reads, what leaves their machine, where its ledger is. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 20 ++- CHANGELOG.md | 18 ++- docs.go | 2 +- docs/console.md | 8 ++ docs/knowledge.md | 243 ++++++++++++++++++++++++++++++++++++ internal/cli/tui_console.go | 21 +++- internal/cli/tui_kb.go | 217 ++++++++++++++++++++++++++++++++ internal/cli/tui_kb_test.go | 62 +++++++++ 8 files changed, 586 insertions(+), 5 deletions(-) create mode 100644 docs/knowledge.md create mode 100644 internal/cli/tui_kb.go create mode 100644 internal/cli/tui_kb_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 93e4386..86bdeff 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -865,7 +865,7 @@ pgha on a running cluster — install by hand or add an upgrade step; and SQLite has none); tests need `LLMGW_TEST_POSTGRES` pointing at a pgvector database as a non-superuser (a superuser bypasses RLS). -### F2. Ingest on the user's machine (L) — redesigned 2026-10-08 +### F2. Ingest on the user's machine (L) — done 2026-10-08 (branch `rag`) Uploading whole files to the control plane was dropped: a PDF is 20–100× its text, and the work belongs where the files are. Instead a small client, `llmgw-kb`, run by the person on their own machine with their own key: @@ -910,6 +910,24 @@ its text, and the work belongs where the files are. Instead a small client, - Trade-off accepted: vectors come from the client, so a writer could store vectors that do not match their text — only in collections they may already write. +Built: `internal/kb/extract` (formats above; helpers found on PATH and in +each system's install places), `internal/kb/chunk` (≈1800 characters under +the heading path, a paragraph of overlap, sections kept apart once a chunk +is a third full), `internal/kb/kbsync` (ledger in SQLite at +os.UserConfigDir/llmgw-kb/ledger.db: documents keyed by gateway, collection +and name with file, size, mtime, digest, state, reason; runs), `cmd/llmgw-kb` +(login, collections, sync, status, history, formats), `internal/router/kb.go` +(the four gateway endpoints, `X-Llmgw-Project` for personal keys, audit), +`internal/control/adminapi/kb.go` + `llmgw ctl kb` + console Knowledge +screen, `docs/knowledge.md`; CI builds the client for all six targets and +releases attach it. Tested: every format from generated fixtures and a +Turkish PDF (OCR path with Tesseract here), sync against a fake gateway and +end to end against the router on pgvector. +Left: never run against the PoC (needs pgvector there and an embedding +model in the catalog); no LibreOffice on the dev machine, so the legacy +Office path is untested beyond its error; the client is unsigned (Gatekeeper +and SmartScreen warn); Outlook `.msg` is not read; one document over ~60 MiB +of JSON (a few thousand pages) is refused rather than split. ### F3. Retrieval on the request path (M) - `llmgw.knowledge: [collections]` in the body (stripped before the diff --git a/CHANGELOG.md b/CHANGELOG.md index b5657c5..fe7fe23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,22 @@ APIs change between minors without a deprecation period. cluster set up before needs `postgresql--pgvector` on every database machine and `CREATE EXTENSION vector` in llmgw's database. A project, team or organization that owns collections is not deleted; a - person's collections go with their account. Nothing reads or writes them - yet from a command or a request. + person's collections go with their account. +- **Knowledge bases filled from people's own machines** (BACKLOG F2): + `llmgw-kb`, its own download for Windows, macOS and Linux, reads documents + where they are, cuts them into chunks under their headings, has the + gateway embed them with the person's key, and sends text and vectors — + never the files — for what changed only, remembered in a SQLite ledger on + the machine. It reads PDF, Word, Excel, PowerPoint, OpenDocument, RTF, + EPUB, HTML, MHTML, mail with attachments, notebooks, subtitles, ZIP + archives and text and source in any common encoding; legacy Office files + with LibreOffice and scanned pages with Tesseract when installed. + `--dry-run` says what would leave the machine; `--delete` removes only + what that machine sent. The gateway takes them at `/v1/kb/…` with the + key's own access; administrators make, grant and search collections with + `llmgw ctl kb`, `/api/v1/kb` and the console's Knowledge screen + ([docs/knowledge.md](docs/knowledge.md)). Answering requests from them is + next (F3). ## v0.13.0 — 2026-10-08 diff --git a/docs.go b/docs.go index ff3b449..c794813 100644 --- a/docs.go +++ b/docs.go @@ -39,7 +39,7 @@ func (d Doc) Source() string { // than with either reader, because the console's manual and the gateway's // docs portal both list them, and two lists drift. var docOrder = []string{"console", "quick-install", "install", "node-onboarding", "onboarding", - "clients", "guardrails", "choosing-models", "fabrics", "facility-aware-placement", "data", + "clients", "guardrails", "knowledge", "choosing-models", "fabrics", "facility-aware-placement", "data", "reference", "development"} // Manual is every document, in reading order: those docOrder names, then any diff --git a/docs/console.md b/docs/console.md index e9cb6ca..de4ea0c 100644 --- a/docs/console.md +++ b/docs/console.md @@ -164,6 +164,14 @@ which): requests, tokens, errors and spend. team or project is guaranteed and may at most hold, and what it holds now. **A** adds one; on a row, change or remove it. +**Knowledge** lists the knowledge-base collections: each one's owner, the +model that embeds it, how many documents and chunks it holds and who else it +is granted to. **A** makes one. On a row: its **Documents**, **Grant…** a +project, team, organization, person or key read or write access, **Revoke** +a grant, **Try a search…** as an administrator, or **Delete…** it. People +fill collections from their own machines with `llmgw-kb` +([knowledge.md](knowledge.md)). Off on a SQLite store, and says why. + **Mirror** is what the model mirror holds, one row for each version of each model — a file, or the set of files a model of several is: shards, a vision projector, a config and tokenizer — with how far its download has got and diff --git a/docs/knowledge.md b/docs/knowledge.md new file mode 100644 index 0000000..99f84d7 --- /dev/null +++ b/docs/knowledge.md @@ -0,0 +1,243 @@ +# Knowledge bases + +A knowledge base is a collection of a company's documents that the cluster +can answer from. People put their documents into it from their own machines +with `llmgw-kb`. The tool reads the files on that machine, cuts them into +passages ("chunks"), has the gateway embed each passage with the person's own +key, and sends the text and the vectors. The files themselves never leave +the machine. + +> **Status.** Collections, access and filling them work now. Answering +> requests from them (the gateway finding the passages a question needs and +> putting them in front of the model) is the next step and is not in this +> build yet. Until then a collection can be searched by an administrator +> (`llmgw ctl kb search`) to check what it holds. + +## Why retrieval, and not training + +Retrieval is the tool for *knowledge*: policies, procedures, contracts. The +answer can name its source, a changed document is sent again in minutes, and +a passage only ever reaches people allowed to read its collection. + +Training a model on the documents (LoRA) changes how it *writes*, not what it +reliably *knows*. It has no access control: whatever goes into the weights +is available to everyone who may use them. + +## What it needs + +- **The PostgreSQL store with pgvector.** On a SQLite store knowledge bases + are off, and `llmgw ctl kb list` and the console say why. + - `llmgw server init` and the high-availability database setup install + pgvector and enable it. + - A cluster set up before this build needs it by hand, once: + 1. Install the package on every database machine: + `sudo apt-get install postgresql-16-pgvector` (use the server's + version number). + 2. On the leader, enable it in llmgw's database: + `sudo -u postgres psql -d llmgw -c 'CREATE EXTENSION vector'`. + 3. Restart the control planes: `llmgw restart`. +- **An embedding model in the catalog**, with `capabilities.embeddings`. + Two good choices handle Turkish and run on CPUs: `BAAI/bge-m3` (1024 + dimensions) and `Qwen/Qwen3-Embedding-0.6B`. + + Add it as any model is added (**Serve a model** in the console, or the + catalog), with its entry saying that it embeds: + + ```yaml + capabilities: + embeddings: true + ``` + + A collection stays with the model it was made with: vectors from two + models cannot be compared. + +## For administrators + +### Make a collection + +A collection belongs to one part of the tenancy tree: an organization, a +team, a project or a person. Whoever is in the owner, at any level below it, +may read it and add to it. + +```sh +llmgw ctl kb create team:Acme/hr policies bge-m3 +llmgw ctl kb create user:USER-ID notes bge-m3 --org Acme +``` + +The model's dimension is asked of the model itself, so the model has to be +able to serve when the collection is made. `--dims 1024` gives it instead. + +In the console: **Knowledge**, then **A**. + +### Who may read and add + +The owner, and everything below it, may read and add. A **grant** gives the +same to another part of the same organization, to a person, or to one key: + +```sh +llmgw ctl kb grant Acme/policies project:Acme/sales/crm # read +llmgw ctl kb grant Acme/policies team:Acme/legal --write # read and add +llmgw ctl kb revoke Acme/policies project:Acme/sales/crm +``` + +How access works: + +- **Grants add up down the tree.** A grant to a team reaches its projects + and their keys. +- **Nothing crosses organizations.** A grant to another organization's + team, project, key or person is refused. +- **A person's own key** (a key with no project) reads nothing by itself. + The person names the project they act in (`llmgw-kb login` asks for it, + or `--project`) and must be a member of it. Joining a second team + therefore does not quietly widen what every key of theirs can read. +- **No access looks like no collection.** A collection a key may not read + is answered exactly as one that does not exist, so even its name does not + leak. + +### Look after it + +```sh +llmgw ctl kb list # collections, owners, documents, grants +llmgw ctl kb docs Acme/policies # its documents, who added each and when +llmgw ctl kb rm Acme/policies old/form.pdf # one document out +llmgw ctl kb search Acme/policies how many days of annual leave +llmgw ctl kb delete Acme/policies # the collection, its documents and grants +``` + +Rules that apply: + +- A project, team or organization that owns a collection is not deleted + until its collections are. This is the same rule as for one with members. +- A person's own collections go with their account (`users delete`, + `users forget`). +- Every document added or removed is in the audit log (`llmgw ctl audit`), + by name. The text is never recorded. + +## For everyone: `llmgw-kb` + +### Get it + +Download `llmgw-kb` for your system from the release page, next to `llmgw`: + +| System | File | +|---|---| +| Windows (Intel/AMD) | `llmgw-kb-VERSION-windows-amd64.exe` | +| Windows (ARM) | `llmgw-kb-VERSION-windows-arm64.exe` | +| macOS (Apple silicon) | `llmgw-kb-VERSION-darwin-arm64` | +| macOS (Intel) | `llmgw-kb-VERSION-darwin-amd64` | +| Linux | `llmgw-kb-VERSION-linux-amd64` / `-arm64` | + +Rename it to `llmgw-kb` (`llmgw-kb.exe` on Windows) and put it somewhere on +your PATH. You do not need to install anything else. + +On macOS, a downloaded program that is not signed is stopped the first time. +Either right-click it and choose Open, or run +`xattr -d com.apple.quarantine llmgw-kb`. + +### Log in once + +```sh +llmgw-kb login +``` + +It asks for three things, checks them, and saves them where only you can +read them: + +- **The gateway's address.** +- **Your API key.** It is typed without being shown. +- **A project.** Needed only for a personal key. + +Instead of logging in you can set `LLMGW_URL`, `LLMGW_API_KEY` and +`LLMGW_PROJECT`. + +```sh +llmgw-kb collections # what your key may read, and which you may add to +``` + +### Send your documents + +```sh +llmgw-kb sync policies ~/Documents/HR +llmgw-kb sync policies "C:\Users\ayse\Documents\İK" --include "*.pdf,*.docx" +llmgw-kb sync policies ./handbook ./forms --exclude "drafts/**" +``` + +Run it as often as you like. The first run sends everything. Later runs +send only what changed, and do not even read files whose size and time have +not changed. + +| Option | What it does | +|---|---| +| `--dry-run` | Reads and cuts everything, sends nothing, and lists what would leave the machine and how much. | +| `--delete` | Also removes documents whose files are gone. Only those this machine sent, from these folders; never anything someone else sent. | +| `--retry` | Reads again files skipped before, for example after installing LibreOffice. | +| `--include`, `--exclude` | Glob patterns. `*.pdf` matches a file name; `reports/**/*.xlsx` matches a path. | +| `--hidden` | Reads hidden files and folders too (left out by default). | +| `-v` | Lists unchanged files too. | + +Documents are named by their path under the folder, with forward slashes, +on every system. With several folders, each folder's own name comes first. + +```sh +llmgw-kb status policies # what this machine sent and skipped, and why +llmgw-kb history # the latest runs +llmgw-kb formats # what can be read here +``` + +### What it reads + +| Read here | Formats | +|---|---| +| Documents | PDF (text), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`, with speaker notes), OpenDocument (`.odt`, `.ods`, `.odp`), RTF, EPUB | +| Web and mail | HTML, MHTML, e-mail (`.eml`, with its attachments) | +| Text | Markdown, plain text, CSV, JSON, YAML, XML, subtitles, notebooks (`.ipynb`), source code | +| Archives | ZIP, with every file inside it read in turn | + +| Read with a helper on your machine | Helper | +|---|---| +| Word, Excel and PowerPoint 97–2003 (`.doc`, `.xls`, `.ppt`), Pages, Numbers, Keynote and others | [LibreOffice](https://www.libreoffice.org) | +| Images and scanned PDFs | [Tesseract](https://tesseract-ocr.github.io) (with the Turkish language data), plus Poppler's `pdftoppm` for PDFs | +| PDFs the built-in reader cannot parse | Poppler's `pdftotext` | + +`llmgw-kb formats` says which helpers it found. Text in UTF-8, UTF-16, or +the Windows Turkish and Western code pages is read correctly. + +A file that cannot be read is skipped with the reason, and the run goes on. +Reasons include a locked Office file, a format with no helper, and a +scanned page with no OCR. + +### What leaves your machine + +For each new or changed document, two things leave the machine, both over +the gateway's TLS connection: + +- **The passages, to be embedded.** They go through `/v1/embeddings` like + any request of yours. Your key's limits, budget and guardrails apply. +- **The document itself.** Its name, its file's SHA-256 digest and size, + and the passages with their vectors. + +The file itself is never sent. `--dry-run` shows exactly what would be. + +### The ledger + +`llmgw-kb` keeps a small SQLite database of what it did with every +document: the file it came from, its size, time and digest, and whether it +was sent, skipped or failed, with the reason. + +| System | Where it is | +|---|---| +| Windows | `%AppData%\llmgw-kb\ledger.db` | +| macOS | `~/Library/Application Support/llmgw-kb/ledger.db` | +| Linux | `~/.config/llmgw-kb/ledger.db` | + +Deleting the ledger is safe: the next run hashes every file once, finds what +the collection already has, and sends nothing that is unchanged. `--ledger` +uses another file. + +## How it is kept apart + +Passages live in PostgreSQL, partitioned by organization, with an index per +collection. Every search and every write names the organization and the +collections the caller may use. On top of that, row-level security on the +table checks the same thing again, so a query that forgot its filter +returns nothing rather than another tenant's passages. diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 8164c25..bb88cb4 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -45,6 +45,7 @@ const ( cUsage cQuotas cGuard + cKB cFederation cMirror cJobs @@ -68,7 +69,7 @@ const ( cCount ) -var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual", +var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual", "server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster", "Restart all replicas", "Back up now", "Wipe this machine", "Leave"} @@ -177,6 +178,7 @@ type consoleModel struct { quotas []adminapi.QuotaView quotasErr error guard guardState + kb kbState st *storeHolder people peopleData @@ -323,6 +325,9 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.menu == cGuard { cmds = append(cmds, m.fetchGuard()) } + if m.menu == cKB { + cmds = append(cmds, m.fetchKB()) + } if m.menu == cMirror { cmds = append(cmds, m.fetchMirror()) } @@ -381,6 +386,8 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.people = peopleData(msg) case consoleQuotasMsg: m.quotas, m.quotasErr = msg.rows, msg.err + case consoleKBMsg: + m.kb = kbState{view: msg.view, err: msg.err, loaded: true} case consoleGuardMsg: window := m.guard.window m.guard = guardState{view: msg.view, events: msg.events, err: msg.err, window: window} @@ -539,6 +546,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.menuD = m.quotaMenu(m.quotas[m.row]) case cGuard: m.menuD = m.guardAttachmentMenu(m.guard.view.Attachments[m.row]) + case cKB: + m.menuD = m.kbMenu(m.kb.view.Collections[m.row]) case cMirror: m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row]) case cJobs: @@ -596,6 +605,10 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.quotaForm(adminapi.QuotaView{}) case cGuard: m.guardAttachForm() + case cKB: + if m.kb.view.Available { + m.kbCreateForm() + } case cModels: m.addModelForm() case cMirror: @@ -651,6 +664,8 @@ func (m consoleModel) onEnter() tea.Cmd { return m.fetchQuotas() case cGuard: return m.fetchGuard() + case cKB: + return m.fetchKB() case cMirror: return m.fetchMirror() case cMonitor: @@ -1027,6 +1042,8 @@ func (m consoleModel) rowCount() int { return len(m.quotas) case cGuard: return len(m.guard.view.Attachments) + case cKB: + return len(m.kb.view.Collections) case cMirror: return len(m.mirrorGroups()) case cJobs: @@ -1260,6 +1277,8 @@ func (m consoleModel) panel() (title, body, keys string) { return m.quotasPanel(pw) case cGuard: return m.guardPanel(pw) + case cKB: + return m.kbPanel(pw) case cAccounts, cKeys, cOrgs: return m.peoplePanel(pw) case cManual: diff --git a/internal/cli/tui_kb.go b/internal/cli/tui_kb.go new file mode 100644 index 0000000..4859ce0 --- /dev/null +++ b/internal/cli/tui_kb.go @@ -0,0 +1,217 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// Knowledge bases, as the console shows them: each collection with its +// owner, model, documents and grants; Enter for what can be done to one, A +// to make one. People fill them from their own machines with llmgw-kb. + +type kbState struct { + view adminapi.KBView + err error + // loaded is set once an answer came, so "none yet" is not shown while + // asking. + loaded bool +} + +type consoleKBMsg struct { + view adminapi.KBView + err error +} + +func (m consoleModel) fetchKB() tea.Cmd { + c := m.client + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var msg consoleKBMsg + msg.err = c.get(ctx, "/api/v1/kb", &msg.view) + return msg + } +} + +func (m consoleModel) kbPanel(pw int) (string, string, string) { + keys := "→ or Enter open · ↑↓ choose · Q leave" + if m.inPanel { + keys = "↑↓ collection · Enter actions · A new collection · ← back · Q leave" + } + k := m.kb + var b strings.Builder + b.WriteString("\n " + sDim.Render("Documents requests are answered from; people add them with llmgw-kb.") + "\n\n") + switch { + case k.err != nil: + b.WriteString(" " + sBad.Render(k.err.Error())) + return "Knowledge", b.String(), keys + case !k.loaded: + b.WriteString(" " + sDim.Render("asking the control plane…")) + return "Knowledge", b.String(), keys + case !k.view.Available: + b.WriteString(" " + sBad.Render("Off: ") + k.view.Reason) + return "Knowledge", b.String(), keys + case len(k.view.Collections) == 0: + b.WriteString(" " + sDim.Render("No collections yet. A to make one.")) + return "Knowledge", b.String(), keys + } + var rows [][]string + for _, c := range k.view.Collections { + var grants []string + for _, g := range c.Grants { + grants = append(grants, or(g.Name, g.Target)+" "+g.Perm) + } + rows = append(rows, []string{or(c.OrgName, c.Org) + "/" + c.Name, or(c.OwnerName, c.Owner), + fmt.Sprintf("%s (%d)", c.EmbedModel, c.Dims), fmt.Sprint(c.Documents), fmt.Sprint(c.Chunks), dash(strings.Join(grants, ", "))}) + } + b.WriteString(table([]string{"COLLECTION", "OWNER", "MODEL", "DOCUMENTS", "CHUNKS", "GRANTED TO"}, rows, m.row, pw, m.inPanel)) + b.WriteString("\n\n " + sDim.Render("On a person's machine: llmgw-kb sync COLLECTION FOLDER")) + return "Knowledge", b.String(), keys +} + +// kbRef is how the admin API is told which collection: by its ID. +func kbRef(c adminapi.KBCollectionView) string { return url.PathEscape(c.ID) } + +// kbCreateForm makes a collection. +func (m *consoleModel) kbCreateForm() { + model := "" + for _, md := range m.data.models { + n := strings.ToLower(md.Name) + if strings.Contains(n, "embed") || strings.Contains(n, "bge") || strings.Contains(n, "e5") { + model = md.Name + break + } + } + m.openForm(&formDialog{title: "New knowledge-base collection", + note: "Owned by part of the tree: it and everything below it may read and add to it.\n" + + "The model must embed (capabilities.embeddings) and be able to serve now.", + fields: []formField{ + choiceF("owner", []string{"project", "team", "org", "user"}, "team"), + textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"), + textF("org", "", "for a person's collection only"), + textF("name", "", "how people name it to llmgw-kb"), + textF("model", model, "the catalog model that embeds it"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + for i, what := range map[int]string{1: "which " + v[0], 3: "a name", 4: "a model"} { + if strings.TrimSpace(v[i]) == "" { + return nil, fmt.Errorf("%s?", what) + } + } + req := adminapi.KBCreateRequest{Owner: v[0] + ":" + v[1], Org: v[2], Name: v[3], Model: v[4]} + return m.call("New collection "+v[3], http.MethodPost, "/api/v1/kb/collections", req, func(raw json.RawMessage) string { + var c adminapi.KBCollectionView + _ = json.Unmarshal(raw, &c) + return fmt.Sprintf("Collection %s, embedded by %s in %d dimensions.\n\nPeople with access fill it with:\n llmgw-kb sync %s FOLDER", + c.Name, c.EmbedModel, c.Dims, c.Name) + }), nil + }}) +} + +// kbMenu is what can be done to one collection. +func (m consoleModel) kbMenu(c adminapi.KBCollectionView) *menuDialog { + name := or(c.OrgName, c.Org) + "/" + c.Name + items := []menuItem{ + {"Documents", func(m *consoleModel) tea.Cmd { return m.kbDocuments(c) }}, + {"Grant…", func(m *consoleModel) tea.Cmd { + m.openForm(&formDialog{title: "Grant " + name, + note: "It and everything below it may then read, or with write add to, this collection.\nNever outside " + or(c.OrgName, c.Org) + ".", + fields: []formField{ + choiceF("to", []string{"project", "team", "org", "user", "key"}, "project"), + textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"), + choiceF("may", []string{"read", "write"}, "read"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if strings.TrimSpace(v[1]) == "" { + return nil, fmt.Errorf("which %s?", v[0]) + } + req := adminapi.KBGrantRequest{Collection: c.ID, Target: v[0] + ":" + strings.TrimSpace(v[1]), Perm: v[2]} + return m.call("Grant "+c.Name, http.MethodPost, "/api/v1/kb/grants", req, nil), nil + }}) + return nil + }}, + } + for _, g := range c.Grants { + items = append(items, menuItem{"Revoke " + or(g.Name, g.Target) + " (" + g.Perm + ")…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Revoke a grant on " + name, + text: or(g.Name, g.Target) + " may then no longer " + g.Perm + " it, unless it owns it.", + yes: func(m *consoleModel) tea.Cmd { + return m.call("Revoke", http.MethodDelete, "/api/v1/kb/grants", + adminapi.KBGrantRequest{Collection: c.ID, Target: g.Target}, nil) + }} + return nil + }}) + } + items = append(items, + menuItem{"Try a search…", func(m *consoleModel) tea.Cmd { + m.openForm(&formDialog{title: "Search " + name, + note: "As an administrator: no tenancy applies. Shows what a question would find.", + fields: []formField{textF("question", "", ""), textF("results", "5", "how many chunks")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if v[0] == "" { + return nil, fmt.Errorf("what question?") + } + k, _ := strconv.Atoi(v[1]) + req := adminapi.KBSearchRequest{Collection: c.ID, Query: v[0], K: k} + return m.call("Search "+c.Name, http.MethodPost, "/api/v1/kb/search", req, func(raw json.RawMessage) string { + var hits []adminapi.KBHitView + _ = json.Unmarshal(raw, &hits) + if len(hits) == 0 { + return "Nothing found." + } + var b strings.Builder + for i, h := range hits { + text := h.Text + if r := []rune(text); len(r) > 400 { + text = string(r[:400]) + "…" + } + fmt.Fprintf(&b, "%d. %s #%d\n%s\n\n", i+1, h.Document, h.Ord, indent(strings.TrimSpace(text), " ")) + } + return strings.TrimRight(b.String(), "\n") + }), nil + }}) + return nil + }}, + menuItem{"Delete…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Delete " + name, + text: fmt.Sprintf("Its %d documents and %d chunks, and every grant, go with it. People's own files are not touched.", + c.Documents, c.Chunks), + yes: func(m *consoleModel) tea.Cmd { + return m.call("Delete "+c.Name, http.MethodDelete, "/api/v1/kb/collections/"+kbRef(c), nil, nil) + }} + return nil + }}, + ) + return &menuDialog{title: name, items: items} +} + +// kbDocuments shows a collection's documents in a dialog. +func (m *consoleModel) kbDocuments(c adminapi.KBCollectionView) tea.Cmd { + return m.call("Documents of "+c.Name, http.MethodGet, "/api/v1/kb/collections/"+kbRef(c)+"/documents", nil, + func(raw json.RawMessage) string { + var docs []adminapi.KBDocumentView + _ = json.Unmarshal(raw, &docs) + if len(docs) == 0 { + return "None yet. On a person's machine: llmgw-kb sync " + c.Name + " FOLDER" + } + var b strings.Builder + for i, d := range docs { + if i == 40 { + fmt.Fprintf(&b, "… and %d more (llmgw ctl kb docs %s)", len(docs)-40, c.ID) + break + } + fmt.Fprintf(&b, "%-48s %5d chunks %s %s\n", d.Name, d.Chunks, d.AddedAt.Local().Format("2006-01-02 15:04"), d.AddedBy) + } + return strings.TrimRight(b.String(), "\n") + }) +} diff --git a/internal/cli/tui_kb_test.go b/internal/cli/tui_kb_test.go new file mode 100644 index 0000000..b8845b2 --- /dev/null +++ b/internal/cli/tui_kb_test.go @@ -0,0 +1,62 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +func TestACollectionIsMadeAndGrantedFromTheConsole(t *testing.T) { + m, s := consoleWith(t, cKB) + m.data.models = append(m.data.models, adminapi.ModelView{Name: "bge-m3"}) + m.kb = kbState{loaded: true, view: adminapi.KBView{Available: true, Collections: []adminapi.KBCollectionView{{ + ID: "kb_aaaaaaaaaaaaaaaa", Name: "policies", Org: "o1", OrgName: "Acme", Owner: "team:t1", OwnerName: "Acme / hr", + EmbedModel: "bge-m3", Dims: 1024, Documents: 3, Chunks: 40, + Grants: []adminapi.KBGrantView{{Target: "project:p9", Name: "Acme/sales/crm", Perm: "read"}}, + }}}} + var tm tea.Model = m + view := tm.View() + for _, want := range []string{"Knowledge", "Acme/policies", "Acme / hr", "bge-m3 (1024)", "Acme/s", "llmgw-kb sync COLLECTION FOLDER"} { + if !strings.Contains(view, want) { + t.Errorf("missing %q in:\n%s", want, view) + } + } + + // A: the form, prefilled with the embedding model. + tm = press(tm, "a") + f := tm.(consoleModel).formD + if f == nil || f.fields[4].value() != "bge-m3" { + t.Fatalf("no form, or the model not offered: %+v", f) + } + f.fields[1].text.SetValue("Acme/hr") + f.fields[3].text.SetValue("handbook") + tm = pressRun(tm, "tab", "tab", "tab", "tab", "tab", "enter") + var req adminapi.KBCreateRequest + got := s.last() + if !strings.HasPrefix(got, "POST /api/v1/kb/collections ") || + json.Unmarshal([]byte(strings.SplitN(got, " ", 3)[2]), &req) != nil || + req.Owner != "team:Acme/hr" || req.Name != "handbook" || req.Model != "bge-m3" { + t.Fatalf("sent %q", got) + } + + // Enter on a row: grant to a project for writing. + m.formD = nil + tm = m + tm = press(tm, "enter") + if md := tm.(consoleModel).menuD; md == nil || !strings.Contains(md.items[2].label, "Revoke Acme/sales/crm") { + t.Fatalf("menu %+v", md) + } + tm = press(tm, "down", "enter") + f = tm.(consoleModel).formD + f.fields[1].text.SetValue("Acme/sales/crm") + f.fields[2].opt = 1 // write + tm = pressRun(tm, "tab", "tab", "tab", "enter") + if got := s.last(); got != `POST /api/v1/kb/grants {"collection":"kb_aaaaaaaaaaaaaaaa","target":"project:Acme/sales/crm","perm":"write"}` { + t.Fatalf("sent %q", got) + } + _ = tm +}