diff --git a/BACKLOG.md b/BACKLOG.md index 191caae..93e4386 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -865,16 +865,51 @@ pgha on a running cluster — install by hand or add an upgrade step; and SQLite has none); tests need `LLMGW_TEST_POSTGRES` pointing at a pgvector database as a non-superuser (a superuser bypasses RLS). -### F2. Ingest (M) -- `llmgw ctl kb create|list|delete|grant|revoke|docs`, `ctl kb add - `, admin API under `/api/v1/kb`, audit events. -- Text from PDF (text layer), DOCX, HTML, Markdown, plain text — in Go, no - Python. Split on headings/paragraphs, ~500 tokens with overlap. Embedded - through the router at the bronze band, accounted to the collection's - owner. One transaction per document; same digest skipped, changed digest - replaces. OCR out of scope. -- Console: a Knowledge screen (collections, documents, grants; add, grant, - revoke, delete); `docs/console.md`, `docs/knowledge.md`. +### F2. Ingest on the user's machine (L) — redesigned 2026-10-08 +Uploading whole files to the control plane was dropped: a PDF is 20–100× +its text, and the work belongs where the files are. Instead a small client, +`llmgw-kb`, run by the person on their own machine with their own key: +- **Its own binary** (`cmd/llmgw-kb`), for Windows, macOS and Linux on amd64 + and arm64. `llmgw` itself does not build on Windows (agent syscalls), and + employees' laptops should not carry the server, agent and setup screens. A + test keeps server packages out of its imports. +- `llmgw-kb sync [--include/--exclude globs] + [--dry-run] [--delete]`: scan → local catalog (path, size, mtime, sha256; + cache under os.UserCacheDir, digests not recomputed for unchanged files) + → the collection's manifest from the gateway (name → digest only) → diff + → for new/changed files only: extract, chunk, embed through the gateway's + `/v1/embeddings` with the person's key (its accounting, limits, guard and + band apply), and send name, digest, chunk texts and vectors; `--delete` + removes documents whose files are gone. `--dry-run` lists what would leave + the machine and how much. A file that cannot be read (locked, unsupported, + no text) is skipped with the reason; the run goes on. +- **Formats, as many as possible.** Built in, pure Go: PDF (text layer), + DOCX/XLSX/PPTX, ODT/ODS/ODP, RTF, EPUB, HTML/XHTML, EML (text and HTML + parts), Jupyter notebooks, subtitles (SRT/VTT), CSV/TSV, JSON/YAML/XML, + Markdown/RST/AsciiDoc/Org, plain text and source code; ZIP archives + recursed into. With a helper found on the machine: legacy DOC/XLS/PPT, RTF + and anything else LibreOffice opens (`soffice --headless --convert-to`), + images and scanned PDFs through Tesseract (`tur+eng`). Text in UTF-8, + UTF-16 (BOM) or Windows-1254/1252 is converted to UTF-8. +- **Across systems:** document names relative to the synced root, `/` + separated, Unicode NFC (macOS hands out NFD), a warning for two names that + differ only in case; CRLF → LF; key from `LLMGW_API_KEY` or `llmgw-kb + login` (os.UserConfigDir, 0600 where that means something); the system's + CA store, `--ca` for another. +- **Gateway endpoints** (client listener, person's key): + `GET /v1/kb/collections` (what this key may read/write), + `GET /v1/kb/collections/{id}/manifest`, `PUT + /v1/kb/collections/{id}/documents` (one document per request: name, + digest, bytes, chunks with text and embedding; dims checked, chunk count + and size bounded), `DELETE …/documents?name=`. Write needs a write grant + or ownership on the key's chain (`CollectionsFor(…, write)`); otherwise + 404. +- **Administration** stays with operators: `llmgw ctl kb + create|list|delete|grant|revoke|docs`, admin API `/api/v1/kb`, audit, + console screen; `docs/knowledge.md`. +- Trade-off accepted: vectors come from the client, so a writer could store + vectors that do not match their text — only in collections they may + already write. ### F3. Retrieval on the request path (M) - `llmgw.knowledge: [collections]` in the body (stripped before the diff --git a/go.mod b/go.mod index 5734aec..ac921bb 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/knadh/koanf/providers/env/v2 v2.0.1 github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/v2 v2.3.6 + github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 github.com/mattn/go-isatty v0.0.24 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 diff --git a/go.sum b/go.sum index 4690486..f47ac0c 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,8 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo= github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 h1:7Q+xNAZFmnfYOMweHN3c/PDFUKKfY1pVJ26K++QvVfU= +github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= diff --git a/internal/kb/chunk/chunk.go b/internal/kb/chunk/chunk.go new file mode 100644 index 0000000..a60042a --- /dev/null +++ b/internal/kb/chunk/chunk.go @@ -0,0 +1,248 @@ +// Package chunk splits extracted text into the pieces a knowledge base keeps +// and searches (BACKLOG F2). +// +// A chunk is a few paragraphs under the headings they sit beneath: about 500 +// tokens, which an embedding model reads whole and a prompt can carry several +// of. Paragraphs are not cut while they fit; a section starts a new chunk +// when the current one is already a fair size, so a chunk rarely mixes two +// topics; and the last paragraph of a chunk is repeated at the start of the +// next, so an answer that spans the boundary is found whole in one of them. +package chunk + +import ( + "strings" + "unicode" + "unicode/utf8" +) + +// Options size the chunks. Zero values take the defaults. +type Options struct { + // MaxChars is the most characters in a chunk's body: 1800, about 500 + // tokens of Turkish or English. + MaxChars int + // Overlap is the most characters of the previous chunk repeated at the + // start of the next: 300. + Overlap int +} + +func (o Options) max() int { + if o.MaxChars > 0 { + return o.MaxChars + } + return 1800 +} + +func (o Options) overlap() int { + if o.Overlap >= 0 && o.MaxChars > 0 { + return min(o.Overlap, o.max()/2) + } + return 300 +} + +// Chunk is one piece: the headings above it and its text. +type Chunk struct { + // Headings is the path of headings the chunk starts under, widest first. + Headings []string + Body string +} + +// Text is what is stored and embedded: the heading path, then the body. The +// path is part of it so that "how many days?" under "Annual leave" finds the +// paragraph that says "twenty days" without naming leave. +func (c Chunk) Text() string { + if len(c.Headings) == 0 { + return c.Body + } + return strings.Join(c.Headings, " › ") + "\n\n" + c.Body +} + +// Split cuts Markdown-shaped text into chunks. +func Split(text string, o Options) []Chunk { + max, overlap := o.max(), o.overlap() + var out []Chunk + type head struct { + level int + title string + } + var stack []head // the headings in force, widest first + titles := func() []string { + out := make([]string, len(stack)) + for i, h := range stack { + out[i] = h.title + } + return out + } + var cur []string // blocks of the chunk being built + var curHeads []string + size := 0 + // carried is the overlap at the start of cur: it does not count as the + // chunk having content of its own. + carried := 0 + + emit := func(keepOverlap bool) { + if size-carried <= 0 { + cur, size, carried = nil, 0, 0 + return + } + out = append(out, Chunk{Headings: curHeads, Body: strings.Join(cur, "\n\n")}) + last := cur[len(cur)-1] + cur, size, carried = nil, 0, 0 + if keepOverlap && overlap > 0 && runes(last) <= overlap { + cur, size, carried = []string{last}, runes(last), runes(last) + } + } + add := func(b string) { + if len(cur) == 0 || size == carried { + curHeads = titles() + } + if size > 0 && size+2+runes(b) > max { + emit(true) + curHeads = titles() + } + cur = append(cur, b) + size += runes(b) + 2 + } + + for _, b := range blocks(text) { + if lvl, title := heading(b); lvl > 0 { + // A new section starts a new chunk once this one is worth + // keeping by itself; otherwise the heading goes into it. + if size-carried >= max/3 { + emit(false) + } + for len(stack) > 0 && stack[len(stack)-1].level >= lvl { + stack = stack[:len(stack)-1] + } + stack = append(stack, head{lvl, title}) + if size-carried > 0 { + add(b) + } else { + // Nothing of its own yet: the chunk starts under this heading. + cur, size, carried = nil, 0, 0 + curHeads = titles() + } + continue + } + for _, piece := range fit(b, max) { + add(piece) + } + } + emit(false) + return out +} + +func runes(s string) int { return utf8.RuneCountInString(s) } + +// heading reads a Markdown ATX heading line: its level and title. +func heading(b string) (int, string) { + if strings.Contains(b, "\n") { + return 0, "" + } + n := 0 + for n < len(b) && b[n] == '#' { + n++ + } + if n == 0 || n > 6 || n >= len(b) || b[n] != ' ' { + return 0, "" + } + return n, strings.TrimSpace(strings.TrimRight(strings.TrimSpace(b[n:]), "#")) +} + +// blocks splits text on blank lines, keeping fenced code blocks whole. +func blocks(text string) []string { + var out []string + var b []string + fenced := false + flush := func() { + s := strings.TrimSpace(strings.Join(b, "\n")) + if s != "" { + out = append(out, s) + } + b = nil + } + for _, line := range strings.Split(text, "\n") { + t := strings.TrimSpace(line) + if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") { + fenced = !fenced + } + if !fenced && t == "" { + flush() + continue + } + // A heading line is a block of its own even without blank lines + // around it. + if !fenced && len(b) > 0 { + if lvl, _ := heading(t); lvl > 0 { + flush() + } + } + b = append(b, strings.TrimRight(line, " \t")) + if !fenced { + if lvl, _ := heading(t); lvl > 0 { + flush() + } + } + } + flush() + return out +} + +// fit cuts a block longer than max: at line ends, then sentence ends, then +// spaces, and only as a last resort mid-word. +func fit(b string, max int) []string { + if runes(b) <= max { + return []string{b} + } + for _, split := range []func(string) []string{lines, sentences, words} { + parts := split(b) + if len(parts) < 2 { + continue + } + var out []string + var cur strings.Builder + for _, p := range parts { + if cur.Len() > 0 && runes(cur.String())+runes(p) > max { + out = append(out, strings.TrimSpace(cur.String())) + cur.Reset() + } + if runes(p) > max { + out = append(out, fit(strings.TrimSpace(p), max)...) + continue + } + cur.WriteString(p) + } + if strings.TrimSpace(cur.String()) != "" { + out = append(out, strings.TrimSpace(cur.String())) + } + return out + } + // One word longer than a chunk: a hash, a line of base64. + r := []rune(b) + var out []string + for len(r) > max { + out, r = append(out, string(r[:max])), r[max:] + } + return append(out, string(r)) +} + +// lines splits keeping each line's newline. +func lines(s string) []string { return strings.SplitAfter(s, "\n") } + +// sentences splits after ., ? or ! followed by a space, keeping both. +func sentences(s string) []string { + var out []string + start := 0 + r := []rune(s) + for i := 0; i < len(r)-1; i++ { + if (r[i] == '.' || r[i] == '?' || r[i] == '!' || r[i] == '…') && unicode.IsSpace(r[i+1]) { + out = append(out, string(r[start:i+2])) + start = i + 2 + } + } + if start < len(r) { + out = append(out, string(r[start:])) + } + return out +} + +func words(s string) []string { return strings.SplitAfter(s, " ") } diff --git a/internal/kb/chunk/chunk_test.go b/internal/kb/chunk/chunk_test.go new file mode 100644 index 0000000..86a2e06 --- /dev/null +++ b/internal/kb/chunk/chunk_test.go @@ -0,0 +1,106 @@ +package chunk + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func para(word string, n int) string { return strings.TrimSpace(strings.Repeat(word+" ", n)) } + +func TestShortTextIsOneChunkUnderItsHeadings(t *testing.T) { + cs := Split("# İzin\n\n## Yıllık\n\nYirmi gün.\n\nBayramlar ayrı.\n", Options{}) + if len(cs) != 1 { + t.Fatalf("chunks %+v", cs) + } + if got := cs[0].Text(); got != "İzin › Yıllık\n\nYirmi gün.\n\nBayramlar ayrı." { + t.Errorf("text %q", got) + } +} + +func TestChunksStayUnderTheLimitAndOverlap(t *testing.T) { + var b strings.Builder + for i := range 20 { + b.WriteString(para("kelime"+string(rune('a'+i)), 30) + "\n\n") + } + cs := Split(b.String(), Options{MaxChars: 600, Overlap: 300}) + if len(cs) < 5 { + t.Fatalf("only %d chunks", len(cs)) + } + for i, c := range cs { + if n := utf8.RuneCountInString(c.Body); n > 600 { + t.Errorf("chunk %d has %d characters", i, n) + } + if i > 0 { + prev := strings.Split(cs[i-1].Body, "\n\n") + if !strings.HasPrefix(c.Body, prev[len(prev)-1]) { + t.Errorf("chunk %d does not start with the end of chunk %d", i, i-1) + } + } + } +} + +func TestASectionStartsAChunkOnceTheLastIsWorthKeeping(t *testing.T) { + text := "# A\n\n" + para("a", 400) + "\n\n# B\n\nkısa\n\n# C\n\nde kısa\n" + cs := Split(text, Options{MaxChars: 1800}) + if len(cs) != 2 { + t.Fatalf("chunks %+v", cs) + } + if cs[1].Headings[0] != "B" || !strings.Contains(cs[1].Body, "# C") || !strings.Contains(cs[1].Body, "de kısa") { + t.Errorf("small sections were not kept together: %+v", cs[1]) + } + // The overlap does not run into a new section. + if strings.HasPrefix(cs[1].Body, "a a") { + t.Errorf("section B starts with section A's text") + } +} + +func TestHeadingLevelsReplaceTheirSiblings(t *testing.T) { + text := "## A\n\n### A1\n\n" + para("x", 300) + "\n\n## B\n\n" + para("y", 300) + "\n" + cs := Split(text, Options{MaxChars: 700}) + last := cs[len(cs)-1] + if strings.Join(last.Headings, "/") != "B" { + t.Errorf("headings after a sibling: %v", last.Headings) + } + if strings.Join(cs[0].Headings, "/") != "A/A1" { + t.Errorf("first headings %v", cs[0].Headings) + } +} + +func TestLongParagraphsAreCutAtSentencesThenWords(t *testing.T) { + long := strings.Repeat("Bu bir cümledir. ", 200) + for _, c := range Split(long, Options{MaxChars: 500}) { + if utf8.RuneCountInString(c.Body) > 500 { + t.Errorf("chunk of %d", utf8.RuneCountInString(c.Body)) + } + if !strings.HasSuffix(c.Body, ".") { + t.Errorf("cut mid-sentence: …%q", c.Body[len(c.Body)-20:]) + } + } + word := strings.Repeat("ş", 1200) + cs := Split(word, Options{MaxChars: 500}) + total := 0 + for _, c := range cs { + total += utf8.RuneCountInString(c.Body) + if !utf8.ValidString(c.Body) { + t.Error("a multibyte character was cut") + } + } + if total != 1200 { + t.Errorf("a long word lost characters: %d", total) + } +} + +func TestCodeFencesStayWhole(t *testing.T) { + text := "Örnek:\n\n```go\nfunc a() {\n\n\treturn\n}\n```\n\nSon.\n" + cs := Split(text, Options{}) + if len(cs) != 1 || !strings.Contains(cs[0].Body, "func a() {\n\n\treturn\n}") { + t.Errorf("chunks %+v", cs) + } +} + +func TestNothingToSayIsNoChunk(t *testing.T) { + if cs := Split("# Başlık\n\n## Alt\n", Options{}); len(cs) != 0 { + t.Errorf("headings alone made %+v", cs) + } +} diff --git a/internal/kb/extract/containers.go b/internal/kb/extract/containers.go new file mode 100644 index 0000000..2bfa387 --- /dev/null +++ b/internal/kb/extract/containers.go @@ -0,0 +1,215 @@ +package extract + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "mime/quotedprintable" + "net/mail" + "path" + "strings" +) + +// maxArchiveMembers is the most members of one archive read; past it the +// rest are reported skipped. +const maxArchiveMembers = 10000 + +// maxArchiveBytes is the most an archive's members are inflated to in all. +const maxArchiveBytes = 2 << 30 + +// zipParts reads every member of a zip archive this can read, each a part +// named by its path inside. +func zipParts(ctx context.Context, name string, b []byte, o Options, depth int) ([]Part, error) { + zr, err := openZip(b) + if err != nil { + return nil, err + } + var parts []Part + var total int64 + for i, f := range zr.File { + if err := ctx.Err(); err != nil { + return parts, err + } + if f.FileInfo().IsDir() || skipName(f.Name) { + continue + } + mname := memberName(f.Name) + if i >= maxArchiveMembers { + parts = append(parts, Part{Name: mname, Err: fmt.Errorf("%w: past the first %d members", ErrUnsupported, maxArchiveMembers)}) + continue + } + if !Known(f.Name) { + continue + } + if int64(f.UncompressedSize64) > o.maxBytes() { + parts = append(parts, Part{Name: mname, Err: fmt.Errorf("%w: %d MiB is over the limit", ErrUnsupported, f.UncompressedSize64>>20)}) + continue + } + total += int64(f.UncompressedSize64) + if total > maxArchiveBytes { + parts = append(parts, Part{Name: mname, Err: fmt.Errorf("%w: the archive inflates past %d GiB", ErrUnsupported, maxArchiveBytes>>30)}) + break + } + data, err := readMember(f) + if err != nil { + parts = append(parts, Part{Name: mname, Err: err}) + continue + } + inner, err := extractNested(ctx, mname, data, o, depth+1) + parts = append(parts, nest(mname, inner, err)...) + } + if len(parts) == 0 { + return nil, ErrNoText + } + return parts, nil +} + +func extractNested(ctx context.Context, name string, data []byte, o Options, depth int) ([]Part, error) { + return extractDepth(ctx, name, data, o, depth) +} + +// nest names the parts of a member after it: "a/b.pdf", "a/c.zip!/d.txt". +func nest(name string, parts []Part, err error) []Part { + if err != nil { + return []Part{{Name: name, Err: err}} + } + out := make([]Part, 0, len(parts)) + for _, p := range parts { + if p.Name == "" { + p.Name = name + } else { + p.Name = name + "!/" + p.Name + } + out = append(out, p) + } + return out +} + +// memberName is a member's path made safe to name a document: forward +// slashes, no leading slash or dot segments. +func memberName(n string) string { + n = strings.ReplaceAll(n, `\`, "/") + n = path.Clean("/" + n) + return strings.TrimPrefix(n, "/") +} + +// skipName is what systems leave in archives that is not anybody's +// document: macOS resource forks, Office lock files, thumbnails. +func skipName(n string) bool { + base := path.Base(strings.ReplaceAll(n, `\`, "/")) + return strings.HasPrefix(n, "__MACOSX/") || strings.HasPrefix(base, "._") || strings.HasPrefix(base, "~$") || + base == ".DS_Store" || base == "Thumbs.db" || base == "desktop.ini" +} + +// mailParts reads an e-mail (.eml): the message — its headers and the plain +// text, or the HTML when there is no plain text — and each attachment this +// can read, named by its file name. +func mailParts(ctx context.Context, name string, b []byte, o Options, depth int) ([]Part, error) { + m, err := mail.ReadMessage(bytes.NewReader(b)) + if err != nil { + return nil, err + } + dec := new(mime.WordDecoder) + dec.CharsetReader = func(cs string, in io.Reader) (io.Reader, error) { + all, err := io.ReadAll(in) + if err != nil { + return nil, err + } + return strings.NewReader(decodeCharset(all, cs)), nil + } + header := func(k string) string { + v := m.Header.Get(k) + if d, err := dec.DecodeHeader(v); err == nil { + return d + } + return v + } + var plain, htmlBody []string + var attachments []Part + err = walkMIME(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, 0, + func(ctype, filename string, params map[string]string, body []byte) { + if filename != "" { + if d, err := dec.DecodeHeader(filename); err == nil { + filename = d + } + if Known(filename) { + inner, err := extractNested(ctx, filename, body, o, depth+1) + attachments = append(attachments, nest(memberName(filename), inner, err)...) + } + return + } + switch ctype { + case "text/plain": + plain = append(plain, decodeCharset(body, params["charset"])) + case "text/html": + if t, err := htmlFrom(strings.NewReader(decodeCharset(body, params["charset"]))); err == nil { + htmlBody = append(htmlBody, t) + } + case "message/rfc822": + if inner, err := mailParts(ctx, "", body, o, depth+1); err == nil { + attachments = append(attachments, nest("message", inner, nil)...) + } + } + }) + if err != nil && len(plain) == 0 && len(htmlBody) == 0 { + return nil, err + } + var text strings.Builder + if s := header("Subject"); s != "" { + text.WriteString("# " + s + "\n\n") + } + for _, k := range []string{"From", "To", "Cc", "Date"} { + if v := header(k); v != "" { + text.WriteString(k + ": " + v + "\n") + } + } + text.WriteString("\n") + body := plain + if len(body) == 0 { + body = htmlBody + } + for _, p := range body { + text.WriteString(p + "\n\n") + } + parts := []Part{{Text: Clean(text.String())}} + parts = append(parts, attachments...) + return parts, nil +} + +// transferDecoder undoes a MIME part's Content-Transfer-Encoding. +func transferDecoder(cte string, r io.Reader) io.Reader { + switch strings.ToLower(strings.TrimSpace(cte)) { + case "base64": + return base64.NewDecoder(base64.StdEncoding, newlineStripper{r}) + case "quoted-printable": + return quotedprintable.NewReader(r) + } + return r +} + +// newlineStripper drops CR and LF, which base64 in mail is wrapped with and +// encoding/base64 does not skip in every position. +type newlineStripper struct{ r io.Reader } + +func (n newlineStripper) Read(p []byte) (int, error) { + for { + k, err := n.r.Read(p) + j := 0 + for _, c := range p[:k] { + if c != '\r' && c != '\n' { + p[j] = c + j++ + } + } + if j > 0 || err != nil { + if errors.Is(err, io.EOF) && j > 0 { + return j, nil + } + return j, err + } + } +} diff --git a/internal/kb/extract/extract.go b/internal/kb/extract/extract.go new file mode 100644 index 0000000..2f8e619 --- /dev/null +++ b/internal/kb/extract/extract.go @@ -0,0 +1,296 @@ +// Package extract turns documents into plain text for knowledge bases +// (BACKLOG F2), on the machine that holds them. +// +// The text it produces is Markdown-shaped: headings as "# ", "## ", list +// items as "- ", paragraphs separated by a blank line. The chunker splits on +// exactly those, so a format that knows its headings (DOCX styles, HTML h1–h6, +// ODT outline levels, slides, sheets) says so, and its chunks carry them. +// +// Built-in formats are pure Go, so the client builds for every system without +// a C toolchain. What Go cannot read — Word 97 files, scanned pages — goes to +// a helper when one is on the machine (LibreOffice, Tesseract, Poppler), and +// is otherwise skipped with a sentence saying which helper would read it. +package extract + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +// ErrUnsupported is a file this cannot read, with the reason wrapped around +// it ("a .doc needs LibreOffice"). +var ErrUnsupported = errors.New("unsupported") + +// ErrNoText is a file that was read and held no text: a scanned PDF without +// an OCR helper, an empty sheet. +var ErrNoText = errors.New("no text in it") + +// Part is the text of one document. A file is one part; an archive or a mail +// with attachments is several, each named by its path inside, after the +// outer file's name and "!/". +type Part struct { + // Name is "" for the file itself, or the path inside it. + Name string + Text string + // Err is why this part was skipped, when it was; Text is then empty. + Err error +} + +// Options are the limits and helpers extraction uses. +type Options struct { + Helpers Helpers + // MaxBytes is the largest file, or member of an archive, read; 0 is + // 256 MiB. + MaxBytes int64 + // OCRLanguages are Tesseract's languages; "" is "tur+eng". + OCRLanguages string +} + +func (o Options) maxBytes() int64 { + if o.MaxBytes > 0 { + return o.MaxBytes + } + return 256 << 20 +} + +// File extracts a file on disk. +func File(ctx context.Context, name string, o Options) ([]Part, error) { + st, err := os.Stat(name) + if err != nil { + return nil, err + } + if st.Size() > o.maxBytes() { + return nil, fmt.Errorf("%w: %d MiB is over the %d MiB limit", ErrUnsupported, st.Size()>>20, o.maxBytes()>>20) + } + data, err := os.ReadFile(name) + if err != nil { + return nil, err + } + return Bytes(ctx, filepath.Base(name), data, o) +} + +// Bytes extracts a document held in memory; name gives its format. +func Bytes(ctx context.Context, name string, data []byte, o Options) ([]Part, error) { + return extractDepth(ctx, name, data, o, 0) +} + +// maxDepth is how many archives deep members are followed: a zip in a mail +// in a zip, and no further. +const maxDepth = 3 + +func extractDepth(ctx context.Context, name string, data []byte, o Options, depth int) ([]Part, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ext := Ext(name) + if f, ok := containers[ext]; ok { + if depth >= maxDepth { + return nil, fmt.Errorf("%w: archives nested more than %d deep", ErrUnsupported, maxDepth) + } + return f(ctx, name, data, o, depth) + } + text, err := one(ctx, name, ext, data, o) + if err != nil { + return nil, err + } + text = Clean(text) + if strings.TrimSpace(text) == "" { + return nil, ErrNoText + } + return []Part{{Text: text}}, nil +} + +// one extracts a single document. +func one(ctx context.Context, name, ext string, data []byte, o Options) (string, error) { + if f, ok := builtin[ext]; ok { + text, err := f(data) + if err == nil && strings.TrimSpace(text) != "" { + return text, nil + } + // A PDF the Go reader cannot parse, or one with no text layer, is + // what the helpers are for. + if alt, ok := fallback[ext]; ok { + if t, aerr := alt(ctx, name, data, o); aerr == nil { + return t, nil + } else if err == nil { + err = aerr + } + } + if err == nil { + err = ErrNoText + } + return "", err + } + if alt, ok := fallback[ext]; ok { + return alt(ctx, name, data, o) + } + if looksLikeText(data) { + return decodeText(data), nil + } + if o.Helpers.Office != "" { + return o.Helpers.officeToText(ctx, name, data, o) + } + return "", fmt.Errorf("%w: no reader for %q files (LibreOffice, if installed, may read it)", ErrUnsupported, ext) +} + +// Ext is a name's extension, lower case and without the dot. +func Ext(name string) string { + return strings.TrimPrefix(strings.ToLower(path.Ext(strings.ReplaceAll(name, `\`, "/"))), ".") +} + +// Supported lists the extensions read without a helper, and those read with +// one, for --help and the docs. +func Supported() (builtIn, withHelper []string) { + for e := range builtin { + builtIn = append(builtIn, e) + } + for e := range containers { + builtIn = append(builtIn, e) + } + for e := range textExts { + builtIn = append(builtIn, e) + } + for e := range fallback { + if _, ok := builtin[e]; !ok { + withHelper = append(withHelper, e) + } + } + sort.Strings(builtIn) + sort.Strings(withHelper) + return builtIn, withHelper +} + +// Known reports whether a name's extension is one this reads, with or without +// a helper, or one of the text formats. +func Known(name string) bool { + e := Ext(name) + _, a := builtin[e] + _, b := containers[e] + _, c := fallback[e] + return a || b || c || textExts[e] +} + +var ( + builtin map[string]func([]byte) (string, error) + containers map[string]func(context.Context, string, []byte, Options, int) ([]Part, error) + fallback map[string]func(context.Context, string, []byte, Options) (string, error) +) + +func init() { + builtin = map[string]func([]byte) (string, error){ + "pdf": pdfText, + "html": htmlText, "htm": htmlText, "xhtml": htmlText, "mht": mhtmlText, "mhtml": mhtmlText, + "docx": docxText, "docm": docxText, "dotx": docxText, "dotm": docxText, + "xlsx": xlsxText, "xlsm": xlsxText, "xltx": xlsxText, "xltm": xlsxText, + "pptx": pptxText, "pptm": pptxText, "ppsx": pptxText, "ppsm": pptxText, "potx": pptxText, + "odt": odfText, "ott": odfText, "ods": odfText, "ots": odfText, "odp": odfText, "otp": odfText, + "fodt": flatODFText, "fods": flatODFText, "fodp": flatODFText, + "rtf": rtfText, + "epub": epubText, + "ipynb": notebookText, + "srt": subtitleText, "vtt": subtitleText, + } + for e := range textExts { + builtin[e] = func(b []byte) (string, error) { return decodeText(b), nil } + } + containers = map[string]func(context.Context, string, []byte, Options, int) ([]Part, error){ + "zip": zipParts, + "eml": mailParts, + } + fallback = map[string]func(context.Context, string, []byte, Options) (string, error){ + "pdf": pdfWithHelpers, + "rtf": viaOffice, + } + for _, e := range officeOnly { + fallback[e] = viaOffice + } + for _, e := range imageExts { + fallback[e] = viaOCR + } +} + +// textExts are formats that are text already: read as they are, decoded. +var textExts = map[string]bool{ + "txt": true, "text": true, "md": true, "markdown": true, "mdx": true, "rst": true, "adoc": true, + "asciidoc": true, "org": true, "tex": true, "log": true, "csv": true, "tsv": true, "json": true, + "jsonl": true, "ndjson": true, "yaml": true, "yml": true, "toml": true, "ini": true, "cfg": true, + "conf": true, "properties": true, "env": true, "xml": true, "svg": true, "sql": true, + "go": true, "py": true, "js": true, "mjs": true, "cjs": true, "ts": true, "tsx": true, "jsx": true, + "java": true, "kt": true, "kts": true, "scala": true, "c": true, "h": true, "cc": true, "cpp": true, + "cxx": true, "hpp": true, "hh": true, "cs": true, "fs": true, "vb": true, "rs": true, "swift": true, + "m": true, "mm": true, "rb": true, "php": true, "pl": true, "pm": true, "lua": true, "r": true, + "jl": true, "dart": true, "ex": true, "exs": true, "erl": true, "hs": true, "clj": true, + "sh": true, "bash": true, "zsh": true, "fish": true, "ps1": true, "psm1": true, "bat": true, + "cmd": true, "groovy": true, "gradle": true, "tf": true, "hcl": true, "proto": true, + "graphql": true, "gql": true, "css": true, "scss": true, "sass": true, "less": true, + "vue": true, "svelte": true, "dockerfile": true, "makefile": true, "mk": true, "cmake": true, + "bib": true, "nfo": true, "diff": true, "patch": true, +} + +// officeOnly are formats only LibreOffice reads here: the binary Office +// formats before 2007, and others it opens. +var officeOnly = []string{ + "doc", "dot", "xls", "xlt", "xlsb", "ppt", "pps", "pot", "wpd", "wps", "wri", + "pages", "numbers", "key", "sxw", "sxc", "sxi", "hwp", "pub", "vsd", "vsdx", +} + +// imageExts are read by OCR. +var imageExts = []string{"png", "jpg", "jpeg", "tif", "tiff", "bmp", "gif", "webp", "jp2", "pnm"} + +// looksLikeText reports whether bytes are text: no NUL in the first 8 KiB +// (UTF-16 has its BOM checked first), and mostly printable. +func looksLikeText(b []byte) bool { + if hasUTF16BOM(b) { + return true + } + head := b[:min(len(b), 8192)] + if bytes.IndexByte(head, 0) >= 0 { + return false + } + if utf8.Valid(head) { + return true + } + // A single-byte code page: allow it when control characters are rare. + ctl := 0 + for _, c := range head { + if c < 0x20 && c != '\n' && c != '\r' && c != '\t' && c != '\f' { + ctl++ + } + } + return ctl*100 < len(head) +} + +var ( + spaceRun = regexp.MustCompile(`[ \t\x{00A0}]+\n`) + blankRun = regexp.MustCompile(`\n{3,}`) +) + +// Clean normalises extracted text: LF line ends, no trailing spaces, no more +// than one blank line in a row, no control characters but tab and newline. +func Clean(s string) string { + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + s = strings.Map(func(r rune) rune { + switch { + case r == '\n' || r == '\t': + return r + case r == '\f' || r == '\v': + return '\n' + case r < 0x20 || r == 0x7f || r == utf8.RuneError || r == '\uFEFF': + return -1 + } + return r + }, s) + s = spaceRun.ReplaceAllString(s, "\n") + s = blankRun.ReplaceAllString(s, "\n\n") + return strings.TrimSpace(s) + "\n" +} diff --git a/internal/kb/extract/extract_test.go b/internal/kb/extract/extract_test.go new file mode 100644 index 0000000..73157ab --- /dev/null +++ b/internal/kb/extract/extract_test.go @@ -0,0 +1,323 @@ +package extract + +import ( + "archive/zip" + "bytes" + "context" + "encoding/base64" + "errors" + "os" + "strings" + "testing" + + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/unicode" +) + +// zipOf builds a zip from name → content, in the order given. +func zipOf(t *testing.T, files ...string) []byte { + t.Helper() + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for i := 0; i+1 < len(files); i += 2 { + f, err := w.Create(files[i]) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(files[i+1])); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func extractOne(t *testing.T, name string, data []byte) string { + t.Helper() + parts, err := Bytes(context.Background(), name, data, Options{}) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if len(parts) != 1 || parts[0].Err != nil { + t.Fatalf("%s: parts %+v", name, parts) + } + return parts[0].Text +} + +func wants(t *testing.T, got string, want ...string) { + t.Helper() + for _, w := range want { + if !strings.Contains(got, w) { + t.Errorf("missing %q in:\n%s", w, got) + } + } +} + +func TestTextInEveryEncodingBecomesUTF8(t *testing.T) { + const s = "Yıllık izin: Şubat'ta ğüşİöç" + u16, _ := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(s)) + cp, _ := charmap.Windows1254.NewEncoder().Bytes([]byte(s)) + for name, b := range map[string][]byte{ + "utf8": []byte(s), + "utf8-bom": append([]byte{0xEF, 0xBB, 0xBF}, s...), + "utf16": u16, + "cp1254": cp, + "crlf": []byte(strings.ReplaceAll(s+"\nikinci", "\n", "\r\n")), + } { + got := extractOne(t, "a.txt", b) + if !strings.HasPrefix(got, s) { + t.Errorf("%s: got %q", name, got) + } + if strings.Contains(got, "\r") { + t.Errorf("%s: kept a CR", name) + } + } + // macOS writes names and sometimes text decomposed; the text comes out + // composed, so the same word is the same bytes everywhere. + if got := extractOne(t, "a.txt", []byte("Şirket")); !strings.HasPrefix(got, "Şirket") { + t.Errorf("NFD text came out as %q", got) + } +} + +func TestHTMLKeepsHeadingsAndDropsScripts(t *testing.T) { + page := `T +

İzin Politikası

Yıllık izin yirmi gün.

+
  • Bir
  • İki
ab
+

Merhaba

` + got := extractOne(t, "a.html", []byte(page)) + wants(t, got, "# İzin Politikası", "Yıllık izin yirmi gün.", "- Bir", "- İki", "a\tb", "Merhaba") + for _, no := range []string{"gizli", "menü"} { + if strings.Contains(got, no) { + t.Errorf("%q leaked into:\n%s", no, got) + } + } + // Declared charset honoured. + l5, _ := charmap.ISO8859_9.NewEncoder().Bytes([]byte(`

Şirket

`)) + wants(t, extractOne(t, "b.htm", l5), "Şirket") +} + +func TestWordDocument(t *testing.T) { + doc := ` + +Yıllık İzin +Çalışanlar yirmi günotuzizin alır. +Madde bir +Hücre1Hücre2 +Masraflar +` + got := extractOne(t, "a.docx", zipOf(t, "word/document.xml", doc, + "word/footnotes.xml", `Dipnot`)) + wants(t, got, "# Yıllık İzin", "Çalışanlar yirmi gün\tizin alır.", "- Madde bir", "Hücre1\tHücre2", "## Masraflar", "Dipnot") + if strings.Contains(got, "otuz") { + t.Errorf("a tracked deletion is in the text:\n%s", got) + } +} + +func TestWorkbook(t *testing.T) { + got := extractOne(t, "a.xlsx", zipOf(t, + "xl/workbook.xml", ``, + "xl/_rels/workbook.xml.rels", ``, + "xl/sharedStrings.xml", `KalemTutarKira`, + "xl/worksheets/sheet1.xml", `0121500.5not`, + "xl/worksheets/sheet2.xml", ``, + )) + wants(t, got, "## Bütçe", "Kalem\tTutar", "Kira\t1500.5\tnot") +} + +func TestPresentationInItsOwnOrderWithNotes(t *testing.T) { + slide := func(text string) string { + return `` + text + + `9` + } + got := extractOne(t, "a.pptx", zipOf(t, + "ppt/presentation.xml", ``, + "ppt/_rels/presentation.xml.rels", ``, + "ppt/slides/slide1.xml", slide("İkinci slayt"), + "ppt/slides/slide2.xml", slide("Birinci slayt"), + "ppt/slides/_rels/slide2.xml.rels", ``, + "ppt/notesSlides/notesSlide7.xml", slide("Konuşmacı notu"), + )) + first, second := strings.Index(got, "Birinci slayt"), strings.Index(got, "İkinci slayt") + if first < 0 || second < 0 || first > second { + t.Errorf("slides out of the presentation's order:\n%s", got) + } + wants(t, got, "## Slide 1", "Notes:", "Konuşmacı notu") + if strings.Contains(got, "9") { + t.Errorf("a slide number field is in the text:\n%s", got) + } +} + +func TestOpenDocument(t *testing.T) { + content := ` +BölümBirikiüç +madde +xy +` + got := extractOne(t, "a.odt", zipOf(t, "mimetype", "application/vnd.oasis.opendocument.text", "content.xml", content)) + wants(t, got, "## Bölüm", "Bir iki\tüç", "- madde", "## Tablo1", "x\ty") + wants(t, extractOne(t, "a.fodt", []byte(content)), "## Bölüm") +} + +func TestRichText(t *testing.T) { + rtf := `{\rtf1\ansi\ansicpg1254\deff0{\fonttbl{\f0 Calibri;}}{\colortbl;\red0\green0\blue0;}` + + `{\*\generator Riched20;}\pard Y\'fdll\'fdk izin\par ` + + `\u350?irket \u-3913?\'3f ok\tab sonra\par{\header gizli}\cell b\row }` + got := extractOne(t, "a.rtf", []byte(rtf)) + wants(t, got, "Yıllık izin", "Şirket", "ok\tsonra") + for _, no := range []string{"Calibri", "Riched", "gizli"} { + if strings.Contains(got, no) { + t.Errorf("%q leaked into:\n%s", no, got) + } + } +} + +func TestEbookInSpineOrder(t *testing.T) { + got := extractOne(t, "a.epub", zipOf(t, + "mimetype", "application/epub+zip", + "META-INF/container.xml", ``, + "OEBPS/content.opf", ``, + "OEBPS/a.xhtml", `

Önce

`, + "OEBPS/b.xhtml", `

Sonra

`, + )) + if a, b := strings.Index(got, "Önce"), strings.Index(got, "Sonra"); a < 0 || b < a { + t.Errorf("chapters out of spine order:\n%s", got) + } +} + +func TestNotebookAndSubtitles(t *testing.T) { + nb := `{"metadata":{"kernelspec":{"language":"python"}},"cells":[{"cell_type":"markdown","source":["# Analiz\n","Veri"]}, + {"cell_type":"code","source":"print(1)","outputs":[{"text":"çıktı"}]}]}` + got := extractOne(t, "a.ipynb", []byte(nb)) + wants(t, got, "# Analiz", "```python\nprint(1)\n```") + if strings.Contains(got, "çıktı") { + t.Errorf("an output is in the text:\n%s", got) + } + srt := "1\n00:00:01,000 --> 00:00:02,000\nMerhaba\n\n2\n00:00:02,000 --> 00:00:03,000\nMerhaba\nDünya\n" + if got := extractOne(t, "a.srt", []byte(srt)); got != "Merhaba\nDünya\n" { + t.Errorf("subtitles came out as %q", got) + } +} + +func TestPDFTextLayer(t *testing.T) { + b, err := os.ReadFile("testdata/turkish.pdf") + if err != nil { + t.Fatal(err) + } + wants(t, extractOne(t, "a.pdf", b), "Yıllık İzin Politikası", "Masraflar finans ekibine iletilir.") + // Garbage is a failure, not a panic. + if _, err := Bytes(context.Background(), "bad.pdf", []byte("%PDF-1.4 nonsense"), Options{}); err == nil { + t.Error("a broken PDF was read") + } +} + +func TestArchivesAreReadMemberByMember(t *testing.T) { + inner := zipOf(t, "deep/c.md", "# Derin") + archive := zipOf(t, + "docs/a.txt", "Birinci", + `docs\b.html`, "

İkinci

", + "__MACOSX/docs/._a.txt", "çöp", + "docs/~$kilit.docx", "çöp", + "inner.zip", string(inner), + "image.bin", "\x00\x01", + ) + parts, err := Bytes(context.Background(), "arsiv.zip", archive, Options{}) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, p := range parts { + if p.Err != nil { + t.Errorf("%s: %v", p.Name, p.Err) + } + got[p.Name] = p.Text + } + if len(got) != 3 || !strings.Contains(got["docs/a.txt"], "Birinci") || !strings.Contains(got["docs/b.html"], "İkinci") || + !strings.Contains(got["inner.zip!/deep/c.md"], "# Derin") { + t.Errorf("archive parts %v", got) + } +} + +func TestMailWithAttachments(t *testing.T) { + att := zipOf(t, "x.txt", "ekteki metin") + enc := func(b []byte) string { + s := base64.StdEncoding.EncodeToString(b) + var lines []string + for len(s) > 76 { + lines, s = append(lines, s[:76]), s[76:] + } + return strings.Join(append(lines, s), "\r\n") + } + msg := "From: =?UTF-8?B?QXnFn2U=?= \r\n" + + "Subject: =?iso-8859-9?Q?=DEubat_raporu?=\r\n" + + "Content-Type: multipart/mixed; boundary=B\r\n\r\n" + + "--B\r\nContent-Type: multipart/alternative; boundary=C\r\n\r\n" + + "--C\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" + + "Rapor ekte, te=C5=9Fekk=C3=BCrler.\r\n" + + "--C\r\nContent-Type: text/html; charset=utf-8\r\n\r\n

HTML hali

\r\n--C--\r\n" + + "--B\r\nContent-Type: application/zip; name=\"ek.zip\"\r\nContent-Transfer-Encoding: base64\r\n" + + "Content-Disposition: attachment; filename=\"ek.zip\"\r\n\r\n" + enc(att) + "\r\n--B--\r\n" + parts, err := Bytes(context.Background(), "a.eml", []byte(msg), Options{}) + if err != nil { + t.Fatal(err) + } + if len(parts) != 2 { + t.Fatalf("parts %+v", parts) + } + wants(t, parts[0].Text, "# Şubat raporu", "From: Ayşe", "Rapor ekte, teşekkürler.") + if strings.Contains(parts[0].Text, "HTML hali") { + t.Errorf("the HTML alternative was read beside the plain text:\n%s", parts[0].Text) + } + if parts[1].Name != "ek.zip!/x.txt" || !strings.Contains(parts[1].Text, "ekteki metin") { + t.Errorf("attachment %+v", parts[1]) + } +} + +func TestUnreadableFilesSayWhy(t *testing.T) { + _, err := Bytes(context.Background(), "a.doc", []byte{0xD0, 0xCF, 0x11, 0xE0, 0}, Options{}) + if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "LibreOffice") { + t.Errorf("a .doc without LibreOffice gave %v", err) + } + _, err = Bytes(context.Background(), "a.png", []byte{0x89, 'P', 'N', 'G', 0}, Options{}) + if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "Tesseract") { + t.Errorf("an image without Tesseract gave %v", err) + } + _, err = Bytes(context.Background(), "a.bin", []byte{0, 1, 2, 3}, Options{}) + if !errors.Is(err, ErrUnsupported) { + t.Errorf("binary gave %v", err) + } + _, err = Bytes(context.Background(), "a.txt", []byte(" \n"), Options{}) + if !errors.Is(err, ErrNoText) { + t.Errorf("an empty file gave %v", err) + } +} + +func TestHelpersWhenInstalled(t *testing.T) { + h := FindHelpers() + ctx := context.Background() + if h.Tesseract != "" && h.PdfToPPM != "" { + // The text-layer PDF read by OCR instead: what a scanned one gets. + b, _ := os.ReadFile("testdata/turkish.pdf") + got, err := pdfWithHelpers(ctx, "a.pdf", b, Options{Helpers: Helpers{Tesseract: h.Tesseract, PdfToPPM: h.PdfToPPM}}) + if err != nil { + t.Fatal(err) + } + wants(t, got, "Masraflar") + } else { + t.Log("no Tesseract with pdftoppm here; OCR not tried") + } + if h.Office == "" { + t.Log("no LibreOffice here; legacy Office formats not tried") + return + } + // A Word document converted to the legacy format and back would need a + // fixture made by Word; an ODT through the Office path proves the + // conversion runs and its result is read. + content := `LibreOffice yolu` + got, err := h.officeToText(ctx, "a.fodt", []byte(``+content), Options{}) + if err != nil { + t.Fatal(err) + } + wants(t, got, "LibreOffice yolu") +} diff --git a/internal/kb/extract/helpers.go b/internal/kb/extract/helpers.go new file mode 100644 index 0000000..385da37 --- /dev/null +++ b/internal/kb/extract/helpers.go @@ -0,0 +1,277 @@ +package extract + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// Helpers are programs on this machine that read what Go here cannot. Any +// may be "", and the formats that need it are then skipped with a reason. +type Helpers struct { + // Office is LibreOffice's soffice: the binary Office formats and others. + Office string + // Tesseract reads images and scanned pages. + Tesseract string + // PdfToText and PdfToPPM are Poppler's: a second PDF reader, and pages + // rendered for Tesseract. + PdfToText string + PdfToPPM string +} + +// helperTimeout bounds one helper run; a conversion that hangs is a file +// skipped, not a sync stopped. +const helperTimeout = 3 * time.Minute + +// FindHelpers looks for the helpers on PATH and where their installers put +// them on each system. +func FindHelpers() Helpers { + return Helpers{ + Office: find("soffice", officePlaces()...), + Tesseract: find("tesseract", tesseractPlaces()...), + PdfToText: find("pdftotext"), + PdfToPPM: find("pdftoppm"), + } +} + +// Describe says which helpers were found, and what each missing one would +// read. +func (h Helpers) Describe() []string { + line := func(have, name, reads string) string { + if have != "" { + return fmt.Sprintf("%-11s %s", name, have) + } + return fmt.Sprintf("%-11s not found — %s are skipped", name, reads) + } + return []string{ + line(h.Office, "LibreOffice", "Word 97–2003, Excel 97–2003, PowerPoint 97–2003, Pages, Numbers, Keynote and similar files"), + line(h.Tesseract, "Tesseract", "images and scanned PDFs"), + line(h.PdfToText, "pdftotext", "PDFs the built-in reader cannot parse"), + line(h.PdfToPPM, "pdftoppm", "scanned PDFs (Tesseract needs their pages as images)"), + } +} + +func find(name string, places ...string) string { + exe := name + if runtime.GOOS == "windows" { + exe += ".exe" + } + if p, err := exec.LookPath(exe); err == nil { + return p + } + if runtime.GOOS != "windows" { + if p, err := exec.LookPath(name); err == nil { + return p + } + } + for _, p := range places { + if matches, _ := filepath.Glob(p); len(matches) > 0 { + sort.Strings(matches) + return matches[len(matches)-1] + } + } + return "" +} + +func officePlaces() []string { + switch runtime.GOOS { + case "windows": + return []string{ + `C:\Program Files\LibreOffice*\program\soffice.exe`, + `C:\Program Files (x86)\LibreOffice*\program\soffice.exe`, + } + case "darwin": + return []string{"/Applications/LibreOffice.app/Contents/MacOS/soffice"} + } + return []string{"/usr/bin/libreoffice", "/usr/lib/libreoffice/program/soffice", "/opt/libreoffice*/program/soffice", + "/snap/bin/libreoffice"} +} + +func tesseractPlaces() []string { + switch runtime.GOOS { + case "windows": + return []string{`C:\Program Files\Tesseract-OCR\tesseract.exe`, `C:\Program Files (x86)\Tesseract-OCR\tesseract.exe`} + case "darwin": + return []string{"/opt/homebrew/bin/tesseract", "/usr/local/bin/tesseract"} + } + return nil +} + +// workDir is a temporary directory for one helper run, removed after. +func workDir() (string, func(), error) { + d, err := os.MkdirTemp("", "llmgw-kb-") + if err != nil { + return "", nil, err + } + return d, func() { _ = os.RemoveAll(d) }, nil +} + +// writeInput puts a document where a helper can read it, under a safe name +// that keeps its extension. +func writeInput(dir, name string, data []byte) (string, error) { + p := filepath.Join(dir, "input."+Ext(name)) + return p, os.WriteFile(p, data, 0o600) +} + +func run(ctx context.Context, dir string, bin string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + out, err := cmd.Output() + if ctx.Err() != nil { + return nil, fmt.Errorf("%s took over %s", filepath.Base(bin), helperTimeout) + } + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) && len(ee.Stderr) > 0 { + return nil, fmt.Errorf("%s: %v: %s", filepath.Base(bin), err, lastLine(ee.Stderr)) + } + return nil, fmt.Errorf("%s: %w", filepath.Base(bin), err) + } + return out, nil +} + +func lastLine(b []byte) string { + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} + +// fileURL is a path as LibreOffice takes its profile directory: a file URL, +// with a drive letter on Windows. +func fileURL(p string) string { + p = filepath.ToSlash(p) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return "file://" + p +} + +var spreadsheets = map[string]bool{"xls": true, "xlt": true, "xlsb": true, "numbers": true, "sxc": true} +var presentations = map[string]bool{"ppt": true, "pps": true, "pot": true, "key": true, "sxi": true} + +func viaOffice(ctx context.Context, name string, data []byte, o Options) (string, error) { + if o.Helpers.Office == "" { + return "", fmt.Errorf("%w: a .%s file needs LibreOffice, which is not installed here", ErrUnsupported, Ext(name)) + } + return o.Helpers.officeToText(ctx, name, data, o) +} + +// officeToText has LibreOffice convert a document to the matching Office +// Open XML format, then reads that: headings, sheets and slides survive, +// which a conversion to plain text would lose. +func (h Helpers) officeToText(ctx context.Context, name string, data []byte, _ Options) (string, error) { + dir, done, err := workDir() + if err != nil { + return "", err + } + defer done() + in, err := writeInput(dir, name, data) + if err != nil { + return "", err + } + ext := Ext(name) + to, read := "docx", docxText + switch { + case spreadsheets[ext]: + to, read = "xlsx", xlsxText + case presentations[ext]: + to, read = "pptx", pptxText + } + // A profile of its own: two conversions at once with the user's + // profile wait on each other's lock, or fail. + profile := fileURL(filepath.Join(dir, "profile")) + if _, err := run(ctx, dir, h.Office, "-env:UserInstallation="+profile, "--headless", "--norestore", + "--convert-to", to, "--outdir", dir, in); err != nil { + return "", err + } + converted, err := os.ReadFile(filepath.Join(dir, "input."+to)) + if err != nil { + return "", fmt.Errorf("LibreOffice did not convert it: %w", err) + } + return read(converted) +} + +func viaOCR(ctx context.Context, name string, data []byte, o Options) (string, error) { + if o.Helpers.Tesseract == "" { + return "", fmt.Errorf("%w: an image needs Tesseract to be read, which is not installed here", ErrUnsupported) + } + dir, done, err := workDir() + if err != nil { + return "", err + } + defer done() + in, err := writeInput(dir, name, data) + if err != nil { + return "", err + } + return ocr(ctx, dir, o, in) +} + +func ocr(ctx context.Context, dir string, o Options, file string) (string, error) { + langs := o.OCRLanguages + if langs == "" { + langs = "tur+eng" + } + out, err := run(ctx, dir, o.Helpers.Tesseract, file, "stdout", "-l", langs) + if err != nil && langs != "eng" { + // A Tesseract without the Turkish data still reads English. + out, err = run(ctx, dir, o.Helpers.Tesseract, file, "stdout", "-l", "eng") + } + return string(out), err +} + +// maxOCRPages bounds how many pages of one scanned PDF are read. +const maxOCRPages = 500 + +// pdfWithHelpers reads a PDF the built-in reader could not: with pdftotext, +// and when that finds no text either, page by page through Tesseract. +func pdfWithHelpers(ctx context.Context, name string, data []byte, o Options) (string, error) { + h := o.Helpers + if h.PdfToText == "" && (h.Tesseract == "" || h.PdfToPPM == "") { + return "", fmt.Errorf("%w: no text layer the built-in reader can use; Poppler (pdftotext) or "+ + "Tesseract with pdftoppm would read it", ErrNoText) + } + dir, done, err := workDir() + if err != nil { + return "", err + } + defer done() + in, err := writeInput(dir, name, data) + if err != nil { + return "", err + } + if h.PdfToText != "" { + if out, err := run(ctx, dir, h.PdfToText, "-enc", "UTF-8", in, "-"); err == nil && strings.TrimSpace(string(out)) != "" { + return string(out), nil + } + } + if h.Tesseract == "" || h.PdfToPPM == "" { + return "", fmt.Errorf("%w: a scanned PDF needs Tesseract and pdftoppm", ErrNoText) + } + if _, err := run(ctx, dir, h.PdfToPPM, "-r", "300", "-gray", "-png", "-l", fmt.Sprint(maxOCRPages), in, + filepath.Join(dir, "page")); err != nil { + return "", err + } + pages, _ := filepath.Glob(filepath.Join(dir, "page-*.png")) + // pdftoppm pads page numbers to the width of the last one, so the + // names sort in page order. + sort.Strings(pages) + var all strings.Builder + for _, p := range pages { + t, err := ocr(ctx, dir, o, p) + if err != nil { + return "", err + } + all.WriteString(t) + all.WriteString("\n\n") + } + return all.String(), nil +} diff --git a/internal/kb/extract/html.go b/internal/kb/extract/html.go new file mode 100644 index 0000000..f9b7d81 --- /dev/null +++ b/internal/kb/extract/html.go @@ -0,0 +1,277 @@ +package extract + +import ( + "bytes" + "io" + "mime" + "mime/multipart" + "net/mail" + "strings" + + "golang.org/x/net/html" + "golang.org/x/net/html/charset" +) + +// out builds Markdown-shaped text: paragraphs, headings, list items, table +// rows, with whitespace collapsed inside them. +type out struct { + b strings.Builder + // line is the text of the line being built, whitespace collapsed. + line strings.Builder + prefix string +} + +// text adds running text; runs of whitespace become one space, and none is +// added where the source had none ("Merhaba" is one word). +func (o *out) text(s string) { + if s == "" { + return + } + r := []rune(s) + lead, trail := isSpace(r[0]), isSpace(r[len(r)-1]) + for i, f := range strings.FieldsFunc(s, isSpace) { + if i > 0 || lead { + o.space() + } + o.line.WriteString(f) + } + if trail { + o.space() + } +} + +// space adds one space unless the line is empty or already ends in one. +func (o *out) space() { + l := o.line.String() + if l != "" && !strings.HasSuffix(l, " ") && !strings.HasSuffix(l, "\t") { + o.line.WriteByte(' ') + } +} + +// raw adds text as it is, for preformatted blocks. +func (o *out) raw(s string) { o.line.WriteString(s) } + +// tab separates cells. +func (o *out) tab() { + o.trimLine() + o.line.WriteString("\t") +} + +func (o *out) trimLine() { + s := strings.TrimRight(o.line.String(), " ") + o.line.Reset() + o.line.WriteString(s) +} + +// newline ends the line being built, if it has anything. +func (o *out) newline() { + s := strings.TrimSpace(o.line.String()) + o.line.Reset() + if s == "" { + return + } + o.b.WriteString(o.prefix) + o.b.WriteString(s) + o.b.WriteByte('\n') + o.prefix = "" +} + +// para ends a paragraph: the line, and a blank line after it. +func (o *out) para() { + o.newline() + if o.b.Len() > 0 && !strings.HasSuffix(o.b.String(), "\n\n") { + o.b.WriteByte('\n') + } +} + +// heading starts a heading of a level, 1 to 6. +func (o *out) heading(level int) { + o.para() + level = max(1, min(level, 6)) + o.prefix = strings.Repeat("#", level) + " " +} + +// item starts a list item. +func (o *out) item() { + o.newline() + o.prefix = "- " +} + +func (o *out) String() string { + o.newline() + return o.b.String() +} + +func isSpace(r rune) bool { + return r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f' || r == ' ' +} + +// htmlSkip are elements whose content is not text a reader sees. +var htmlSkip = map[string]bool{ + "script": true, "style": true, "noscript": true, "template": true, "svg": true, + "head": true, "nav": true, "iframe": true, "object": true, "canvas": true, "button": true, "select": true, +} + +// htmlBlock are elements that end a paragraph. +var htmlBlock = map[string]bool{ + "p": true, "div": true, "section": true, "article": true, "main": true, "header": true, + "footer": true, "aside": true, "blockquote": true, "pre": true, "table": true, "ul": true, + "ol": true, "dl": true, "figure": true, "figcaption": true, "address": true, "hr": true, + "form": true, "fieldset": true, "details": true, "summary": true, "body": true, "dt": true, "dd": true, + "caption": true, +} + +// htmlText reads an HTML page, in the charset it declares. +func htmlText(b []byte) (string, error) { + r, err := charset.NewReader(bytes.NewReader(b), "text/html") + if err != nil { + r = bytes.NewReader(b) + } + return htmlFrom(r) +} + +// htmlFrom reads HTML already decoded to UTF-8. +func htmlFrom(r io.Reader) (string, error) { + z := html.NewTokenizer(r) + var o out + skip, pre := 0, 0 + for { + tt := z.Next() + switch tt { + case html.ErrorToken: + if z.Err() == io.EOF { + return o.String(), nil + } + return o.String(), z.Err() + case html.TextToken: + if skip > 0 { + continue + } + if pre > 0 { + o.raw(string(z.Text())) + } else { + o.text(string(z.Text())) + } + case html.StartTagToken, html.SelfClosingTagToken, html.EndTagToken: + name, _ := z.TagName() + tag := string(name) + end := tt == html.EndTagToken + if htmlSkip[tag] { + if tt == html.StartTagToken { + skip++ + } else if end && skip > 0 { + skip-- + } + continue + } + if skip > 0 { + continue + } + switch { + case len(tag) == 2 && tag[0] == 'h' && tag[1] >= '1' && tag[1] <= '6': + if end { + o.para() + } else { + o.heading(int(tag[1] - '0')) + } + case tag == "title" && !end: + // Inside head, which is skipped; a title outside it is a + // heading. + o.heading(1) + case tag == "li": + if !end { + o.item() + } else { + o.newline() + } + case tag == "br": + o.newline() + case tag == "tr": + o.newline() + case tag == "td" || tag == "th": + if end { + o.tab() + } + case tag == "pre": + if end { + pre-- + } else { + pre++ + } + o.para() + case htmlBlock[tag]: + o.para() + } + } + } +} + +// mhtmlText reads a web page saved as one file (MHTML): a MIME message whose +// HTML part is the page. +func mhtmlText(b []byte) (string, error) { + m, err := mail.ReadMessage(bytes.NewReader(b)) + if err != nil { + return "", err + } + var best string + err = walkMIME(m.Header.Get("Content-Type"), m.Header.Get("Content-Transfer-Encoding"), m.Body, 0, + func(ctype, _ string, params map[string]string, body []byte) { + if best != "" { + return + } + switch ctype { + case "text/html": + best, _ = htmlFrom(strings.NewReader(decodeCharset(body, params["charset"]))) + } + }) + return best, err +} + +// walkMIME calls leaf for every non-multipart part of a MIME body, decoded +// from its transfer encoding. +func walkMIME(contentType, cte string, body io.Reader, depth int, + leaf func(ctype, filename string, params map[string]string, body []byte), +) error { + if depth > 8 { + return nil + } + ctype, params, err := mime.ParseMediaType(contentType) + if err != nil { + ctype, params = "text/plain", map[string]string{} + } + if strings.HasPrefix(ctype, "multipart/") { + mr := multipart.NewReader(body, params["boundary"]) + for { + p, err := mr.NextRawPart() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err := walkMIME(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p, depth+1, + func(ct, fn string, pr map[string]string, b []byte) { + if fn == "" { + fn = filenameOf(p.Header.Get("Content-Disposition")) + } + leaf(ct, fn, pr, b) + }); err != nil { + return err + } + } + } + data, err := io.ReadAll(transferDecoder(cte, body)) + if err != nil { + return err + } + leaf(ctype, params["name"], params, data) + return nil +} + +func filenameOf(disposition string) string { + _, p, err := mime.ParseMediaType(disposition) + if err != nil { + return "" + } + return p["filename"] +} diff --git a/internal/kb/extract/misc.go b/internal/kb/extract/misc.go new file mode 100644 index 0000000..377eca8 --- /dev/null +++ b/internal/kb/extract/misc.go @@ -0,0 +1,172 @@ +package extract + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "path" + "regexp" + "strings" + + "github.com/ledongthuc/pdf" +) + +// epubText reads an e-book: its chapters in reading order (the spine). +func epubText(b []byte) (string, error) { + zr, err := openZip(b) + if err != nil { + return "", err + } + container, _ := member(zr, "META-INF/container.xml") + if container == nil { + return "", errors.New("not an EPUB: no META-INF/container.xml") + } + var opf string + _ = walkXML(container, func(e xmlEvent) { + if e.start && e.name == "rootfile" && opf == "" { + opf = e.get("full-path") + } + }) + pkg, _ := member(zr, opf) + if pkg == nil { + return "", fmt.Errorf("not an EPUB: no package document %q", opf) + } + dir := path.Dir(opf) + items := map[string]string{} + var spine []string + _ = walkXML(pkg, func(e xmlEvent) { + switch { + case e.start && e.name == "item": + items[e.get("id")] = path.Clean(path.Join(dir, e.get("href"))) + case e.start && e.name == "itemref": + spine = append(spine, e.get("idref")) + } + }) + var all strings.Builder + for _, id := range spine { + page, _ := member(zr, items[id]) + if page == nil { + continue + } + t, err := htmlText(page) + if err != nil { + continue + } + all.WriteString(t) + all.WriteString("\n\n") + } + return all.String(), nil +} + +// notebookText reads a Jupyter notebook: Markdown cells as they are, code +// cells fenced; outputs left out, since they are mostly numbers and images. +func notebookText(b []byte) (string, error) { + var nb struct { + Cells []struct { + CellType string `json:"cell_type"` + Source json.RawMessage `json:"source"` + } `json:"cells"` + Metadata struct { + Kernel struct { + Language string `json:"language"` + } `json:"kernelspec"` + } `json:"metadata"` + } + if err := json.Unmarshal(b, &nb); err != nil { + return "", err + } + var out strings.Builder + for _, c := range nb.Cells { + src := jsonText(c.Source) + if strings.TrimSpace(src) == "" { + continue + } + switch c.CellType { + case "code": + out.WriteString("```" + nb.Metadata.Kernel.Language + "\n" + strings.TrimRight(src, "\n") + "\n```\n\n") + default: + out.WriteString(strings.TrimRight(src, "\n") + "\n\n") + } + } + return out.String(), nil +} + +// jsonText is a notebook source: a string, or a list of lines. +func jsonText(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var lines []string + if json.Unmarshal(raw, &lines) == nil { + return strings.Join(lines, "") + } + return "" +} + +var ( + subtitleTiming = regexp.MustCompile(`^\d{1,2}:\d{2}(:\d{2})?[.,]\d{3}\s+-->`) + subtitleIndex = regexp.MustCompile(`^\d+$`) + subtitleTag = regexp.MustCompile(`<[^>]+>`) +) + +// subtitleText reads SRT and WebVTT: the spoken lines, without numbers, +// timings and styling. +func subtitleText(b []byte) (string, error) { + var out strings.Builder + last := "" + for _, line := range strings.Split(decodeText(b), "\n") { + line = strings.TrimSpace(line) + switch { + case line == "", line == "WEBVTT", strings.HasPrefix(line, "NOTE"), + subtitleIndex.MatchString(line), subtitleTiming.MatchString(line): + continue + } + line = strings.TrimSpace(subtitleTag.ReplaceAllString(line, "")) + // Captions repeat a line across cues as it scrolls. + if line == "" || line == last { + continue + } + last = line + out.WriteString(line + "\n") + } + return out.String(), nil +} + +// pdfText reads a PDF's text layer, page by page, rows in reading order. +// A scanned PDF has none; pdfWithHelpers is then tried. +func pdfText(b []byte) (text string, err error) { + // The reader panics on some malformed files rather than failing; a bad + // PDF is one skipped file, not a stopped sync. + defer func() { + if r := recover(); r != nil { + text, err = "", fmt.Errorf("the PDF could not be parsed: %v", r) + } + }() + r, err := pdf.NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + return "", err + } + var o out + for i := 1; i <= r.NumPage(); i++ { + p := r.Page(i) + if p.V.IsNull() { + continue + } + rows, err := p.GetTextByRow() + if err != nil { + continue + } + for _, row := range rows { + var line strings.Builder + for _, w := range row.Content { + line.WriteString(w.S) + } + o.text(line.String()) + o.newline() + } + o.para() + } + return o.String(), nil +} diff --git a/internal/kb/extract/odf.go b/internal/kb/extract/odf.go new file mode 100644 index 0000000..84860ea --- /dev/null +++ b/internal/kb/extract/odf.go @@ -0,0 +1,117 @@ +package extract + +import ( + "errors" + "fmt" + "strconv" +) + +// odfText reads an OpenDocument file (LibreOffice's ODT, ODS, ODP): its +// content.xml. +func odfText(b []byte) (string, error) { + zr, err := openZip(b) + if err != nil { + return "", err + } + content, err := member(zr, "content.xml") + if err != nil { + return "", err + } + if content == nil { + return "", errors.New("not an OpenDocument file: no content.xml") + } + return odfXML(content) +} + +// flatODFText reads the single-file XML form (.fodt, .fods, .fodp). +func flatODFText(b []byte) (string, error) { return odfXML(b) } + +// odfXML reads OpenDocument's text model, which all three kinds share: +// text:h with an outline level is a heading, text:p a paragraph, list items, +// table rows and cells, slides (draw:page) and sheets (table:table) named as +// headings. +func odfXML(b []byte) (string, error) { + var o out + skip, slide := 0, 0 + // Paragraphs inside a cell or a list item do not end the row or the list. + inCell, inItem := 0, 0 + err := walkXML(b, func(e xmlEvent) { + switch { + case e.start: + switch e.name { + case "tracked-changes", "notes-configuration", "annotation", "frame-name": + skip++ + case "h": + lvl, _ := strconv.Atoi(e.get("outline-level")) + o.heading(max(lvl, 1)) + case "list-item": + inItem++ + o.item() + case "table-cell": + inCell++ + case "tab": + o.raw("\t") + case "s": + n, _ := strconv.Atoi(e.get("c")) + for range max(n, 1) { + o.raw(" ") + } + case "line-break": + o.newline() + case "page": + slide++ + o.heading(2) + name := e.get("name") + if name == "" || isDefaultPageName(name) { + name = fmt.Sprintf("Slide %d", slide) + } + o.text(name) + o.para() + case "table": + if n := e.get("name"); n != "" { + o.heading(2) + o.text(n) + o.para() + } + } + case e.end: + switch e.name { + case "tracked-changes", "notes-configuration", "annotation", "frame-name": + skip-- + case "h", "p": + switch { + case inCell > 0: + o.text(" ") + case inItem > 0: + o.newline() + default: + o.para() + } + case "list-item": + inItem-- + o.newline() + case "table-cell": + inCell-- + o.tab() + case "table-row": + o.newline() + case "table", "page": + o.para() + } + default: + if skip == 0 { + o.raw(e.text) + } + } + }) + return o.String(), err +} + +// isDefaultPageName is LibreOffice's own name for an unnamed slide, "page1". +func isDefaultPageName(s string) bool { + if len(s) < 5 || s[:4] != "page" { + return false + } + _, err := strconv.Atoi(s[4:]) + return err == nil +} diff --git a/internal/kb/extract/ooxml.go b/internal/kb/extract/ooxml.go new file mode 100644 index 0000000..8ca4eb4 --- /dev/null +++ b/internal/kb/extract/ooxml.go @@ -0,0 +1,463 @@ +package extract + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strconv" + "strings" +) + +// maxMember is the most one member of an Office file or EPUB is inflated +// to: a zip bomb stops here rather than at the machine's memory. +const maxMember = 512 << 20 + +func openZip(b []byte) (*zip.Reader, error) { + return zip.NewReader(bytes.NewReader(b), int64(len(b))) +} + +// member reads one file inside a zip, by its exact name; nil when absent. +func member(zr *zip.Reader, name string) ([]byte, error) { + for _, f := range zr.File { + if f.Name == name { + return readMember(f) + } + } + return nil, nil +} + +func readMember(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + b, err := io.ReadAll(io.LimitReader(rc, maxMember+1)) + if err != nil { + return nil, err + } + if len(b) > maxMember { + return nil, fmt.Errorf("%s inflates past %d MiB", f.Name, maxMember>>20) + } + return b, nil +} + +// relID is an element's r:id, the namespaced id that names a relationship — +// not its own plain id, which sldId has as well. +func (e xmlEvent) relID() string { + for _, a := range e.attr { + if a.Name.Local == "id" && a.Name.Space != "" { + return a.Value + } + } + return "" +} + +// xmlEvent is one step of walking an XML document: a start, an end, or +// character data, with the element's local name and attributes. +type xmlEvent struct { + start, end bool + name string + attr []xml.Attr + text string +} + +func (e xmlEvent) get(local string) string { + for _, a := range e.attr { + if a.Name.Local == local { + return a.Value + } + } + return "" +} + +// walkXML calls f for each element start and end and each run of character +// data. Namespaces are dropped: OOXML and ODF are read by local names. +func walkXML(b []byte, f func(xmlEvent)) error { + d := xml.NewDecoder(bytes.NewReader(b)) + d.Strict = false + d.CharsetReader = func(cs string, in io.Reader) (io.Reader, error) { + all, err := io.ReadAll(in) + if err != nil { + return nil, err + } + return strings.NewReader(decodeCharset(all, cs)), nil + } + for { + tok, err := d.Token() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + switch t := tok.(type) { + case xml.StartElement: + f(xmlEvent{start: true, name: t.Name.Local, attr: t.Attr}) + case xml.EndElement: + f(xmlEvent{end: true, name: t.Name.Local}) + case xml.CharData: + f(xmlEvent{text: string(t)}) + } + } +} + +// headingLevel reads a heading level out of a paragraph style: Word's +// "Heading1", "Heading 2", "heading3", its Turkish "Balk1" (the id Word gives +// "Başlık 1"), "Title" and "Subtitle". +func headingLevel(style string) int { + s := strings.ToLower(strings.ReplaceAll(style, " ", "")) + switch s { + case "title", "konubaşlığı", "konubal": + return 1 + case "subtitle", "altkonubaşlığı", "altkonubal": + return 2 + } + for _, p := range []string{"heading", "balk", "başlık", "berschrift", "titre", "título", "titolo", "kop"} { + if rest, ok := strings.CutPrefix(s, p); ok { + if n, err := strconv.Atoi(rest); err == nil && n >= 1 && n <= 9 { + return min(n, 6) + } + } + } + return 0 +} + +// docxText reads a Word document: the body, then footnotes and endnotes. +// Paragraph styles give headings; tables are rows of tab-separated cells. +func docxText(b []byte) (string, error) { + zr, err := openZip(b) + if err != nil { + return "", err + } + body, err := member(zr, "word/document.xml") + if err != nil { + return "", err + } + if body == nil { + return "", errors.New("not a Word document: no word/document.xml") + } + var o out + if err := wordML(body, &o); err != nil { + return "", err + } + for _, extra := range []string{"word/footnotes.xml", "word/endnotes.xml"} { + if x, _ := member(zr, extra); x != nil { + _ = wordML(x, &o) + } + } + return o.String(), nil +} + +func wordML(b []byte, o *out) error { + inText, inDel, inCell := false, 0, 0 + level, list := 0, false + return walkXML(b, func(e xmlEvent) { + switch { + case e.start: + switch e.name { + case "p": + level, list = 0, false + case "pStyle": + level = headingLevel(e.get("val")) + if strings.Contains(strings.ToLower(e.get("val")), "list") { + list = true + } + case "numPr": + list = true + case "t": + inText = true + if level > 0 && inCell == 0 && o.line.Len() == 0 && o.prefix == "" { + o.heading(level) + } else if list && inCell == 0 && o.line.Len() == 0 && o.prefix == "" { + o.item() + } + case "delText": + // Tracked deletions are not the document's text. + inDel++ + case "tc": + inCell++ + case "tab": + o.raw("\t") + case "br", "cr": + o.newline() + } + case e.end: + switch e.name { + case "t": + inText = false + case "delText": + inDel-- + case "p": + switch { + case inCell > 0: + // A cell's paragraphs stay in its row. + o.text(" ") + case list: + o.newline() + default: + o.para() + } + case "tc": + inCell-- + o.tab() + case "tr": + o.newline() + case "tbl": + o.para() + } + default: + if inText && inDel == 0 { + o.raw(e.text) + } + } + }) +} + +// xlsxText reads a workbook: each sheet under its name as a heading, each +// row a line of tab-separated cells, shared strings resolved. +func xlsxText(b []byte) (string, error) { + zr, err := openZip(b) + if err != nil { + return "", err + } + var shared []string + if ss, _ := member(zr, "xl/sharedStrings.xml"); ss != nil { + var cur strings.Builder + inT, inRPh := false, 0 + _ = walkXML(ss, func(e xmlEvent) { + switch { + case e.start && e.name == "si": + cur.Reset() + case e.end && e.name == "si": + shared = append(shared, cur.String()) + case e.start && e.name == "rPh": + inRPh++ // phonetic guides, not the cell's text + case e.end && e.name == "rPh": + inRPh-- + case e.start && e.name == "t": + inT = true + case e.end && e.name == "t": + inT = false + case !e.start && !e.end && inT && inRPh == 0: + cur.WriteString(e.text) + } + }) + } + sheets, err := xlsxSheets(zr) + if err != nil { + return "", err + } + var o out + for _, sh := range sheets { + data, err := member(zr, sh.path) + if err != nil || data == nil { + continue + } + o.heading(2) + o.text(sh.name) + o.para() + var cell strings.Builder + cellType, inV, inIs := "", false, false + first := true + _ = walkXML(data, func(e xmlEvent) { + switch { + case e.start && e.name == "row": + first = true + case e.end && e.name == "row": + o.newline() + case e.start && e.name == "c": + cellType = e.get("t") + cell.Reset() + case e.end && e.name == "c": + v := cell.String() + if cellType == "s" { + if i, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && i >= 0 && i < len(shared) { + v = shared[i] + } + } + v = strings.TrimSpace(strings.ReplaceAll(v, "\n", " ")) + if v != "" { + if !first { + o.tab() + } + o.text(v) + first = false + } + case e.start && e.name == "v": + inV = true + case e.end && e.name == "v": + inV = false + case e.start && e.name == "is": + inIs = true + case e.end && e.name == "is": + inIs = false + case !e.start && !e.end && (inV || inIs): + cell.WriteString(e.text) + } + }) + o.para() + } + return o.String(), nil +} + +type sheetRef struct{ name, path string } + +// xlsxSheets lists a workbook's sheets in their order, with the part each is +// kept in. +func xlsxSheets(zr *zip.Reader) ([]sheetRef, error) { + wb, err := member(zr, "xl/workbook.xml") + if err != nil || wb == nil { + return nil, errors.New("not a workbook: no xl/workbook.xml") + } + rels := map[string]string{} + if r, _ := member(zr, "xl/_rels/workbook.xml.rels"); r != nil { + _ = walkXML(r, func(e xmlEvent) { + if e.start && e.name == "Relationship" { + rels[e.get("Id")] = e.get("Target") + } + }) + } + var out []sheetRef + _ = walkXML(wb, func(e xmlEvent) { + if !e.start || e.name != "sheet" { + return + } + target := rels[e.relID()] + if target == "" { + return + } + p := path.Clean(path.Join("xl", target)) + if strings.HasPrefix(target, "/") { + p = strings.TrimPrefix(target, "/") + } + out = append(out, sheetRef{name: e.get("name"), path: p}) + }) + return out, nil +} + +var slideNum = regexp.MustCompile(`(\d+)\.xml$`) + +func numbered(zr *zip.Reader, prefix string) []*zip.File { + var fs []*zip.File + for _, f := range zr.File { + if strings.HasPrefix(f.Name, prefix) && slideNum.MatchString(f.Name) && !strings.Contains(f.Name[len(prefix):], "/") { + fs = append(fs, f) + } + } + n := func(f *zip.File) int { + v, _ := strconv.Atoi(slideNum.FindStringSubmatch(f.Name)[1]) + return v + } + sort.Slice(fs, func(i, j int) bool { return n(fs[i]) < n(fs[j]) }) + return fs +} + +// rels reads a part's relationships: id → target, and type → target, with +// targets resolved against the part's directory. +func rels(zr *zip.Reader, part string) (byID, byType map[string]string) { + byID, byType = map[string]string{}, map[string]string{} + dir, base := path.Split(part) + r, _ := member(zr, dir+"_rels/"+base+".rels") + if r == nil { + return byID, byType + } + _ = walkXML(r, func(e xmlEvent) { + if !e.start || e.name != "Relationship" || e.get("TargetMode") == "External" { + return + } + t := e.get("Target") + if strings.HasPrefix(t, "/") { + t = strings.TrimPrefix(t, "/") + } else { + t = path.Clean(path.Join(dir, t)) + } + byID[e.get("Id")] = t + ty := e.get("Type") + byType[ty[strings.LastIndex(ty, "/")+1:]] = t + }) + return byID, byType +} + +// pptxText reads a presentation: each slide, in the presentation's order, as +// a heading, its text, and its speaker notes. +func pptxText(b []byte) (string, error) { + zr, err := openZip(b) + if err != nil { + return "", err + } + var order []string + if pres, _ := member(zr, "ppt/presentation.xml"); pres != nil { + ids, _ := rels(zr, "ppt/presentation.xml") + _ = walkXML(pres, func(e xmlEvent) { + if e.start && e.name == "sldId" { + if t := ids[e.relID()]; t != "" { + order = append(order, t) + } + } + }) + } + if len(order) == 0 { + for _, f := range numbered(zr, "ppt/slides/slide") { + order = append(order, f.Name) + } + } + if len(order) == 0 { + return "", errors.New("not a presentation: no slides") + } + var o out + drawing := func(d []byte) { + inT, inField := false, 0 + _ = walkXML(d, func(e xmlEvent) { + switch { + case e.start && e.name == "fld": + // Slide numbers and dates, not the slide's text. + inField++ + case e.end && e.name == "fld": + inField-- + case e.start && e.name == "t": + inT = true + case e.end && e.name == "t": + inT = false + case e.end && e.name == "p": + o.newline() + case e.start && e.name == "br": + o.newline() + case e.end && e.name == "sp": + o.para() + case e.end && e.name == "tc": + o.tab() + case e.end && e.name == "tr": + o.newline() + case !e.start && !e.end && inT && inField == 0: + o.raw(e.text) + } + }) + } + for i, part := range order { + d, err := member(zr, part) + if err != nil || d == nil { + continue + } + o.heading(2) + o.text(fmt.Sprintf("Slide %d", i+1)) + o.para() + drawing(d) + if _, byType := rels(zr, part); byType["notesSlide"] != "" { + if nd, _ := member(zr, byType["notesSlide"]); nd != nil { + o.para() + o.text("Notes:") + o.newline() + drawing(nd) + } + } + o.para() + } + return o.String(), nil +} diff --git a/internal/kb/extract/rtf.go b/internal/kb/extract/rtf.go new file mode 100644 index 0000000..f30d42c --- /dev/null +++ b/internal/kb/extract/rtf.go @@ -0,0 +1,207 @@ +package extract + +import ( + "errors" + "strconv" + "strings" + "unicode/utf16" + + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/unicode/norm" +) + +// rtfSkip are destinations whose content is not the document's text. +var rtfSkip = map[string]bool{ + "fonttbl": true, "colortbl": true, "stylesheet": true, "info": true, "pict": true, + "header": true, "headerl": true, "headerr": true, "headerf": true, "footer": true, + "footerl": true, "footerr": true, "footerf": true, "object": true, "xmlnstbl": true, + "listtable": true, "listoverridetable": true, "rsidtbl": true, "generator": true, + "themedata": true, "colorschememapping": true, "latentstyles": true, "datastore": true, + "filetbl": true, "revtbl": true, "pgdsctbl": true, "fldinst": true, "bkmkstart": true, + "bkmkend": true, "shppict": true, "nonshppict": true, "datafield": true, "blipuid": true, +} + +// rtfText reads Rich Text Format: text, paragraphs, tabs and table cells, +// characters in the document's code page (\ansicpg) or as \u escapes; +// font tables, pictures, headers and other destinations skipped. +func rtfText(b []byte) (string, error) { + s := string(b) + if !strings.HasPrefix(strings.TrimSpace(s), "{\\rtf") { + return "", errors.New("not RTF: no {\\rtf header") + } + type state struct { + skip bool + uc int // how many fallback characters follow a \u + } + cp := charmap.Windows1252 + stack := []state{{uc: 1}} + var o out + var pending []byte // \'hh bytes, decoded in runs so a code page's pairs stay together + var surrogate rune + flush := func() { + if len(pending) == 0 { + return + } + d, _ := cp.NewDecoder().Bytes(pending) + o.raw(string(d)) + pending = pending[:0] + } + skipChars := 0 + cur := func() *state { return &stack[len(stack)-1] } + for i := 0; i < len(s); i++ { + c := s[i] + switch c { + case '{': + flush() + stack = append(stack, *cur()) + case '}': + flush() + if len(stack) > 1 { + stack = stack[:len(stack)-1] + } + case '\\': + if i+1 >= len(s) { + break + } + n := s[i+1] + switch { + case n == '\'' && i+3 < len(s): + v, err := strconv.ParseUint(s[i+2:i+4], 16, 8) + i += 3 + if err != nil || cur().skip { + continue + } + if skipChars > 0 { + skipChars-- + continue + } + pending = append(pending, byte(v)) + case n == '*': + i++ + cur().skip = true + case n == '\\' || n == '{' || n == '}': + i++ + if !cur().skip { + flush() + o.raw(string(n)) + } + case n == '~': + i++ + if !cur().skip { + o.raw(" ") + } + case n == '-' || n == '_': + i++ + case n == '\n' || n == '\r': + i++ + if !cur().skip { + flush() + o.para() + } + case isLetter(n): + j := i + 1 + for j < len(s) && isLetter(s[j]) { + j++ + } + word := s[i+1 : j] + k := j + if k < len(s) && (s[k] == '-' || (s[k] >= '0' && s[k] <= '9')) { + k++ + for k < len(s) && s[k] >= '0' && s[k] <= '9' { + k++ + } + } + arg, hasArg := 0, k > j + if hasArg { + arg, _ = strconv.Atoi(s[j:k]) + } + if k < len(s) && s[k] == ' ' { + k++ + } + i = k - 1 + flush() + if rtfSkip[word] { + cur().skip = true + continue + } + switch word { + case "ansicpg": + cp = codePage(arg) + case "uc": + cur().uc = arg + case "u": + if cur().skip { + continue + } + r := rune(arg) + if r < 0 { + r += 65536 + } + switch { + case utf16.IsSurrogate(r) && r < 0xDC00: + surrogate = r + case utf16.IsSurrogate(r): + o.raw(string(utf16.DecodeRune(surrogate, r))) + surrogate = 0 + default: + o.raw(string(r)) + } + skipChars = cur().uc + case "par", "sect", "page": + if !cur().skip { + o.para() + } + case "line": + if !cur().skip { + o.newline() + } + case "tab": + if !cur().skip { + o.raw("\t") + } + case "cell": + if !cur().skip { + o.tab() + } + case "row": + if !cur().skip { + o.newline() + } + case "emdash": + o.raw("—") + case "endash": + o.raw("–") + case "bullet": + o.raw("•") + case "lquote", "rquote": + o.raw("'") + case "ldblquote", "rdblquote": + o.raw("\"") + } + default: + i++ + } + case '\r', '\n': + // Line breaks in RTF source are not text. + default: + if cur().skip { + continue + } + if skipChars > 0 { + skipChars-- + continue + } + if c >= 0x80 { + // Unescaped 8-bit text, as some writers leave it. + pending = append(pending, c) + continue + } + flush() + o.raw(string(c)) + } + } + flush() + return norm.NFC.String(o.String()), nil +} + +func isLetter(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') } diff --git a/internal/kb/extract/testdata/turkish.pdf b/internal/kb/extract/testdata/turkish.pdf new file mode 100644 index 0000000..1a12aa0 Binary files /dev/null and b/internal/kb/extract/testdata/turkish.pdf differ diff --git a/internal/kb/extract/text.go b/internal/kb/extract/text.go new file mode 100644 index 0000000..c5ac00f --- /dev/null +++ b/internal/kb/extract/text.go @@ -0,0 +1,95 @@ +package extract + +import ( + "bytes" + "strings" + "unicode/utf8" + + "golang.org/x/net/html/charset" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/unicode/norm" +) + +func hasUTF16BOM(b []byte) bool { + return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF)) +} + +// decodeText turns a text file into UTF-8, NFC. +// +// UTF-8 is taken as it is (its BOM dropped); UTF-16 by its BOM, which is how +// Windows' Notepad writes "Unicode". Anything else is a single-byte code +// page, and which one cannot be known from the bytes; Windows-1254 is +// assumed, since it is what a Turkish Windows writes and it agrees with +// Windows-1252 on every letter but six (ğĞıİşŞ in place of ðÐýÝþÞ) — which +// are rare in the western languages 1252 serves. +func decodeText(b []byte) string { + var s string + switch { + case bytes.HasPrefix(b, []byte{0xEF, 0xBB, 0xBF}): + s = string(b[3:]) + case hasUTF16BOM(b): + out, err := unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder().Bytes(b) + if err != nil { + return "" + } + s = string(out) + case utf8.Valid(b): + s = string(b) + default: + out, err := charmap.Windows1254.NewDecoder().Bytes(b) + if err != nil { + return "" + } + s = string(out) + } + return norm.NFC.String(strings.ReplaceAll(s, "\r\n", "\n")) +} + +// decodeCharset decodes bytes declared to be in a charset, as mail and web +// pages declare theirs, by any of the WHATWG labels; an unknown or missing +// one is treated as decodeText would. +func decodeCharset(b []byte, cs string) string { + cs = strings.ToLower(strings.TrimSpace(cs)) + if cs == "" || cs == "utf-8" || cs == "utf8" || cs == "us-ascii" { + return decodeText(b) + } + enc, _ := charset.Lookup(cs) + if enc == nil { + return decodeText(b) + } + out, err := enc.NewDecoder().Bytes(b) + if err != nil { + return decodeText(b) + } + return norm.NFC.String(strings.ReplaceAll(string(out), "\r\n", "\n")) +} + +// codePage is the decoder for an RTF \ansicpgN or a Windows code page number. +func codePage(n int) *charmap.Charmap { + switch n { + case 1250: + return charmap.Windows1250 + case 1251: + return charmap.Windows1251 + case 1252: + return charmap.Windows1252 + case 1253: + return charmap.Windows1253 + case 1254: + return charmap.Windows1254 + case 1255: + return charmap.Windows1255 + case 1256: + return charmap.Windows1256 + case 1257: + return charmap.Windows1257 + case 1258: + return charmap.Windows1258 + case 850: + return charmap.CodePage850 + case 437: + return charmap.CodePage437 + } + return charmap.Windows1252 +}