A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
347 lines
15 KiB
Go
347 lines
15 KiB
Go
package extract
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"errors"
|
||
"os"
|
||
"reflect"
|
||
"strings"
|
||
"testing"
|
||
|
||
"golang.org/x/text/encoding/charmap"
|
||
"golang.org/x/text/encoding/unicode"
|
||
)
|
||
|
||
// zipOf builds a zip from name → content, in the order given.
|
||
func zipOf(t *testing.T, files ...string) []byte {
|
||
t.Helper()
|
||
var buf bytes.Buffer
|
||
w := zip.NewWriter(&buf)
|
||
for i := 0; i+1 < len(files); i += 2 {
|
||
f, err := w.Create(files[i])
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := f.Write([]byte(files[i+1])); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
if err := w.Close(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return buf.Bytes()
|
||
}
|
||
|
||
func extractOne(t *testing.T, name string, data []byte) string {
|
||
t.Helper()
|
||
parts, err := Bytes(context.Background(), name, data, Options{})
|
||
if err != nil {
|
||
t.Fatalf("%s: %v", name, err)
|
||
}
|
||
if len(parts) != 1 || parts[0].Err != nil {
|
||
t.Fatalf("%s: parts %+v", name, parts)
|
||
}
|
||
return parts[0].Text
|
||
}
|
||
|
||
func wants(t *testing.T, got string, want ...string) {
|
||
t.Helper()
|
||
for _, w := range want {
|
||
if !strings.Contains(got, w) {
|
||
t.Errorf("missing %q in:\n%s", w, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTextInEveryEncodingBecomesUTF8(t *testing.T) {
|
||
const s = "Yıllık izin: Şubat'ta ğüşİöç"
|
||
u16, _ := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(s))
|
||
cp, _ := charmap.Windows1254.NewEncoder().Bytes([]byte(s))
|
||
for name, b := range map[string][]byte{
|
||
"utf8": []byte(s),
|
||
"utf8-bom": append([]byte{0xEF, 0xBB, 0xBF}, s...),
|
||
"utf16": u16,
|
||
"cp1254": cp,
|
||
"crlf": []byte(strings.ReplaceAll(s+"\nikinci", "\n", "\r\n")),
|
||
} {
|
||
got := extractOne(t, "a.txt", b)
|
||
if !strings.HasPrefix(got, s) {
|
||
t.Errorf("%s: got %q", name, got)
|
||
}
|
||
if strings.Contains(got, "\r") {
|
||
t.Errorf("%s: kept a CR", name)
|
||
}
|
||
}
|
||
// macOS writes names and sometimes text decomposed; the text comes out
|
||
// composed, so the same word is the same bytes everywhere.
|
||
if got := extractOne(t, "a.txt", []byte("Şirket")); !strings.HasPrefix(got, "Şirket") {
|
||
t.Errorf("NFD text came out as %q", got)
|
||
}
|
||
}
|
||
|
||
func TestHTMLKeepsHeadingsAndDropsScripts(t *testing.T) {
|
||
page := `<html><head><title>T</title><script>var x="gizli";</script></head><body>
|
||
<nav>menü</nav><h1>İzin Politikası</h1><p>Yıllık <b>izin</b> yirmi gün.</p>
|
||
<ul><li>Bir</li><li>İki</li></ul><table><tr><td>a</td><td>b</td></tr></table>
|
||
<p><b>Mer</b>haba</p></body></html>`
|
||
got := extractOne(t, "a.html", []byte(page))
|
||
wants(t, got, "# İzin Politikası", "Yıllık izin yirmi gün.", "- Bir", "- İki", "a\tb", "Merhaba")
|
||
for _, no := range []string{"gizli", "menü"} {
|
||
if strings.Contains(got, no) {
|
||
t.Errorf("%q leaked into:\n%s", no, got)
|
||
}
|
||
}
|
||
// Declared charset honoured.
|
||
l5, _ := charmap.ISO8859_9.NewEncoder().Bytes([]byte(`<meta charset="iso-8859-9"><p>Şirket</p>`))
|
||
wants(t, extractOne(t, "b.htm", l5), "Şirket")
|
||
}
|
||
|
||
func TestWordDocument(t *testing.T) {
|
||
doc := `<?xml version="1.0" encoding="UTF-8"?>
|
||
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>
|
||
<w:p><w:pPr><w:pStyle w:val="Balk1"/></w:pPr><w:r><w:t>Yıllık İzin</w:t></w:r></w:p>
|
||
<w:p><w:r><w:t xml:space="preserve">Çalışanlar </w:t></w:r><w:r><w:t>yirmi gün</w:t></w:r><w:del><w:r><w:delText>otuz</w:delText></w:r></w:del><w:r><w:tab/><w:t>izin alır.</w:t></w:r></w:p>
|
||
<w:p><w:pPr><w:numPr><w:ilvl w:val="0"/></w:numPr></w:pPr><w:r><w:t>Madde bir</w:t></w:r></w:p>
|
||
<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Hücre1</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Hücre2</w:t></w:r></w:p></w:tc></w:tr></w:tbl>
|
||
<w:p><w:pPr><w:pStyle w:val="Heading2"/></w:pPr><w:r><w:t>Masraflar</w:t></w:r></w:p>
|
||
</w:body></w:document>`
|
||
got := extractOne(t, "a.docx", zipOf(t, "word/document.xml", doc,
|
||
"word/footnotes.xml", `<w:footnotes xmlns:w="x"><w:footnote><w:p><w:r><w:t>Dipnot</w:t></w:r></w:p></w:footnote></w:footnotes>`))
|
||
wants(t, got, "# Yıllık İzin", "Çalışanlar yirmi gün\tizin alır.", "- Madde bir", "Hücre1\tHücre2", "## Masraflar", "Dipnot")
|
||
if strings.Contains(got, "otuz") {
|
||
t.Errorf("a tracked deletion is in the text:\n%s", got)
|
||
}
|
||
}
|
||
|
||
func TestWorkbook(t *testing.T) {
|
||
got := extractOne(t, "a.xlsx", zipOf(t,
|
||
"xl/workbook.xml", `<workbook xmlns:r="r"><sheets><sheet name="Bütçe" sheetId="1" r:id="rId1"/><sheet name="Boş" sheetId="2" r:id="rId2"/></sheets></workbook>`,
|
||
"xl/_rels/workbook.xml.rels", `<Relationships><Relationship Id="rId1" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Target="worksheets/sheet2.xml"/></Relationships>`,
|
||
"xl/sharedStrings.xml", `<sst><si><t>Kalem</t></si><si><r><t>Tu</t></r><r><t>tar</t></r></si><si><t>Kira</t></si></sst>`,
|
||
"xl/worksheets/sheet1.xml", `<worksheet><sheetData><row><c t="s"><v>0</v></c><c t="s"><v>1</v></c></row><row><c t="s"><v>2</v></c><c><v>1500.5</v></c><c t="inlineStr"><is><t>not</t></is></c></row></sheetData></worksheet>`,
|
||
"xl/worksheets/sheet2.xml", `<worksheet><sheetData/></worksheet>`,
|
||
))
|
||
wants(t, got, "## Bütçe", "Kalem\tTutar", "Kira\t1500.5\tnot")
|
||
}
|
||
|
||
func TestPresentationInItsOwnOrderWithNotes(t *testing.T) {
|
||
slide := func(text string) string {
|
||
return `<p:sld xmlns:a="a" xmlns:p="p"><p:cSld><p:spTree><p:sp><p:txBody><a:p><a:r><a:t>` + text +
|
||
`</a:t></a:r></a:p><a:p><a:fld type="slidenum"><a:t>9</a:t></a:fld></a:p></p:txBody></p:sp></p:spTree></p:cSld></p:sld>`
|
||
}
|
||
got := extractOne(t, "a.pptx", zipOf(t,
|
||
"ppt/presentation.xml", `<p:presentation xmlns:p="p" xmlns:r="r"><p:sldIdLst><p:sldId id="256" r:id="rId3"/><p:sldId id="257" r:id="rId2"/></p:sldIdLst></p:presentation>`,
|
||
"ppt/_rels/presentation.xml.rels", `<Relationships><Relationship Id="rId2" Target="slides/slide1.xml"/><Relationship Id="rId3" Target="slides/slide2.xml"/></Relationships>`,
|
||
"ppt/slides/slide1.xml", slide("İkinci slayt"),
|
||
"ppt/slides/slide2.xml", slide("Birinci slayt"),
|
||
"ppt/slides/_rels/slide2.xml.rels", `<Relationships><Relationship Id="rId1" Type="http://x/notesSlide" Target="../notesSlides/notesSlide7.xml"/></Relationships>`,
|
||
"ppt/notesSlides/notesSlide7.xml", slide("Konuşmacı notu"),
|
||
))
|
||
first, second := strings.Index(got, "Birinci slayt"), strings.Index(got, "İkinci slayt")
|
||
if first < 0 || second < 0 || first > second {
|
||
t.Errorf("slides out of the presentation's order:\n%s", got)
|
||
}
|
||
wants(t, got, "## Slide 1", "Notes:", "Konuşmacı notu")
|
||
if strings.Contains(got, "9") {
|
||
t.Errorf("a slide number field is in the text:\n%s", got)
|
||
}
|
||
}
|
||
|
||
func TestOpenDocument(t *testing.T) {
|
||
content := `<office:document-content xmlns:office="o" xmlns:text="t" xmlns:table="tb"><office:body><office:text>
|
||
<text:h text:outline-level="2">Bölüm</text:h><text:p>Bir<text:s text:c="2"/>iki<text:tab/>üç</text:p>
|
||
<text:list><text:list-item><text:p>madde</text:p></text:list-item></text:list>
|
||
<table:table table:name="Tablo1"><table:table-row><table:table-cell><text:p>x</text:p></table:table-cell><table:table-cell><text:p>y</text:p></table:table-cell></table:table-row></table:table>
|
||
</office:text></office:body></office:document-content>`
|
||
got := extractOne(t, "a.odt", zipOf(t, "mimetype", "application/vnd.oasis.opendocument.text", "content.xml", content))
|
||
wants(t, got, "## Bölüm", "Bir iki\tüç", "- madde", "## Tablo1", "x\ty")
|
||
wants(t, extractOne(t, "a.fodt", []byte(content)), "## Bölüm")
|
||
}
|
||
|
||
func TestRichText(t *testing.T) {
|
||
rtf := `{\rtf1\ansi\ansicpg1254\deff0{\fonttbl{\f0 Calibri;}}{\colortbl;\red0\green0\blue0;}` +
|
||
`{\*\generator Riched20;}\pard Y\'fdll\'fdk izin\par ` +
|
||
`\u350?irket \u-3913?\'3f ok\tab sonra\par{\header gizli}\cell b\row }`
|
||
got := extractOne(t, "a.rtf", []byte(rtf))
|
||
wants(t, got, "Yıllık izin", "Şirket", "ok\tsonra")
|
||
for _, no := range []string{"Calibri", "Riched", "gizli"} {
|
||
if strings.Contains(got, no) {
|
||
t.Errorf("%q leaked into:\n%s", no, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestEbookInSpineOrder(t *testing.T) {
|
||
got := extractOne(t, "a.epub", zipOf(t,
|
||
"mimetype", "application/epub+zip",
|
||
"META-INF/container.xml", `<container><rootfiles><rootfile full-path="OEBPS/content.opf"/></rootfiles></container>`,
|
||
"OEBPS/content.opf", `<package><manifest><item id="c1" href="b.xhtml"/><item id="c2" href="a.xhtml"/></manifest><spine><itemref idref="c2"/><itemref idref="c1"/></spine></package>`,
|
||
"OEBPS/a.xhtml", `<html><body><h1>Önce</h1></body></html>`,
|
||
"OEBPS/b.xhtml", `<html><body><h1>Sonra</h1></body></html>`,
|
||
))
|
||
if a, b := strings.Index(got, "Önce"), strings.Index(got, "Sonra"); a < 0 || b < a {
|
||
t.Errorf("chapters out of spine order:\n%s", got)
|
||
}
|
||
}
|
||
|
||
func TestNotebookAndSubtitles(t *testing.T) {
|
||
nb := `{"metadata":{"kernelspec":{"language":"python"}},"cells":[{"cell_type":"markdown","source":["# Analiz\n","Veri"]},
|
||
{"cell_type":"code","source":"print(1)","outputs":[{"text":"çıktı"}]}]}`
|
||
got := extractOne(t, "a.ipynb", []byte(nb))
|
||
wants(t, got, "# Analiz", "```python\nprint(1)\n```")
|
||
if strings.Contains(got, "çıktı") {
|
||
t.Errorf("an output is in the text:\n%s", got)
|
||
}
|
||
srt := "1\n00:00:01,000 --> 00:00:02,000\n<i>Merhaba</i>\n\n2\n00:00:02,000 --> 00:00:03,000\nMerhaba\nDünya\n"
|
||
if got := extractOne(t, "a.srt", []byte(srt)); got != "Merhaba\nDünya\n" {
|
||
t.Errorf("subtitles came out as %q", got)
|
||
}
|
||
}
|
||
|
||
func TestPDFTextLayer(t *testing.T) {
|
||
b, err := os.ReadFile("testdata/turkish.pdf")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wants(t, extractOne(t, "a.pdf", b), "Yıllık İzin Politikası", "Masraflar finans ekibine iletilir.")
|
||
// Garbage is a failure, not a panic.
|
||
if _, err := Bytes(context.Background(), "bad.pdf", []byte("%PDF-1.4 nonsense"), Options{}); err == nil {
|
||
t.Error("a broken PDF was read")
|
||
}
|
||
}
|
||
|
||
func TestArchivesAreReadMemberByMember(t *testing.T) {
|
||
inner := zipOf(t, "deep/c.md", "# Derin")
|
||
archive := zipOf(t,
|
||
"docs/a.txt", "Birinci",
|
||
`docs\b.html`, "<p>İkinci</p>",
|
||
"__MACOSX/docs/._a.txt", "çöp",
|
||
"docs/~$kilit.docx", "çöp",
|
||
"inner.zip", string(inner),
|
||
"image.bin", "\x00\x01",
|
||
)
|
||
parts, err := Bytes(context.Background(), "arsiv.zip", archive, Options{})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got := map[string]string{}
|
||
for _, p := range parts {
|
||
if p.Err != nil {
|
||
t.Errorf("%s: %v", p.Name, p.Err)
|
||
}
|
||
got[p.Name] = p.Text
|
||
}
|
||
if len(got) != 3 || !strings.Contains(got["docs/a.txt"], "Birinci") || !strings.Contains(got["docs/b.html"], "İkinci") ||
|
||
!strings.Contains(got["inner.zip!/deep/c.md"], "# Derin") {
|
||
t.Errorf("archive parts %v", got)
|
||
}
|
||
}
|
||
|
||
func TestMailWithAttachments(t *testing.T) {
|
||
att := zipOf(t, "x.txt", "ekteki metin")
|
||
enc := func(b []byte) string {
|
||
s := base64.StdEncoding.EncodeToString(b)
|
||
var lines []string
|
||
for len(s) > 76 {
|
||
lines, s = append(lines, s[:76]), s[76:]
|
||
}
|
||
return strings.Join(append(lines, s), "\r\n")
|
||
}
|
||
msg := "From: =?UTF-8?B?QXnFn2U=?= <a@example.com>\r\n" +
|
||
"Subject: =?iso-8859-9?Q?=DEubat_raporu?=\r\n" +
|
||
"Content-Type: multipart/mixed; boundary=B\r\n\r\n" +
|
||
"--B\r\nContent-Type: multipart/alternative; boundary=C\r\n\r\n" +
|
||
"--C\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" +
|
||
"Rapor ekte, te=C5=9Fekk=C3=BCrler.\r\n" +
|
||
"--C\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<p>HTML hali</p>\r\n--C--\r\n" +
|
||
"--B\r\nContent-Type: application/zip; name=\"ek.zip\"\r\nContent-Transfer-Encoding: base64\r\n" +
|
||
"Content-Disposition: attachment; filename=\"ek.zip\"\r\n\r\n" + enc(att) + "\r\n--B--\r\n"
|
||
parts, err := Bytes(context.Background(), "a.eml", []byte(msg), Options{})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(parts) != 2 {
|
||
t.Fatalf("parts %+v", parts)
|
||
}
|
||
wants(t, parts[0].Text, "# Şubat raporu", "From: Ayşe", "Rapor ekte, teşekkürler.")
|
||
if strings.Contains(parts[0].Text, "HTML hali") {
|
||
t.Errorf("the HTML alternative was read beside the plain text:\n%s", parts[0].Text)
|
||
}
|
||
if parts[1].Name != "ek.zip!/x.txt" || !strings.Contains(parts[1].Text, "ekteki metin") {
|
||
t.Errorf("attachment %+v", parts[1])
|
||
}
|
||
}
|
||
|
||
func TestUnreadableFilesSayWhy(t *testing.T) {
|
||
_, err := Bytes(context.Background(), "a.doc", []byte{0xD0, 0xCF, 0x11, 0xE0, 0}, Options{})
|
||
if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "LibreOffice") {
|
||
t.Errorf("a .doc without LibreOffice gave %v", err)
|
||
}
|
||
_, err = Bytes(context.Background(), "a.png", []byte{0x89, 'P', 'N', 'G', 0}, Options{})
|
||
if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "Tesseract") {
|
||
t.Errorf("an image without Tesseract gave %v", err)
|
||
}
|
||
_, err = Bytes(context.Background(), "a.bin", []byte{0, 1, 2, 3}, Options{})
|
||
if !errors.Is(err, ErrUnsupported) {
|
||
t.Errorf("binary gave %v", err)
|
||
}
|
||
_, err = Bytes(context.Background(), "a.txt", []byte(" \n"), Options{})
|
||
if !errors.Is(err, ErrNoText) {
|
||
t.Errorf("an empty file gave %v", err)
|
||
}
|
||
}
|
||
|
||
func TestHelpersWhenInstalled(t *testing.T) {
|
||
h := FindHelpers()
|
||
ctx := context.Background()
|
||
if h.Tesseract != "" && h.PdfToPPM != "" {
|
||
// The text-layer PDF read by OCR instead: what a scanned one gets.
|
||
b, _ := os.ReadFile("testdata/turkish.pdf")
|
||
got, err := pdfWithHelpers(ctx, "a.pdf", b, Options{Helpers: Helpers{Tesseract: h.Tesseract, PdfToPPM: h.PdfToPPM}})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wants(t, got, "Masraflar")
|
||
} else {
|
||
t.Log("no Tesseract with pdftoppm here; OCR not tried")
|
||
}
|
||
if h.Office == "" {
|
||
t.Log("no LibreOffice here; legacy Office formats not tried")
|
||
return
|
||
}
|
||
// A Word document converted to the legacy format and back would need a
|
||
// fixture made by Word; an ODT through the Office path proves the
|
||
// conversion runs and its result is read.
|
||
content := `<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"><office:body><office:text><text:p>LibreOffice yolu</text:p></office:text></office:body></office:document-content>`
|
||
got, err := h.officeToText(ctx, "a.fodt", []byte(`<?xml version="1.0"?>`+content), Options{})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
wants(t, got, "LibreOffice yolu")
|
||
}
|
||
|
||
func TestWorkbookRowsKeepTheirColumns(t *testing.T) {
|
||
b := zipOf(t,
|
||
"xl/workbook.xml", `<workbook xmlns:r="r"><sheets><sheet name="Sorular" sheetId="1" r:id="rId1"/></sheets></workbook>`,
|
||
"xl/_rels/workbook.xml.rels", `<Relationships><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>`,
|
||
"xl/sharedStrings.xml", `<sst><si><t>Soru</t></si><si><t>Cevap</t></si><si><t>İzin kaç gün?</t></si></sst>`,
|
||
"xl/worksheets/sheet1.xml", `<worksheet><sheetData>`+
|
||
`<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>`+
|
||
`<row r="2"/>`+
|
||
`<row r="3"><c r="A3" t="s"><v>2</v></c><c r="C3" t="inlineStr"><is><t>not</t></is></c></row>`+
|
||
`<row r="4"><c r="B4"><v>20</v></c></row>`+
|
||
`</sheetData></worksheet>`,
|
||
)
|
||
rows, err := XLSXRows(b)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
want := [][]string{{"Soru", "Cevap"}, {"İzin kaç gün?", "", "not"}, {"", "20"}}
|
||
if !reflect.DeepEqual(rows, want) {
|
||
t.Errorf("rows = %q, want %q", rows, want)
|
||
}
|
||
}
|