package extract import ( "archive/zip" "bytes" "context" "encoding/base64" "errors" "os" "strings" "testing" "golang.org/x/text/encoding/charmap" "golang.org/x/text/encoding/unicode" ) // zipOf builds a zip from name → content, in the order given. func zipOf(t *testing.T, files ...string) []byte { t.Helper() var buf bytes.Buffer w := zip.NewWriter(&buf) for i := 0; i+1 < len(files); i += 2 { f, err := w.Create(files[i]) if err != nil { t.Fatal(err) } if _, err := f.Write([]byte(files[i+1])); err != nil { t.Fatal(err) } } if err := w.Close(); err != nil { t.Fatal(err) } return buf.Bytes() } func extractOne(t *testing.T, name string, data []byte) string { t.Helper() parts, err := Bytes(context.Background(), name, data, Options{}) if err != nil { t.Fatalf("%s: %v", name, err) } if len(parts) != 1 || parts[0].Err != nil { t.Fatalf("%s: parts %+v", name, parts) } return parts[0].Text } func wants(t *testing.T, got string, want ...string) { t.Helper() for _, w := range want { if !strings.Contains(got, w) { t.Errorf("missing %q in:\n%s", w, got) } } } func TestTextInEveryEncodingBecomesUTF8(t *testing.T) { const s = "Yıllık izin: Şubat'ta ğüşİöç" u16, _ := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(s)) cp, _ := charmap.Windows1254.NewEncoder().Bytes([]byte(s)) for name, b := range map[string][]byte{ "utf8": []byte(s), "utf8-bom": append([]byte{0xEF, 0xBB, 0xBF}, s...), "utf16": u16, "cp1254": cp, "crlf": []byte(strings.ReplaceAll(s+"\nikinci", "\n", "\r\n")), } { got := extractOne(t, "a.txt", b) if !strings.HasPrefix(got, s) { t.Errorf("%s: got %q", name, got) } if strings.Contains(got, "\r") { t.Errorf("%s: kept a CR", name) } } // macOS writes names and sometimes text decomposed; the text comes out // composed, so the same word is the same bytes everywhere. if got := extractOne(t, "a.txt", []byte("Şirket")); !strings.HasPrefix(got, "Şirket") { t.Errorf("NFD text came out as %q", got) } } func TestHTMLKeepsHeadingsAndDropsScripts(t *testing.T) { page := `T

İzin Politikası

Yıllık izin yirmi gün.

ab

Merhaba

` got := extractOne(t, "a.html", []byte(page)) wants(t, got, "# İzin Politikası", "Yıllık izin yirmi gün.", "- Bir", "- İki", "a\tb", "Merhaba") for _, no := range []string{"gizli", "menü"} { if strings.Contains(got, no) { t.Errorf("%q leaked into:\n%s", no, got) } } // Declared charset honoured. l5, _ := charmap.ISO8859_9.NewEncoder().Bytes([]byte(`

Şirket

`)) wants(t, extractOne(t, "b.htm", l5), "Şirket") } func TestWordDocument(t *testing.T) { doc := ` Yıllık İzin Çalışanlar yirmi günotuzizin alır. Madde bir Hücre1Hücre2 Masraflar ` got := extractOne(t, "a.docx", zipOf(t, "word/document.xml", doc, "word/footnotes.xml", `Dipnot`)) wants(t, got, "# Yıllık İzin", "Çalışanlar yirmi gün\tizin alır.", "- Madde bir", "Hücre1\tHücre2", "## Masraflar", "Dipnot") if strings.Contains(got, "otuz") { t.Errorf("a tracked deletion is in the text:\n%s", got) } } func TestWorkbook(t *testing.T) { got := extractOne(t, "a.xlsx", zipOf(t, "xl/workbook.xml", ``, "xl/_rels/workbook.xml.rels", ``, "xl/sharedStrings.xml", `KalemTutarKira`, "xl/worksheets/sheet1.xml", `0121500.5not`, "xl/worksheets/sheet2.xml", ``, )) wants(t, got, "## Bütçe", "Kalem\tTutar", "Kira\t1500.5\tnot") } func TestPresentationInItsOwnOrderWithNotes(t *testing.T) { slide := func(text string) string { return `` + text + `9` } got := extractOne(t, "a.pptx", zipOf(t, "ppt/presentation.xml", ``, "ppt/_rels/presentation.xml.rels", ``, "ppt/slides/slide1.xml", slide("İkinci slayt"), "ppt/slides/slide2.xml", slide("Birinci slayt"), "ppt/slides/_rels/slide2.xml.rels", ``, "ppt/notesSlides/notesSlide7.xml", slide("Konuşmacı notu"), )) first, second := strings.Index(got, "Birinci slayt"), strings.Index(got, "İkinci slayt") if first < 0 || second < 0 || first > second { t.Errorf("slides out of the presentation's order:\n%s", got) } wants(t, got, "## Slide 1", "Notes:", "Konuşmacı notu") if strings.Contains(got, "9") { t.Errorf("a slide number field is in the text:\n%s", got) } } func TestOpenDocument(t *testing.T) { content := ` BölümBirikiüç madde xy ` got := extractOne(t, "a.odt", zipOf(t, "mimetype", "application/vnd.oasis.opendocument.text", "content.xml", content)) wants(t, got, "## Bölüm", "Bir iki\tüç", "- madde", "## Tablo1", "x\ty") wants(t, extractOne(t, "a.fodt", []byte(content)), "## Bölüm") } func TestRichText(t *testing.T) { rtf := `{\rtf1\ansi\ansicpg1254\deff0{\fonttbl{\f0 Calibri;}}{\colortbl;\red0\green0\blue0;}` + `{\*\generator Riched20;}\pard Y\'fdll\'fdk izin\par ` + `\u350?irket \u-3913?\'3f ok\tab sonra\par{\header gizli}\cell b\row }` got := extractOne(t, "a.rtf", []byte(rtf)) wants(t, got, "Yıllık izin", "Şirket", "ok\tsonra") for _, no := range []string{"Calibri", "Riched", "gizli"} { if strings.Contains(got, no) { t.Errorf("%q leaked into:\n%s", no, got) } } } func TestEbookInSpineOrder(t *testing.T) { got := extractOne(t, "a.epub", zipOf(t, "mimetype", "application/epub+zip", "META-INF/container.xml", ``, "OEBPS/content.opf", ``, "OEBPS/a.xhtml", `

Önce

`, "OEBPS/b.xhtml", `

Sonra

`, )) if a, b := strings.Index(got, "Önce"), strings.Index(got, "Sonra"); a < 0 || b < a { t.Errorf("chapters out of spine order:\n%s", got) } } func TestNotebookAndSubtitles(t *testing.T) { nb := `{"metadata":{"kernelspec":{"language":"python"}},"cells":[{"cell_type":"markdown","source":["# Analiz\n","Veri"]}, {"cell_type":"code","source":"print(1)","outputs":[{"text":"çıktı"}]}]}` got := extractOne(t, "a.ipynb", []byte(nb)) wants(t, got, "# Analiz", "```python\nprint(1)\n```") if strings.Contains(got, "çıktı") { t.Errorf("an output is in the text:\n%s", got) } srt := "1\n00:00:01,000 --> 00:00:02,000\nMerhaba\n\n2\n00:00:02,000 --> 00:00:03,000\nMerhaba\nDünya\n" if got := extractOne(t, "a.srt", []byte(srt)); got != "Merhaba\nDünya\n" { t.Errorf("subtitles came out as %q", got) } } func TestPDFTextLayer(t *testing.T) { b, err := os.ReadFile("testdata/turkish.pdf") if err != nil { t.Fatal(err) } wants(t, extractOne(t, "a.pdf", b), "Yıllık İzin Politikası", "Masraflar finans ekibine iletilir.") // Garbage is a failure, not a panic. if _, err := Bytes(context.Background(), "bad.pdf", []byte("%PDF-1.4 nonsense"), Options{}); err == nil { t.Error("a broken PDF was read") } } func TestArchivesAreReadMemberByMember(t *testing.T) { inner := zipOf(t, "deep/c.md", "# Derin") archive := zipOf(t, "docs/a.txt", "Birinci", `docs\b.html`, "

İkinci

", "__MACOSX/docs/._a.txt", "çöp", "docs/~$kilit.docx", "çöp", "inner.zip", string(inner), "image.bin", "\x00\x01", ) parts, err := Bytes(context.Background(), "arsiv.zip", archive, Options{}) if err != nil { t.Fatal(err) } got := map[string]string{} for _, p := range parts { if p.Err != nil { t.Errorf("%s: %v", p.Name, p.Err) } got[p.Name] = p.Text } if len(got) != 3 || !strings.Contains(got["docs/a.txt"], "Birinci") || !strings.Contains(got["docs/b.html"], "İkinci") || !strings.Contains(got["inner.zip!/deep/c.md"], "# Derin") { t.Errorf("archive parts %v", got) } } func TestMailWithAttachments(t *testing.T) { att := zipOf(t, "x.txt", "ekteki metin") enc := func(b []byte) string { s := base64.StdEncoding.EncodeToString(b) var lines []string for len(s) > 76 { lines, s = append(lines, s[:76]), s[76:] } return strings.Join(append(lines, s), "\r\n") } msg := "From: =?UTF-8?B?QXnFn2U=?= \r\n" + "Subject: =?iso-8859-9?Q?=DEubat_raporu?=\r\n" + "Content-Type: multipart/mixed; boundary=B\r\n\r\n" + "--B\r\nContent-Type: multipart/alternative; boundary=C\r\n\r\n" + "--C\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" + "Rapor ekte, te=C5=9Fekk=C3=BCrler.\r\n" + "--C\r\nContent-Type: text/html; charset=utf-8\r\n\r\n

HTML hali

\r\n--C--\r\n" + "--B\r\nContent-Type: application/zip; name=\"ek.zip\"\r\nContent-Transfer-Encoding: base64\r\n" + "Content-Disposition: attachment; filename=\"ek.zip\"\r\n\r\n" + enc(att) + "\r\n--B--\r\n" parts, err := Bytes(context.Background(), "a.eml", []byte(msg), Options{}) if err != nil { t.Fatal(err) } if len(parts) != 2 { t.Fatalf("parts %+v", parts) } wants(t, parts[0].Text, "# Şubat raporu", "From: Ayşe", "Rapor ekte, teşekkürler.") if strings.Contains(parts[0].Text, "HTML hali") { t.Errorf("the HTML alternative was read beside the plain text:\n%s", parts[0].Text) } if parts[1].Name != "ek.zip!/x.txt" || !strings.Contains(parts[1].Text, "ekteki metin") { t.Errorf("attachment %+v", parts[1]) } } func TestUnreadableFilesSayWhy(t *testing.T) { _, err := Bytes(context.Background(), "a.doc", []byte{0xD0, 0xCF, 0x11, 0xE0, 0}, Options{}) if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "LibreOffice") { t.Errorf("a .doc without LibreOffice gave %v", err) } _, err = Bytes(context.Background(), "a.png", []byte{0x89, 'P', 'N', 'G', 0}, Options{}) if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "Tesseract") { t.Errorf("an image without Tesseract gave %v", err) } _, err = Bytes(context.Background(), "a.bin", []byte{0, 1, 2, 3}, Options{}) if !errors.Is(err, ErrUnsupported) { t.Errorf("binary gave %v", err) } _, err = Bytes(context.Background(), "a.txt", []byte(" \n"), Options{}) if !errors.Is(err, ErrNoText) { t.Errorf("an empty file gave %v", err) } } func TestHelpersWhenInstalled(t *testing.T) { h := FindHelpers() ctx := context.Background() if h.Tesseract != "" && h.PdfToPPM != "" { // The text-layer PDF read by OCR instead: what a scanned one gets. b, _ := os.ReadFile("testdata/turkish.pdf") got, err := pdfWithHelpers(ctx, "a.pdf", b, Options{Helpers: Helpers{Tesseract: h.Tesseract, PdfToPPM: h.PdfToPPM}}) if err != nil { t.Fatal(err) } wants(t, got, "Masraflar") } else { t.Log("no Tesseract with pdftoppm here; OCR not tried") } if h.Office == "" { t.Log("no LibreOffice here; legacy Office formats not tried") return } // A Word document converted to the legacy format and back would need a // fixture made by Word; an ODT through the Office path proves the // conversion runs and its result is read. content := `LibreOffice yolu` got, err := h.officeToText(ctx, "a.fodt", []byte(``+content), Options{}) if err != nil { t.Fatal(err) } wants(t, got, "LibreOffice yolu") }