package extract import ( "archive/zip" "bytes" "context" "encoding/base64" "errors" "os" "strings" "testing" "golang.org/x/text/encoding/charmap" "golang.org/x/text/encoding/unicode" ) // zipOf builds a zip from name → content, in the order given. func zipOf(t *testing.T, files ...string) []byte { t.Helper() var buf bytes.Buffer w := zip.NewWriter(&buf) for i := 0; i+1 < len(files); i += 2 { f, err := w.Create(files[i]) if err != nil { t.Fatal(err) } if _, err := f.Write([]byte(files[i+1])); err != nil { t.Fatal(err) } } if err := w.Close(); err != nil { t.Fatal(err) } return buf.Bytes() } func extractOne(t *testing.T, name string, data []byte) string { t.Helper() parts, err := Bytes(context.Background(), name, data, Options{}) if err != nil { t.Fatalf("%s: %v", name, err) } if len(parts) != 1 || parts[0].Err != nil { t.Fatalf("%s: parts %+v", name, parts) } return parts[0].Text } func wants(t *testing.T, got string, want ...string) { t.Helper() for _, w := range want { if !strings.Contains(got, w) { t.Errorf("missing %q in:\n%s", w, got) } } } func TestTextInEveryEncodingBecomesUTF8(t *testing.T) { const s = "Yıllık izin: Şubat'ta ğüşİöç" u16, _ := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(s)) cp, _ := charmap.Windows1254.NewEncoder().Bytes([]byte(s)) for name, b := range map[string][]byte{ "utf8": []byte(s), "utf8-bom": append([]byte{0xEF, 0xBB, 0xBF}, s...), "utf16": u16, "cp1254": cp, "crlf": []byte(strings.ReplaceAll(s+"\nikinci", "\n", "\r\n")), } { got := extractOne(t, "a.txt", b) if !strings.HasPrefix(got, s) { t.Errorf("%s: got %q", name, got) } if strings.Contains(got, "\r") { t.Errorf("%s: kept a CR", name) } } // macOS writes names and sometimes text decomposed; the text comes out // composed, so the same word is the same bytes everywhere. if got := extractOne(t, "a.txt", []byte("Şirket")); !strings.HasPrefix(got, "Şirket") { t.Errorf("NFD text came out as %q", got) } } func TestHTMLKeepsHeadingsAndDropsScripts(t *testing.T) { page := `
Yıllık izin yirmi gün.
| a | b |
Merhaba
` got := extractOne(t, "a.html", []byte(page)) wants(t, got, "# İzin Politikası", "Yıllık izin yirmi gün.", "- Bir", "- İki", "a\tb", "Merhaba") for _, no := range []string{"gizli", "menü"} { if strings.Contains(got, no) { t.Errorf("%q leaked into:\n%s", no, got) } } // Declared charset honoured. l5, _ := charmap.ISO8859_9.NewEncoder().Bytes([]byte(`Şirket
`)) wants(t, extractOne(t, "b.htm", l5), "Şirket") } func TestWordDocument(t *testing.T) { doc := `İkinci
", "__MACOSX/docs/._a.txt", "çöp", "docs/~$kilit.docx", "çöp", "inner.zip", string(inner), "image.bin", "\x00\x01", ) parts, err := Bytes(context.Background(), "arsiv.zip", archive, Options{}) if err != nil { t.Fatal(err) } got := map[string]string{} for _, p := range parts { if p.Err != nil { t.Errorf("%s: %v", p.Name, p.Err) } got[p.Name] = p.Text } if len(got) != 3 || !strings.Contains(got["docs/a.txt"], "Birinci") || !strings.Contains(got["docs/b.html"], "İkinci") || !strings.Contains(got["inner.zip!/deep/c.md"], "# Derin") { t.Errorf("archive parts %v", got) } } func TestMailWithAttachments(t *testing.T) { att := zipOf(t, "x.txt", "ekteki metin") enc := func(b []byte) string { s := base64.StdEncoding.EncodeToString(b) var lines []string for len(s) > 76 { lines, s = append(lines, s[:76]), s[76:] } return strings.Join(append(lines, s), "\r\n") } msg := "From: =?UTF-8?B?QXnFn2U=?= \r\n" + "Subject: =?iso-8859-9?Q?=DEubat_raporu?=\r\n" + "Content-Type: multipart/mixed; boundary=B\r\n\r\n" + "--B\r\nContent-Type: multipart/alternative; boundary=C\r\n\r\n" + "--C\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" + "Rapor ekte, te=C5=9Fekk=C3=BCrler.\r\n" + "--C\r\nContent-Type: text/html; charset=utf-8\r\n\r\nHTML hali
\r\n--C--\r\n" + "--B\r\nContent-Type: application/zip; name=\"ek.zip\"\r\nContent-Transfer-Encoding: base64\r\n" + "Content-Disposition: attachment; filename=\"ek.zip\"\r\n\r\n" + enc(att) + "\r\n--B--\r\n" parts, err := Bytes(context.Background(), "a.eml", []byte(msg), Options{}) if err != nil { t.Fatal(err) } if len(parts) != 2 { t.Fatalf("parts %+v", parts) } wants(t, parts[0].Text, "# Şubat raporu", "From: Ayşe", "Rapor ekte, teşekkürler.") if strings.Contains(parts[0].Text, "HTML hali") { t.Errorf("the HTML alternative was read beside the plain text:\n%s", parts[0].Text) } if parts[1].Name != "ek.zip!/x.txt" || !strings.Contains(parts[1].Text, "ekteki metin") { t.Errorf("attachment %+v", parts[1]) } } func TestUnreadableFilesSayWhy(t *testing.T) { _, err := Bytes(context.Background(), "a.doc", []byte{0xD0, 0xCF, 0x11, 0xE0, 0}, Options{}) if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "LibreOffice") { t.Errorf("a .doc without LibreOffice gave %v", err) } _, err = Bytes(context.Background(), "a.png", []byte{0x89, 'P', 'N', 'G', 0}, Options{}) if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "Tesseract") { t.Errorf("an image without Tesseract gave %v", err) } _, err = Bytes(context.Background(), "a.bin", []byte{0, 1, 2, 3}, Options{}) if !errors.Is(err, ErrUnsupported) { t.Errorf("binary gave %v", err) } _, err = Bytes(context.Background(), "a.txt", []byte(" \n"), Options{}) if !errors.Is(err, ErrNoText) { t.Errorf("an empty file gave %v", err) } } func TestHelpersWhenInstalled(t *testing.T) { h := FindHelpers() ctx := context.Background() if h.Tesseract != "" && h.PdfToPPM != "" { // The text-layer PDF read by OCR instead: what a scanned one gets. b, _ := os.ReadFile("testdata/turkish.pdf") got, err := pdfWithHelpers(ctx, "a.pdf", b, Options{Helpers: Helpers{Tesseract: h.Tesseract, PdfToPPM: h.PdfToPPM}}) if err != nil { t.Fatal(err) } wants(t, got, "Masraflar") } else { t.Log("no Tesseract with pdftoppm here; OCR not tried") } if h.Office == "" { t.Log("no LibreOffice here; legacy Office formats not tried") return } // A Word document converted to the legacy format and back would need a // fixture made by Word; an ODT through the Office path proves the // conversion runs and its result is read. content := `