Ilker ManapandClaude Opus 5.5
a0cfafe772
PDF, Word, Excel, PowerPoint, OpenDocument, RTF, EPUB, HTML, MHTML, mail with its attachments, notebooks, subtitles, zip archives and text and source files in any common encoding, in pure Go; legacy Office formats through LibreOffice and scanned pages through Tesseract when they are installed. Chunks of about 500 tokens under their headings, overlapping by a paragraph. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
96 lines
2.5 KiB
Go
96 lines
2.5 KiB
Go
package extract
|
|
|
|
import (
|
|
"bytes"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"golang.org/x/net/html/charset"
|
|
"golang.org/x/text/encoding/charmap"
|
|
"golang.org/x/text/encoding/unicode"
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
func hasUTF16BOM(b []byte) bool {
|
|
return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF))
|
|
}
|
|
|
|
// decodeText turns a text file into UTF-8, NFC.
|
|
//
|
|
// UTF-8 is taken as it is (its BOM dropped); UTF-16 by its BOM, which is how
|
|
// Windows' Notepad writes "Unicode". Anything else is a single-byte code
|
|
// page, and which one cannot be known from the bytes; Windows-1254 is
|
|
// assumed, since it is what a Turkish Windows writes and it agrees with
|
|
// Windows-1252 on every letter but six (ğĞıİşŞ in place of ðÐýÝþÞ) — which
|
|
// are rare in the western languages 1252 serves.
|
|
func decodeText(b []byte) string {
|
|
var s string
|
|
switch {
|
|
case bytes.HasPrefix(b, []byte{0xEF, 0xBB, 0xBF}):
|
|
s = string(b[3:])
|
|
case hasUTF16BOM(b):
|
|
out, err := unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder().Bytes(b)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
s = string(out)
|
|
case utf8.Valid(b):
|
|
s = string(b)
|
|
default:
|
|
out, err := charmap.Windows1254.NewDecoder().Bytes(b)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
s = string(out)
|
|
}
|
|
return norm.NFC.String(strings.ReplaceAll(s, "\r\n", "\n"))
|
|
}
|
|
|
|
// decodeCharset decodes bytes declared to be in a charset, as mail and web
|
|
// pages declare theirs, by any of the WHATWG labels; an unknown or missing
|
|
// one is treated as decodeText would.
|
|
func decodeCharset(b []byte, cs string) string {
|
|
cs = strings.ToLower(strings.TrimSpace(cs))
|
|
if cs == "" || cs == "utf-8" || cs == "utf8" || cs == "us-ascii" {
|
|
return decodeText(b)
|
|
}
|
|
enc, _ := charset.Lookup(cs)
|
|
if enc == nil {
|
|
return decodeText(b)
|
|
}
|
|
out, err := enc.NewDecoder().Bytes(b)
|
|
if err != nil {
|
|
return decodeText(b)
|
|
}
|
|
return norm.NFC.String(strings.ReplaceAll(string(out), "\r\n", "\n"))
|
|
}
|
|
|
|
// codePage is the decoder for an RTF \ansicpgN or a Windows code page number.
|
|
func codePage(n int) *charmap.Charmap {
|
|
switch n {
|
|
case 1250:
|
|
return charmap.Windows1250
|
|
case 1251:
|
|
return charmap.Windows1251
|
|
case 1252:
|
|
return charmap.Windows1252
|
|
case 1253:
|
|
return charmap.Windows1253
|
|
case 1254:
|
|
return charmap.Windows1254
|
|
case 1255:
|
|
return charmap.Windows1255
|
|
case 1256:
|
|
return charmap.Windows1256
|
|
case 1257:
|
|
return charmap.Windows1257
|
|
case 1258:
|
|
return charmap.Windows1258
|
|
case 850:
|
|
return charmap.CodePage850
|
|
case 437:
|
|
return charmap.CodePage437
|
|
}
|
|
return charmap.Windows1252
|
|
}
|