Ilker ManapandClaude Opus 5.5
a0cfafe772
PDF, Word, Excel, PowerPoint, OpenDocument, RTF, EPUB, HTML, MHTML, mail with its attachments, notebooks, subtitles, zip archives and text and source files in any common encoding, in pure Go; legacy Office formats through LibreOffice and scanned pages through Tesseract when they are installed. Chunks of about 500 tokens under their headings, overlapping by a paragraph. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
107 lines
3.2 KiB
Go
107 lines
3.2 KiB
Go
package chunk
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
func para(word string, n int) string { return strings.TrimSpace(strings.Repeat(word+" ", n)) }
|
||
|
||
func TestShortTextIsOneChunkUnderItsHeadings(t *testing.T) {
|
||
cs := Split("# İzin\n\n## Yıllık\n\nYirmi gün.\n\nBayramlar ayrı.\n", Options{})
|
||
if len(cs) != 1 {
|
||
t.Fatalf("chunks %+v", cs)
|
||
}
|
||
if got := cs[0].Text(); got != "İzin › Yıllık\n\nYirmi gün.\n\nBayramlar ayrı." {
|
||
t.Errorf("text %q", got)
|
||
}
|
||
}
|
||
|
||
func TestChunksStayUnderTheLimitAndOverlap(t *testing.T) {
|
||
var b strings.Builder
|
||
for i := range 20 {
|
||
b.WriteString(para("kelime"+string(rune('a'+i)), 30) + "\n\n")
|
||
}
|
||
cs := Split(b.String(), Options{MaxChars: 600, Overlap: 300})
|
||
if len(cs) < 5 {
|
||
t.Fatalf("only %d chunks", len(cs))
|
||
}
|
||
for i, c := range cs {
|
||
if n := utf8.RuneCountInString(c.Body); n > 600 {
|
||
t.Errorf("chunk %d has %d characters", i, n)
|
||
}
|
||
if i > 0 {
|
||
prev := strings.Split(cs[i-1].Body, "\n\n")
|
||
if !strings.HasPrefix(c.Body, prev[len(prev)-1]) {
|
||
t.Errorf("chunk %d does not start with the end of chunk %d", i, i-1)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestASectionStartsAChunkOnceTheLastIsWorthKeeping(t *testing.T) {
|
||
text := "# A\n\n" + para("a", 400) + "\n\n# B\n\nkısa\n\n# C\n\nde kısa\n"
|
||
cs := Split(text, Options{MaxChars: 1800})
|
||
if len(cs) != 2 {
|
||
t.Fatalf("chunks %+v", cs)
|
||
}
|
||
if cs[1].Headings[0] != "B" || !strings.Contains(cs[1].Body, "# C") || !strings.Contains(cs[1].Body, "de kısa") {
|
||
t.Errorf("small sections were not kept together: %+v", cs[1])
|
||
}
|
||
// The overlap does not run into a new section.
|
||
if strings.HasPrefix(cs[1].Body, "a a") {
|
||
t.Errorf("section B starts with section A's text")
|
||
}
|
||
}
|
||
|
||
func TestHeadingLevelsReplaceTheirSiblings(t *testing.T) {
|
||
text := "## A\n\n### A1\n\n" + para("x", 300) + "\n\n## B\n\n" + para("y", 300) + "\n"
|
||
cs := Split(text, Options{MaxChars: 700})
|
||
last := cs[len(cs)-1]
|
||
if strings.Join(last.Headings, "/") != "B" {
|
||
t.Errorf("headings after a sibling: %v", last.Headings)
|
||
}
|
||
if strings.Join(cs[0].Headings, "/") != "A/A1" {
|
||
t.Errorf("first headings %v", cs[0].Headings)
|
||
}
|
||
}
|
||
|
||
func TestLongParagraphsAreCutAtSentencesThenWords(t *testing.T) {
|
||
long := strings.Repeat("Bu bir cümledir. ", 200)
|
||
for _, c := range Split(long, Options{MaxChars: 500}) {
|
||
if utf8.RuneCountInString(c.Body) > 500 {
|
||
t.Errorf("chunk of %d", utf8.RuneCountInString(c.Body))
|
||
}
|
||
if !strings.HasSuffix(c.Body, ".") {
|
||
t.Errorf("cut mid-sentence: …%q", c.Body[len(c.Body)-20:])
|
||
}
|
||
}
|
||
word := strings.Repeat("ş", 1200)
|
||
cs := Split(word, Options{MaxChars: 500})
|
||
total := 0
|
||
for _, c := range cs {
|
||
total += utf8.RuneCountInString(c.Body)
|
||
if !utf8.ValidString(c.Body) {
|
||
t.Error("a multibyte character was cut")
|
||
}
|
||
}
|
||
if total != 1200 {
|
||
t.Errorf("a long word lost characters: %d", total)
|
||
}
|
||
}
|
||
|
||
func TestCodeFencesStayWhole(t *testing.T) {
|
||
text := "Örnek:\n\n```go\nfunc a() {\n\n\treturn\n}\n```\n\nSon.\n"
|
||
cs := Split(text, Options{})
|
||
if len(cs) != 1 || !strings.Contains(cs[0].Body, "func a() {\n\n\treturn\n}") {
|
||
t.Errorf("chunks %+v", cs)
|
||
}
|
||
}
|
||
|
||
func TestNothingToSayIsNoChunk(t *testing.T) {
|
||
if cs := Split("# Başlık\n\n## Alt\n", Options{}); len(cs) != 0 {
|
||
t.Errorf("headings alone made %+v", cs)
|
||
}
|
||
}
|