package chunk import ( "strings" "testing" "unicode/utf8" ) func para(word string, n int) string { return strings.TrimSpace(strings.Repeat(word+" ", n)) } func TestShortTextIsOneChunkUnderItsHeadings(t *testing.T) { cs := Split("# İzin\n\n## Yıllık\n\nYirmi gün.\n\nBayramlar ayrı.\n", Options{}) if len(cs) != 1 { t.Fatalf("chunks %+v", cs) } if got := cs[0].Text(); got != "İzin › Yıllık\n\nYirmi gün.\n\nBayramlar ayrı." { t.Errorf("text %q", got) } } func TestChunksStayUnderTheLimitAndOverlap(t *testing.T) { var b strings.Builder for i := range 20 { b.WriteString(para("kelime"+string(rune('a'+i)), 30) + "\n\n") } cs := Split(b.String(), Options{MaxChars: 600, Overlap: 300}) if len(cs) < 5 { t.Fatalf("only %d chunks", len(cs)) } for i, c := range cs { if n := utf8.RuneCountInString(c.Body); n > 600 { t.Errorf("chunk %d has %d characters", i, n) } if i > 0 { prev := strings.Split(cs[i-1].Body, "\n\n") if !strings.HasPrefix(c.Body, prev[len(prev)-1]) { t.Errorf("chunk %d does not start with the end of chunk %d", i, i-1) } } } } func TestASectionStartsAChunkOnceTheLastIsWorthKeeping(t *testing.T) { text := "# A\n\n" + para("a", 400) + "\n\n# B\n\nkısa\n\n# C\n\nde kısa\n" cs := Split(text, Options{MaxChars: 1800}) if len(cs) != 2 { t.Fatalf("chunks %+v", cs) } if cs[1].Headings[0] != "B" || !strings.Contains(cs[1].Body, "# C") || !strings.Contains(cs[1].Body, "de kısa") { t.Errorf("small sections were not kept together: %+v", cs[1]) } // The overlap does not run into a new section. if strings.HasPrefix(cs[1].Body, "a a") { t.Errorf("section B starts with section A's text") } } func TestHeadingLevelsReplaceTheirSiblings(t *testing.T) { text := "## A\n\n### A1\n\n" + para("x", 300) + "\n\n## B\n\n" + para("y", 300) + "\n" cs := Split(text, Options{MaxChars: 700}) last := cs[len(cs)-1] if strings.Join(last.Headings, "/") != "B" { t.Errorf("headings after a sibling: %v", last.Headings) } if strings.Join(cs[0].Headings, "/") != "A/A1" { t.Errorf("first headings %v", cs[0].Headings) } } func TestLongParagraphsAreCutAtSentencesThenWords(t *testing.T) { long := strings.Repeat("Bu bir cümledir. ", 200) for _, c := range Split(long, Options{MaxChars: 500}) { if utf8.RuneCountInString(c.Body) > 500 { t.Errorf("chunk of %d", utf8.RuneCountInString(c.Body)) } if !strings.HasSuffix(c.Body, ".") { t.Errorf("cut mid-sentence: …%q", c.Body[len(c.Body)-20:]) } } word := strings.Repeat("ş", 1200) cs := Split(word, Options{MaxChars: 500}) total := 0 for _, c := range cs { total += utf8.RuneCountInString(c.Body) if !utf8.ValidString(c.Body) { t.Error("a multibyte character was cut") } } if total != 1200 { t.Errorf("a long word lost characters: %d", total) } } func TestCodeFencesStayWhole(t *testing.T) { text := "Örnek:\n\n```go\nfunc a() {\n\n\treturn\n}\n```\n\nSon.\n" cs := Split(text, Options{}) if len(cs) != 1 || !strings.Contains(cs[0].Body, "func a() {\n\n\treturn\n}") { t.Errorf("chunks %+v", cs) } } func TestNothingToSayIsNoChunk(t *testing.T) { if cs := Split("# Başlık\n\n## Alt\n", Options{}); len(cs) != 0 { t.Errorf("headings alone made %+v", cs) } }