package sim import "testing" func TestTokenizeIsPrefixStable(t *testing.T) { a := Tokenize("merhaba") b := Tokenize("merhaba dunya") if len(a) > len(b) { t.Fatal("longer text produced fewer tokens") } for i := range a[:len(a)-1] { // the last token may be partial if a[i] != b[i] { t.Fatalf("token %d is not prefix stable", i) } } } func TestCountTokensMatchesTokenize(t *testing.T) { for _, s := range []string{"", "a", "abcd", "abcde", "a somewhat longer prompt"} { if got, want := CountTokens(s), len(Tokenize(s)); got != want { t.Errorf("%q: CountTokens=%d Tokenize=%d", s, got, want) } } }