ilkerandClaude Opus 5
28029eb43b
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
25 lines
622 B
Go
25 lines
622 B
Go
package sim
|
|
|
|
import "testing"
|
|
|
|
func TestTokenizeIsPrefixStable(t *testing.T) {
|
|
a := Tokenize("merhaba")
|
|
b := Tokenize("merhaba dunya")
|
|
if len(a) > len(b) {
|
|
t.Fatal("longer text produced fewer tokens")
|
|
}
|
|
for i := range a[:len(a)-1] { // the last token may be partial
|
|
if a[i] != b[i] {
|
|
t.Fatalf("token %d is not prefix stable", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCountTokensMatchesTokenize(t *testing.T) {
|
|
for _, s := range []string{"", "a", "abcd", "abcde", "a somewhat longer prompt"} {
|
|
if got, want := CountTokens(s), len(Tokenize(s)); got != want {
|
|
t.Errorf("%q: CountTokens=%d Tokenize=%d", s, got, want)
|
|
}
|
|
}
|
|
}
|