ilkerandClaude Opus 5
28029eb43b
CI / test (push) Has been cancelled
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
25 lines
622 B
Go
25 lines
622 B
Go
package sim
|
|
|
|
import "testing"
|
|
|
|
func TestTokenizeIsPrefixStable(t *testing.T) {
|
|
a := Tokenize("merhaba")
|
|
b := Tokenize("merhaba dunya")
|
|
if len(a) > len(b) {
|
|
t.Fatal("longer text produced fewer tokens")
|
|
}
|
|
for i := range a[:len(a)-1] { // the last token may be partial
|
|
if a[i] != b[i] {
|
|
t.Fatalf("token %d is not prefix stable", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCountTokensMatchesTokenize(t *testing.T) {
|
|
for _, s := range []string{"", "a", "abcd", "abcde", "a somewhat longer prompt"} {
|
|
if got, want := CountTokens(s), len(Tokenize(s)); got != want {
|
|
t.Errorf("%q: CountTokens=%d Tokenize=%d", s, got, want)
|
|
}
|
|
}
|
|
}
|