ilkerandClaude Opus 5
28029eb43b
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
38 lines
1.2 KiB
Go
38 lines
1.2 KiB
Go
package sim
|
|
|
|
import "hash/fnv"
|
|
|
|
// bytesPerToken is how many bytes the fake tokenizer folds into one token.
|
|
// A crude approximation of real BPE tokenizers; all the simulation needs is
|
|
// for it to be deterministic and prefix-stable.
|
|
const bytesPerToken = 4
|
|
|
|
// Token is a token id produced by the fake tokenizer.
|
|
type Token uint32
|
|
|
|
// Tokenize splits text into tokens deterministically and prefix-stably.
|
|
//
|
|
// Prefix stability is essential: two texts sharing a prefix must produce token
|
|
// sequences sharing that prefix, otherwise the prefix cache simulation is
|
|
// meaningless. That is why splitting happens at fixed byte offsets from the
|
|
// start rather than content-dependent (BPE-style) boundaries.
|
|
func Tokenize(s string) []Token {
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
n := (len(s) + bytesPerToken - 1) / bytesPerToken
|
|
out := make([]Token, 0, n)
|
|
for i := 0; i < len(s); i += bytesPerToken {
|
|
end := min(i+bytesPerToken, len(s))
|
|
h := fnv.New32a()
|
|
_, _ = h.Write([]byte(s[i:end]))
|
|
out = append(out, Token(h.Sum32()))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// CountTokens returns the token count for a text without tokenizing it.
|
|
func CountTokens(s string) int {
|
|
return (len(s) + bytesPerToken - 1) / bytesPerToken
|
|
}
|