package sim import "hash/fnv" // bytesPerToken is how many bytes the fake tokenizer folds into one token. // A crude approximation of real BPE tokenizers; all the simulation needs is // for it to be deterministic and prefix-stable. const bytesPerToken = 4 // Token is a token id produced by the fake tokenizer. type Token uint32 // Tokenize splits text into tokens deterministically and prefix-stably. // // Prefix stability is essential: two texts sharing a prefix must produce token // sequences sharing that prefix, otherwise the prefix cache simulation is // meaningless. That is why splitting happens at fixed byte offsets from the // start rather than content-dependent (BPE-style) boundaries. func Tokenize(s string) []Token { if s == "" { return nil } n := (len(s) + bytesPerToken - 1) / bytesPerToken out := make([]Token, 0, n) for i := 0; i < len(s); i += bytesPerToken { end := min(i+bytesPerToken, len(s)) h := fnv.New32a() _, _ = h.Write([]byte(s[i:end])) out = append(out, Token(h.Sum32())) } return out } // CountTokens returns the token count for a text without tokenizing it. func CountTokens(s string) int { return (len(s) + bytesPerToken - 1) / bytesPerToken }