ilkerandClaude Opus 5
28029eb43b
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
78 lines
2.5 KiB
Go
78 lines
2.5 KiB
Go
package sim
|
|
|
|
import "testing"
|
|
|
|
func TestBlockHashesAreChained(t *testing.T) {
|
|
// The same content following a different prefix must hash differently.
|
|
a := BlockHashes(Tokenize("AAAAAAAABBBBBBBB"), 2)
|
|
b := BlockHashes(Tokenize("CCCCCCCCBBBBBBBB"), 2)
|
|
if len(a) != 2 || len(b) != 2 {
|
|
t.Fatalf("expected 2 blocks each, got a=%d b=%d", len(a), len(b))
|
|
}
|
|
if a[1] == b[1] {
|
|
t.Error("identical block after different prefixes hashed the same; chaining is broken")
|
|
}
|
|
}
|
|
|
|
func TestBlockHashesArePrefixStable(t *testing.T) {
|
|
// A prefix aligned to a block boundary must hash identically in a longer
|
|
// text. Alignment matters: a differing trailing partial token changes the
|
|
// block containing it — the same boundary effect real tokenizers have,
|
|
// which is why cache hits are always measured in whole blocks.
|
|
const aligned = "merhaba " // 8 bytes = 2 tokens = 1 full block at blockSize 2
|
|
short := BlockHashes(Tokenize(aligned), 2)
|
|
long := BlockHashes(Tokenize(aligned+"dunya nasilsin"), 2)
|
|
if len(short) != 1 {
|
|
t.Fatalf("expected 1 block, got %d", len(short))
|
|
}
|
|
if short[0] != long[0] {
|
|
t.Fatal("aligned prefix produced a different block hash; not prefix stable")
|
|
}
|
|
}
|
|
|
|
func TestBlockHashesDropPartialBlock(t *testing.T) {
|
|
// 3 tokens at blockSize 2 -> 1 full block
|
|
if got := len(BlockHashes(make([]Token, 3), 2)); got != 1 {
|
|
t.Errorf("expected 1 block, got %d", got)
|
|
}
|
|
if got := len(BlockHashes(make([]Token, 1), 2)); got != 0 {
|
|
t.Errorf("produced %d blocks with no full block available", got)
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheReturnsContiguousPrefix(t *testing.T) {
|
|
c := NewPrefixCache(10)
|
|
h := []BlockHash{1, 2, 3, 4}
|
|
c.Insert(h)
|
|
|
|
if n := c.Lookup(h); n != 4 {
|
|
t.Fatalf("expected a full match, got %d", n)
|
|
}
|
|
// With a gap in the middle only the first block counts.
|
|
if n := c.Lookup([]BlockHash{1, 99, 3}); n != 1 {
|
|
t.Errorf("contiguous prefix should have been 1, got %d", n)
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheEvictsLRU(t *testing.T) {
|
|
c := NewPrefixCache(2)
|
|
c.Insert([]BlockHash{1, 2})
|
|
c.Lookup([]BlockHash{1}) // refresh 1
|
|
c.Insert([]BlockHash{3}) // over capacity: the oldest (2) must go
|
|
|
|
if n := c.Lookup([]BlockHash{1}); n != 1 {
|
|
t.Error("recently used block was evicted")
|
|
}
|
|
if n := c.Lookup([]BlockHash{2}); n != 0 {
|
|
t.Error("least recently used block should have been evicted")
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheDisabled(t *testing.T) {
|
|
c := NewPrefixCache(0)
|
|
c.Insert([]BlockHash{1, 2})
|
|
if n := c.Lookup([]BlockHash{1}); n != 0 {
|
|
t.Errorf("disabled cache reported a hit: %d", n)
|
|
}
|
|
}
|