ilkerandClaude Opus 5
28029eb43b
CI / test (push) Has been cancelled
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
78 lines
2.5 KiB
Go
78 lines
2.5 KiB
Go
package sim
|
|
|
|
import "testing"
|
|
|
|
func TestBlockHashesAreChained(t *testing.T) {
|
|
// The same content following a different prefix must hash differently.
|
|
a := BlockHashes(Tokenize("AAAAAAAABBBBBBBB"), 2)
|
|
b := BlockHashes(Tokenize("CCCCCCCCBBBBBBBB"), 2)
|
|
if len(a) != 2 || len(b) != 2 {
|
|
t.Fatalf("expected 2 blocks each, got a=%d b=%d", len(a), len(b))
|
|
}
|
|
if a[1] == b[1] {
|
|
t.Error("identical block after different prefixes hashed the same; chaining is broken")
|
|
}
|
|
}
|
|
|
|
func TestBlockHashesArePrefixStable(t *testing.T) {
|
|
// A prefix aligned to a block boundary must hash identically in a longer
|
|
// text. Alignment matters: a differing trailing partial token changes the
|
|
// block containing it — the same boundary effect real tokenizers have,
|
|
// which is why cache hits are always measured in whole blocks.
|
|
const aligned = "merhaba " // 8 bytes = 2 tokens = 1 full block at blockSize 2
|
|
short := BlockHashes(Tokenize(aligned), 2)
|
|
long := BlockHashes(Tokenize(aligned+"dunya nasilsin"), 2)
|
|
if len(short) != 1 {
|
|
t.Fatalf("expected 1 block, got %d", len(short))
|
|
}
|
|
if short[0] != long[0] {
|
|
t.Fatal("aligned prefix produced a different block hash; not prefix stable")
|
|
}
|
|
}
|
|
|
|
func TestBlockHashesDropPartialBlock(t *testing.T) {
|
|
// 3 tokens at blockSize 2 -> 1 full block
|
|
if got := len(BlockHashes(make([]Token, 3), 2)); got != 1 {
|
|
t.Errorf("expected 1 block, got %d", got)
|
|
}
|
|
if got := len(BlockHashes(make([]Token, 1), 2)); got != 0 {
|
|
t.Errorf("produced %d blocks with no full block available", got)
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheReturnsContiguousPrefix(t *testing.T) {
|
|
c := NewPrefixCache(10)
|
|
h := []BlockHash{1, 2, 3, 4}
|
|
c.Insert(h)
|
|
|
|
if n := c.Lookup(h); n != 4 {
|
|
t.Fatalf("expected a full match, got %d", n)
|
|
}
|
|
// With a gap in the middle only the first block counts.
|
|
if n := c.Lookup([]BlockHash{1, 99, 3}); n != 1 {
|
|
t.Errorf("contiguous prefix should have been 1, got %d", n)
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheEvictsLRU(t *testing.T) {
|
|
c := NewPrefixCache(2)
|
|
c.Insert([]BlockHash{1, 2})
|
|
c.Lookup([]BlockHash{1}) // refresh 1
|
|
c.Insert([]BlockHash{3}) // over capacity: the oldest (2) must go
|
|
|
|
if n := c.Lookup([]BlockHash{1}); n != 1 {
|
|
t.Error("recently used block was evicted")
|
|
}
|
|
if n := c.Lookup([]BlockHash{2}); n != 0 {
|
|
t.Error("least recently used block should have been evicted")
|
|
}
|
|
}
|
|
|
|
func TestPrefixCacheDisabled(t *testing.T) {
|
|
c := NewPrefixCache(0)
|
|
c.Insert([]BlockHash{1, 2})
|
|
if n := c.Lookup([]BlockHash{1}); n != 0 {
|
|
t.Errorf("disabled cache reported a hit: %d", n)
|
|
}
|
|
}
|