ilkerandClaude Opus 5
28029eb43b
Comments, doc comments, test names, error and log messages, CLI help, README, Makefile, CI step names and the design document are now all in English. No behaviour changes; the full suite passes under the race detector. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
129 lines
3.3 KiB
Go
129 lines
3.3 KiB
Go
package sim
|
|
|
|
import (
|
|
"container/list"
|
|
"encoding/binary"
|
|
"hash/fnv"
|
|
)
|
|
|
|
// BlockHash identifies one KV cache block, the unit of cache accounting.
|
|
type BlockHash uint64
|
|
|
|
// BlockHashes computes chained block hashes over a token sequence.
|
|
//
|
|
// Chaining (h_i = hash(h_{i-1}, block_i)) mirrors what vLLM does and is
|
|
// required: a block's identity must represent not only its own contents but
|
|
// every block before it. Otherwise two requests containing the same sentence
|
|
// in different contexts would falsely count as a cache hit.
|
|
//
|
|
// A trailing partial block is ignored — half blocks are never cached.
|
|
func BlockHashes(tokens []Token, blockSize int) []BlockHash {
|
|
if blockSize <= 0 || len(tokens) < blockSize {
|
|
return nil
|
|
}
|
|
n := len(tokens) / blockSize
|
|
out := make([]BlockHash, 0, n)
|
|
var prev BlockHash
|
|
buf := make([]byte, 8)
|
|
for i := range n {
|
|
h := fnv.New64a()
|
|
binary.LittleEndian.PutUint64(buf, uint64(prev))
|
|
_, _ = h.Write(buf)
|
|
for _, t := range tokens[i*blockSize : (i+1)*blockSize] {
|
|
binary.LittleEndian.PutUint32(buf[:4], uint32(t))
|
|
_, _ = h.Write(buf[:4])
|
|
}
|
|
prev = BlockHash(h.Sum64())
|
|
out = append(out, prev)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// PrefixCache models an engine's automatic prefix cache as a block-level LRU.
|
|
//
|
|
// A real engine uses a radix tree, but what matters for measuring routing
|
|
// quality is which blocks are still resident, and LRU models that faithfully.
|
|
type PrefixCache struct {
|
|
capacity int // in blocks
|
|
ll *list.List
|
|
m map[BlockHash]*list.Element
|
|
|
|
queries int64
|
|
hits int64
|
|
}
|
|
|
|
// NewPrefixCache creates a cache holding at most capacity blocks.
|
|
// A capacity of zero or less disables the cache (every lookup misses).
|
|
func NewPrefixCache(capacity int) *PrefixCache {
|
|
return &PrefixCache{
|
|
capacity: capacity,
|
|
ll: list.New(),
|
|
m: make(map[BlockHash]*list.Element),
|
|
}
|
|
}
|
|
|
|
// Enabled reports whether the cache is active.
|
|
func (c *PrefixCache) Enabled() bool { return c != nil && c.capacity > 0 }
|
|
|
|
// Lookup returns the length, in blocks, of the longest **contiguous prefix**
|
|
// of hashes present in the cache, refreshing those blocks in the LRU.
|
|
//
|
|
// Contiguity matters: if a block in the middle was evicted, everything after
|
|
// it must be recomputed too, because attention depends on all preceding blocks.
|
|
func (c *PrefixCache) Lookup(hashes []BlockHash) int {
|
|
if !c.Enabled() {
|
|
return 0
|
|
}
|
|
c.queries += int64(len(hashes))
|
|
n := 0
|
|
for _, h := range hashes {
|
|
el, ok := c.m[h]
|
|
if !ok {
|
|
break
|
|
}
|
|
c.ll.MoveToFront(el)
|
|
n++
|
|
}
|
|
c.hits += int64(n)
|
|
return n
|
|
}
|
|
|
|
// Insert stores blocks at the front of the LRU, evicting the oldest ones once
|
|
// capacity is exceeded.
|
|
func (c *PrefixCache) Insert(hashes []BlockHash) {
|
|
if !c.Enabled() {
|
|
return
|
|
}
|
|
for _, h := range hashes {
|
|
if el, ok := c.m[h]; ok {
|
|
c.ll.MoveToFront(el)
|
|
continue
|
|
}
|
|
c.m[h] = c.ll.PushFront(h)
|
|
}
|
|
for c.ll.Len() > c.capacity {
|
|
el := c.ll.Back()
|
|
if el == nil {
|
|
break
|
|
}
|
|
c.ll.Remove(el)
|
|
delete(c.m, el.Value.(BlockHash))
|
|
}
|
|
}
|
|
|
|
// Stats returns cumulative query and hit counts, in blocks.
|
|
func (c *PrefixCache) Stats() (queries, hits int64) {
|
|
if c == nil {
|
|
return 0, 0
|
|
}
|
|
return c.queries, c.hits
|
|
}
|
|
|
|
// Len returns the number of blocks currently cached.
|
|
func (c *PrefixCache) Len() int {
|
|
if !c.Enabled() {
|
|
return 0
|
|
}
|
|
return c.ll.Len()
|
|
}
|