// Package kvindex knows which blocks of KV cache each replica holds, from the // events its engine publishes (plan ยง7.5 tier 2). // // Tier 1 guesses from what the router sent where. This is what the engines // say they hold, so it also knows what they have evicted, and what reached // them some other way. It is precise only about the replicas it hears from; // for the rest the router falls back to the guess. package kvindex import ( "sync" "gitea.mynodes.xyz/ilker/llmgw/internal/kvevents" ) // Index is the block hashes each replica holds. type Index struct { mu sync.RWMutex replicas map[string]*replica } type replica struct { blockSize int blocks map[uint64]struct{} problem string } // New returns an empty index. func New() *Index { return &Index{replicas: map[string]*replica{}} } // Apply folds in one update from an agent. func (x *Index) Apply(u kvevents.Update) { if u.Replica == "" { return } x.mu.Lock() defer x.mu.Unlock() r, ok := x.replicas[u.Replica] if !ok { r = &replica{blocks: map[uint64]struct{}{}} x.replicas[u.Replica] = r } if u.Reset { r.blocks = map[uint64]struct{}{} } if u.BlockSize > 0 { r.blockSize = u.BlockSize } r.problem = u.Problem if r.problem != "" { r.blocks = map[uint64]struct{}{} return } for _, h := range u.Removed { delete(r.blocks, h) } for _, h := range u.Stored { r.blocks[h] = struct{}{} } } // Forget drops a replica: it has gone, or the stream that described it has. func (x *Index) Forget(id string) { x.mu.Lock() defer x.mu.Unlock() delete(x.replicas, id) } // BlockSizes are the distinct block sizes of the replicas among ids the index // can say something about. Empty means it can say nothing, and the request // is not worth tokenizing. func (x *Index) BlockSizes(ids []string) []int { x.mu.RLock() defer x.mu.RUnlock() var out []int for _, id := range ids { r, ok := x.replicas[id] if !ok || r.problem != "" || r.blockSize <= 0 { continue } dup := false for _, s := range out { dup = dup || s == r.blockSize } if !dup { out = append(out, r.blockSize) } } return out } // Leading reports how many of a prompt's leading blocks a replica holds, and // whether the index knows the replica at all. // // Leading, not total: a block is only reusable when every block before it is, // because its hash is chained from theirs. vLLM stops at the first miss, and // so does this. hashes is keyed by block size, since a replica's blocks are // only comparable with hashes cut at its own. func (x *Index) Leading(id string, hashes map[int][]uint64) (matched, total int, known bool) { x.mu.RLock() defer x.mu.RUnlock() r, ok := x.replicas[id] if !ok || r.problem != "" || r.blockSize <= 0 { return 0, 0, false } hs := hashes[r.blockSize] for _, h := range hs { if _, ok := r.blocks[h]; !ok { break } matched++ } return matched, len(hs), true } // Stat is what the index holds of one replica, for the metrics. type Stat struct { Replica string Blocks int Problem string } // Stats lists every replica the index has heard of. func (x *Index) Stats() []Stat { x.mu.RLock() defer x.mu.RUnlock() out := make([]Stat, 0, len(x.replicas)) for id, r := range x.replicas { out = append(out, Stat{Replica: id, Blocks: len(r.blocks), Problem: r.problem}) } return out }