package kvevents import ( "maps" "slices" "sync" ) // Update is a change to what one replica's cache holds, as it travels from // the agent to the router. // // It is a net effect, not a log. Between two sends a block can be stored, // evicted and stored again; only where it ended up matters, and sending the // history would cost the stream for nothing. type Update struct { Replica string // BlockSize is how many tokens make one of this replica's blocks. BlockSize int // Reset says to forget everything known of the replica before applying // the rest: the engine cleared its cache or restarted, events were lost, // or the stream is new and this is the start of a full copy. Reset bool // Stored and Removed are block hashes, in external form. Stored []uint64 Removed []uint64 // Problem, when set, says the replica's events cannot be used, and why. // Its index is empty until the problem goes, and requests for it are // routed as though it had none. Problem string } // maxPerUpdate bounds the hashes in one update, so a full copy of a large // cache goes up the stream in pieces rather than as one message the size of // the cache. const maxPerUpdate = 32 << 10 // mismatchLimit is how many first blocks must hash differently, with none // hashing alike, before a replica's hashing is declared wrong. Not one: a // block can carry a cache salt that older engines do not report, and one such // prompt must not switch the index off. const mismatchLimit = 8 // Mirror holds what each engine's cache holds, as its events describe it, and // what each stream to the control plane has yet to be told. // // It keeps the whole set rather than forwarding events as they come because // the stream is not permanent. A control plane restarts, an agent fails over // to another: the new one knows nothing, and the only way to tell it the // truth is a full copy. Holding the set costs eight bytes a block, and an // engine has some tens of thousands. type Mirror struct { mu sync.Mutex replicas map[string]*mirrored streams map[*Stream]struct{} } type mirrored struct { blockSize int blocks map[uint64]struct{} // matched and mismatched count first blocks checked against llmgw's // hashing; problem is set once the check has failed. matched, mismatched int problem string } // Stream is one consumer of a mirror's changes: one session with a control // plane. type Stream struct { m *Mirror pending map[string]*delta } type delta struct { reset bool blockSize int net map[uint64]bool // true stored, false removed problem string } // NewMirror returns an empty mirror. func NewMirror() *Mirror { return &Mirror{replicas: map[string]*mirrored{}, streams: map[*Stream]struct{}{}} } // Open starts a stream whose first changes are a full copy of every replica. func (m *Mirror) Open() *Stream { m.mu.Lock() defer m.mu.Unlock() s := &Stream{m: m, pending: map[string]*delta{}} for id, r := range m.replicas { d := &delta{reset: true, blockSize: r.blockSize, problem: r.problem, net: make(map[uint64]bool, len(r.blocks))} for h := range r.blocks { d.net[h] = true } s.pending[id] = d } m.streams[s] = struct{}{} return s } // Close stops a stream; its pending changes are dropped. func (s *Stream) Close() { s.m.mu.Lock() defer s.m.mu.Unlock() delete(s.m.streams, s) } // Drain takes everything the stream has yet to send, as updates of bounded // size. A replica's reset, when there is one, is in its first update. func (s *Stream) Drain() []Update { s.m.mu.Lock() pending := s.pending s.pending = map[string]*delta{} s.m.mu.Unlock() var out []Update for _, id := range slices.Sorted(maps.Keys(pending)) { d := pending[id] u := Update{Replica: id, BlockSize: d.blockSize, Reset: d.reset, Problem: d.problem} for h, stored := range d.net { if stored { u.Stored = append(u.Stored, h) } else { u.Removed = append(u.Removed, h) } } out = append(out, split(u)...) } return out } // split cuts an update into pieces of at most maxPerUpdate hashes, keeping the // reset on the first so later pieces add to it rather than wiping it. func split(u Update) []Update { if len(u.Stored)+len(u.Removed) <= maxPerUpdate { return []Update{u} } var out []Update stored, removed := u.Stored, u.Removed first := true for first || len(stored) > 0 || len(removed) > 0 { p := Update{Replica: u.Replica, BlockSize: u.BlockSize, Problem: u.Problem, Reset: first && u.Reset} first = false room := maxPerUpdate n := min(room, len(removed)) p.Removed, removed = removed[:n], removed[n:] room -= n n = min(room, len(stored)) p.Stored, stored = stored[:n], stored[n:] out = append(out, p) } return out } // Apply folds one batch of a replica's events into the mirror. // // Events about copies outside the GPU, or about a second KV cache group, are // ignored: a block offloaded to CPU memory is a slower hit the router cannot // weigh, and a hybrid model's groups would need a set each. The index then // under-counts, which costs a missed preference and never a wrong one. func (m *Mirror) Apply(replica string, b Batch) { m.mu.Lock() defer m.mu.Unlock() r := m.replica(replica) for _, e := range b.Events { if e.Kind == Cleared { m.resetLocked(replica, r) continue } if e.Elsewhere { continue } if e.Kind == Stored { m.check(replica, r, e) } if r.problem != "" { continue } if e.Kind == Stored && e.BlockSize > r.blockSize { // The largest seen, not the latest: a partial block is published // with its own, smaller size. r.blockSize = e.BlockSize for s := range m.streams { s.delta(replica).blockSize = r.blockSize } } for _, h := range e.Hashes { if e.Kind == Stored { r.blocks[h] = struct{}{} } else { delete(r.blocks, h) } for s := range m.streams { s.delta(replica).net[h] = e.Kind == Stored } } } } // check compares the first block of a prompt with llmgw's hashing of its // tokens. // // This is the one thing that can go silently wrong with an index: an engine // started with another hash algorithm or seed — a profile argument overriding // llmgw's, a vLLM that changed its scheme — publishes hashes the router will // never compute, and the index fills with blocks that match nothing. Better to // say so, and route without it. func (m *Mirror) check(replica string, r *mirrored, e Event) { if e.HasParent || e.Extra || len(e.Hashes) == 0 || e.BlockSize <= 0 || len(e.Tokens) < e.BlockSize || r.problem != "" { return } if FirstBlockHash(e.Tokens[:e.BlockSize]) == e.Hashes[0] { r.matched++ return } r.mismatched++ if r.matched == 0 && r.mismatched >= mismatchLimit { r.problem = "the engine's block hashes are not " + HashAlgo + " with PYTHONHASHSEED=" + HashSeed + "; remove any --prefix-caching-hash-algo from the profile's arguments, and check the vLLM version" r.blocks = map[uint64]struct{}{} for s := range m.streams { d := s.delta(replica) d.reset = true d.net = map[uint64]bool{} } } } // Reset forgets what a replica's cache holds, keeping the replica: its events // were interrupted, and what was missed cannot be known. func (m *Mirror) Reset(replica string) { m.mu.Lock() defer m.mu.Unlock() m.resetLocked(replica, m.replica(replica)) } func (m *Mirror) resetLocked(replica string, r *mirrored) { r.blocks = map[uint64]struct{}{} for s := range m.streams { d := s.delta(replica) d.reset = true d.net = map[uint64]bool{} } } // Remove drops a replica whose engine has stopped. The streams are told it // holds nothing, which is true. func (m *Mirror) Remove(replica string) { m.mu.Lock() defer m.mu.Unlock() if _, ok := m.replicas[replica]; !ok { return } delete(m.replicas, replica) for s := range m.streams { d := s.delta(replica) d.reset = true d.net = map[uint64]bool{} } } // Blocks is how many blocks the mirror believes a replica holds. func (m *Mirror) Blocks(replica string) int { m.mu.Lock() defer m.mu.Unlock() if r, ok := m.replicas[replica]; ok { return len(r.blocks) } return 0 } // Problem is why a replica's events are not used, or "". func (m *Mirror) Problem(replica string) string { m.mu.Lock() defer m.mu.Unlock() if r, ok := m.replicas[replica]; ok { return r.problem } return "" } func (m *Mirror) replica(id string) *mirrored { r, ok := m.replicas[id] if !ok { r = &mirrored{blocks: map[uint64]struct{}{}} m.replicas[id] = r } return r } func (s *Stream) delta(replica string) *delta { d, ok := s.pending[replica] if !ok { d = &delta{net: map[uint64]bool{}} s.pending[replica] = d } if r, ok := s.m.replicas[replica]; ok { if d.blockSize == 0 { d.blockSize = r.blockSize } // Every update carries the replica's standing problem, so one sent // after it was found cannot read as its having gone away. d.problem = r.problem } return d }