Files
llmgw/internal/kvevents/mirror.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

309 lines
8.7 KiB
Go

package kvevents
import (
"maps"
"slices"
"sync"
)
// Update is a change to what one replica's cache holds, as it travels from
// the agent to the router.
//
// It is a net effect, not a log. Between two sends a block can be stored,
// evicted and stored again; only where it ended up matters, and sending the
// history would cost the stream for nothing.
type Update struct {
Replica string
// BlockSize is how many tokens make one of this replica's blocks.
BlockSize int
// Reset says to forget everything known of the replica before applying
// the rest: the engine cleared its cache or restarted, events were lost,
// or the stream is new and this is the start of a full copy.
Reset bool
// Stored and Removed are block hashes, in external form.
Stored []uint64
Removed []uint64
// Problem, when set, says the replica's events cannot be used, and why.
// Its index is empty until the problem goes, and requests for it are
// routed as though it had none.
Problem string
}
// maxPerUpdate bounds the hashes in one update, so a full copy of a large
// cache goes up the stream in pieces rather than as one message the size of
// the cache.
const maxPerUpdate = 32 << 10
// mismatchLimit is how many first blocks must hash differently, with none
// hashing alike, before a replica's hashing is declared wrong. Not one: a
// block can carry a cache salt that older engines do not report, and one such
// prompt must not switch the index off.
const mismatchLimit = 8
// Mirror holds what each engine's cache holds, as its events describe it, and
// what each stream to the control plane has yet to be told.
//
// It keeps the whole set rather than forwarding events as they come because
// the stream is not permanent. A control plane restarts, an agent fails over
// to another: the new one knows nothing, and the only way to tell it the
// truth is a full copy. Holding the set costs eight bytes a block, and an
// engine has some tens of thousands.
type Mirror struct {
mu sync.Mutex
replicas map[string]*mirrored
streams map[*Stream]struct{}
}
type mirrored struct {
blockSize int
blocks map[uint64]struct{}
// matched and mismatched count first blocks checked against llmgw's
// hashing; problem is set once the check has failed.
matched, mismatched int
problem string
}
// Stream is one consumer of a mirror's changes: one session with a control
// plane.
type Stream struct {
m *Mirror
pending map[string]*delta
}
type delta struct {
reset bool
blockSize int
net map[uint64]bool // true stored, false removed
problem string
}
// NewMirror returns an empty mirror.
func NewMirror() *Mirror {
return &Mirror{replicas: map[string]*mirrored{}, streams: map[*Stream]struct{}{}}
}
// Open starts a stream whose first changes are a full copy of every replica.
func (m *Mirror) Open() *Stream {
m.mu.Lock()
defer m.mu.Unlock()
s := &Stream{m: m, pending: map[string]*delta{}}
for id, r := range m.replicas {
d := &delta{reset: true, blockSize: r.blockSize, problem: r.problem,
net: make(map[uint64]bool, len(r.blocks))}
for h := range r.blocks {
d.net[h] = true
}
s.pending[id] = d
}
m.streams[s] = struct{}{}
return s
}
// Close stops a stream; its pending changes are dropped.
func (s *Stream) Close() {
s.m.mu.Lock()
defer s.m.mu.Unlock()
delete(s.m.streams, s)
}
// Drain takes everything the stream has yet to send, as updates of bounded
// size. A replica's reset, when there is one, is in its first update.
func (s *Stream) Drain() []Update {
s.m.mu.Lock()
pending := s.pending
s.pending = map[string]*delta{}
s.m.mu.Unlock()
var out []Update
for _, id := range slices.Sorted(maps.Keys(pending)) {
d := pending[id]
u := Update{Replica: id, BlockSize: d.blockSize, Reset: d.reset, Problem: d.problem}
for h, stored := range d.net {
if stored {
u.Stored = append(u.Stored, h)
} else {
u.Removed = append(u.Removed, h)
}
}
out = append(out, split(u)...)
}
return out
}
// split cuts an update into pieces of at most maxPerUpdate hashes, keeping the
// reset on the first so later pieces add to it rather than wiping it.
func split(u Update) []Update {
if len(u.Stored)+len(u.Removed) <= maxPerUpdate {
return []Update{u}
}
var out []Update
stored, removed := u.Stored, u.Removed
first := true
for first || len(stored) > 0 || len(removed) > 0 {
p := Update{Replica: u.Replica, BlockSize: u.BlockSize, Problem: u.Problem, Reset: first && u.Reset}
first = false
room := maxPerUpdate
n := min(room, len(removed))
p.Removed, removed = removed[:n], removed[n:]
room -= n
n = min(room, len(stored))
p.Stored, stored = stored[:n], stored[n:]
out = append(out, p)
}
return out
}
// Apply folds one batch of a replica's events into the mirror.
//
// Events about copies outside the GPU, or about a second KV cache group, are
// ignored: a block offloaded to CPU memory is a slower hit the router cannot
// weigh, and a hybrid model's groups would need a set each. The index then
// under-counts, which costs a missed preference and never a wrong one.
func (m *Mirror) Apply(replica string, b Batch) {
m.mu.Lock()
defer m.mu.Unlock()
r := m.replica(replica)
for _, e := range b.Events {
if e.Kind == Cleared {
m.resetLocked(replica, r)
continue
}
if e.Elsewhere {
continue
}
if e.Kind == Stored {
m.check(replica, r, e)
}
if r.problem != "" {
continue
}
if e.Kind == Stored && e.BlockSize > r.blockSize {
// The largest seen, not the latest: a partial block is published
// with its own, smaller size.
r.blockSize = e.BlockSize
for s := range m.streams {
s.delta(replica).blockSize = r.blockSize
}
}
for _, h := range e.Hashes {
if e.Kind == Stored {
r.blocks[h] = struct{}{}
} else {
delete(r.blocks, h)
}
for s := range m.streams {
s.delta(replica).net[h] = e.Kind == Stored
}
}
}
}
// check compares the first block of a prompt with llmgw's hashing of its
// tokens.
//
// This is the one thing that can go silently wrong with an index: an engine
// started with another hash algorithm or seed — a profile argument overriding
// llmgw's, a vLLM that changed its scheme — publishes hashes the router will
// never compute, and the index fills with blocks that match nothing. Better to
// say so, and route without it.
func (m *Mirror) check(replica string, r *mirrored, e Event) {
if e.HasParent || e.Extra || len(e.Hashes) == 0 || e.BlockSize <= 0 ||
len(e.Tokens) < e.BlockSize || r.problem != "" {
return
}
if FirstBlockHash(e.Tokens[:e.BlockSize]) == e.Hashes[0] {
r.matched++
return
}
r.mismatched++
if r.matched == 0 && r.mismatched >= mismatchLimit {
r.problem = "the engine's block hashes are not " + HashAlgo + " with PYTHONHASHSEED=" + HashSeed +
"; remove any --prefix-caching-hash-algo from the profile's arguments, and check the vLLM version"
r.blocks = map[uint64]struct{}{}
for s := range m.streams {
d := s.delta(replica)
d.reset = true
d.net = map[uint64]bool{}
}
}
}
// Reset forgets what a replica's cache holds, keeping the replica: its events
// were interrupted, and what was missed cannot be known.
func (m *Mirror) Reset(replica string) {
m.mu.Lock()
defer m.mu.Unlock()
m.resetLocked(replica, m.replica(replica))
}
func (m *Mirror) resetLocked(replica string, r *mirrored) {
r.blocks = map[uint64]struct{}{}
for s := range m.streams {
d := s.delta(replica)
d.reset = true
d.net = map[uint64]bool{}
}
}
// Remove drops a replica whose engine has stopped. The streams are told it
// holds nothing, which is true.
func (m *Mirror) Remove(replica string) {
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.replicas[replica]; !ok {
return
}
delete(m.replicas, replica)
for s := range m.streams {
d := s.delta(replica)
d.reset = true
d.net = map[uint64]bool{}
}
}
// Blocks is how many blocks the mirror believes a replica holds.
func (m *Mirror) Blocks(replica string) int {
m.mu.Lock()
defer m.mu.Unlock()
if r, ok := m.replicas[replica]; ok {
return len(r.blocks)
}
return 0
}
// Problem is why a replica's events are not used, or "".
func (m *Mirror) Problem(replica string) string {
m.mu.Lock()
defer m.mu.Unlock()
if r, ok := m.replicas[replica]; ok {
return r.problem
}
return ""
}
func (m *Mirror) replica(id string) *mirrored {
r, ok := m.replicas[id]
if !ok {
r = &mirrored{blocks: map[uint64]struct{}{}}
m.replicas[id] = r
}
return r
}
func (s *Stream) delta(replica string) *delta {
d, ok := s.pending[replica]
if !ok {
d = &delta{net: map[uint64]bool{}}
s.pending[replica] = d
}
if r, ok := s.m.replicas[replica]; ok {
if d.blockSize == 0 {
d.blockSize = r.blockSize
}
// Every update carries the replica's standing problem, so one sent
// after it was found cannot read as its having gone away.
d.problem = r.problem
}
return d
}