Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
309 lines
8.7 KiB
Go
309 lines
8.7 KiB
Go
package kvevents
|
|
|
|
import (
|
|
"maps"
|
|
"slices"
|
|
"sync"
|
|
)
|
|
|
|
// Update is a change to what one replica's cache holds, as it travels from
|
|
// the agent to the router.
|
|
//
|
|
// It is a net effect, not a log. Between two sends a block can be stored,
|
|
// evicted and stored again; only where it ended up matters, and sending the
|
|
// history would cost the stream for nothing.
|
|
type Update struct {
|
|
Replica string
|
|
// BlockSize is how many tokens make one of this replica's blocks.
|
|
BlockSize int
|
|
// Reset says to forget everything known of the replica before applying
|
|
// the rest: the engine cleared its cache or restarted, events were lost,
|
|
// or the stream is new and this is the start of a full copy.
|
|
Reset bool
|
|
// Stored and Removed are block hashes, in external form.
|
|
Stored []uint64
|
|
Removed []uint64
|
|
// Problem, when set, says the replica's events cannot be used, and why.
|
|
// Its index is empty until the problem goes, and requests for it are
|
|
// routed as though it had none.
|
|
Problem string
|
|
}
|
|
|
|
// maxPerUpdate bounds the hashes in one update, so a full copy of a large
|
|
// cache goes up the stream in pieces rather than as one message the size of
|
|
// the cache.
|
|
const maxPerUpdate = 32 << 10
|
|
|
|
// mismatchLimit is how many first blocks must hash differently, with none
|
|
// hashing alike, before a replica's hashing is declared wrong. Not one: a
|
|
// block can carry a cache salt that older engines do not report, and one such
|
|
// prompt must not switch the index off.
|
|
const mismatchLimit = 8
|
|
|
|
// Mirror holds what each engine's cache holds, as its events describe it, and
|
|
// what each stream to the control plane has yet to be told.
|
|
//
|
|
// It keeps the whole set rather than forwarding events as they come because
|
|
// the stream is not permanent. A control plane restarts, an agent fails over
|
|
// to another: the new one knows nothing, and the only way to tell it the
|
|
// truth is a full copy. Holding the set costs eight bytes a block, and an
|
|
// engine has some tens of thousands.
|
|
type Mirror struct {
|
|
mu sync.Mutex
|
|
replicas map[string]*mirrored
|
|
streams map[*Stream]struct{}
|
|
}
|
|
|
|
type mirrored struct {
|
|
blockSize int
|
|
blocks map[uint64]struct{}
|
|
// matched and mismatched count first blocks checked against llmgw's
|
|
// hashing; problem is set once the check has failed.
|
|
matched, mismatched int
|
|
problem string
|
|
}
|
|
|
|
// Stream is one consumer of a mirror's changes: one session with a control
|
|
// plane.
|
|
type Stream struct {
|
|
m *Mirror
|
|
pending map[string]*delta
|
|
}
|
|
|
|
type delta struct {
|
|
reset bool
|
|
blockSize int
|
|
net map[uint64]bool // true stored, false removed
|
|
problem string
|
|
}
|
|
|
|
// NewMirror returns an empty mirror.
|
|
func NewMirror() *Mirror {
|
|
return &Mirror{replicas: map[string]*mirrored{}, streams: map[*Stream]struct{}{}}
|
|
}
|
|
|
|
// Open starts a stream whose first changes are a full copy of every replica.
|
|
func (m *Mirror) Open() *Stream {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
s := &Stream{m: m, pending: map[string]*delta{}}
|
|
for id, r := range m.replicas {
|
|
d := &delta{reset: true, blockSize: r.blockSize, problem: r.problem,
|
|
net: make(map[uint64]bool, len(r.blocks))}
|
|
for h := range r.blocks {
|
|
d.net[h] = true
|
|
}
|
|
s.pending[id] = d
|
|
}
|
|
m.streams[s] = struct{}{}
|
|
return s
|
|
}
|
|
|
|
// Close stops a stream; its pending changes are dropped.
|
|
func (s *Stream) Close() {
|
|
s.m.mu.Lock()
|
|
defer s.m.mu.Unlock()
|
|
delete(s.m.streams, s)
|
|
}
|
|
|
|
// Drain takes everything the stream has yet to send, as updates of bounded
|
|
// size. A replica's reset, when there is one, is in its first update.
|
|
func (s *Stream) Drain() []Update {
|
|
s.m.mu.Lock()
|
|
pending := s.pending
|
|
s.pending = map[string]*delta{}
|
|
s.m.mu.Unlock()
|
|
|
|
var out []Update
|
|
for _, id := range slices.Sorted(maps.Keys(pending)) {
|
|
d := pending[id]
|
|
u := Update{Replica: id, BlockSize: d.blockSize, Reset: d.reset, Problem: d.problem}
|
|
for h, stored := range d.net {
|
|
if stored {
|
|
u.Stored = append(u.Stored, h)
|
|
} else {
|
|
u.Removed = append(u.Removed, h)
|
|
}
|
|
}
|
|
out = append(out, split(u)...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// split cuts an update into pieces of at most maxPerUpdate hashes, keeping the
|
|
// reset on the first so later pieces add to it rather than wiping it.
|
|
func split(u Update) []Update {
|
|
if len(u.Stored)+len(u.Removed) <= maxPerUpdate {
|
|
return []Update{u}
|
|
}
|
|
var out []Update
|
|
stored, removed := u.Stored, u.Removed
|
|
first := true
|
|
for first || len(stored) > 0 || len(removed) > 0 {
|
|
p := Update{Replica: u.Replica, BlockSize: u.BlockSize, Problem: u.Problem, Reset: first && u.Reset}
|
|
first = false
|
|
room := maxPerUpdate
|
|
n := min(room, len(removed))
|
|
p.Removed, removed = removed[:n], removed[n:]
|
|
room -= n
|
|
n = min(room, len(stored))
|
|
p.Stored, stored = stored[:n], stored[n:]
|
|
out = append(out, p)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Apply folds one batch of a replica's events into the mirror.
|
|
//
|
|
// Events about copies outside the GPU, or about a second KV cache group, are
|
|
// ignored: a block offloaded to CPU memory is a slower hit the router cannot
|
|
// weigh, and a hybrid model's groups would need a set each. The index then
|
|
// under-counts, which costs a missed preference and never a wrong one.
|
|
func (m *Mirror) Apply(replica string, b Batch) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
r := m.replica(replica)
|
|
for _, e := range b.Events {
|
|
if e.Kind == Cleared {
|
|
m.resetLocked(replica, r)
|
|
continue
|
|
}
|
|
if e.Elsewhere {
|
|
continue
|
|
}
|
|
if e.Kind == Stored {
|
|
m.check(replica, r, e)
|
|
}
|
|
if r.problem != "" {
|
|
continue
|
|
}
|
|
if e.Kind == Stored && e.BlockSize > r.blockSize {
|
|
// The largest seen, not the latest: a partial block is published
|
|
// with its own, smaller size.
|
|
r.blockSize = e.BlockSize
|
|
for s := range m.streams {
|
|
s.delta(replica).blockSize = r.blockSize
|
|
}
|
|
}
|
|
for _, h := range e.Hashes {
|
|
if e.Kind == Stored {
|
|
r.blocks[h] = struct{}{}
|
|
} else {
|
|
delete(r.blocks, h)
|
|
}
|
|
for s := range m.streams {
|
|
s.delta(replica).net[h] = e.Kind == Stored
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// check compares the first block of a prompt with llmgw's hashing of its
|
|
// tokens.
|
|
//
|
|
// This is the one thing that can go silently wrong with an index: an engine
|
|
// started with another hash algorithm or seed — a profile argument overriding
|
|
// llmgw's, a vLLM that changed its scheme — publishes hashes the router will
|
|
// never compute, and the index fills with blocks that match nothing. Better to
|
|
// say so, and route without it.
|
|
func (m *Mirror) check(replica string, r *mirrored, e Event) {
|
|
if e.HasParent || e.Extra || len(e.Hashes) == 0 || e.BlockSize <= 0 ||
|
|
len(e.Tokens) < e.BlockSize || r.problem != "" {
|
|
return
|
|
}
|
|
if FirstBlockHash(e.Tokens[:e.BlockSize]) == e.Hashes[0] {
|
|
r.matched++
|
|
return
|
|
}
|
|
r.mismatched++
|
|
if r.matched == 0 && r.mismatched >= mismatchLimit {
|
|
r.problem = "the engine's block hashes are not " + HashAlgo + " with PYTHONHASHSEED=" + HashSeed +
|
|
"; remove any --prefix-caching-hash-algo from the profile's arguments, and check the vLLM version"
|
|
r.blocks = map[uint64]struct{}{}
|
|
for s := range m.streams {
|
|
d := s.delta(replica)
|
|
d.reset = true
|
|
d.net = map[uint64]bool{}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Reset forgets what a replica's cache holds, keeping the replica: its events
|
|
// were interrupted, and what was missed cannot be known.
|
|
func (m *Mirror) Reset(replica string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.resetLocked(replica, m.replica(replica))
|
|
}
|
|
|
|
func (m *Mirror) resetLocked(replica string, r *mirrored) {
|
|
r.blocks = map[uint64]struct{}{}
|
|
for s := range m.streams {
|
|
d := s.delta(replica)
|
|
d.reset = true
|
|
d.net = map[uint64]bool{}
|
|
}
|
|
}
|
|
|
|
// Remove drops a replica whose engine has stopped. The streams are told it
|
|
// holds nothing, which is true.
|
|
func (m *Mirror) Remove(replica string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if _, ok := m.replicas[replica]; !ok {
|
|
return
|
|
}
|
|
delete(m.replicas, replica)
|
|
for s := range m.streams {
|
|
d := s.delta(replica)
|
|
d.reset = true
|
|
d.net = map[uint64]bool{}
|
|
}
|
|
}
|
|
|
|
// Blocks is how many blocks the mirror believes a replica holds.
|
|
func (m *Mirror) Blocks(replica string) int {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if r, ok := m.replicas[replica]; ok {
|
|
return len(r.blocks)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// Problem is why a replica's events are not used, or "".
|
|
func (m *Mirror) Problem(replica string) string {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if r, ok := m.replicas[replica]; ok {
|
|
return r.problem
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (m *Mirror) replica(id string) *mirrored {
|
|
r, ok := m.replicas[id]
|
|
if !ok {
|
|
r = &mirrored{blocks: map[uint64]struct{}{}}
|
|
m.replicas[id] = r
|
|
}
|
|
return r
|
|
}
|
|
|
|
func (s *Stream) delta(replica string) *delta {
|
|
d, ok := s.pending[replica]
|
|
if !ok {
|
|
d = &delta{net: map[uint64]bool{}}
|
|
s.pending[replica] = d
|
|
}
|
|
if r, ok := s.m.replicas[replica]; ok {
|
|
if d.blockSize == 0 {
|
|
d.blockSize = r.blockSize
|
|
}
|
|
// Every update carries the replica's standing problem, so one sent
|
|
// after it was found cannot read as its having gone away.
|
|
d.problem = r.problem
|
|
}
|
|
return d
|
|
}
|