Files
llmgw/internal/router/kvindex/index.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

133 lines
3.3 KiB
Go

// Package kvindex knows which blocks of KV cache each replica holds, from the
// events its engine publishes (plan §7.5 tier 2).
//
// Tier 1 guesses from what the router sent where. This is what the engines
// say they hold, so it also knows what they have evicted, and what reached
// them some other way. It is precise only about the replicas it hears from;
// for the rest the router falls back to the guess.
package kvindex
import (
"sync"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
// Index is the block hashes each replica holds.
type Index struct {
mu sync.RWMutex
replicas map[string]*replica
}
type replica struct {
blockSize int
blocks map[uint64]struct{}
problem string
}
// New returns an empty index.
func New() *Index { return &Index{replicas: map[string]*replica{}} }
// Apply folds in one update from an agent.
func (x *Index) Apply(u kvevents.Update) {
if u.Replica == "" {
return
}
x.mu.Lock()
defer x.mu.Unlock()
r, ok := x.replicas[u.Replica]
if !ok {
r = &replica{blocks: map[uint64]struct{}{}}
x.replicas[u.Replica] = r
}
if u.Reset {
r.blocks = map[uint64]struct{}{}
}
if u.BlockSize > 0 {
r.blockSize = u.BlockSize
}
r.problem = u.Problem
if r.problem != "" {
r.blocks = map[uint64]struct{}{}
return
}
for _, h := range u.Removed {
delete(r.blocks, h)
}
for _, h := range u.Stored {
r.blocks[h] = struct{}{}
}
}
// Forget drops a replica: it has gone, or the stream that described it has.
func (x *Index) Forget(id string) {
x.mu.Lock()
defer x.mu.Unlock()
delete(x.replicas, id)
}
// BlockSizes are the distinct block sizes of the replicas among ids the index
// can say something about. Empty means it can say nothing, and the request
// is not worth tokenizing.
func (x *Index) BlockSizes(ids []string) []int {
x.mu.RLock()
defer x.mu.RUnlock()
var out []int
for _, id := range ids {
r, ok := x.replicas[id]
if !ok || r.problem != "" || r.blockSize <= 0 {
continue
}
dup := false
for _, s := range out {
dup = dup || s == r.blockSize
}
if !dup {
out = append(out, r.blockSize)
}
}
return out
}
// Leading reports how many of a prompt's leading blocks a replica holds, and
// whether the index knows the replica at all.
//
// Leading, not total: a block is only reusable when every block before it is,
// because its hash is chained from theirs. vLLM stops at the first miss, and
// so does this. hashes is keyed by block size, since a replica's blocks are
// only comparable with hashes cut at its own.
func (x *Index) Leading(id string, hashes map[int][]uint64) (matched, total int, known bool) {
x.mu.RLock()
defer x.mu.RUnlock()
r, ok := x.replicas[id]
if !ok || r.problem != "" || r.blockSize <= 0 {
return 0, 0, false
}
hs := hashes[r.blockSize]
for _, h := range hs {
if _, ok := r.blocks[h]; !ok {
break
}
matched++
}
return matched, len(hs), true
}
// Stat is what the index holds of one replica, for the metrics.
type Stat struct {
Replica string
Blocks int
Problem string
}
// Stats lists every replica the index has heard of.
func (x *Index) Stats() []Stat {
x.mu.RLock()
defer x.mu.RUnlock()
out := make([]Stat, 0, len(x.replicas))
for id, r := range x.replicas {
out = append(out, Stat{Replica: id, Blocks: len(r.blocks), Problem: r.problem})
}
return out
}