Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
133 lines
3.3 KiB
Go
133 lines
3.3 KiB
Go
// Package kvindex knows which blocks of KV cache each replica holds, from the
|
|
// events its engine publishes (plan §7.5 tier 2).
|
|
//
|
|
// Tier 1 guesses from what the router sent where. This is what the engines
|
|
// say they hold, so it also knows what they have evicted, and what reached
|
|
// them some other way. It is precise only about the replicas it hears from;
|
|
// for the rest the router falls back to the guess.
|
|
package kvindex
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
|
)
|
|
|
|
// Index is the block hashes each replica holds.
|
|
type Index struct {
|
|
mu sync.RWMutex
|
|
replicas map[string]*replica
|
|
}
|
|
|
|
type replica struct {
|
|
blockSize int
|
|
blocks map[uint64]struct{}
|
|
problem string
|
|
}
|
|
|
|
// New returns an empty index.
|
|
func New() *Index { return &Index{replicas: map[string]*replica{}} }
|
|
|
|
// Apply folds in one update from an agent.
|
|
func (x *Index) Apply(u kvevents.Update) {
|
|
if u.Replica == "" {
|
|
return
|
|
}
|
|
x.mu.Lock()
|
|
defer x.mu.Unlock()
|
|
r, ok := x.replicas[u.Replica]
|
|
if !ok {
|
|
r = &replica{blocks: map[uint64]struct{}{}}
|
|
x.replicas[u.Replica] = r
|
|
}
|
|
if u.Reset {
|
|
r.blocks = map[uint64]struct{}{}
|
|
}
|
|
if u.BlockSize > 0 {
|
|
r.blockSize = u.BlockSize
|
|
}
|
|
r.problem = u.Problem
|
|
if r.problem != "" {
|
|
r.blocks = map[uint64]struct{}{}
|
|
return
|
|
}
|
|
for _, h := range u.Removed {
|
|
delete(r.blocks, h)
|
|
}
|
|
for _, h := range u.Stored {
|
|
r.blocks[h] = struct{}{}
|
|
}
|
|
}
|
|
|
|
// Forget drops a replica: it has gone, or the stream that described it has.
|
|
func (x *Index) Forget(id string) {
|
|
x.mu.Lock()
|
|
defer x.mu.Unlock()
|
|
delete(x.replicas, id)
|
|
}
|
|
|
|
// BlockSizes are the distinct block sizes of the replicas among ids the index
|
|
// can say something about. Empty means it can say nothing, and the request
|
|
// is not worth tokenizing.
|
|
func (x *Index) BlockSizes(ids []string) []int {
|
|
x.mu.RLock()
|
|
defer x.mu.RUnlock()
|
|
var out []int
|
|
for _, id := range ids {
|
|
r, ok := x.replicas[id]
|
|
if !ok || r.problem != "" || r.blockSize <= 0 {
|
|
continue
|
|
}
|
|
dup := false
|
|
for _, s := range out {
|
|
dup = dup || s == r.blockSize
|
|
}
|
|
if !dup {
|
|
out = append(out, r.blockSize)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Leading reports how many of a prompt's leading blocks a replica holds, and
|
|
// whether the index knows the replica at all.
|
|
//
|
|
// Leading, not total: a block is only reusable when every block before it is,
|
|
// because its hash is chained from theirs. vLLM stops at the first miss, and
|
|
// so does this. hashes is keyed by block size, since a replica's blocks are
|
|
// only comparable with hashes cut at its own.
|
|
func (x *Index) Leading(id string, hashes map[int][]uint64) (matched, total int, known bool) {
|
|
x.mu.RLock()
|
|
defer x.mu.RUnlock()
|
|
r, ok := x.replicas[id]
|
|
if !ok || r.problem != "" || r.blockSize <= 0 {
|
|
return 0, 0, false
|
|
}
|
|
hs := hashes[r.blockSize]
|
|
for _, h := range hs {
|
|
if _, ok := r.blocks[h]; !ok {
|
|
break
|
|
}
|
|
matched++
|
|
}
|
|
return matched, len(hs), true
|
|
}
|
|
|
|
// Stat is what the index holds of one replica, for the metrics.
|
|
type Stat struct {
|
|
Replica string
|
|
Blocks int
|
|
Problem string
|
|
}
|
|
|
|
// Stats lists every replica the index has heard of.
|
|
func (x *Index) Stats() []Stat {
|
|
x.mu.RLock()
|
|
defer x.mu.RUnlock()
|
|
out := make([]Stat, 0, len(x.replicas))
|
|
for id, r := range x.replicas {
|
|
out = append(out, Stat{Replica: id, Blocks: len(r.blocks), Problem: r.problem})
|
|
}
|
|
return out
|
|
}
|