Files
llmgw/internal/kvevents/kvevents_test.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

387 lines
13 KiB
Go

package kvevents
import (
"context"
"encoding/binary"
"encoding/hex"
"slices"
"strings"
"testing"
"time"
"github.com/go-zeromq/zmq4"
"github.com/vmihailenco/msgpack/v5"
)
// The expected values below were computed by Python with cbor2 and hashlib,
// exactly as vLLM's sha256_cbor and hash_block_tokens do it, under
// PYTHONHASHSEED=1729:
//
// NONE = sha256(cbor2.dumps("1729", canonical=True))
// h_i = sha256(cbor2.dumps((h_{i-1}, tuple(block_i), None), canonical=True))
// external = int.from_bytes(h_i, "big") & (2**64 - 1)
func TestBlockHashesAreVLLMsSha256CborChain(t *testing.T) {
tokens := []int{1, 23, 24, 255, 256, 65535, 65536, 151643, 4294967296}
for i := 100; i < 140; i++ {
tokens = append(tokens, i)
}
got := BlockHashes(tokens, 16, 0)
want := []uint64{2557436066646899323, 14514142130713257781, 480514406790448453}
if !slices.Equal(got, want) {
t.Fatalf("hashes %v, want %v", got, want)
}
if got := BlockHashes([]int{0, 1, 2, 3, 4}, 4, 0); !slices.Equal(got, []uint64{2734566579713004419}) {
t.Fatalf("a partial last block must not be hashed: %v", got)
}
if got := BlockHashes(tokens, 16, 2); len(got) != 2 {
t.Fatalf("max should bound the blocks hashed, got %d", len(got))
}
if FirstBlockHash(tokens[:16]) != want[0] {
t.Fatal("the first block's hash should need nothing but its tokens")
}
}
func TestTheSeedIsTheOneTheGoldenValuesWereMadeWith(t *testing.T) {
sum := noneHash()
if hex.EncodeToString(sum[:]) != "25699066d86cd786fe44373a87613dbf15e2386a5d945a476bacb4e72d1130a6" {
t.Fatalf("NONE_HASH changed: %x — the router and running engines would no longer agree", sum)
}
}
// Encoded by msgspec from vLLM's own struct definitions (main branch, 2026):
// tagged-map events, a GPU store of two blocks, a CPU store, a removal and a
// clear.
const goldenNew = "93cb3ff80000000000009489a474797065ab426c6f636b53746f726564ac626c6f636b5f68617368657392cf237dd68d8042ba7bcfc96c975e60b1e735b1706172656e745f626c6f636b5f68617368c0a9746f6b656e5f696473dc0020000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0faa626c6f636b5f73697a6510a76c6f72615f6964c0a66d656469756da3475055a96c6f72615f6e616d65c0a967726f75705f6964780088a474797065ab426c6f636b53746f726564ac626c6f636b5f68617368657391c4200000000000000000000000000000000000000000000000000000000000000007b1706172656e745f626c6f636b5f68617368cfc96c975e60b1e735a9746f6b656e5f696473dc0010000102030405060708090a0b0c0d0e0faa626c6f636b5f73697a6510a76c6f72615f6964c0a66d656469756da3435055a96c6f72615f6e616d65c083a474797065ac426c6f636b52656d6f766564ac626c6f636b5f68617368657391cfc96c975e60b1e735a66d656469756da347505581a474797065b0416c6c426c6f636b73436c656172656400"
// The same with the older array_like events.
const goldenOld = "93cb40000000000000009397ab426c6f636b53746f72656491cf237dd68d8042ba7bc0dc0010000102030405060708090a0b0c0d0e0f10c0c093ac426c6f636b52656d6f76656491cf237dd68d8042ba7bc091b0416c6c426c6f636b73436c6561726564c0"
func TestABatchInVLLMsCurrentEncodingIsRead(t *testing.T) {
raw, _ := hex.DecodeString(goldenNew)
b, err := DecodeBatch(raw)
if err != nil {
t.Fatal(err)
}
if b.TS != 1.5 || len(b.Events) != 4 {
t.Fatalf("batch %+v", b)
}
st := b.Events[0]
if st.Kind != Stored || !slices.Equal(st.Hashes, []uint64{2557436066646899323, 14514142130713257781}) ||
st.HasParent || st.BlockSize != 16 || len(st.Tokens) != 32 || st.Elsewhere || st.Extra {
t.Fatalf("stored %+v", st)
}
cpu := b.Events[1]
if !cpu.Elsewhere || cpu.Hashes[0] != 7 || !cpu.HasParent || cpu.Parent != 14514142130713257781 {
t.Fatalf("a CPU copy, hashed as bytes, should be read and marked elsewhere: %+v", cpu)
}
if b.Events[2].Kind != Removed || b.Events[2].Hashes[0] != 14514142130713257781 {
t.Fatalf("removed %+v", b.Events[2])
}
if b.Events[3].Kind != Cleared {
t.Fatalf("cleared %+v", b.Events[3])
}
}
func TestABatchInVLLMsOlderArrayEncodingIsRead(t *testing.T) {
raw, _ := hex.DecodeString(goldenOld)
b, err := DecodeBatch(raw)
if err != nil {
t.Fatal(err)
}
if len(b.Events) != 3 || b.Events[0].Kind != Stored || b.Events[0].Hashes[0] != 2557436066646899323 ||
b.Events[0].BlockSize != 16 || len(b.Events[0].Tokens) != 16 ||
b.Events[1].Kind != Removed || b.Events[2].Kind != Cleared {
t.Fatalf("batch %+v", b)
}
}
func TestAnUnknownEventKindIsSkippedNotRefused(t *testing.T) {
raw, _ := msgpack.Marshal([]any{1.0, []any{
map[string]any{"type": "SomethingNew", "x": 1},
map[string]any{"type": "AllBlocksCleared"},
}})
b, err := DecodeBatch(raw)
if err != nil || len(b.Events) != 1 || b.Events[0].Kind != Cleared {
t.Fatalf("batch %+v, err %v", b, err)
}
if _, err := DecodeBatch([]byte{0xc1}); err == nil {
t.Fatal("garbage should be refused")
}
}
// stored builds a BlockStored event for a chain of blocks, hashed as vLLM
// would, in the current encoding.
func stored(tokens []int, blockSize int) map[string]any {
hs := BlockHashes(tokens, blockSize, 0)
anyHs := make([]any, len(hs))
for i, h := range hs {
anyHs[i] = h
}
return map[string]any{
"type": "BlockStored", "block_hashes": anyHs, "parent_block_hash": nil,
"token_ids": tokens[:len(hs)*blockSize], "block_size": blockSize,
"lora_id": nil, "medium": "GPU", "lora_name": nil,
}
}
func removed(hs ...uint64) map[string]any {
anyHs := make([]any, len(hs))
for i, h := range hs {
anyHs[i] = h
}
return map[string]any{"type": "BlockRemoved", "block_hashes": anyHs, "medium": "GPU"}
}
func batch(t *testing.T, events ...any) Batch {
t.Helper()
raw, err := msgpack.Marshal([]any{float64(time.Now().Unix()), events})
if err != nil {
t.Fatal(err)
}
b, err := DecodeBatch(raw)
if err != nil {
t.Fatal(err)
}
return b
}
func seq(n int) []int {
out := make([]int, n)
for i := range out {
out[i] = 1000 + i
}
return out
}
func TestTheMirrorFollowsStoresRemovalsAndClears(t *testing.T) {
m := NewMirror()
s := m.Open()
defer s.Close()
toks := seq(64)
hs := BlockHashes(toks, 16, 0)
m.Apply("r1", batch(t, stored(toks, 16)))
if m.Blocks("r1") != 4 {
t.Fatalf("blocks %d, want 4", m.Blocks("r1"))
}
m.Apply("r1", batch(t, removed(hs[3])))
ups := s.Drain()
if len(ups) != 1 || ups[0].Reset || ups[0].BlockSize != 16 || len(ups[0].Stored) != 3 || len(ups[0].Removed) != 1 {
t.Fatalf("a store then a removal should arrive as their net effect: %+v", ups)
}
if len(s.Drain()) != 0 {
t.Fatal("drained changes must not be sent twice")
}
m.Apply("r1", batch(t, map[string]any{"type": "AllBlocksCleared"}))
ups = s.Drain()
if m.Blocks("r1") != 0 || len(ups) != 1 || !ups[0].Reset || len(ups[0].Stored) != 0 {
t.Fatalf("a clear should empty the replica and say so: %+v", ups)
}
}
func TestANewStreamStartsWithAFullCopy(t *testing.T) {
m := NewMirror()
m.Apply("r1", batch(t, stored(seq(48), 16)))
m.Apply("r2", batch(t, stored(seq(16), 16)))
s := m.Open()
defer s.Close()
ups := s.Drain()
if len(ups) != 2 || !ups[0].Reset || len(ups[0].Stored) != 3 || ups[1].Replica != "r2" || len(ups[1].Stored) != 1 {
t.Fatalf("a control plane that knows nothing should be told everything: %+v", ups)
}
}
func TestAFullCopyOfALargeCacheGoesInPieces(t *testing.T) {
m := NewMirror()
toks := seq(16 * (maxPerUpdate + 10))
m.Apply("r1", batch(t, stored(toks, 16)))
s := m.Open()
defer s.Close()
ups := s.Drain()
if len(ups) != 2 || !ups[0].Reset || ups[1].Reset || len(ups[0].Stored)+len(ups[1].Stored) != maxPerUpdate+10 {
t.Fatalf("pieces: %d, first reset %v", len(ups), len(ups) > 0 && ups[0].Reset)
}
}
func TestCopiesOutsideTheGPUAreNotIndexed(t *testing.T) {
m := NewMirror()
ev := stored(seq(16), 16)
ev["medium"] = "CPU"
m.Apply("r1", batch(t, ev))
if m.Blocks("r1") != 0 {
t.Fatal("a block offloaded to CPU memory is not one the GPU holds")
}
}
func TestAnEngineHashingAnotherWayIsCaughtAndItsIndexEmptied(t *testing.T) {
m := NewMirror()
s := m.Open()
defer s.Close()
for i := range mismatchLimit {
ev := stored(seq(16+i), 16)
ev["block_hashes"] = []any{uint64(i + 1)} // not what sha256_cbor gives
m.Apply("r1", batch(t, ev))
}
if m.Problem("r1") == "" {
t.Fatal("first blocks that never hash alike should be reported")
}
if m.Blocks("r1") != 0 {
t.Fatal("an index of hashes the router cannot compute is worse than none")
}
ups := s.Drain()
if len(ups) != 1 || ups[0].Problem == "" || !ups[0].Reset {
t.Fatalf("the problem should travel: %+v", ups)
}
// One bad first block among good ones is a cache salt, not a wrong seed.
m2 := NewMirror()
m2.Apply("r1", batch(t, stored(seq(16), 16)))
for i := range mismatchLimit {
ev := stored(seq(17+i), 16)
ev["block_hashes"] = []any{uint64(i + 1)}
m2.Apply("r1", batch(t, ev))
}
if m2.Problem("r1") != "" {
t.Fatal("an engine seen to hash alike should not be condemned by odd blocks")
}
}
// publisher is a stand-in for vLLM's ZmqEventPublisher: a PUB socket that
// connects to the endpoint it is given — as vLLM does for an endpoint naming
// a host — and sends (topic, 8-byte big-endian sequence, msgpack batch).
type publisher struct {
t *testing.T
sock zmq4.Socket
seq uint64
}
func newPublisher(t *testing.T, endpoint string) *publisher {
t.Helper()
sock := zmq4.NewPub(context.Background())
if err := sock.Dial(endpoint); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sock.Close() })
return &publisher{t: t, sock: sock}
}
func (p *publisher) send(events ...any) {
p.t.Helper()
raw, err := msgpack.Marshal([]any{float64(time.Now().UnixNano()) / 1e9, events})
if err != nil {
p.t.Fatal(err)
}
var s [8]byte
binary.BigEndian.PutUint64(s[:], p.seq)
p.seq++
if err := p.sock.Send(zmq4.NewMsgFrom([]byte("kv-events"), s[:], raw)); err != nil {
p.t.Fatal(err)
}
}
func waitFor(t *testing.T, what string, ok func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !ok() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(10 * time.Millisecond)
}
}
// waitSubscribed sends a probe until it arrives: a subscription travels to
// the publisher after the connection, and a PUB socket drops what is sent
// before it does, as libzmq's does.
func waitSubscribed(t *testing.T, p *publisher, m *Mirror, replica string) {
t.Helper()
probe := seq(16)
deadline := time.Now().Add(5 * time.Second)
for m.Blocks(replica) == 0 {
if time.Now().After(deadline) {
t.Fatal("the subscriber never received anything")
}
p.send(stored(probe, 16))
time.Sleep(20 * time.Millisecond)
}
p.send(map[string]any{"type": "AllBlocksCleared"})
waitFor(t, "the probe to be cleared", func() bool { return m.Blocks(replica) == 0 })
}
func TestTheSubscriberReceivesAnEnginesEventsOverZMQ(t *testing.T) {
m := NewMirror()
sub, err := Listen("r1", 0, m, nil)
if err != nil {
t.Fatal(err)
}
defer sub.Close()
if sub.Port() == 0 {
t.Fatal("listening on port 0 should report the port chosen")
}
pub := newPublisher(t, sub.Endpoint())
waitSubscribed(t, pub, m, "r1")
toks := seq(80)
pub.send(stored(toks, 16))
waitFor(t, "five blocks", func() bool { return m.Blocks("r1") == 5 })
hs := BlockHashes(toks, 16, 0)
pub.send(removed(hs[4], hs[3]))
waitFor(t, "three blocks", func() bool { return m.Blocks("r1") == 3 })
}
func TestLostEventsStartTheIndexAgain(t *testing.T) {
m := NewMirror()
sub, err := Listen("r1", 0, m, nil)
if err != nil {
t.Fatal(err)
}
defer sub.Close()
pub := newPublisher(t, sub.Endpoint())
waitSubscribed(t, pub, m, "r1")
pub.send(stored(seq(64), 16))
waitFor(t, "four blocks", func() bool { return m.Blocks("r1") == 4 })
pub.seq += 3 // three batches never arrive; one may have removed blocks
other := seq(16)
for i := range other {
other[i] += 5000
}
pub.send(stored(other, 16))
waitFor(t, "only the block after the gap", func() bool { return m.Blocks("r1") == 1 })
}
func TestARestartedEngineStartsTheIndexAgain(t *testing.T) {
m := NewMirror()
sub, err := Listen("r1", 0, m, nil)
if err != nil {
t.Fatal(err)
}
defer sub.Close()
pub := newPublisher(t, sub.Endpoint())
waitSubscribed(t, pub, m, "r1")
pub.send(stored(seq(64), 16))
waitFor(t, "four blocks", func() bool { return m.Blocks("r1") == 4 })
pub.seq = 0 // a new process numbers from zero again
pub.send(stored(seq(16), 16))
waitFor(t, "the new engine's one block", func() bool { return m.Blocks("r1") == 1 })
}
func TestTheEngineIsToldToHashTheWayTheRouterDoes(t *testing.T) {
args := VLLMArgs("tcp://127.0.0.1:5999")
if !slices.Contains(args, "sha256_cbor") || !slices.Contains(args, "--kv-events-config") {
t.Fatalf("args %v", args)
}
cfg := args[slices.Index(args, "--kv-events-config")+1]
if want := `"endpoint":"tcp://127.0.0.1:5999"`; !strings.Contains(cfg, want) {
t.Fatalf("config %s should name the loopback endpoint, so vLLM connects rather than binding every address", cfg)
}
if VLLMEnv()["PYTHONHASHSEED"] != HashSeed {
t.Fatal("the seed must be the one the router hashes with")
}
}