Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
387 lines
13 KiB
Go
387 lines
13 KiB
Go
package kvevents
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-zeromq/zmq4"
|
|
"github.com/vmihailenco/msgpack/v5"
|
|
)
|
|
|
|
// The expected values below were computed by Python with cbor2 and hashlib,
|
|
// exactly as vLLM's sha256_cbor and hash_block_tokens do it, under
|
|
// PYTHONHASHSEED=1729:
|
|
//
|
|
// NONE = sha256(cbor2.dumps("1729", canonical=True))
|
|
// h_i = sha256(cbor2.dumps((h_{i-1}, tuple(block_i), None), canonical=True))
|
|
// external = int.from_bytes(h_i, "big") & (2**64 - 1)
|
|
func TestBlockHashesAreVLLMsSha256CborChain(t *testing.T) {
|
|
tokens := []int{1, 23, 24, 255, 256, 65535, 65536, 151643, 4294967296}
|
|
for i := 100; i < 140; i++ {
|
|
tokens = append(tokens, i)
|
|
}
|
|
got := BlockHashes(tokens, 16, 0)
|
|
want := []uint64{2557436066646899323, 14514142130713257781, 480514406790448453}
|
|
if !slices.Equal(got, want) {
|
|
t.Fatalf("hashes %v, want %v", got, want)
|
|
}
|
|
if got := BlockHashes([]int{0, 1, 2, 3, 4}, 4, 0); !slices.Equal(got, []uint64{2734566579713004419}) {
|
|
t.Fatalf("a partial last block must not be hashed: %v", got)
|
|
}
|
|
if got := BlockHashes(tokens, 16, 2); len(got) != 2 {
|
|
t.Fatalf("max should bound the blocks hashed, got %d", len(got))
|
|
}
|
|
if FirstBlockHash(tokens[:16]) != want[0] {
|
|
t.Fatal("the first block's hash should need nothing but its tokens")
|
|
}
|
|
}
|
|
|
|
func TestTheSeedIsTheOneTheGoldenValuesWereMadeWith(t *testing.T) {
|
|
sum := noneHash()
|
|
if hex.EncodeToString(sum[:]) != "25699066d86cd786fe44373a87613dbf15e2386a5d945a476bacb4e72d1130a6" {
|
|
t.Fatalf("NONE_HASH changed: %x — the router and running engines would no longer agree", sum)
|
|
}
|
|
}
|
|
|
|
// Encoded by msgspec from vLLM's own struct definitions (main branch, 2026):
|
|
// tagged-map events, a GPU store of two blocks, a CPU store, a removal and a
|
|
// clear.
|
|
const goldenNew = "93cb3ff80000000000009489a474797065ab426c6f636b53746f726564ac626c6f636b5f68617368657392cf237dd68d8042ba7bcfc96c975e60b1e735b1706172656e745f626c6f636b5f68617368c0a9746f6b656e5f696473dc0020000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0faa626c6f636b5f73697a6510a76c6f72615f6964c0a66d656469756da3475055a96c6f72615f6e616d65c0a967726f75705f6964780088a474797065ab426c6f636b53746f726564ac626c6f636b5f68617368657391c4200000000000000000000000000000000000000000000000000000000000000007b1706172656e745f626c6f636b5f68617368cfc96c975e60b1e735a9746f6b656e5f696473dc0010000102030405060708090a0b0c0d0e0faa626c6f636b5f73697a6510a76c6f72615f6964c0a66d656469756da3435055a96c6f72615f6e616d65c083a474797065ac426c6f636b52656d6f766564ac626c6f636b5f68617368657391cfc96c975e60b1e735a66d656469756da347505581a474797065b0416c6c426c6f636b73436c656172656400"
|
|
|
|
// The same with the older array_like events.
|
|
const goldenOld = "93cb40000000000000009397ab426c6f636b53746f72656491cf237dd68d8042ba7bc0dc0010000102030405060708090a0b0c0d0e0f10c0c093ac426c6f636b52656d6f76656491cf237dd68d8042ba7bc091b0416c6c426c6f636b73436c6561726564c0"
|
|
|
|
func TestABatchInVLLMsCurrentEncodingIsRead(t *testing.T) {
|
|
raw, _ := hex.DecodeString(goldenNew)
|
|
b, err := DecodeBatch(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if b.TS != 1.5 || len(b.Events) != 4 {
|
|
t.Fatalf("batch %+v", b)
|
|
}
|
|
st := b.Events[0]
|
|
if st.Kind != Stored || !slices.Equal(st.Hashes, []uint64{2557436066646899323, 14514142130713257781}) ||
|
|
st.HasParent || st.BlockSize != 16 || len(st.Tokens) != 32 || st.Elsewhere || st.Extra {
|
|
t.Fatalf("stored %+v", st)
|
|
}
|
|
cpu := b.Events[1]
|
|
if !cpu.Elsewhere || cpu.Hashes[0] != 7 || !cpu.HasParent || cpu.Parent != 14514142130713257781 {
|
|
t.Fatalf("a CPU copy, hashed as bytes, should be read and marked elsewhere: %+v", cpu)
|
|
}
|
|
if b.Events[2].Kind != Removed || b.Events[2].Hashes[0] != 14514142130713257781 {
|
|
t.Fatalf("removed %+v", b.Events[2])
|
|
}
|
|
if b.Events[3].Kind != Cleared {
|
|
t.Fatalf("cleared %+v", b.Events[3])
|
|
}
|
|
}
|
|
|
|
func TestABatchInVLLMsOlderArrayEncodingIsRead(t *testing.T) {
|
|
raw, _ := hex.DecodeString(goldenOld)
|
|
b, err := DecodeBatch(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(b.Events) != 3 || b.Events[0].Kind != Stored || b.Events[0].Hashes[0] != 2557436066646899323 ||
|
|
b.Events[0].BlockSize != 16 || len(b.Events[0].Tokens) != 16 ||
|
|
b.Events[1].Kind != Removed || b.Events[2].Kind != Cleared {
|
|
t.Fatalf("batch %+v", b)
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownEventKindIsSkippedNotRefused(t *testing.T) {
|
|
raw, _ := msgpack.Marshal([]any{1.0, []any{
|
|
map[string]any{"type": "SomethingNew", "x": 1},
|
|
map[string]any{"type": "AllBlocksCleared"},
|
|
}})
|
|
b, err := DecodeBatch(raw)
|
|
if err != nil || len(b.Events) != 1 || b.Events[0].Kind != Cleared {
|
|
t.Fatalf("batch %+v, err %v", b, err)
|
|
}
|
|
if _, err := DecodeBatch([]byte{0xc1}); err == nil {
|
|
t.Fatal("garbage should be refused")
|
|
}
|
|
}
|
|
|
|
// stored builds a BlockStored event for a chain of blocks, hashed as vLLM
|
|
// would, in the current encoding.
|
|
func stored(tokens []int, blockSize int) map[string]any {
|
|
hs := BlockHashes(tokens, blockSize, 0)
|
|
anyHs := make([]any, len(hs))
|
|
for i, h := range hs {
|
|
anyHs[i] = h
|
|
}
|
|
return map[string]any{
|
|
"type": "BlockStored", "block_hashes": anyHs, "parent_block_hash": nil,
|
|
"token_ids": tokens[:len(hs)*blockSize], "block_size": blockSize,
|
|
"lora_id": nil, "medium": "GPU", "lora_name": nil,
|
|
}
|
|
}
|
|
|
|
func removed(hs ...uint64) map[string]any {
|
|
anyHs := make([]any, len(hs))
|
|
for i, h := range hs {
|
|
anyHs[i] = h
|
|
}
|
|
return map[string]any{"type": "BlockRemoved", "block_hashes": anyHs, "medium": "GPU"}
|
|
}
|
|
|
|
func batch(t *testing.T, events ...any) Batch {
|
|
t.Helper()
|
|
raw, err := msgpack.Marshal([]any{float64(time.Now().Unix()), events})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := DecodeBatch(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func seq(n int) []int {
|
|
out := make([]int, n)
|
|
for i := range out {
|
|
out[i] = 1000 + i
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestTheMirrorFollowsStoresRemovalsAndClears(t *testing.T) {
|
|
m := NewMirror()
|
|
s := m.Open()
|
|
defer s.Close()
|
|
toks := seq(64)
|
|
hs := BlockHashes(toks, 16, 0)
|
|
|
|
m.Apply("r1", batch(t, stored(toks, 16)))
|
|
if m.Blocks("r1") != 4 {
|
|
t.Fatalf("blocks %d, want 4", m.Blocks("r1"))
|
|
}
|
|
m.Apply("r1", batch(t, removed(hs[3])))
|
|
ups := s.Drain()
|
|
if len(ups) != 1 || ups[0].Reset || ups[0].BlockSize != 16 || len(ups[0].Stored) != 3 || len(ups[0].Removed) != 1 {
|
|
t.Fatalf("a store then a removal should arrive as their net effect: %+v", ups)
|
|
}
|
|
if len(s.Drain()) != 0 {
|
|
t.Fatal("drained changes must not be sent twice")
|
|
}
|
|
|
|
m.Apply("r1", batch(t, map[string]any{"type": "AllBlocksCleared"}))
|
|
ups = s.Drain()
|
|
if m.Blocks("r1") != 0 || len(ups) != 1 || !ups[0].Reset || len(ups[0].Stored) != 0 {
|
|
t.Fatalf("a clear should empty the replica and say so: %+v", ups)
|
|
}
|
|
}
|
|
|
|
func TestANewStreamStartsWithAFullCopy(t *testing.T) {
|
|
m := NewMirror()
|
|
m.Apply("r1", batch(t, stored(seq(48), 16)))
|
|
m.Apply("r2", batch(t, stored(seq(16), 16)))
|
|
|
|
s := m.Open()
|
|
defer s.Close()
|
|
ups := s.Drain()
|
|
if len(ups) != 2 || !ups[0].Reset || len(ups[0].Stored) != 3 || ups[1].Replica != "r2" || len(ups[1].Stored) != 1 {
|
|
t.Fatalf("a control plane that knows nothing should be told everything: %+v", ups)
|
|
}
|
|
}
|
|
|
|
func TestAFullCopyOfALargeCacheGoesInPieces(t *testing.T) {
|
|
m := NewMirror()
|
|
toks := seq(16 * (maxPerUpdate + 10))
|
|
m.Apply("r1", batch(t, stored(toks, 16)))
|
|
s := m.Open()
|
|
defer s.Close()
|
|
ups := s.Drain()
|
|
if len(ups) != 2 || !ups[0].Reset || ups[1].Reset || len(ups[0].Stored)+len(ups[1].Stored) != maxPerUpdate+10 {
|
|
t.Fatalf("pieces: %d, first reset %v", len(ups), len(ups) > 0 && ups[0].Reset)
|
|
}
|
|
}
|
|
|
|
func TestCopiesOutsideTheGPUAreNotIndexed(t *testing.T) {
|
|
m := NewMirror()
|
|
ev := stored(seq(16), 16)
|
|
ev["medium"] = "CPU"
|
|
m.Apply("r1", batch(t, ev))
|
|
if m.Blocks("r1") != 0 {
|
|
t.Fatal("a block offloaded to CPU memory is not one the GPU holds")
|
|
}
|
|
}
|
|
|
|
func TestAnEngineHashingAnotherWayIsCaughtAndItsIndexEmptied(t *testing.T) {
|
|
m := NewMirror()
|
|
s := m.Open()
|
|
defer s.Close()
|
|
for i := range mismatchLimit {
|
|
ev := stored(seq(16+i), 16)
|
|
ev["block_hashes"] = []any{uint64(i + 1)} // not what sha256_cbor gives
|
|
m.Apply("r1", batch(t, ev))
|
|
}
|
|
if m.Problem("r1") == "" {
|
|
t.Fatal("first blocks that never hash alike should be reported")
|
|
}
|
|
if m.Blocks("r1") != 0 {
|
|
t.Fatal("an index of hashes the router cannot compute is worse than none")
|
|
}
|
|
ups := s.Drain()
|
|
if len(ups) != 1 || ups[0].Problem == "" || !ups[0].Reset {
|
|
t.Fatalf("the problem should travel: %+v", ups)
|
|
}
|
|
|
|
// One bad first block among good ones is a cache salt, not a wrong seed.
|
|
m2 := NewMirror()
|
|
m2.Apply("r1", batch(t, stored(seq(16), 16)))
|
|
for i := range mismatchLimit {
|
|
ev := stored(seq(17+i), 16)
|
|
ev["block_hashes"] = []any{uint64(i + 1)}
|
|
m2.Apply("r1", batch(t, ev))
|
|
}
|
|
if m2.Problem("r1") != "" {
|
|
t.Fatal("an engine seen to hash alike should not be condemned by odd blocks")
|
|
}
|
|
}
|
|
|
|
// publisher is a stand-in for vLLM's ZmqEventPublisher: a PUB socket that
|
|
// connects to the endpoint it is given — as vLLM does for an endpoint naming
|
|
// a host — and sends (topic, 8-byte big-endian sequence, msgpack batch).
|
|
type publisher struct {
|
|
t *testing.T
|
|
sock zmq4.Socket
|
|
seq uint64
|
|
}
|
|
|
|
func newPublisher(t *testing.T, endpoint string) *publisher {
|
|
t.Helper()
|
|
sock := zmq4.NewPub(context.Background())
|
|
if err := sock.Dial(endpoint); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = sock.Close() })
|
|
return &publisher{t: t, sock: sock}
|
|
}
|
|
|
|
func (p *publisher) send(events ...any) {
|
|
p.t.Helper()
|
|
raw, err := msgpack.Marshal([]any{float64(time.Now().UnixNano()) / 1e9, events})
|
|
if err != nil {
|
|
p.t.Fatal(err)
|
|
}
|
|
var s [8]byte
|
|
binary.BigEndian.PutUint64(s[:], p.seq)
|
|
p.seq++
|
|
if err := p.sock.Send(zmq4.NewMsgFrom([]byte("kv-events"), s[:], raw)); err != nil {
|
|
p.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func waitFor(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// waitSubscribed sends a probe until it arrives: a subscription travels to
|
|
// the publisher after the connection, and a PUB socket drops what is sent
|
|
// before it does, as libzmq's does.
|
|
func waitSubscribed(t *testing.T, p *publisher, m *Mirror, replica string) {
|
|
t.Helper()
|
|
probe := seq(16)
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for m.Blocks(replica) == 0 {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the subscriber never received anything")
|
|
}
|
|
p.send(stored(probe, 16))
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
p.send(map[string]any{"type": "AllBlocksCleared"})
|
|
waitFor(t, "the probe to be cleared", func() bool { return m.Blocks(replica) == 0 })
|
|
}
|
|
|
|
func TestTheSubscriberReceivesAnEnginesEventsOverZMQ(t *testing.T) {
|
|
m := NewMirror()
|
|
sub, err := Listen("r1", 0, m, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sub.Close()
|
|
if sub.Port() == 0 {
|
|
t.Fatal("listening on port 0 should report the port chosen")
|
|
}
|
|
pub := newPublisher(t, sub.Endpoint())
|
|
waitSubscribed(t, pub, m, "r1")
|
|
|
|
toks := seq(80)
|
|
pub.send(stored(toks, 16))
|
|
waitFor(t, "five blocks", func() bool { return m.Blocks("r1") == 5 })
|
|
|
|
hs := BlockHashes(toks, 16, 0)
|
|
pub.send(removed(hs[4], hs[3]))
|
|
waitFor(t, "three blocks", func() bool { return m.Blocks("r1") == 3 })
|
|
}
|
|
|
|
func TestLostEventsStartTheIndexAgain(t *testing.T) {
|
|
m := NewMirror()
|
|
sub, err := Listen("r1", 0, m, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sub.Close()
|
|
pub := newPublisher(t, sub.Endpoint())
|
|
waitSubscribed(t, pub, m, "r1")
|
|
|
|
pub.send(stored(seq(64), 16))
|
|
waitFor(t, "four blocks", func() bool { return m.Blocks("r1") == 4 })
|
|
|
|
pub.seq += 3 // three batches never arrive; one may have removed blocks
|
|
other := seq(16)
|
|
for i := range other {
|
|
other[i] += 5000
|
|
}
|
|
pub.send(stored(other, 16))
|
|
waitFor(t, "only the block after the gap", func() bool { return m.Blocks("r1") == 1 })
|
|
}
|
|
|
|
func TestARestartedEngineStartsTheIndexAgain(t *testing.T) {
|
|
m := NewMirror()
|
|
sub, err := Listen("r1", 0, m, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sub.Close()
|
|
pub := newPublisher(t, sub.Endpoint())
|
|
waitSubscribed(t, pub, m, "r1")
|
|
pub.send(stored(seq(64), 16))
|
|
waitFor(t, "four blocks", func() bool { return m.Blocks("r1") == 4 })
|
|
|
|
pub.seq = 0 // a new process numbers from zero again
|
|
pub.send(stored(seq(16), 16))
|
|
waitFor(t, "the new engine's one block", func() bool { return m.Blocks("r1") == 1 })
|
|
}
|
|
|
|
func TestTheEngineIsToldToHashTheWayTheRouterDoes(t *testing.T) {
|
|
args := VLLMArgs("tcp://127.0.0.1:5999")
|
|
if !slices.Contains(args, "sha256_cbor") || !slices.Contains(args, "--kv-events-config") {
|
|
t.Fatalf("args %v", args)
|
|
}
|
|
cfg := args[slices.Index(args, "--kv-events-config")+1]
|
|
if want := `"endpoint":"tcp://127.0.0.1:5999"`; !strings.Contains(cfg, want) {
|
|
t.Fatalf("config %s should name the loopback endpoint, so vLLM connects rather than binding every address", cfg)
|
|
}
|
|
if VLLMEnv()["PYTHONHASHSEED"] != HashSeed {
|
|
t.Fatal("the seed must be the one the router hashes with")
|
|
}
|
|
}
|