Files
llmgw/internal/agent/engines/kvevents_test.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

48 lines
1.6 KiB
Go

package engines
import (
"slices"
"strings"
"testing"
)
func TestAVLLMShardWithKVEventsPublishesToTheAgentOnLoopback(t *testing.T) {
s := singleNodeSpec()
s.Profile.KVEvents = true
s.KVEventsPort = 5999
s.Profile.Args = []string{"--max-model-len", "8192"}
cmd := buildOne(t, &VLLM{}, s)
i := slices.Index(cmd.Args, "--kv-events-config")
if i < 0 || !strings.Contains(cmd.Args[i+1], `"endpoint":"tcp://127.0.0.1:5999"`) {
t.Fatalf("args %v should name the agent's loopback port", cmd.Args)
}
if j := slices.Index(cmd.Args, "--prefix-caching-hash-algo"); j < 0 || cmd.Args[j+1] != "sha256_cbor" {
t.Fatalf("args %v should pin the hashing the router computes", cmd.Args)
}
if slices.Index(cmd.Args, "--max-model-len") < i {
t.Fatal("profile arguments come last, so an operator can still override")
}
if !slices.Contains(cmd.Env, "PYTHONHASHSEED=1729") {
t.Fatalf("env %v should fix the hash seed", cmd.Env)
}
}
func TestNoKVEventsWithoutAPortOrOffTheLeader(t *testing.T) {
s := singleNodeSpec()
s.Profile.KVEvents = true // but the agent could not listen
if cmd := buildOne(t, &VLLM{}, s); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("an engine told to publish where nobody listens only fills a queue")
}
w := multiNodeSpec(1)
w.Profile.KVEvents, w.KVEventsPort = true, 5999
if cmd := buildEngine(t, &VLLM{}, w); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("a headless worker has no scheduler to publish from")
}
off := singleNodeSpec()
off.KVEventsPort = 5999
if cmd := buildOne(t, &VLLM{}, off); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("kv_events is opt-in")
}
}