Files
llmgw/internal/control/catalog/kvevents_test.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

41 lines
1.1 KiB
Go

package catalog
import (
"strings"
"testing"
)
func TestKVEventsReachTheEngineAndTheRouter(t *testing.T) {
c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 1, KVEvents: true})})
if err != nil {
t.Fatal(err)
}
m, _ := c.Resolve("m")
if !m.KVIndex() {
t.Fatal("a model with a profile publishing events should be routed by them")
}
if ep, _ := m.EngineProfile("p"); !ep.KVEvents {
t.Fatal("the engine has to be told to publish")
}
plain, _ := New([]Model{model("n")})
if n, _ := plain.Resolve("n"); n.KVIndex() {
t.Fatal("off by default")
}
}
func TestKVEventsAreVLLMsOnly(t *testing.T) {
_, err := New([]Model{model("m", Profile{Name: "p", Engine: "llamacpp", KVEvents: true})})
if err == nil || !strings.Contains(err.Error(), "kv_events") {
t.Fatalf("err %v", err)
}
}
func TestTurningKVEventsOnMarksRunningReplicasStale(t *testing.T) {
p := Profile{Name: "p", GPUClass: "h100-80g", TP: 1}
q := p
q.KVEvents = true
if p.Fingerprint() == q.Fingerprint() {
t.Fatal("replicas started without events do not publish until replaced, and should be shown as stale")
}
}