Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
41 lines
1.1 KiB
Go
41 lines
1.1 KiB
Go
package catalog
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestKVEventsReachTheEngineAndTheRouter(t *testing.T) {
|
|
c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 1, KVEvents: true})})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, _ := c.Resolve("m")
|
|
if !m.KVIndex() {
|
|
t.Fatal("a model with a profile publishing events should be routed by them")
|
|
}
|
|
if ep, _ := m.EngineProfile("p"); !ep.KVEvents {
|
|
t.Fatal("the engine has to be told to publish")
|
|
}
|
|
plain, _ := New([]Model{model("n")})
|
|
if n, _ := plain.Resolve("n"); n.KVIndex() {
|
|
t.Fatal("off by default")
|
|
}
|
|
}
|
|
|
|
func TestKVEventsAreVLLMsOnly(t *testing.T) {
|
|
_, err := New([]Model{model("m", Profile{Name: "p", Engine: "llamacpp", KVEvents: true})})
|
|
if err == nil || !strings.Contains(err.Error(), "kv_events") {
|
|
t.Fatalf("err %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTurningKVEventsOnMarksRunningReplicasStale(t *testing.T) {
|
|
p := Profile{Name: "p", GPUClass: "h100-80g", TP: 1}
|
|
q := p
|
|
q.KVEvents = true
|
|
if p.Fingerprint() == q.Fingerprint() {
|
|
t.Fatal("replicas started without events do not publish until replaced, and should be shown as stale")
|
|
}
|
|
}
|