Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
48 lines
1.6 KiB
Go
48 lines
1.6 KiB
Go
package engines
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestAVLLMShardWithKVEventsPublishesToTheAgentOnLoopback(t *testing.T) {
|
|
s := singleNodeSpec()
|
|
s.Profile.KVEvents = true
|
|
s.KVEventsPort = 5999
|
|
s.Profile.Args = []string{"--max-model-len", "8192"}
|
|
cmd := buildOne(t, &VLLM{}, s)
|
|
|
|
i := slices.Index(cmd.Args, "--kv-events-config")
|
|
if i < 0 || !strings.Contains(cmd.Args[i+1], `"endpoint":"tcp://127.0.0.1:5999"`) {
|
|
t.Fatalf("args %v should name the agent's loopback port", cmd.Args)
|
|
}
|
|
if j := slices.Index(cmd.Args, "--prefix-caching-hash-algo"); j < 0 || cmd.Args[j+1] != "sha256_cbor" {
|
|
t.Fatalf("args %v should pin the hashing the router computes", cmd.Args)
|
|
}
|
|
if slices.Index(cmd.Args, "--max-model-len") < i {
|
|
t.Fatal("profile arguments come last, so an operator can still override")
|
|
}
|
|
if !slices.Contains(cmd.Env, "PYTHONHASHSEED=1729") {
|
|
t.Fatalf("env %v should fix the hash seed", cmd.Env)
|
|
}
|
|
}
|
|
|
|
func TestNoKVEventsWithoutAPortOrOffTheLeader(t *testing.T) {
|
|
s := singleNodeSpec()
|
|
s.Profile.KVEvents = true // but the agent could not listen
|
|
if cmd := buildOne(t, &VLLM{}, s); slices.Contains(cmd.Args, "--kv-events-config") {
|
|
t.Fatal("an engine told to publish where nobody listens only fills a queue")
|
|
}
|
|
w := multiNodeSpec(1)
|
|
w.Profile.KVEvents, w.KVEventsPort = true, 5999
|
|
if cmd := buildEngine(t, &VLLM{}, w); slices.Contains(cmd.Args, "--kv-events-config") {
|
|
t.Fatal("a headless worker has no scheduler to publish from")
|
|
}
|
|
off := singleNodeSpec()
|
|
off.KVEventsPort = 5999
|
|
if cmd := buildOne(t, &VLLM{}, off); slices.Contains(cmd.Args, "--kv-events-config") {
|
|
t.Fatal("kv_events is opt-in")
|
|
}
|
|
}
|