Files
llmgw/internal/router/kvindex/kvindex_test.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

68 lines
2.5 KiB
Go

package kvindex
import (
"errors"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
func TestLeadingStopsAtTheFirstMissingBlock(t *testing.T) {
x := New()
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1, 2, 4}})
m, total, ok := x.Leading("a", map[int][]uint64{16: {1, 2, 3, 4}})
if !ok || m != 2 || total != 4 {
t.Fatalf("matched %d of %d (known %v); block 4 is useless without block 3", m, total, ok)
}
if _, _, ok := x.Leading("b", nil); ok {
t.Fatal("a replica never heard from is unknown, not empty")
}
}
func TestAResetReplacesWhatWasKnown(t *testing.T) {
x := New()
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1, 2}})
x.Apply(kvevents.Update{Replica: "a", Reset: true, Stored: []uint64{9}})
if m, _, _ := x.Leading("a", map[int][]uint64{16: {1}}); m != 0 {
t.Fatal("a reset must forget the blocks before it")
}
if x.Stats()[0].Blocks != 1 {
t.Fatal("and keep what came with it")
}
}
func TestAProblemReplicaIsNotIndexed(t *testing.T) {
x := New()
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1}})
x.Apply(kvevents.Update{Replica: "a", Problem: "hashes differ"})
if len(x.BlockSizes([]string{"a"})) != 0 {
t.Fatal("a replica whose hashing is wrong must not be asked about")
}
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Reset: true, Stored: []uint64{1}})
if len(x.BlockSizes([]string{"a"})) != 1 {
t.Fatal("once the problem is gone the replica is indexed again")
}
}
func TestOnlyWhatChangesTheTokensIsSentToTokenize(t *testing.T) {
cases := map[string]struct {
path, body string
ok bool
}{
"chat": {"/v1/chat/completions", `{"model":"x","messages":[{"role":"user","content":"hi"}]}`, true},
"text parts": {"/v1/chat/completions", `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, true},
"image": {"/v1/chat/completions", `{"messages":[{"role":"user","content":[{"type":"image_url"}]}]}`, false},
"cache salt": {"/v1/chat/completions", `{"messages":[],"cache_salt":"s"}`, false},
"completion": {"/v1/completions", `{"prompt":"once upon"}`, true},
"prompt list": {"/v1/completions", `{"prompt":["a","b"]}`, false},
"embeddings": {"/v1/embeddings", `{"input":"x"}`, false},
"not json at all": {"/v1/chat/completions", `{`, false},
}
for name, c := range cases {
_, err := TokenizeBody(c.path, []byte(c.body), "served")
if c.ok != (err == nil) || (err != nil && !errors.Is(err, ErrUnsupported)) {
t.Errorf("%s: err %v", name, err)
}
}
}