Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
68 lines
2.5 KiB
Go
68 lines
2.5 KiB
Go
package kvindex
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
|
)
|
|
|
|
func TestLeadingStopsAtTheFirstMissingBlock(t *testing.T) {
|
|
x := New()
|
|
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1, 2, 4}})
|
|
m, total, ok := x.Leading("a", map[int][]uint64{16: {1, 2, 3, 4}})
|
|
if !ok || m != 2 || total != 4 {
|
|
t.Fatalf("matched %d of %d (known %v); block 4 is useless without block 3", m, total, ok)
|
|
}
|
|
if _, _, ok := x.Leading("b", nil); ok {
|
|
t.Fatal("a replica never heard from is unknown, not empty")
|
|
}
|
|
}
|
|
|
|
func TestAResetReplacesWhatWasKnown(t *testing.T) {
|
|
x := New()
|
|
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1, 2}})
|
|
x.Apply(kvevents.Update{Replica: "a", Reset: true, Stored: []uint64{9}})
|
|
if m, _, _ := x.Leading("a", map[int][]uint64{16: {1}}); m != 0 {
|
|
t.Fatal("a reset must forget the blocks before it")
|
|
}
|
|
if x.Stats()[0].Blocks != 1 {
|
|
t.Fatal("and keep what came with it")
|
|
}
|
|
}
|
|
|
|
func TestAProblemReplicaIsNotIndexed(t *testing.T) {
|
|
x := New()
|
|
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Stored: []uint64{1}})
|
|
x.Apply(kvevents.Update{Replica: "a", Problem: "hashes differ"})
|
|
if len(x.BlockSizes([]string{"a"})) != 0 {
|
|
t.Fatal("a replica whose hashing is wrong must not be asked about")
|
|
}
|
|
x.Apply(kvevents.Update{Replica: "a", BlockSize: 16, Reset: true, Stored: []uint64{1}})
|
|
if len(x.BlockSizes([]string{"a"})) != 1 {
|
|
t.Fatal("once the problem is gone the replica is indexed again")
|
|
}
|
|
}
|
|
|
|
func TestOnlyWhatChangesTheTokensIsSentToTokenize(t *testing.T) {
|
|
cases := map[string]struct {
|
|
path, body string
|
|
ok bool
|
|
}{
|
|
"chat": {"/v1/chat/completions", `{"model":"x","messages":[{"role":"user","content":"hi"}]}`, true},
|
|
"text parts": {"/v1/chat/completions", `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`, true},
|
|
"image": {"/v1/chat/completions", `{"messages":[{"role":"user","content":[{"type":"image_url"}]}]}`, false},
|
|
"cache salt": {"/v1/chat/completions", `{"messages":[],"cache_salt":"s"}`, false},
|
|
"completion": {"/v1/completions", `{"prompt":"once upon"}`, true},
|
|
"prompt list": {"/v1/completions", `{"prompt":["a","b"]}`, false},
|
|
"embeddings": {"/v1/embeddings", `{"input":"x"}`, false},
|
|
"not json at all": {"/v1/chat/completions", `{`, false},
|
|
}
|
|
for name, c := range cases {
|
|
_, err := TokenizeBody(c.path, []byte(c.body), "served")
|
|
if c.ok != (err == nil) || (err != nil && !errors.Is(err, ErrUnsupported)) {
|
|
t.Errorf("%s: err %v", name, err)
|
|
}
|
|
}
|
|
}
|