Ilker ManapandClaude Opus 5.5
117f417ef5
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
30 lines
1020 B
Go
30 lines
1020 B
Go
package agentsvc
|
|
|
|
import (
|
|
"log/slog"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
|
)
|
|
|
|
func TestKVEventsAreTakenOnlyForTheNodesOwnReplicas(t *testing.T) {
|
|
s := &Service{log: slog.New(slog.DiscardHandler)}
|
|
var got []kvevents.Update
|
|
s.SetKVSink(func(u kvevents.Update) { got = append(got, u) })
|
|
c := &conn{nodeID: "n1"}
|
|
c.own("r1")
|
|
|
|
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r1", BlockSize: 16, Reset: true, Stored: []uint64{7}}).GetKvEvents())
|
|
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r-elsewhere", Stored: []uint64{8}}).GetKvEvents())
|
|
if len(got) != 1 || got[0].Replica != "r1" || !got[0].Reset || got[0].Stored[0] != 7 || got[0].BlockSize != 16 {
|
|
t.Fatalf("got %+v; a node may describe its own replicas' caches and no others", got)
|
|
}
|
|
|
|
// The stream closing empties what it described: no more evictions will
|
|
// arrive by it.
|
|
s.kvGone(c)
|
|
if len(got) != 2 || got[1].Replica != "r1" || !got[1].Reset || len(got[1].Stored) != 0 {
|
|
t.Fatalf("got %+v", got)
|
|
}
|
|
}
|