Ilker ManapandClaude Opus 5.5
117f417ef5
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
50 lines
1.5 KiB
Go
50 lines
1.5 KiB
Go
package kvevents
|
|
|
|
import (
|
|
"os"
|
|
"os/exec"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// TestInteropWithPyZMQ checks the subscriber against libzmq and msgspec, the
|
|
// libraries vLLM publishes with, by running testdata/pypub.py under uv. Off
|
|
// unless LLMGW_PYZMQ_INTEROP is set: it needs uv and the network to fetch
|
|
// pyzmq and msgspec. It is not vLLM — no engine runs — but it is the wire
|
|
// format from the libraries vLLM uses, with vLLM's struct definitions.
|
|
func TestInteropWithPyZMQ(t *testing.T) {
|
|
if os.Getenv("LLMGW_PYZMQ_INTEROP") == "" {
|
|
t.Skip("set LLMGW_PYZMQ_INTEROP=1 to run against pyzmq and msgspec (needs uv)")
|
|
}
|
|
m := NewMirror()
|
|
sub, err := Listen("r1", 0, m, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sub.Close()
|
|
cmd := exec.Command("uv", "run", "--quiet", "--with", "pyzmq", "--with", "msgspec",
|
|
"python", "testdata/pypub.py", sub.Endpoint())
|
|
cmd.Stderr = os.Stderr
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = cmd.Wait() }()
|
|
|
|
waitForWithin(t, 60*time.Second, "pyzmq's events", func() bool { return m.Blocks("r1") == 3 })
|
|
if m.Problem("r1") != "" {
|
|
t.Fatalf("hashes made by the golden chain should verify: %s", m.Problem("r1"))
|
|
}
|
|
waitForWithin(t, 10*time.Second, "the removal", func() bool { return m.Blocks("r1") == 1 })
|
|
}
|
|
|
|
func waitForWithin(t *testing.T, d time.Duration, what string, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(d)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|