Files
llmgw/internal/router/scheduler/kv_test.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

43 lines
1.6 KiB
Go

package scheduler
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/kvindex"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
)
func kvScorer(got *[]kvindex.Decision) *PrefixScorer {
return NewPrefixScorer(nil, prefix.DefaultThresholds(), nil).
WithKV(2, func(d kvindex.Decision) { *got = append(*got, d) })
}
func TestReplicasAreScoredByTheShareOfThePromptTheyHold(t *testing.T) {
var got []kvindex.Decision
q := &KVQuery{Matched: map[string]int{"a": 2, "b": 8}, Total: map[string]int{"a": 8, "b": 8}}
s := kvScorer(&got).Score(newCtx(Request{KV: q}), []endpoint.Snapshot{ep("a"), ep("b"), ep("c")})
if s[0] != 0.25 || s[1] != 1 || s[2] != 0 || got[0] != kvindex.Hit {
t.Fatalf("scores %v, decisions %v", s, got)
}
}
func TestACertainHitIsStillIgnoredWhenLoadIsLopsided(t *testing.T) {
var got []kvindex.Decision
q := &KVQuery{Matched: map[string]int{"a": 8}, Total: map[string]int{"a": 8, "b": 8}}
busy := ep("a", func(s *endpoint.Snapshot) { s.Stats.Waiting = 40 })
s := kvScorer(&got).Score(newCtx(Request{KV: q}), []endpoint.Snapshot{busy, ep("b")})
if s[0] != 0 || got[0] != kvindex.LoadOverride {
t.Fatalf("scores %v, decisions %v", s, got)
}
}
func TestAFallbackWithTier1OffScoresNothing(t *testing.T) {
var got []kvindex.Decision
q := &KVQuery{Fallback: kvindex.FallbackNoIndex}
s := kvScorer(&got).Score(newCtx(Request{KV: q, PromptPrefix: "x"}), []endpoint.Snapshot{ep("a"), ep("b")})
if s[0] != 0 || s[1] != 0 || len(got) != 1 || got[0] != kvindex.FallbackNoIndex {
t.Fatalf("scores %v, decisions %v", s, got)
}
}