Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
43 lines
1.6 KiB
Go
43 lines
1.6 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/kvindex"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
)
|
|
|
|
func kvScorer(got *[]kvindex.Decision) *PrefixScorer {
|
|
return NewPrefixScorer(nil, prefix.DefaultThresholds(), nil).
|
|
WithKV(2, func(d kvindex.Decision) { *got = append(*got, d) })
|
|
}
|
|
|
|
func TestReplicasAreScoredByTheShareOfThePromptTheyHold(t *testing.T) {
|
|
var got []kvindex.Decision
|
|
q := &KVQuery{Matched: map[string]int{"a": 2, "b": 8}, Total: map[string]int{"a": 8, "b": 8}}
|
|
s := kvScorer(&got).Score(newCtx(Request{KV: q}), []endpoint.Snapshot{ep("a"), ep("b"), ep("c")})
|
|
if s[0] != 0.25 || s[1] != 1 || s[2] != 0 || got[0] != kvindex.Hit {
|
|
t.Fatalf("scores %v, decisions %v", s, got)
|
|
}
|
|
}
|
|
|
|
func TestACertainHitIsStillIgnoredWhenLoadIsLopsided(t *testing.T) {
|
|
var got []kvindex.Decision
|
|
q := &KVQuery{Matched: map[string]int{"a": 8}, Total: map[string]int{"a": 8, "b": 8}}
|
|
busy := ep("a", func(s *endpoint.Snapshot) { s.Stats.Waiting = 40 })
|
|
s := kvScorer(&got).Score(newCtx(Request{KV: q}), []endpoint.Snapshot{busy, ep("b")})
|
|
if s[0] != 0 || got[0] != kvindex.LoadOverride {
|
|
t.Fatalf("scores %v, decisions %v", s, got)
|
|
}
|
|
}
|
|
|
|
func TestAFallbackWithTier1OffScoresNothing(t *testing.T) {
|
|
var got []kvindex.Decision
|
|
q := &KVQuery{Fallback: kvindex.FallbackNoIndex}
|
|
s := kvScorer(&got).Score(newCtx(Request{KV: q, PromptPrefix: "x"}), []endpoint.Snapshot{ep("a"), ep("b")})
|
|
if s[0] != 0 || s[1] != 0 || len(got) != 1 || got[0] != kvindex.FallbackNoIndex {
|
|
t.Fatalf("scores %v, decisions %v", s, got)
|
|
}
|
|
}
|