Files
llmgw/internal/router/prefix/scorer.go
T
Ilker ManapandClaude Opus 5.5 117f417ef5 Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM
publishing KV cache events to a loopback port it listens on (pure-Go zmq4,
msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed
PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the
existing stream, with a full copy on every new one and a reset on gaps,
restarts and clears. The control plane hands them to the router, for the
sending node's own replicas only. The router tokenizes through a replica's
/tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib
golden values), and scores replicas by the share of leading blocks held,
falling back to the approximate tree. New metrics beside
llmgw_prefix_decisions_total. Untested against a running vLLM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:21:11 +02:00

135 lines
4.2 KiB
Go

package prefix
import (
"math"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Decision is which branch a prefix lookup took.
//
// Recorded as a metric because tuning prefix routing without it is guesswork:
// a cache hit rate says nothing about how often affinity was overridden, and
// "affinity is not helping" and "affinity never applied" look identical from
// the outside.
type Decision string
// The branches.
const (
Hit Decision = "cache_hit"
MissMinLoad Decision = "cache_miss_min_load"
LoadOverride Decision = "load_balance_override"
NoTree Decision = "no_tree"
NoCandidates Decision = "no_healthy_worker"
)
// Thresholds decide when the prefix is worth following.
type Thresholds struct {
// MatchRatio is the share of the prompt that must match. Below it, the
// overlap is not enough to save a meaningful prefill.
MatchRatio float64
// BalanceAbs and BalanceRel are the imbalance guard, and both must trip
// before it does. Without this the cache hit rate rises while p99 latency
// collapses: affinity keeps sending a hot prefix to one replica long after
// that replica has stopped being able to serve it quickly.
//
// Absolute alone would fire on an idle cluster where one replica has two
// requests and another none. Relative alone would fire on a busy one where
// 200 against 180 is a rounding error. Together they mean "meaningfully
// more loaded, and by enough to matter".
BalanceAbs float64
BalanceRel float64
}
// DefaultThresholds returns values that work without measurement.
func DefaultThresholds() Thresholds {
return Thresholds{MatchRatio: 0.3, BalanceAbs: 8, BalanceRel: 1.5}
}
func (t *Thresholds) normalize() {
d := DefaultThresholds()
if t.MatchRatio <= 0 {
t.MatchRatio = d.MatchRatio
}
if t.BalanceAbs <= 0 {
t.BalanceAbs = d.BalanceAbs
}
if t.BalanceRel <= 1 {
t.BalanceRel = d.BalanceRel
}
}
// Choose picks the replica a prompt should go to, and says why.
//
// It returns an index into eps, or -1 when the caller should decide on load
// alone — which is the common case and not a failure.
func Choose(t *Tree, prompt string, eps []endpoint.Snapshot, th Thresholds) (int, Decision) {
th.normalize()
if len(eps) == 0 {
return -1, NoCandidates
}
if t == nil || prompt == "" {
return -1, NoTree
}
// The guard runs before the lookup, not after. Checking afterwards would
// mean paying for a match only to discard it, and — worse — would invite
// the shape where a cache hit quietly wins because somebody moved the
// check.
if imbalanced(eps, th) {
return -1, LoadOverride
}
replica, matched := t.Match(prompt)
if replica == "" || float64(matched)/float64(len(prompt)) < th.MatchRatio {
return -1, MissMinLoad
}
for i, e := range eps {
if e.ID == replica {
return i, Hit
}
}
// The match names a replica that is no longer a candidate: gone, draining,
// or its breaker is open. That is a miss, not an error.
return -1, MissMinLoad
}
// Imbalanced reports whether load is lopsided enough to ignore affinity. The
// precise index (tier 2) asks it too: knowing for certain that a replica holds
// a prefix does not make it able to serve one more request quickly.
func Imbalanced(eps []endpoint.Snapshot, th Thresholds) bool {
th.normalize()
return imbalanced(eps, th)
}
// imbalanced reports whether load is lopsided enough to ignore affinity.
func imbalanced(eps []endpoint.Snapshot, th Thresholds) bool {
if len(eps) < 2 {
return false
}
maxL, minL := math.Inf(-1), math.Inf(1)
for _, e := range eps {
l := load(e)
maxL = math.Max(maxL, l)
minL = math.Min(minL, l)
}
if maxL-minL <= th.BalanceAbs {
return false
}
// A ratio against zero is infinite, and an idle replica is exactly where
// the next request should go, so treat it as tripping the guard.
if minL <= 0 {
return true
}
return maxL/minL > th.BalanceRel
}
// load is what the guard compares.
//
// Queue depth plus what the router has dispatched but not seen finish: the
// metric alone lags by a scrape interval, which is long enough for affinity to
// pile a burst onto one replica before anything notices.
func load(e endpoint.Snapshot) float64 {
return float64(e.Stats.Waiting + e.Stats.Running + e.Inflight)
}