Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
135 lines
4.2 KiB
Go
135 lines
4.2 KiB
Go
package prefix
|
|
|
|
import (
|
|
"math"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Decision is which branch a prefix lookup took.
|
|
//
|
|
// Recorded as a metric because tuning prefix routing without it is guesswork:
|
|
// a cache hit rate says nothing about how often affinity was overridden, and
|
|
// "affinity is not helping" and "affinity never applied" look identical from
|
|
// the outside.
|
|
type Decision string
|
|
|
|
// The branches.
|
|
const (
|
|
Hit Decision = "cache_hit"
|
|
MissMinLoad Decision = "cache_miss_min_load"
|
|
LoadOverride Decision = "load_balance_override"
|
|
NoTree Decision = "no_tree"
|
|
NoCandidates Decision = "no_healthy_worker"
|
|
)
|
|
|
|
// Thresholds decide when the prefix is worth following.
|
|
type Thresholds struct {
|
|
// MatchRatio is the share of the prompt that must match. Below it, the
|
|
// overlap is not enough to save a meaningful prefill.
|
|
MatchRatio float64
|
|
// BalanceAbs and BalanceRel are the imbalance guard, and both must trip
|
|
// before it does. Without this the cache hit rate rises while p99 latency
|
|
// collapses: affinity keeps sending a hot prefix to one replica long after
|
|
// that replica has stopped being able to serve it quickly.
|
|
//
|
|
// Absolute alone would fire on an idle cluster where one replica has two
|
|
// requests and another none. Relative alone would fire on a busy one where
|
|
// 200 against 180 is a rounding error. Together they mean "meaningfully
|
|
// more loaded, and by enough to matter".
|
|
BalanceAbs float64
|
|
BalanceRel float64
|
|
}
|
|
|
|
// DefaultThresholds returns values that work without measurement.
|
|
func DefaultThresholds() Thresholds {
|
|
return Thresholds{MatchRatio: 0.3, BalanceAbs: 8, BalanceRel: 1.5}
|
|
}
|
|
|
|
func (t *Thresholds) normalize() {
|
|
d := DefaultThresholds()
|
|
if t.MatchRatio <= 0 {
|
|
t.MatchRatio = d.MatchRatio
|
|
}
|
|
if t.BalanceAbs <= 0 {
|
|
t.BalanceAbs = d.BalanceAbs
|
|
}
|
|
if t.BalanceRel <= 1 {
|
|
t.BalanceRel = d.BalanceRel
|
|
}
|
|
}
|
|
|
|
// Choose picks the replica a prompt should go to, and says why.
|
|
//
|
|
// It returns an index into eps, or -1 when the caller should decide on load
|
|
// alone — which is the common case and not a failure.
|
|
func Choose(t *Tree, prompt string, eps []endpoint.Snapshot, th Thresholds) (int, Decision) {
|
|
th.normalize()
|
|
if len(eps) == 0 {
|
|
return -1, NoCandidates
|
|
}
|
|
if t == nil || prompt == "" {
|
|
return -1, NoTree
|
|
}
|
|
|
|
// The guard runs before the lookup, not after. Checking afterwards would
|
|
// mean paying for a match only to discard it, and — worse — would invite
|
|
// the shape where a cache hit quietly wins because somebody moved the
|
|
// check.
|
|
if imbalanced(eps, th) {
|
|
return -1, LoadOverride
|
|
}
|
|
|
|
replica, matched := t.Match(prompt)
|
|
if replica == "" || float64(matched)/float64(len(prompt)) < th.MatchRatio {
|
|
return -1, MissMinLoad
|
|
}
|
|
for i, e := range eps {
|
|
if e.ID == replica {
|
|
return i, Hit
|
|
}
|
|
}
|
|
// The match names a replica that is no longer a candidate: gone, draining,
|
|
// or its breaker is open. That is a miss, not an error.
|
|
return -1, MissMinLoad
|
|
}
|
|
|
|
// Imbalanced reports whether load is lopsided enough to ignore affinity. The
|
|
// precise index (tier 2) asks it too: knowing for certain that a replica holds
|
|
// a prefix does not make it able to serve one more request quickly.
|
|
func Imbalanced(eps []endpoint.Snapshot, th Thresholds) bool {
|
|
th.normalize()
|
|
return imbalanced(eps, th)
|
|
}
|
|
|
|
// imbalanced reports whether load is lopsided enough to ignore affinity.
|
|
func imbalanced(eps []endpoint.Snapshot, th Thresholds) bool {
|
|
if len(eps) < 2 {
|
|
return false
|
|
}
|
|
maxL, minL := math.Inf(-1), math.Inf(1)
|
|
for _, e := range eps {
|
|
l := load(e)
|
|
maxL = math.Max(maxL, l)
|
|
minL = math.Min(minL, l)
|
|
}
|
|
if maxL-minL <= th.BalanceAbs {
|
|
return false
|
|
}
|
|
// A ratio against zero is infinite, and an idle replica is exactly where
|
|
// the next request should go, so treat it as tripping the guard.
|
|
if minL <= 0 {
|
|
return true
|
|
}
|
|
return maxL/minL > th.BalanceRel
|
|
}
|
|
|
|
// load is what the guard compares.
|
|
//
|
|
// Queue depth plus what the router has dispatched but not seen finish: the
|
|
// metric alone lags by a scrape interval, which is long enough for affinity to
|
|
// pile a burst onto one replica before anything notices.
|
|
func load(e endpoint.Snapshot) float64 {
|
|
return float64(e.Stats.Waiting + e.Stats.Running + e.Inflight)
|
|
}
|