package prefix import ( "math" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // Decision is which branch a prefix lookup took. // // Recorded as a metric because tuning prefix routing without it is guesswork: // a cache hit rate says nothing about how often affinity was overridden, and // "affinity is not helping" and "affinity never applied" look identical from // the outside. type Decision string // The branches. const ( Hit Decision = "cache_hit" MissMinLoad Decision = "cache_miss_min_load" LoadOverride Decision = "load_balance_override" NoTree Decision = "no_tree" NoCandidates Decision = "no_healthy_worker" ) // Thresholds decide when the prefix is worth following. type Thresholds struct { // MatchRatio is the share of the prompt that must match. Below it, the // overlap is not enough to save a meaningful prefill. MatchRatio float64 // BalanceAbs and BalanceRel are the imbalance guard, and both must trip // before it does. Without this the cache hit rate rises while p99 latency // collapses: affinity keeps sending a hot prefix to one replica long after // that replica has stopped being able to serve it quickly. // // Absolute alone would fire on an idle cluster where one replica has two // requests and another none. Relative alone would fire on a busy one where // 200 against 180 is a rounding error. Together they mean "meaningfully // more loaded, and by enough to matter". BalanceAbs float64 BalanceRel float64 } // DefaultThresholds returns values that work without measurement. func DefaultThresholds() Thresholds { return Thresholds{MatchRatio: 0.3, BalanceAbs: 8, BalanceRel: 1.5} } func (t *Thresholds) normalize() { d := DefaultThresholds() if t.MatchRatio <= 0 { t.MatchRatio = d.MatchRatio } if t.BalanceAbs <= 0 { t.BalanceAbs = d.BalanceAbs } if t.BalanceRel <= 1 { t.BalanceRel = d.BalanceRel } } // Choose picks the replica a prompt should go to, and says why. // // It returns an index into eps, or -1 when the caller should decide on load // alone — which is the common case and not a failure. func Choose(t *Tree, prompt string, eps []endpoint.Snapshot, th Thresholds) (int, Decision) { th.normalize() if len(eps) == 0 { return -1, NoCandidates } if t == nil || prompt == "" { return -1, NoTree } // The guard runs before the lookup, not after. Checking afterwards would // mean paying for a match only to discard it, and — worse — would invite // the shape where a cache hit quietly wins because somebody moved the // check. if imbalanced(eps, th) { return -1, LoadOverride } replica, matched := t.Match(prompt) if replica == "" || float64(matched)/float64(len(prompt)) < th.MatchRatio { return -1, MissMinLoad } for i, e := range eps { if e.ID == replica { return i, Hit } } // The match names a replica that is no longer a candidate: gone, draining, // or its breaker is open. That is a miss, not an error. return -1, MissMinLoad } // Imbalanced reports whether load is lopsided enough to ignore affinity. The // precise index (tier 2) asks it too: knowing for certain that a replica holds // a prefix does not make it able to serve one more request quickly. func Imbalanced(eps []endpoint.Snapshot, th Thresholds) bool { th.normalize() return imbalanced(eps, th) } // imbalanced reports whether load is lopsided enough to ignore affinity. func imbalanced(eps []endpoint.Snapshot, th Thresholds) bool { if len(eps) < 2 { return false } maxL, minL := math.Inf(-1), math.Inf(1) for _, e := range eps { l := load(e) maxL = math.Max(maxL, l) minL = math.Min(minL, l) } if maxL-minL <= th.BalanceAbs { return false } // A ratio against zero is infinite, and an idle replica is exactly where // the next request should go, so treat it as tripping the guard. if minL <= 0 { return true } return maxL/minL > th.BalanceRel } // load is what the guard compares. // // Queue depth plus what the router has dispatched but not seen finish: the // metric alone lags by a scrape interval, which is long enough for affinity to // pile a burst onto one replica before anything notices. func load(e endpoint.Snapshot) float64 { return float64(e.Stats.Waiting + e.Stats.Running + e.Inflight) }