Files
llmgw/internal/router/scheduler/scorers.go
T
Ilker ManapandClaude Opus 5.5 73e1bb6733 Chat goes to the replica that has been answering soonest
The router keeps each endpoint's time to first token, smoothed over its last
streamed answers, and the chat profile scores by it beside session, queue,
KV and inflight: the fastest endpoint 1, one twice as slow 0.5, one not yet
measured 1 so a new replica is tried. Load still sees a slow drain from the
engine's own counters; this sees what answers actually took. BACKLOG L4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 17:01:30 +01:00

214 lines
5.5 KiB
Go

package scheduler
import (
"math"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// scorerFunc helps define scorers that rate each endpoint independently.
type scorerFunc struct {
name string
f func(c *Ctx, e endpoint.Snapshot) float64
}
func (s scorerFunc) Name() string { return s.name }
func (s scorerFunc) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
out := make([]float64, len(eps))
for i, e := range eps {
out[i] = clamp01(s.f(c, e))
}
return out
}
func clamp01(v float64) float64 {
if math.IsNaN(v) {
return 0
}
return min(max(v, 0), 1)
}
// Queue scores by queue depth: 1 - min(1, waiting/target).
//
// target is the "healthy buffer" size (plan §7.2.1): enough queued requests
// for continuous batching to stay efficient, but not so many that queueing
// delay creeps back in. Measure it with ctl bench rather than guessing.
func Queue(target float64) Scorer {
if target <= 0 {
target = 4
}
return scorerFunc{
name: "queue",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
return 1 - float64(e.Stats.Waiting)/target
},
}
}
// KV scores by KV cache occupancy: 1 - usage.
//
// Above the ceiling a hard penalty applies: going from 90% to 95% is not the
// same as going from 10% to 15% — a replica near the ceiling starts preempting
// and dropping requests.
func KV(ceiling float64) Scorer {
if ceiling <= 0 || ceiling > 1 {
ceiling = 0.9
}
return scorerFunc{
name: "kv",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
u := e.Stats.KVCacheUsage
s := 1 - u
if u >= ceiling {
s *= 0.1
}
return s
},
}
}
// Inflight scores by the unfinished requests the router itself is tracking.
//
// With a 2s scrape interval the engine metrics are stale at decision time, so
// back-to-back requests all see the same "idle" replica and all pile onto it.
// This scorer closes that gap, which is why every profile keeps it on (§7.6).
func Inflight(target float64) Scorer {
if target <= 0 {
target = 8
}
return scorerFunc{
name: "inflight",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
w := e.Weight
if w <= 0 {
w = 1
}
return 1 - float64(e.Inflight)/(target*w)
},
}
}
// Load is AIBrix's load-balance score: running / drain_rate.
//
// Running alone cannot spot a slow replica — one processing the same number of
// requests at half the speed is twice as loaded. While drain_rate is still
// unmeasured (the first seconds) it returns a neutral 1 and leaves the decision
// to the other scorers.
func Load(targetSeconds float64) Scorer {
if targetSeconds <= 0 {
targetSeconds = 30
}
return scorerFunc{
name: "load",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
if e.DrainRate <= 0 {
return 1
}
// Estimated seconds to drain, normalized against the target.
drainTime := float64(e.Stats.Running) / e.DrainRate
return 1 - drainTime/targetSeconds
},
}
}
// Weight scores by static capacity weight.
//
// In heterogeneous pools (A100s alongside H100s) raw queue length misleads:
// the same queue drains faster on the faster card. The weight comes from
// ctl bench (§6.2.1) and should never be guessed by hand.
func Weight() Scorer {
return weightScorer{}
}
type weightScorer struct{}
func (weightScorer) Name() string { return "weight" }
func (weightScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
maxW := 0.0
for _, e := range eps {
w := e.Weight
if w <= 0 {
w = 1
}
maxW = max(maxW, w)
}
out := make([]float64, len(eps))
if maxW <= 0 {
return out
}
for i, e := range eps {
w := e.Weight
if w <= 0 {
w = 1
}
out[i] = w / maxW
}
return out
}
// Session scores session affinity (plan §7.5, tier 0).
//
// Bounded-load consistent hashing: a session maps to a replica, but if that
// replica carries more than loadFactor times the average it moves on around the
// ring. Unbounded stickiness drowns a single replica: cache hits go up while
// p99 collapses.
//
// With no session id every candidate scores 0, leaving the decision to the
// other scorers.
func Session(loadFactor float64) Scorer {
if loadFactor < 1 {
loadFactor = 1.25
}
return sessionScorer{loadFactor: loadFactor}
}
type sessionScorer struct{ loadFactor float64 }
func (sessionScorer) Name() string { return "session" }
func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
out := make([]float64, len(eps))
if c.Req.SessionID == "" || len(eps) == 0 {
return out
}
if idx := pickBoundedLoad(c.Req.SessionID, eps, s.loadFactor); idx >= 0 {
out[idx] = 1
}
return out
}
// Latency scores by time to first token as this router has seen it: the
// fastest endpoint 1, one twice as slow 0.5. Kong and others balance by it;
// for chat it is what a person waits for. An endpoint not yet measured
// scores 1, so a new replica is tried rather than avoided.
//
// Beside Load, not instead: Load sees a replica draining slowly from its
// own counters, before any answer of this router's has come back from it;
// this sees what answers actually took, a slow card or a busy host included.
func Latency() Scorer { return latencyScorer{} }
type latencyScorer struct{}
func (latencyScorer) Name() string { return "latency" }
func (latencyScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
best := time.Duration(0)
for _, e := range eps {
if e.TTFT > 0 && (best == 0 || e.TTFT < best) {
best = e.TTFT
}
}
out := make([]float64, len(eps))
for i, e := range eps {
if e.TTFT <= 0 || best == 0 {
out[i] = 1
continue
}
out[i] = float64(best) / float64(e.TTFT)
}
return out
}