The router keeps each endpoint's time to first token, smoothed over its last streamed answers, and the chat profile scores by it beside session, queue, KV and inflight: the fastest endpoint 1, one twice as slow 0.5, one not yet measured 1 so a new replica is tried. Load still sees a slow drain from the engine's own counters; this sees what answers actually took. BACKLOG L4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
214 lines
5.5 KiB
Go
214 lines
5.5 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"math"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// scorerFunc helps define scorers that rate each endpoint independently.
|
|
type scorerFunc struct {
|
|
name string
|
|
f func(c *Ctx, e endpoint.Snapshot) float64
|
|
}
|
|
|
|
func (s scorerFunc) Name() string { return s.name }
|
|
|
|
func (s scorerFunc) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
out := make([]float64, len(eps))
|
|
for i, e := range eps {
|
|
out[i] = clamp01(s.f(c, e))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func clamp01(v float64) float64 {
|
|
if math.IsNaN(v) {
|
|
return 0
|
|
}
|
|
return min(max(v, 0), 1)
|
|
}
|
|
|
|
// Queue scores by queue depth: 1 - min(1, waiting/target).
|
|
//
|
|
// target is the "healthy buffer" size (plan §7.2.1): enough queued requests
|
|
// for continuous batching to stay efficient, but not so many that queueing
|
|
// delay creeps back in. Measure it with ctl bench rather than guessing.
|
|
func Queue(target float64) Scorer {
|
|
if target <= 0 {
|
|
target = 4
|
|
}
|
|
return scorerFunc{
|
|
name: "queue",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
return 1 - float64(e.Stats.Waiting)/target
|
|
},
|
|
}
|
|
}
|
|
|
|
// KV scores by KV cache occupancy: 1 - usage.
|
|
//
|
|
// Above the ceiling a hard penalty applies: going from 90% to 95% is not the
|
|
// same as going from 10% to 15% — a replica near the ceiling starts preempting
|
|
// and dropping requests.
|
|
func KV(ceiling float64) Scorer {
|
|
if ceiling <= 0 || ceiling > 1 {
|
|
ceiling = 0.9
|
|
}
|
|
return scorerFunc{
|
|
name: "kv",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
u := e.Stats.KVCacheUsage
|
|
s := 1 - u
|
|
if u >= ceiling {
|
|
s *= 0.1
|
|
}
|
|
return s
|
|
},
|
|
}
|
|
}
|
|
|
|
// Inflight scores by the unfinished requests the router itself is tracking.
|
|
//
|
|
// With a 2s scrape interval the engine metrics are stale at decision time, so
|
|
// back-to-back requests all see the same "idle" replica and all pile onto it.
|
|
// This scorer closes that gap, which is why every profile keeps it on (§7.6).
|
|
func Inflight(target float64) Scorer {
|
|
if target <= 0 {
|
|
target = 8
|
|
}
|
|
return scorerFunc{
|
|
name: "inflight",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
return 1 - float64(e.Inflight)/(target*w)
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load is AIBrix's load-balance score: running / drain_rate.
|
|
//
|
|
// Running alone cannot spot a slow replica — one processing the same number of
|
|
// requests at half the speed is twice as loaded. While drain_rate is still
|
|
// unmeasured (the first seconds) it returns a neutral 1 and leaves the decision
|
|
// to the other scorers.
|
|
func Load(targetSeconds float64) Scorer {
|
|
if targetSeconds <= 0 {
|
|
targetSeconds = 30
|
|
}
|
|
return scorerFunc{
|
|
name: "load",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
if e.DrainRate <= 0 {
|
|
return 1
|
|
}
|
|
// Estimated seconds to drain, normalized against the target.
|
|
drainTime := float64(e.Stats.Running) / e.DrainRate
|
|
return 1 - drainTime/targetSeconds
|
|
},
|
|
}
|
|
}
|
|
|
|
// Weight scores by static capacity weight.
|
|
//
|
|
// In heterogeneous pools (A100s alongside H100s) raw queue length misleads:
|
|
// the same queue drains faster on the faster card. The weight comes from
|
|
// ctl bench (§6.2.1) and should never be guessed by hand.
|
|
func Weight() Scorer {
|
|
return weightScorer{}
|
|
}
|
|
|
|
type weightScorer struct{}
|
|
|
|
func (weightScorer) Name() string { return "weight" }
|
|
|
|
func (weightScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
maxW := 0.0
|
|
for _, e := range eps {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
maxW = max(maxW, w)
|
|
}
|
|
out := make([]float64, len(eps))
|
|
if maxW <= 0 {
|
|
return out
|
|
}
|
|
for i, e := range eps {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
out[i] = w / maxW
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Session scores session affinity (plan §7.5, tier 0).
|
|
//
|
|
// Bounded-load consistent hashing: a session maps to a replica, but if that
|
|
// replica carries more than loadFactor times the average it moves on around the
|
|
// ring. Unbounded stickiness drowns a single replica: cache hits go up while
|
|
// p99 collapses.
|
|
//
|
|
// With no session id every candidate scores 0, leaving the decision to the
|
|
// other scorers.
|
|
func Session(loadFactor float64) Scorer {
|
|
if loadFactor < 1 {
|
|
loadFactor = 1.25
|
|
}
|
|
return sessionScorer{loadFactor: loadFactor}
|
|
}
|
|
|
|
type sessionScorer struct{ loadFactor float64 }
|
|
|
|
func (sessionScorer) Name() string { return "session" }
|
|
|
|
func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
out := make([]float64, len(eps))
|
|
if c.Req.SessionID == "" || len(eps) == 0 {
|
|
return out
|
|
}
|
|
if idx := pickBoundedLoad(c.Req.SessionID, eps, s.loadFactor); idx >= 0 {
|
|
out[idx] = 1
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Latency scores by time to first token as this router has seen it: the
|
|
// fastest endpoint 1, one twice as slow 0.5. Kong and others balance by it;
|
|
// for chat it is what a person waits for. An endpoint not yet measured
|
|
// scores 1, so a new replica is tried rather than avoided.
|
|
//
|
|
// Beside Load, not instead: Load sees a replica draining slowly from its
|
|
// own counters, before any answer of this router's has come back from it;
|
|
// this sees what answers actually took, a slow card or a busy host included.
|
|
func Latency() Scorer { return latencyScorer{} }
|
|
|
|
type latencyScorer struct{}
|
|
|
|
func (latencyScorer) Name() string { return "latency" }
|
|
|
|
func (latencyScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
best := time.Duration(0)
|
|
for _, e := range eps {
|
|
if e.TTFT > 0 && (best == 0 || e.TTFT < best) {
|
|
best = e.TTFT
|
|
}
|
|
}
|
|
out := make([]float64, len(eps))
|
|
for i, e := range eps {
|
|
if e.TTFT <= 0 || best == 0 {
|
|
out[i] = 1
|
|
continue
|
|
}
|
|
out[i] = float64(best) / float64(e.TTFT)
|
|
}
|
|
return out
|
|
}
|