The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
181 lines
4.5 KiB
Go
181 lines
4.5 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"math"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// scorerFunc helps define scorers that rate each endpoint independently.
|
|
type scorerFunc struct {
|
|
name string
|
|
f func(c *Ctx, e endpoint.Snapshot) float64
|
|
}
|
|
|
|
func (s scorerFunc) Name() string { return s.name }
|
|
|
|
func (s scorerFunc) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
out := make([]float64, len(eps))
|
|
for i, e := range eps {
|
|
out[i] = clamp01(s.f(c, e))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func clamp01(v float64) float64 {
|
|
if math.IsNaN(v) {
|
|
return 0
|
|
}
|
|
return min(max(v, 0), 1)
|
|
}
|
|
|
|
// Queue scores by queue depth: 1 - min(1, waiting/target).
|
|
//
|
|
// target is the "healthy buffer" size (plan §7.2.1): enough queued requests
|
|
// for continuous batching to stay efficient, but not so many that queueing
|
|
// delay creeps back in. Measure it with ctl bench rather than guessing.
|
|
func Queue(target float64) Scorer {
|
|
if target <= 0 {
|
|
target = 4
|
|
}
|
|
return scorerFunc{
|
|
name: "queue",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
return 1 - float64(e.Stats.Waiting)/target
|
|
},
|
|
}
|
|
}
|
|
|
|
// KV scores by KV cache occupancy: 1 - usage.
|
|
//
|
|
// Above the ceiling a hard penalty applies: going from 90% to 95% is not the
|
|
// same as going from 10% to 15% — a replica near the ceiling starts preempting
|
|
// and dropping requests.
|
|
func KV(ceiling float64) Scorer {
|
|
if ceiling <= 0 || ceiling > 1 {
|
|
ceiling = 0.9
|
|
}
|
|
return scorerFunc{
|
|
name: "kv",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
u := e.Stats.KVCacheUsage
|
|
s := 1 - u
|
|
if u >= ceiling {
|
|
s *= 0.1
|
|
}
|
|
return s
|
|
},
|
|
}
|
|
}
|
|
|
|
// Inflight scores by the unfinished requests the router itself is tracking.
|
|
//
|
|
// With a 2s scrape interval the engine metrics are stale at decision time, so
|
|
// back-to-back requests all see the same "idle" replica and all pile onto it.
|
|
// This scorer closes that gap, which is why every profile keeps it on (§7.6).
|
|
func Inflight(target float64) Scorer {
|
|
if target <= 0 {
|
|
target = 8
|
|
}
|
|
return scorerFunc{
|
|
name: "inflight",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
return 1 - float64(e.Inflight)/(target*w)
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load is AIBrix's load-balance score: running / drain_rate.
|
|
//
|
|
// Running alone cannot spot a slow replica — one processing the same number of
|
|
// requests at half the speed is twice as loaded. While drain_rate is still
|
|
// unmeasured (the first seconds) it returns a neutral 1 and leaves the decision
|
|
// to the other scorers.
|
|
func Load(targetSeconds float64) Scorer {
|
|
if targetSeconds <= 0 {
|
|
targetSeconds = 30
|
|
}
|
|
return scorerFunc{
|
|
name: "load",
|
|
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
|
|
if e.DrainRate <= 0 {
|
|
return 1
|
|
}
|
|
// Estimated seconds to drain, normalized against the target.
|
|
drainTime := float64(e.Stats.Running) / e.DrainRate
|
|
return 1 - drainTime/targetSeconds
|
|
},
|
|
}
|
|
}
|
|
|
|
// Weight scores by static capacity weight.
|
|
//
|
|
// In heterogeneous pools (A100s alongside H100s) raw queue length misleads:
|
|
// the same queue drains faster on the faster card. The weight comes from
|
|
// ctl bench (§6.2.1) and should never be guessed by hand.
|
|
func Weight() Scorer {
|
|
return weightScorer{}
|
|
}
|
|
|
|
type weightScorer struct{}
|
|
|
|
func (weightScorer) Name() string { return "weight" }
|
|
|
|
func (weightScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
maxW := 0.0
|
|
for _, e := range eps {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
maxW = max(maxW, w)
|
|
}
|
|
out := make([]float64, len(eps))
|
|
if maxW <= 0 {
|
|
return out
|
|
}
|
|
for i, e := range eps {
|
|
w := e.Weight
|
|
if w <= 0 {
|
|
w = 1
|
|
}
|
|
out[i] = w / maxW
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Session scores session affinity (plan §7.5, tier 0).
|
|
//
|
|
// Bounded-load consistent hashing: a session maps to a replica, but if that
|
|
// replica carries more than loadFactor times the average it moves on around the
|
|
// ring. Unbounded stickiness drowns a single replica: cache hits go up while
|
|
// p99 collapses.
|
|
//
|
|
// With no session id every candidate scores 0, leaving the decision to the
|
|
// other scorers.
|
|
func Session(loadFactor float64) Scorer {
|
|
if loadFactor < 1 {
|
|
loadFactor = 1.25
|
|
}
|
|
return sessionScorer{loadFactor: loadFactor}
|
|
}
|
|
|
|
type sessionScorer struct{ loadFactor float64 }
|
|
|
|
func (sessionScorer) Name() string { return "session" }
|
|
|
|
func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
|
|
out := make([]float64, len(eps))
|
|
if c.Req.SessionID == "" || len(eps) == 0 {
|
|
return out
|
|
}
|
|
if idx := pickBoundedLoad(c.Req.SessionID, eps, s.loadFactor); idx >= 0 {
|
|
out[idx] = 1
|
|
}
|
|
return out
|
|
}
|