Files
llmgw/internal/router/scheduler/scorers.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

181 lines
4.5 KiB
Go

package scheduler
import (
"math"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// scorerFunc helps define scorers that rate each endpoint independently.
type scorerFunc struct {
name string
f func(c *Ctx, e endpoint.Snapshot) float64
}
func (s scorerFunc) Name() string { return s.name }
func (s scorerFunc) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
out := make([]float64, len(eps))
for i, e := range eps {
out[i] = clamp01(s.f(c, e))
}
return out
}
func clamp01(v float64) float64 {
if math.IsNaN(v) {
return 0
}
return min(max(v, 0), 1)
}
// Queue scores by queue depth: 1 - min(1, waiting/target).
//
// target is the "healthy buffer" size (plan §7.2.1): enough queued requests
// for continuous batching to stay efficient, but not so many that queueing
// delay creeps back in. Measure it with ctl bench rather than guessing.
func Queue(target float64) Scorer {
if target <= 0 {
target = 4
}
return scorerFunc{
name: "queue",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
return 1 - float64(e.Stats.Waiting)/target
},
}
}
// KV scores by KV cache occupancy: 1 - usage.
//
// Above the ceiling a hard penalty applies: going from 90% to 95% is not the
// same as going from 10% to 15% — a replica near the ceiling starts preempting
// and dropping requests.
func KV(ceiling float64) Scorer {
if ceiling <= 0 || ceiling > 1 {
ceiling = 0.9
}
return scorerFunc{
name: "kv",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
u := e.Stats.KVCacheUsage
s := 1 - u
if u >= ceiling {
s *= 0.1
}
return s
},
}
}
// Inflight scores by the unfinished requests the router itself is tracking.
//
// With a 2s scrape interval the engine metrics are stale at decision time, so
// back-to-back requests all see the same "idle" replica and all pile onto it.
// This scorer closes that gap, which is why every profile keeps it on (§7.6).
func Inflight(target float64) Scorer {
if target <= 0 {
target = 8
}
return scorerFunc{
name: "inflight",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
w := e.Weight
if w <= 0 {
w = 1
}
return 1 - float64(e.Inflight)/(target*w)
},
}
}
// Load is AIBrix's load-balance score: running / drain_rate.
//
// Running alone cannot spot a slow replica — one processing the same number of
// requests at half the speed is twice as loaded. While drain_rate is still
// unmeasured (the first seconds) it returns a neutral 1 and leaves the decision
// to the other scorers.
func Load(targetSeconds float64) Scorer {
if targetSeconds <= 0 {
targetSeconds = 30
}
return scorerFunc{
name: "load",
f: func(_ *Ctx, e endpoint.Snapshot) float64 {
if e.DrainRate <= 0 {
return 1
}
// Estimated seconds to drain, normalized against the target.
drainTime := float64(e.Stats.Running) / e.DrainRate
return 1 - drainTime/targetSeconds
},
}
}
// Weight scores by static capacity weight.
//
// In heterogeneous pools (A100s alongside H100s) raw queue length misleads:
// the same queue drains faster on the faster card. The weight comes from
// ctl bench (§6.2.1) and should never be guessed by hand.
func Weight() Scorer {
return weightScorer{}
}
type weightScorer struct{}
func (weightScorer) Name() string { return "weight" }
func (weightScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
maxW := 0.0
for _, e := range eps {
w := e.Weight
if w <= 0 {
w = 1
}
maxW = max(maxW, w)
}
out := make([]float64, len(eps))
if maxW <= 0 {
return out
}
for i, e := range eps {
w := e.Weight
if w <= 0 {
w = 1
}
out[i] = w / maxW
}
return out
}
// Session scores session affinity (plan §7.5, tier 0).
//
// Bounded-load consistent hashing: a session maps to a replica, but if that
// replica carries more than loadFactor times the average it moves on around the
// ring. Unbounded stickiness drowns a single replica: cache hits go up while
// p99 collapses.
//
// With no session id every candidate scores 0, leaving the decision to the
// other scorers.
func Session(loadFactor float64) Scorer {
if loadFactor < 1 {
loadFactor = 1.25
}
return sessionScorer{loadFactor: loadFactor}
}
type sessionScorer struct{ loadFactor float64 }
func (sessionScorer) Name() string { return "session" }
func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
out := make([]float64, len(eps))
if c.Req.SessionID == "" || len(eps) == 0 {
return out
}
if idx := pickBoundedLoad(c.Req.SessionID, eps, s.loadFactor); idx >= 0 {
out[idx] = 1
}
return out
}