package scheduler import ( "math" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // scorerFunc helps define scorers that rate each endpoint independently. type scorerFunc struct { name string f func(c *Ctx, e endpoint.Snapshot) float64 } func (s scorerFunc) Name() string { return s.name } func (s scorerFunc) Score(c *Ctx, eps []endpoint.Snapshot) []float64 { out := make([]float64, len(eps)) for i, e := range eps { out[i] = clamp01(s.f(c, e)) } return out } func clamp01(v float64) float64 { if math.IsNaN(v) { return 0 } return min(max(v, 0), 1) } // Queue scores by queue depth: 1 - min(1, waiting/target). // // target is the "healthy buffer" size (plan §7.2.1): enough queued requests // for continuous batching to stay efficient, but not so many that queueing // delay creeps back in. Measure it with ctl bench rather than guessing. func Queue(target float64) Scorer { if target <= 0 { target = 4 } return scorerFunc{ name: "queue", f: func(_ *Ctx, e endpoint.Snapshot) float64 { return 1 - float64(e.Stats.Waiting)/target }, } } // KV scores by KV cache occupancy: 1 - usage. // // Above the ceiling a hard penalty applies: going from 90% to 95% is not the // same as going from 10% to 15% — a replica near the ceiling starts preempting // and dropping requests. func KV(ceiling float64) Scorer { if ceiling <= 0 || ceiling > 1 { ceiling = 0.9 } return scorerFunc{ name: "kv", f: func(_ *Ctx, e endpoint.Snapshot) float64 { u := e.Stats.KVCacheUsage s := 1 - u if u >= ceiling { s *= 0.1 } return s }, } } // Inflight scores by the unfinished requests the router itself is tracking. // // With a 2s scrape interval the engine metrics are stale at decision time, so // back-to-back requests all see the same "idle" replica and all pile onto it. // This scorer closes that gap, which is why every profile keeps it on (§7.6). func Inflight(target float64) Scorer { if target <= 0 { target = 8 } return scorerFunc{ name: "inflight", f: func(_ *Ctx, e endpoint.Snapshot) float64 { w := e.Weight if w <= 0 { w = 1 } return 1 - float64(e.Inflight)/(target*w) }, } } // Load is AIBrix's load-balance score: running / drain_rate. // // Running alone cannot spot a slow replica — one processing the same number of // requests at half the speed is twice as loaded. While drain_rate is still // unmeasured (the first seconds) it returns a neutral 1 and leaves the decision // to the other scorers. func Load(targetSeconds float64) Scorer { if targetSeconds <= 0 { targetSeconds = 30 } return scorerFunc{ name: "load", f: func(_ *Ctx, e endpoint.Snapshot) float64 { if e.DrainRate <= 0 { return 1 } // Estimated seconds to drain, normalized against the target. drainTime := float64(e.Stats.Running) / e.DrainRate return 1 - drainTime/targetSeconds }, } } // Weight scores by static capacity weight. // // In heterogeneous pools (A100s alongside H100s) raw queue length misleads: // the same queue drains faster on the faster card. The weight comes from // ctl bench (§6.2.1) and should never be guessed by hand. func Weight() Scorer { return weightScorer{} } type weightScorer struct{} func (weightScorer) Name() string { return "weight" } func (weightScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 { maxW := 0.0 for _, e := range eps { w := e.Weight if w <= 0 { w = 1 } maxW = max(maxW, w) } out := make([]float64, len(eps)) if maxW <= 0 { return out } for i, e := range eps { w := e.Weight if w <= 0 { w = 1 } out[i] = w / maxW } return out } // Session scores session affinity (plan §7.5, tier 0). // // Bounded-load consistent hashing: a session maps to a replica, but if that // replica carries more than loadFactor times the average it moves on around the // ring. Unbounded stickiness drowns a single replica: cache hits go up while // p99 collapses. // // With no session id every candidate scores 0, leaving the decision to the // other scorers. func Session(loadFactor float64) Scorer { if loadFactor < 1 { loadFactor = 1.25 } return sessionScorer{loadFactor: loadFactor} } type sessionScorer struct{ loadFactor float64 } func (sessionScorer) Name() string { return "session" } func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 { out := make([]float64, len(eps)) if c.Req.SessionID == "" || len(eps) == 0 { return out } if idx := pickBoundedLoad(c.Req.SessionID, eps, s.loadFactor); idx >= 0 { out[idx] = 1 } return out }