// Package scheduler answers "where should this request go?" (plan §7.3). // // Instead of one monolithic "strategy" function it is a plugin pipeline: // // filter -> drops candidates (hard constraints) // score -> rates each candidate 0..1, combined by weight // pick -> chooses one from the combined scores // // A "strategy" then reduces to a set of scorer weights: the difference between // the chat and batch profiles is weights, not code. package scheduler import ( "errors" "fmt" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // Request holds the request fields a routing decision depends on. type Request struct { // Model is the resolved model name. Model string // SessionID drives session affinity (x-session-id header or the user field). SessionID string // PromptPrefix is the leading portion of the prompt, for prefix awareness. PromptPrefix string // PromptTokens is the estimated prompt length in tokens. PromptTokens int // MaxTokens is the requested generation length. MaxTokens int // Adapter is the LoRA adapter asked for, when it is one that loads on // demand; empty otherwise. Adapter string // KV is what the precise KV index says about this request (plan §7.5 // tier 2), nil for a model that does not use it. KV *KVQuery } // Ctx is the context of a single scheduling decision. type Ctx struct { Req Request Now time.Time // Adapters is where Req.Adapter is loaded; nil when it names none. Adapters *AdapterView } // Filter drops candidates. type Filter interface { Name() string Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot } // Scorer rates each candidate between 0 and 1. The returned slice must match // eps in both order and length. type Scorer interface { Name() string Score(c *Ctx, eps []endpoint.Snapshot) []float64 } // Picker chooses one candidate from the combined scores and returns its index. type Picker interface { Name() string Pick(c *Ctx, eps []endpoint.Snapshot, scores []float64) int } // WeightedScorer pairs a scorer with its weight in a profile. type WeightedScorer struct { Scorer Scorer Weight float64 } // Profile is a scorer set with weights for one class of request (plan §7.3). type Profile struct { Name string Filters []Filter Scorers []WeightedScorer Picker Picker // StalePenalty multiplies the total score of an endpoint with stale // metrics. Stale endpoints are never dropped — if they were, a fully stale // pool would leave no candidates — but they rank below fresh ones. // Zero means 0.5. StalePenalty float64 } // Uses reports whether the profile has a scorer of this name with a weight. func (p *Profile) Uses(scorer string) bool { for _, ws := range p.Scorers { if ws.Scorer != nil && ws.Weight != 0 && ws.Scorer.Name() == scorer { return true } } return false } // Decision is a routing choice together with its rationale. // // Recording the rationale is not decoration: tuning the prefix and queue // weights is impossible without seeing why decisions came out the way they // did (§7.5). type Decision struct { Endpoint endpoint.Snapshot Score float64 // Scores maps scorer name to raw score for the chosen endpoint. Scores map[string]float64 // Dropped maps filter name to how many candidates it removed. Dropped map[string]int // Considered is how many candidates survived the filters. Considered int } var ( // ErrNoEndpoints means no endpoint serves the requested model. ErrNoEndpoints = errors.New("no endpoint serves this model") // ErrAllFiltered means every candidate was removed by the filters. ErrAllFiltered = errors.New("all candidates were filtered out") ) // Schedule chooses one endpoint from the candidates. func (p *Profile) Schedule(c *Ctx, eps []endpoint.Snapshot) (Decision, error) { if len(eps) == 0 { return Decision{}, ErrNoEndpoints } dropped := make(map[string]int) cand := eps for _, f := range p.Filters { before := len(cand) cand = f.Filter(c, cand) if n := before - len(cand); n > 0 { dropped[f.Name()] = n } if len(cand) == 0 { return Decision{Dropped: dropped}, fmt.Errorf("%w: %v", ErrAllFiltered, dropped) } } total := make([]float64, len(cand)) raw := make([]map[string]float64, len(cand)) for i := range raw { raw[i] = make(map[string]float64, len(p.Scorers)) } for _, ws := range p.Scorers { if ws.Weight == 0 || ws.Scorer == nil { continue } s := ws.Scorer.Score(c, cand) if len(s) != len(cand) { // A misbehaving scorer must not break the whole decision; skip it. continue } name := ws.Scorer.Name() for i, v := range s { raw[i][name] = v total[i] += ws.Weight * v } } penalty := p.StalePenalty if penalty <= 0 { penalty = 0.5 } for i := range cand { if cand[i].Stale { total[i] *= penalty } } picker := p.Picker if picker == nil { picker = MaxScore() } idx := picker.Pick(c, cand, total) if idx < 0 || idx >= len(cand) { idx = 0 } return Decision{ Endpoint: cand[idx], Score: total[idx], Scores: raw[idx], Dropped: dropped, Considered: len(cand), }, nil }