Files
llmgw/internal/router/scheduler/scheduler.go
T
Ilker ManapandClaude Opus 5.5 9e46ee83e4 Merge C24 KV-index routing from vLLM's cache events
Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:24:25 +02:00

189 lines
5.0 KiB
Go

// Package scheduler answers "where should this request go?" (plan §7.3).
//
// Instead of one monolithic "strategy" function it is a plugin pipeline:
//
// filter -> drops candidates (hard constraints)
// score -> rates each candidate 0..1, combined by weight
// pick -> chooses one from the combined scores
//
// A "strategy" then reduces to a set of scorer weights: the difference between
// the chat and batch profiles is weights, not code.
package scheduler
import (
"errors"
"fmt"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Request holds the request fields a routing decision depends on.
type Request struct {
// Model is the resolved model name.
Model string
// SessionID drives session affinity (x-session-id header or the user field).
SessionID string
// PromptPrefix is the leading portion of the prompt, for prefix awareness.
PromptPrefix string
// PromptTokens is the estimated prompt length in tokens.
PromptTokens int
// MaxTokens is the requested generation length.
MaxTokens int
// Adapter is the LoRA adapter asked for, when it is one that loads on
// demand; empty otherwise.
Adapter string
// KV is what the precise KV index says about this request (plan §7.5
// tier 2), nil for a model that does not use it.
KV *KVQuery
}
// Ctx is the context of a single scheduling decision.
type Ctx struct {
Req Request
Now time.Time
// Adapters is where Req.Adapter is loaded; nil when it names none.
Adapters *AdapterView
}
// Filter drops candidates.
type Filter interface {
Name() string
Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot
}
// Scorer rates each candidate between 0 and 1. The returned slice must match
// eps in both order and length.
type Scorer interface {
Name() string
Score(c *Ctx, eps []endpoint.Snapshot) []float64
}
// Picker chooses one candidate from the combined scores and returns its index.
type Picker interface {
Name() string
Pick(c *Ctx, eps []endpoint.Snapshot, scores []float64) int
}
// WeightedScorer pairs a scorer with its weight in a profile.
type WeightedScorer struct {
Scorer Scorer
Weight float64
}
// Profile is a scorer set with weights for one class of request (plan §7.3).
type Profile struct {
Name string
Filters []Filter
Scorers []WeightedScorer
Picker Picker
// StalePenalty multiplies the total score of an endpoint with stale
// metrics. Stale endpoints are never dropped — if they were, a fully stale
// pool would leave no candidates — but they rank below fresh ones.
// Zero means 0.5.
StalePenalty float64
}
// Uses reports whether the profile has a scorer of this name with a weight.
func (p *Profile) Uses(scorer string) bool {
for _, ws := range p.Scorers {
if ws.Scorer != nil && ws.Weight != 0 && ws.Scorer.Name() == scorer {
return true
}
}
return false
}
// Decision is a routing choice together with its rationale.
//
// Recording the rationale is not decoration: tuning the prefix and queue
// weights is impossible without seeing why decisions came out the way they
// did (§7.5).
type Decision struct {
Endpoint endpoint.Snapshot
Score float64
// Scores maps scorer name to raw score for the chosen endpoint.
Scores map[string]float64
// Dropped maps filter name to how many candidates it removed.
Dropped map[string]int
// Considered is how many candidates survived the filters.
Considered int
}
var (
// ErrNoEndpoints means no endpoint serves the requested model.
ErrNoEndpoints = errors.New("no endpoint serves this model")
// ErrAllFiltered means every candidate was removed by the filters.
ErrAllFiltered = errors.New("all candidates were filtered out")
)
// Schedule chooses one endpoint from the candidates.
func (p *Profile) Schedule(c *Ctx, eps []endpoint.Snapshot) (Decision, error) {
if len(eps) == 0 {
return Decision{}, ErrNoEndpoints
}
dropped := make(map[string]int)
cand := eps
for _, f := range p.Filters {
before := len(cand)
cand = f.Filter(c, cand)
if n := before - len(cand); n > 0 {
dropped[f.Name()] = n
}
if len(cand) == 0 {
return Decision{Dropped: dropped}, fmt.Errorf("%w: %v", ErrAllFiltered, dropped)
}
}
total := make([]float64, len(cand))
raw := make([]map[string]float64, len(cand))
for i := range raw {
raw[i] = make(map[string]float64, len(p.Scorers))
}
for _, ws := range p.Scorers {
if ws.Weight == 0 || ws.Scorer == nil {
continue
}
s := ws.Scorer.Score(c, cand)
if len(s) != len(cand) {
// A misbehaving scorer must not break the whole decision; skip it.
continue
}
name := ws.Scorer.Name()
for i, v := range s {
raw[i][name] = v
total[i] += ws.Weight * v
}
}
penalty := p.StalePenalty
if penalty <= 0 {
penalty = 0.5
}
for i := range cand {
if cand[i].Stale {
total[i] *= penalty
}
}
picker := p.Picker
if picker == nil {
picker = MaxScore()
}
idx := picker.Pick(c, cand, total)
if idx < 0 || idx >= len(cand) {
idx = 0
}
return Decision{
Endpoint: cand[idx],
Score: total[idx],
Scores: raw[idx],
Dropped: dropped,
Considered: len(cand),
}, nil
}