Conflicts with the LoRA work: vLLM's arguments, the engine profile, router metrics and the scheduler's request all keep both; agent.pb.go regenerated from the merged proto; the two test fakes of vLLM given separate names. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
189 lines
5.0 KiB
Go
189 lines
5.0 KiB
Go
// Package scheduler answers "where should this request go?" (plan §7.3).
|
|
//
|
|
// Instead of one monolithic "strategy" function it is a plugin pipeline:
|
|
//
|
|
// filter -> drops candidates (hard constraints)
|
|
// score -> rates each candidate 0..1, combined by weight
|
|
// pick -> chooses one from the combined scores
|
|
//
|
|
// A "strategy" then reduces to a set of scorer weights: the difference between
|
|
// the chat and batch profiles is weights, not code.
|
|
package scheduler
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Request holds the request fields a routing decision depends on.
|
|
type Request struct {
|
|
// Model is the resolved model name.
|
|
Model string
|
|
// SessionID drives session affinity (x-session-id header or the user field).
|
|
SessionID string
|
|
// PromptPrefix is the leading portion of the prompt, for prefix awareness.
|
|
PromptPrefix string
|
|
// PromptTokens is the estimated prompt length in tokens.
|
|
PromptTokens int
|
|
// MaxTokens is the requested generation length.
|
|
MaxTokens int
|
|
// Adapter is the LoRA adapter asked for, when it is one that loads on
|
|
// demand; empty otherwise.
|
|
Adapter string
|
|
// KV is what the precise KV index says about this request (plan §7.5
|
|
// tier 2), nil for a model that does not use it.
|
|
KV *KVQuery
|
|
}
|
|
|
|
// Ctx is the context of a single scheduling decision.
|
|
type Ctx struct {
|
|
Req Request
|
|
Now time.Time
|
|
// Adapters is where Req.Adapter is loaded; nil when it names none.
|
|
Adapters *AdapterView
|
|
}
|
|
|
|
// Filter drops candidates.
|
|
type Filter interface {
|
|
Name() string
|
|
Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot
|
|
}
|
|
|
|
// Scorer rates each candidate between 0 and 1. The returned slice must match
|
|
// eps in both order and length.
|
|
type Scorer interface {
|
|
Name() string
|
|
Score(c *Ctx, eps []endpoint.Snapshot) []float64
|
|
}
|
|
|
|
// Picker chooses one candidate from the combined scores and returns its index.
|
|
type Picker interface {
|
|
Name() string
|
|
Pick(c *Ctx, eps []endpoint.Snapshot, scores []float64) int
|
|
}
|
|
|
|
// WeightedScorer pairs a scorer with its weight in a profile.
|
|
type WeightedScorer struct {
|
|
Scorer Scorer
|
|
Weight float64
|
|
}
|
|
|
|
// Profile is a scorer set with weights for one class of request (plan §7.3).
|
|
type Profile struct {
|
|
Name string
|
|
Filters []Filter
|
|
Scorers []WeightedScorer
|
|
Picker Picker
|
|
|
|
// StalePenalty multiplies the total score of an endpoint with stale
|
|
// metrics. Stale endpoints are never dropped — if they were, a fully stale
|
|
// pool would leave no candidates — but they rank below fresh ones.
|
|
// Zero means 0.5.
|
|
StalePenalty float64
|
|
}
|
|
|
|
// Uses reports whether the profile has a scorer of this name with a weight.
|
|
func (p *Profile) Uses(scorer string) bool {
|
|
for _, ws := range p.Scorers {
|
|
if ws.Scorer != nil && ws.Weight != 0 && ws.Scorer.Name() == scorer {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Decision is a routing choice together with its rationale.
|
|
//
|
|
// Recording the rationale is not decoration: tuning the prefix and queue
|
|
// weights is impossible without seeing why decisions came out the way they
|
|
// did (§7.5).
|
|
type Decision struct {
|
|
Endpoint endpoint.Snapshot
|
|
Score float64
|
|
// Scores maps scorer name to raw score for the chosen endpoint.
|
|
Scores map[string]float64
|
|
// Dropped maps filter name to how many candidates it removed.
|
|
Dropped map[string]int
|
|
// Considered is how many candidates survived the filters.
|
|
Considered int
|
|
}
|
|
|
|
var (
|
|
// ErrNoEndpoints means no endpoint serves the requested model.
|
|
ErrNoEndpoints = errors.New("no endpoint serves this model")
|
|
// ErrAllFiltered means every candidate was removed by the filters.
|
|
ErrAllFiltered = errors.New("all candidates were filtered out")
|
|
)
|
|
|
|
// Schedule chooses one endpoint from the candidates.
|
|
func (p *Profile) Schedule(c *Ctx, eps []endpoint.Snapshot) (Decision, error) {
|
|
if len(eps) == 0 {
|
|
return Decision{}, ErrNoEndpoints
|
|
}
|
|
|
|
dropped := make(map[string]int)
|
|
cand := eps
|
|
for _, f := range p.Filters {
|
|
before := len(cand)
|
|
cand = f.Filter(c, cand)
|
|
if n := before - len(cand); n > 0 {
|
|
dropped[f.Name()] = n
|
|
}
|
|
if len(cand) == 0 {
|
|
return Decision{Dropped: dropped}, fmt.Errorf("%w: %v", ErrAllFiltered, dropped)
|
|
}
|
|
}
|
|
|
|
total := make([]float64, len(cand))
|
|
raw := make([]map[string]float64, len(cand))
|
|
for i := range raw {
|
|
raw[i] = make(map[string]float64, len(p.Scorers))
|
|
}
|
|
|
|
for _, ws := range p.Scorers {
|
|
if ws.Weight == 0 || ws.Scorer == nil {
|
|
continue
|
|
}
|
|
s := ws.Scorer.Score(c, cand)
|
|
if len(s) != len(cand) {
|
|
// A misbehaving scorer must not break the whole decision; skip it.
|
|
continue
|
|
}
|
|
name := ws.Scorer.Name()
|
|
for i, v := range s {
|
|
raw[i][name] = v
|
|
total[i] += ws.Weight * v
|
|
}
|
|
}
|
|
|
|
penalty := p.StalePenalty
|
|
if penalty <= 0 {
|
|
penalty = 0.5
|
|
}
|
|
for i := range cand {
|
|
if cand[i].Stale {
|
|
total[i] *= penalty
|
|
}
|
|
}
|
|
|
|
picker := p.Picker
|
|
if picker == nil {
|
|
picker = MaxScore()
|
|
}
|
|
idx := picker.Pick(c, cand, total)
|
|
if idx < 0 || idx >= len(cand) {
|
|
idx = 0
|
|
}
|
|
|
|
return Decision{
|
|
Endpoint: cand[idx],
|
|
Score: total[idx],
|
|
Scores: raw[idx],
|
|
Dropped: dropped,
|
|
Considered: len(cand),
|
|
}, nil
|
|
}
|