Files
llmgw/internal/router/scheduler/scheduler.go
T
Ilker ManapandClaude Opus 5.5 2a1e0e6c4c LoRA adapters on vLLM, loaded on demand with replica affinity: max_loaded_adapters
A vLLM model can carry PEFT adapters, registered at start, or with
max_loaded_adapters held at most N per replica: the router loads an adapter
through vLLM's runtime LoRA API when a request first wants it, sends the
adapter's requests to a replica that holds it while that one is not
saturated, loads each cold adapter once, and unloads the least recently used
idle one to make room. llama.cpp keeps loading every adapter at start.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:14:38 +02:00

176 lines
4.6 KiB
Go

// Package scheduler answers "where should this request go?" (plan §7.3).
//
// Instead of one monolithic "strategy" function it is a plugin pipeline:
//
// filter -> drops candidates (hard constraints)
// score -> rates each candidate 0..1, combined by weight
// pick -> chooses one from the combined scores
//
// A "strategy" then reduces to a set of scorer weights: the difference between
// the chat and batch profiles is weights, not code.
package scheduler
import (
"errors"
"fmt"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Request holds the request fields a routing decision depends on.
type Request struct {
// Model is the resolved model name.
Model string
// SessionID drives session affinity (x-session-id header or the user field).
SessionID string
// PromptPrefix is the leading portion of the prompt, for prefix awareness.
PromptPrefix string
// PromptTokens is the estimated prompt length in tokens.
PromptTokens int
// MaxTokens is the requested generation length.
MaxTokens int
// Adapter is the LoRA adapter asked for, when it is one that loads on
// demand; empty otherwise.
Adapter string
}
// Ctx is the context of a single scheduling decision.
type Ctx struct {
Req Request
Now time.Time
// Adapters is where Req.Adapter is loaded; nil when it names none.
Adapters *AdapterView
}
// Filter drops candidates.
type Filter interface {
Name() string
Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot
}
// Scorer rates each candidate between 0 and 1. The returned slice must match
// eps in both order and length.
type Scorer interface {
Name() string
Score(c *Ctx, eps []endpoint.Snapshot) []float64
}
// Picker chooses one candidate from the combined scores and returns its index.
type Picker interface {
Name() string
Pick(c *Ctx, eps []endpoint.Snapshot, scores []float64) int
}
// WeightedScorer pairs a scorer with its weight in a profile.
type WeightedScorer struct {
Scorer Scorer
Weight float64
}
// Profile is a scorer set with weights for one class of request (plan §7.3).
type Profile struct {
Name string
Filters []Filter
Scorers []WeightedScorer
Picker Picker
// StalePenalty multiplies the total score of an endpoint with stale
// metrics. Stale endpoints are never dropped — if they were, a fully stale
// pool would leave no candidates — but they rank below fresh ones.
// Zero means 0.5.
StalePenalty float64
}
// Decision is a routing choice together with its rationale.
//
// Recording the rationale is not decoration: tuning the prefix and queue
// weights is impossible without seeing why decisions came out the way they
// did (§7.5).
type Decision struct {
Endpoint endpoint.Snapshot
Score float64
// Scores maps scorer name to raw score for the chosen endpoint.
Scores map[string]float64
// Dropped maps filter name to how many candidates it removed.
Dropped map[string]int
// Considered is how many candidates survived the filters.
Considered int
}
var (
// ErrNoEndpoints means no endpoint serves the requested model.
ErrNoEndpoints = errors.New("no endpoint serves this model")
// ErrAllFiltered means every candidate was removed by the filters.
ErrAllFiltered = errors.New("all candidates were filtered out")
)
// Schedule chooses one endpoint from the candidates.
func (p *Profile) Schedule(c *Ctx, eps []endpoint.Snapshot) (Decision, error) {
if len(eps) == 0 {
return Decision{}, ErrNoEndpoints
}
dropped := make(map[string]int)
cand := eps
for _, f := range p.Filters {
before := len(cand)
cand = f.Filter(c, cand)
if n := before - len(cand); n > 0 {
dropped[f.Name()] = n
}
if len(cand) == 0 {
return Decision{Dropped: dropped}, fmt.Errorf("%w: %v", ErrAllFiltered, dropped)
}
}
total := make([]float64, len(cand))
raw := make([]map[string]float64, len(cand))
for i := range raw {
raw[i] = make(map[string]float64, len(p.Scorers))
}
for _, ws := range p.Scorers {
if ws.Weight == 0 || ws.Scorer == nil {
continue
}
s := ws.Scorer.Score(c, cand)
if len(s) != len(cand) {
// A misbehaving scorer must not break the whole decision; skip it.
continue
}
name := ws.Scorer.Name()
for i, v := range s {
raw[i][name] = v
total[i] += ws.Weight * v
}
}
penalty := p.StalePenalty
if penalty <= 0 {
penalty = 0.5
}
for i := range cand {
if cand[i].Stale {
total[i] *= penalty
}
}
picker := p.Picker
if picker == nil {
picker = MaxScore()
}
idx := picker.Pick(c, cand, total)
if idx < 0 || idx >= len(cand) {
idx = 0
}
return Decision{
Endpoint: cand[idx],
Score: total[idx],
Scores: raw[idx],
Dropped: dropped,
Considered: len(cand),
}, nil
}