Files
llmgw/internal/router/scheduler/scheduler.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

171 lines
4.4 KiB
Go

// Package scheduler answers "where should this request go?" (plan §7.3).
//
// Instead of one monolithic "strategy" function it is a plugin pipeline:
//
// filter -> drops candidates (hard constraints)
// score -> rates each candidate 0..1, combined by weight
// pick -> chooses one from the combined scores
//
// A "strategy" then reduces to a set of scorer weights: the difference between
// the chat and batch profiles is weights, not code.
package scheduler
import (
"errors"
"fmt"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Request holds the request fields a routing decision depends on.
type Request struct {
// Model is the resolved model name.
Model string
// SessionID drives session affinity (x-session-id header or the user field).
SessionID string
// PromptPrefix is the leading portion of the prompt, for prefix awareness.
PromptPrefix string
// PromptTokens is the estimated prompt length in tokens.
PromptTokens int
// MaxTokens is the requested generation length.
MaxTokens int
}
// Ctx is the context of a single scheduling decision.
type Ctx struct {
Req Request
Now time.Time
}
// Filter drops candidates.
type Filter interface {
Name() string
Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot
}
// Scorer rates each candidate between 0 and 1. The returned slice must match
// eps in both order and length.
type Scorer interface {
Name() string
Score(c *Ctx, eps []endpoint.Snapshot) []float64
}
// Picker chooses one candidate from the combined scores and returns its index.
type Picker interface {
Name() string
Pick(c *Ctx, eps []endpoint.Snapshot, scores []float64) int
}
// WeightedScorer pairs a scorer with its weight in a profile.
type WeightedScorer struct {
Scorer Scorer
Weight float64
}
// Profile is a scorer set with weights for one class of request (plan §7.3).
type Profile struct {
Name string
Filters []Filter
Scorers []WeightedScorer
Picker Picker
// StalePenalty multiplies the total score of an endpoint with stale
// metrics. Stale endpoints are never dropped — if they were, a fully stale
// pool would leave no candidates — but they rank below fresh ones.
// Zero means 0.5.
StalePenalty float64
}
// Decision is a routing choice together with its rationale.
//
// Recording the rationale is not decoration: tuning the prefix and queue
// weights is impossible without seeing why decisions came out the way they
// did (§7.5).
type Decision struct {
Endpoint endpoint.Snapshot
Score float64
// Scores maps scorer name to raw score for the chosen endpoint.
Scores map[string]float64
// Dropped maps filter name to how many candidates it removed.
Dropped map[string]int
// Considered is how many candidates survived the filters.
Considered int
}
var (
// ErrNoEndpoints means no endpoint serves the requested model.
ErrNoEndpoints = errors.New("no endpoint serves this model")
// ErrAllFiltered means every candidate was removed by the filters.
ErrAllFiltered = errors.New("all candidates were filtered out")
)
// Schedule chooses one endpoint from the candidates.
func (p *Profile) Schedule(c *Ctx, eps []endpoint.Snapshot) (Decision, error) {
if len(eps) == 0 {
return Decision{}, ErrNoEndpoints
}
dropped := make(map[string]int)
cand := eps
for _, f := range p.Filters {
before := len(cand)
cand = f.Filter(c, cand)
if n := before - len(cand); n > 0 {
dropped[f.Name()] = n
}
if len(cand) == 0 {
return Decision{Dropped: dropped}, fmt.Errorf("%w: %v", ErrAllFiltered, dropped)
}
}
total := make([]float64, len(cand))
raw := make([]map[string]float64, len(cand))
for i := range raw {
raw[i] = make(map[string]float64, len(p.Scorers))
}
for _, ws := range p.Scorers {
if ws.Weight == 0 || ws.Scorer == nil {
continue
}
s := ws.Scorer.Score(c, cand)
if len(s) != len(cand) {
// A misbehaving scorer must not break the whole decision; skip it.
continue
}
name := ws.Scorer.Name()
for i, v := range s {
raw[i][name] = v
total[i] += ws.Weight * v
}
}
penalty := p.StalePenalty
if penalty <= 0 {
penalty = 0.5
}
for i := range cand {
if cand[i].Stale {
total[i] *= penalty
}
}
picker := p.Picker
if picker == nil {
picker = MaxScore()
}
idx := picker.Pick(c, cand, total)
if idx < 0 || idx >= len(cand) {
idx = 0
}
return Decision{
Endpoint: cand[idx],
Score: total[idx],
Scores: raw[idx],
Dropped: dropped,
Considered: len(cand),
}, nil
}