Files
llmgw/internal/router/scheduler/profiles.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

119 lines
3.0 KiB
Go

package scheduler
// ProfileConfig holds thresholds shared by the built-in profiles.
type ProfileConfig struct {
// TargetWaiting is the queue scorer's "healthy buffer" target.
TargetWaiting float64
// TargetInflight normalizes the inflight scorer.
TargetInflight float64
// KVCeiling is the KV occupancy ceiling.
KVCeiling float64
// SessionLoadFactor is the load ceiling multiplier for session affinity.
SessionLoadFactor float64
}
// DefaultProfileConfig returns sensible starting values.
// These numbers should be measured with ctl bench (§10); these are only seeds.
func DefaultProfileConfig() ProfileConfig {
return ProfileConfig{
TargetWaiting: 4,
TargetInflight: 8,
KVCeiling: 0.9,
SessionLoadFactor: 1.25,
}
}
func (c *ProfileConfig) normalize() {
d := DefaultProfileConfig()
if c.TargetWaiting <= 0 {
c.TargetWaiting = d.TargetWaiting
}
if c.TargetInflight <= 0 {
c.TargetInflight = d.TargetInflight
}
if c.KVCeiling <= 0 || c.KVCeiling > 1 {
c.KVCeiling = d.KVCeiling
}
if c.SessionLoadFactor < 1 {
c.SessionLoadFactor = d.SessionLoadFactor
}
}
// Chat is the profile for multi-turn conversations and agent workloads.
// Session affinity dominates, to preserve prefix cache reuse.
func Chat(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "chat",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Scorers: []WeightedScorer{
{Session(c.SessionLoadFactor), 3.0},
{Queue(c.TargetWaiting), 1.0},
{KV(c.KVCeiling), 1.0},
{Inflight(c.TargetInflight), 1.0},
{Weight(), 0.5},
},
Picker: P2C(),
}
}
// Batch is the profile for bulk work: throughput matters, TTFT does not, so
// finding the shortest queue outranks stickiness.
func Batch(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "batch",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Scorers: []WeightedScorer{
{Queue(c.TargetWaiting), 3.0},
{KV(c.KVCeiling), 1.0},
{Load(30), 1.0},
{Inflight(c.TargetInflight), 1.0},
{Weight(), 1.0},
},
Picker: MaxScore(),
}
}
// LongContext is the profile for long-prompt workloads. KV pressure is the
// bottleneck there, so the kv scorer dominates.
func LongContext(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "long-context",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Scorers: []WeightedScorer{
{KV(c.KVCeiling), 3.0},
{Session(c.SessionLoadFactor), 2.0},
{Queue(c.TargetWaiting), 1.0},
{Inflight(c.TargetInflight), 1.0},
},
Picker: Softmax(0.2),
}
}
// Debug is a round-robin profile for troubleshooting.
func Debug() *Profile {
return &Profile{
Name: "debug",
Filters: []Filter{BreakerOpen()},
Picker: RoundRobin(),
}
}
// Builtin returns a built-in profile by name, or nil if the name is unknown.
func Builtin(name string, c ProfileConfig) *Profile {
switch name {
case "chat":
return Chat(c)
case "batch":
return Batch(c)
case "long-context":
return LongContext(c)
case "debug":
return Debug()
default:
return nil
}
}