The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
119 lines
3.0 KiB
Go
119 lines
3.0 KiB
Go
package scheduler
|
|
|
|
// ProfileConfig holds thresholds shared by the built-in profiles.
|
|
type ProfileConfig struct {
|
|
// TargetWaiting is the queue scorer's "healthy buffer" target.
|
|
TargetWaiting float64
|
|
// TargetInflight normalizes the inflight scorer.
|
|
TargetInflight float64
|
|
// KVCeiling is the KV occupancy ceiling.
|
|
KVCeiling float64
|
|
// SessionLoadFactor is the load ceiling multiplier for session affinity.
|
|
SessionLoadFactor float64
|
|
}
|
|
|
|
// DefaultProfileConfig returns sensible starting values.
|
|
// These numbers should be measured with ctl bench (§10); these are only seeds.
|
|
func DefaultProfileConfig() ProfileConfig {
|
|
return ProfileConfig{
|
|
TargetWaiting: 4,
|
|
TargetInflight: 8,
|
|
KVCeiling: 0.9,
|
|
SessionLoadFactor: 1.25,
|
|
}
|
|
}
|
|
|
|
func (c *ProfileConfig) normalize() {
|
|
d := DefaultProfileConfig()
|
|
if c.TargetWaiting <= 0 {
|
|
c.TargetWaiting = d.TargetWaiting
|
|
}
|
|
if c.TargetInflight <= 0 {
|
|
c.TargetInflight = d.TargetInflight
|
|
}
|
|
if c.KVCeiling <= 0 || c.KVCeiling > 1 {
|
|
c.KVCeiling = d.KVCeiling
|
|
}
|
|
if c.SessionLoadFactor < 1 {
|
|
c.SessionLoadFactor = d.SessionLoadFactor
|
|
}
|
|
}
|
|
|
|
// Chat is the profile for multi-turn conversations and agent workloads.
|
|
// Session affinity dominates, to preserve prefix cache reuse.
|
|
func Chat(c ProfileConfig) *Profile {
|
|
c.normalize()
|
|
return &Profile{
|
|
Name: "chat",
|
|
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
|
Scorers: []WeightedScorer{
|
|
{Session(c.SessionLoadFactor), 3.0},
|
|
{Queue(c.TargetWaiting), 1.0},
|
|
{KV(c.KVCeiling), 1.0},
|
|
{Inflight(c.TargetInflight), 1.0},
|
|
{Weight(), 0.5},
|
|
},
|
|
Picker: P2C(),
|
|
}
|
|
}
|
|
|
|
// Batch is the profile for bulk work: throughput matters, TTFT does not, so
|
|
// finding the shortest queue outranks stickiness.
|
|
func Batch(c ProfileConfig) *Profile {
|
|
c.normalize()
|
|
return &Profile{
|
|
Name: "batch",
|
|
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
|
Scorers: []WeightedScorer{
|
|
{Queue(c.TargetWaiting), 3.0},
|
|
{KV(c.KVCeiling), 1.0},
|
|
{Load(30), 1.0},
|
|
{Inflight(c.TargetInflight), 1.0},
|
|
{Weight(), 1.0},
|
|
},
|
|
Picker: MaxScore(),
|
|
}
|
|
}
|
|
|
|
// LongContext is the profile for long-prompt workloads. KV pressure is the
|
|
// bottleneck there, so the kv scorer dominates.
|
|
func LongContext(c ProfileConfig) *Profile {
|
|
c.normalize()
|
|
return &Profile{
|
|
Name: "long-context",
|
|
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
|
Scorers: []WeightedScorer{
|
|
{KV(c.KVCeiling), 3.0},
|
|
{Session(c.SessionLoadFactor), 2.0},
|
|
{Queue(c.TargetWaiting), 1.0},
|
|
{Inflight(c.TargetInflight), 1.0},
|
|
},
|
|
Picker: Softmax(0.2),
|
|
}
|
|
}
|
|
|
|
// Debug is a round-robin profile for troubleshooting.
|
|
func Debug() *Profile {
|
|
return &Profile{
|
|
Name: "debug",
|
|
Filters: []Filter{BreakerOpen()},
|
|
Picker: RoundRobin(),
|
|
}
|
|
}
|
|
|
|
// Builtin returns a built-in profile by name, or nil if the name is unknown.
|
|
func Builtin(name string, c ProfileConfig) *Profile {
|
|
switch name {
|
|
case "chat":
|
|
return Chat(c)
|
|
case "batch":
|
|
return Batch(c)
|
|
case "long-context":
|
|
return LongContext(c)
|
|
case "debug":
|
|
return Debug()
|
|
default:
|
|
return nil
|
|
}
|
|
}
|