The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
199 lines
4.5 KiB
Go
199 lines
4.5 KiB
Go
// Package breaker provides a per-endpoint circuit breaker.
|
|
//
|
|
// It folds two concepts from plan §7.7 into a single component:
|
|
//
|
|
// - Cooldown: an endpoint that errors leaves the candidate pool for a while.
|
|
// (Set Threshold to 1 to get exactly this behaviour.)
|
|
// - Circuit breaker: opens once consecutive failures cross a threshold, then
|
|
// moves to half-open probes with exponential backoff.
|
|
//
|
|
// Keeping them apart would mean two timers and two separate answers to
|
|
// "is this endpoint usable right now?"; merging them produces fewer surprises.
|
|
package breaker
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// State is the breaker's current state.
|
|
type State int
|
|
|
|
const (
|
|
// Closed is normal operation: requests pass through.
|
|
Closed State = iota
|
|
// Open means the endpoint is out of service; no requests are sent.
|
|
Open
|
|
// HalfOpen allows a limited number of probe requests.
|
|
HalfOpen
|
|
)
|
|
|
|
func (s State) String() string {
|
|
switch s {
|
|
case Closed:
|
|
return "closed"
|
|
case Open:
|
|
return "open"
|
|
case HalfOpen:
|
|
return "half-open"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Config holds the breaker's settings.
|
|
type Config struct {
|
|
// FailureThreshold is the number of consecutive failures that opens the
|
|
// circuit. 1 means "one error puts the endpoint in cooldown".
|
|
FailureThreshold int
|
|
// BaseCooldown is how long the circuit stays open the first time it trips.
|
|
BaseCooldown time.Duration
|
|
// MaxCooldown caps the exponential backoff.
|
|
MaxCooldown time.Duration
|
|
// HalfOpenProbes is how many concurrent probes are allowed while half-open.
|
|
HalfOpenProbes int
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
FailureThreshold: 3,
|
|
BaseCooldown: 15 * time.Second,
|
|
MaxCooldown: 2 * time.Minute,
|
|
HalfOpenProbes: 1,
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
if c.FailureThreshold <= 0 {
|
|
c.FailureThreshold = 3
|
|
}
|
|
if c.BaseCooldown <= 0 {
|
|
c.BaseCooldown = 15 * time.Second
|
|
}
|
|
if c.MaxCooldown < c.BaseCooldown {
|
|
c.MaxCooldown = max(c.BaseCooldown, 2*time.Minute)
|
|
}
|
|
if c.HalfOpenProbes <= 0 {
|
|
c.HalfOpenProbes = 1
|
|
}
|
|
}
|
|
|
|
// Breaker is one endpoint's circuit breaker. It is safe for concurrent use.
|
|
type Breaker struct {
|
|
cfg Config
|
|
now func() time.Time
|
|
|
|
mu sync.Mutex
|
|
state State
|
|
consecFails int
|
|
openUntil time.Time
|
|
openRounds int // how many times it has opened, for backoff
|
|
probesInUse int
|
|
totalTrips int64
|
|
}
|
|
|
|
// New creates a breaker with the given settings.
|
|
func New(cfg Config) *Breaker {
|
|
cfg.normalize()
|
|
return &Breaker{cfg: cfg, now: time.Now, state: Closed}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (b *Breaker) SetClock(now func() time.Time) {
|
|
b.mu.Lock()
|
|
b.now = now
|
|
b.mu.Unlock()
|
|
}
|
|
|
|
// Allow reports whether a request may be sent to this endpoint.
|
|
// If it returns true, the outcome must be reported via Success or Failure.
|
|
func (b *Breaker) Allow() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
switch b.state {
|
|
case Closed:
|
|
return true
|
|
case Open:
|
|
if b.now().Before(b.openUntil) {
|
|
return false
|
|
}
|
|
// Cooldown elapsed: move to half-open and let this probe through.
|
|
b.state = HalfOpen
|
|
b.probesInUse = 1
|
|
return true
|
|
case HalfOpen:
|
|
if b.probesInUse >= b.cfg.HalfOpenProbes {
|
|
return false
|
|
}
|
|
b.probesInUse++
|
|
return true
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
// Success reports a successful request.
|
|
func (b *Breaker) Success() {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
b.consecFails = 0
|
|
if b.state == HalfOpen {
|
|
b.probesInUse = 0
|
|
b.openRounds = 0
|
|
}
|
|
b.state = Closed
|
|
}
|
|
|
|
// Failure reports a failed request.
|
|
func (b *Breaker) Failure() {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
if b.state == HalfOpen {
|
|
// The probe failed too: reopen and lengthen the backoff.
|
|
b.probesInUse = 0
|
|
b.trip()
|
|
return
|
|
}
|
|
|
|
b.consecFails++
|
|
if b.consecFails >= b.cfg.FailureThreshold {
|
|
b.trip()
|
|
}
|
|
}
|
|
|
|
// trip opens the circuit. The caller must hold the lock.
|
|
func (b *Breaker) trip() {
|
|
cooldown := b.cfg.BaseCooldown << min(b.openRounds, 16)
|
|
if cooldown > b.cfg.MaxCooldown || cooldown <= 0 {
|
|
cooldown = b.cfg.MaxCooldown
|
|
}
|
|
b.state = Open
|
|
b.openUntil = b.now().Add(cooldown)
|
|
b.openRounds++
|
|
b.consecFails = 0
|
|
b.totalTrips++
|
|
}
|
|
|
|
// State returns the current state. An open circuit whose cooldown has elapsed
|
|
// is reported as half-open without changing state: scoring must not have side
|
|
// effects.
|
|
func (b *Breaker) State() State {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if b.state == Open && !b.now().Before(b.openUntil) {
|
|
return HalfOpen
|
|
}
|
|
return b.state
|
|
}
|
|
|
|
// Trips returns how many times the circuit has opened.
|
|
func (b *Breaker) Trips() int64 {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return b.totalTrips
|
|
}
|