Files
llmgw/internal/router/breaker/breaker.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

199 lines
4.5 KiB
Go

// Package breaker provides a per-endpoint circuit breaker.
//
// It folds two concepts from plan §7.7 into a single component:
//
// - Cooldown: an endpoint that errors leaves the candidate pool for a while.
// (Set Threshold to 1 to get exactly this behaviour.)
// - Circuit breaker: opens once consecutive failures cross a threshold, then
// moves to half-open probes with exponential backoff.
//
// Keeping them apart would mean two timers and two separate answers to
// "is this endpoint usable right now?"; merging them produces fewer surprises.
package breaker
import (
"sync"
"time"
)
// State is the breaker's current state.
type State int
const (
// Closed is normal operation: requests pass through.
Closed State = iota
// Open means the endpoint is out of service; no requests are sent.
Open
// HalfOpen allows a limited number of probe requests.
HalfOpen
)
func (s State) String() string {
switch s {
case Closed:
return "closed"
case Open:
return "open"
case HalfOpen:
return "half-open"
default:
return "unknown"
}
}
// Config holds the breaker's settings.
type Config struct {
// FailureThreshold is the number of consecutive failures that opens the
// circuit. 1 means "one error puts the endpoint in cooldown".
FailureThreshold int
// BaseCooldown is how long the circuit stays open the first time it trips.
BaseCooldown time.Duration
// MaxCooldown caps the exponential backoff.
MaxCooldown time.Duration
// HalfOpenProbes is how many concurrent probes are allowed while half-open.
HalfOpenProbes int
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
FailureThreshold: 3,
BaseCooldown: 15 * time.Second,
MaxCooldown: 2 * time.Minute,
HalfOpenProbes: 1,
}
}
func (c *Config) normalize() {
if c.FailureThreshold <= 0 {
c.FailureThreshold = 3
}
if c.BaseCooldown <= 0 {
c.BaseCooldown = 15 * time.Second
}
if c.MaxCooldown < c.BaseCooldown {
c.MaxCooldown = max(c.BaseCooldown, 2*time.Minute)
}
if c.HalfOpenProbes <= 0 {
c.HalfOpenProbes = 1
}
}
// Breaker is one endpoint's circuit breaker. It is safe for concurrent use.
type Breaker struct {
cfg Config
now func() time.Time
mu sync.Mutex
state State
consecFails int
openUntil time.Time
openRounds int // how many times it has opened, for backoff
probesInUse int
totalTrips int64
}
// New creates a breaker with the given settings.
func New(cfg Config) *Breaker {
cfg.normalize()
return &Breaker{cfg: cfg, now: time.Now, state: Closed}
}
// SetClock replaces the time source, for tests.
func (b *Breaker) SetClock(now func() time.Time) {
b.mu.Lock()
b.now = now
b.mu.Unlock()
}
// Allow reports whether a request may be sent to this endpoint.
// If it returns true, the outcome must be reported via Success or Failure.
func (b *Breaker) Allow() bool {
b.mu.Lock()
defer b.mu.Unlock()
switch b.state {
case Closed:
return true
case Open:
if b.now().Before(b.openUntil) {
return false
}
// Cooldown elapsed: move to half-open and let this probe through.
b.state = HalfOpen
b.probesInUse = 1
return true
case HalfOpen:
if b.probesInUse >= b.cfg.HalfOpenProbes {
return false
}
b.probesInUse++
return true
default:
return true
}
}
// Success reports a successful request.
func (b *Breaker) Success() {
b.mu.Lock()
defer b.mu.Unlock()
b.consecFails = 0
if b.state == HalfOpen {
b.probesInUse = 0
b.openRounds = 0
}
b.state = Closed
}
// Failure reports a failed request.
func (b *Breaker) Failure() {
b.mu.Lock()
defer b.mu.Unlock()
if b.state == HalfOpen {
// The probe failed too: reopen and lengthen the backoff.
b.probesInUse = 0
b.trip()
return
}
b.consecFails++
if b.consecFails >= b.cfg.FailureThreshold {
b.trip()
}
}
// trip opens the circuit. The caller must hold the lock.
func (b *Breaker) trip() {
cooldown := b.cfg.BaseCooldown << min(b.openRounds, 16)
if cooldown > b.cfg.MaxCooldown || cooldown <= 0 {
cooldown = b.cfg.MaxCooldown
}
b.state = Open
b.openUntil = b.now().Add(cooldown)
b.openRounds++
b.consecFails = 0
b.totalTrips++
}
// State returns the current state. An open circuit whose cooldown has elapsed
// is reported as half-open without changing state: scoring must not have side
// effects.
func (b *Breaker) State() State {
b.mu.Lock()
defer b.mu.Unlock()
if b.state == Open && !b.now().Before(b.openUntil) {
return HalfOpen
}
return b.state
}
// Trips returns how many times the circuit has opened.
func (b *Breaker) Trips() int64 {
b.mu.Lock()
defer b.mu.Unlock()
return b.totalTrips
}