// Package breaker provides a per-endpoint circuit breaker. // // It folds two concepts from plan ยง7.7 into a single component: // // - Cooldown: an endpoint that errors leaves the candidate pool for a while. // (Set Threshold to 1 to get exactly this behaviour.) // - Circuit breaker: opens once consecutive failures cross a threshold, then // moves to half-open probes with exponential backoff. // // Keeping them apart would mean two timers and two separate answers to // "is this endpoint usable right now?"; merging them produces fewer surprises. package breaker import ( "sync" "time" ) // State is the breaker's current state. type State int const ( // Closed is normal operation: requests pass through. Closed State = iota // Open means the endpoint is out of service; no requests are sent. Open // HalfOpen allows a limited number of probe requests. HalfOpen ) func (s State) String() string { switch s { case Closed: return "closed" case Open: return "open" case HalfOpen: return "half-open" default: return "unknown" } } // Config holds the breaker's settings. type Config struct { // FailureThreshold is the number of consecutive failures that opens the // circuit. 1 means "one error puts the endpoint in cooldown". FailureThreshold int // BaseCooldown is how long the circuit stays open the first time it trips. BaseCooldown time.Duration // MaxCooldown caps the exponential backoff. MaxCooldown time.Duration // HalfOpenProbes is how many concurrent probes are allowed while half-open. HalfOpenProbes int } // DefaultConfig returns sensible defaults. func DefaultConfig() Config { return Config{ FailureThreshold: 3, BaseCooldown: 15 * time.Second, MaxCooldown: 2 * time.Minute, HalfOpenProbes: 1, } } func (c *Config) normalize() { if c.FailureThreshold <= 0 { c.FailureThreshold = 3 } if c.BaseCooldown <= 0 { c.BaseCooldown = 15 * time.Second } if c.MaxCooldown < c.BaseCooldown { c.MaxCooldown = max(c.BaseCooldown, 2*time.Minute) } if c.HalfOpenProbes <= 0 { c.HalfOpenProbes = 1 } } // Breaker is one endpoint's circuit breaker. It is safe for concurrent use. type Breaker struct { cfg Config now func() time.Time mu sync.Mutex state State consecFails int openUntil time.Time openRounds int // how many times it has opened, for backoff probesInUse int totalTrips int64 } // New creates a breaker with the given settings. func New(cfg Config) *Breaker { cfg.normalize() return &Breaker{cfg: cfg, now: time.Now, state: Closed} } // SetClock replaces the time source, for tests. func (b *Breaker) SetClock(now func() time.Time) { b.mu.Lock() b.now = now b.mu.Unlock() } // Allow reports whether a request may be sent to this endpoint. // If it returns true, the outcome must be reported via Success or Failure. func (b *Breaker) Allow() bool { b.mu.Lock() defer b.mu.Unlock() switch b.state { case Closed: return true case Open: if b.now().Before(b.openUntil) { return false } // Cooldown elapsed: move to half-open and let this probe through. b.state = HalfOpen b.probesInUse = 1 return true case HalfOpen: if b.probesInUse >= b.cfg.HalfOpenProbes { return false } b.probesInUse++ return true default: return true } } // Success reports a successful request. func (b *Breaker) Success() { b.mu.Lock() defer b.mu.Unlock() b.consecFails = 0 if b.state == HalfOpen { b.probesInUse = 0 b.openRounds = 0 } b.state = Closed } // Failure reports a failed request. func (b *Breaker) Failure() { b.mu.Lock() defer b.mu.Unlock() if b.state == HalfOpen { // The probe failed too: reopen and lengthen the backoff. b.probesInUse = 0 b.trip() return } b.consecFails++ if b.consecFails >= b.cfg.FailureThreshold { b.trip() } } // trip opens the circuit. The caller must hold the lock. func (b *Breaker) trip() { cooldown := b.cfg.BaseCooldown << min(b.openRounds, 16) if cooldown > b.cfg.MaxCooldown || cooldown <= 0 { cooldown = b.cfg.MaxCooldown } b.state = Open b.openUntil = b.now().Add(cooldown) b.openRounds++ b.consecFails = 0 b.totalTrips++ } // State returns the current state. An open circuit whose cooldown has elapsed // is reported as half-open without changing state: scoring must not have side // effects. func (b *Breaker) State() State { b.mu.Lock() defer b.mu.Unlock() if b.state == Open && !b.now().Before(b.openUntil) { return HalfOpen } return b.state } // Trips returns how many times the circuit has opened. func (b *Breaker) Trips() int64 { b.mu.Lock() defer b.mu.Unlock() return b.totalTrips }