The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
179 lines
3.7 KiB
Go
179 lines
3.7 KiB
Go
package breaker
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fakeClock lets tests advance time by hand.
|
|
type fakeClock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *fakeClock {
|
|
return &fakeClock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
|
|
func (c *fakeClock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
|
|
func (c *fakeClock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func newTestBreaker(cfg Config) (*Breaker, *fakeClock) {
|
|
b := New(cfg)
|
|
c := newClock()
|
|
b.SetClock(c.now)
|
|
return b, c
|
|
}
|
|
|
|
func TestOpensOnceThresholdIsCrossed(t *testing.T) {
|
|
b, _ := newTestBreaker(Config{FailureThreshold: 3, BaseCooldown: time.Minute})
|
|
|
|
for i := range 2 {
|
|
b.Failure()
|
|
if !b.Allow() {
|
|
t.Fatalf("circuit opened after %d failures; threshold is 3", i+1)
|
|
}
|
|
}
|
|
b.Failure()
|
|
|
|
if b.Allow() {
|
|
t.Error("threshold crossed but the circuit is still closed")
|
|
}
|
|
if got := b.State(); got != Open {
|
|
t.Errorf("state %v, expected open", got)
|
|
}
|
|
}
|
|
|
|
func TestSuccessResetsFailureCount(t *testing.T) {
|
|
b, _ := newTestBreaker(Config{FailureThreshold: 3, BaseCooldown: time.Minute})
|
|
|
|
b.Failure()
|
|
b.Failure()
|
|
b.Success() // must reset the counter
|
|
b.Failure()
|
|
b.Failure()
|
|
|
|
if !b.Allow() {
|
|
t.Error("consecutive counter survived an intervening success")
|
|
}
|
|
}
|
|
|
|
func TestSingleFailureCooldown(t *testing.T) {
|
|
// LiteLLM-style cooldown: one error takes the endpoint out for a while.
|
|
b, clock := newTestBreaker(Config{FailureThreshold: 1, BaseCooldown: 15 * time.Second})
|
|
|
|
b.Failure()
|
|
if b.Allow() {
|
|
t.Fatal("a single failure did not trigger cooldown")
|
|
}
|
|
|
|
clock.advance(14 * time.Second)
|
|
if b.Allow() {
|
|
t.Error("allowed a request before the cooldown elapsed")
|
|
}
|
|
|
|
clock.advance(2 * time.Second)
|
|
if !b.Allow() {
|
|
t.Error("cooldown elapsed but requests are still blocked")
|
|
}
|
|
}
|
|
|
|
func TestHalfOpenClosesOnSuccess(t *testing.T) {
|
|
b, clock := newTestBreaker(Config{FailureThreshold: 1, BaseCooldown: time.Second, HalfOpenProbes: 1})
|
|
|
|
b.Failure()
|
|
clock.advance(2 * time.Second)
|
|
|
|
if !b.Allow() {
|
|
t.Fatal("half-open probe was not allowed")
|
|
}
|
|
// A second concurrent probe must be blocked.
|
|
if b.Allow() {
|
|
t.Error("second probe allowed despite HalfOpenProbes=1")
|
|
}
|
|
|
|
b.Success()
|
|
if got := b.State(); got != Closed {
|
|
t.Errorf("state is %v after a successful probe", got)
|
|
}
|
|
if !b.Allow() {
|
|
t.Error("requests still blocked after the circuit closed")
|
|
}
|
|
}
|
|
|
|
func TestHalfOpenFailureBacksOffExponentially(t *testing.T) {
|
|
b, clock := newTestBreaker(Config{
|
|
FailureThreshold: 1,
|
|
BaseCooldown: time.Second,
|
|
MaxCooldown: time.Minute,
|
|
})
|
|
|
|
b.Failure() // first trip: 1s
|
|
clock.advance(1100 * time.Millisecond)
|
|
|
|
if !b.Allow() {
|
|
t.Fatal("no probe allowed after the first cooldown")
|
|
}
|
|
b.Failure() // probe failed: the second trip should last 2s
|
|
|
|
clock.advance(1100 * time.Millisecond)
|
|
if b.Allow() {
|
|
t.Error("backoff did not grow; allowed again after 1s")
|
|
}
|
|
clock.advance(time.Second)
|
|
if !b.Allow() {
|
|
t.Error("2s backoff elapsed but requests are still blocked")
|
|
}
|
|
}
|
|
|
|
func TestBackoffIsCapped(t *testing.T) {
|
|
b, clock := newTestBreaker(Config{
|
|
FailureThreshold: 1,
|
|
BaseCooldown: time.Second,
|
|
MaxCooldown: 4 * time.Second,
|
|
})
|
|
|
|
for range 10 {
|
|
b.Failure()
|
|
clock.advance(5 * time.Second)
|
|
b.Allow()
|
|
}
|
|
b.Failure()
|
|
|
|
clock.advance(5 * time.Second)
|
|
if !b.Allow() {
|
|
t.Error("cooldown exceeded the MaxCooldown cap")
|
|
}
|
|
}
|
|
|
|
func TestConcurrentUse(t *testing.T) {
|
|
b := New(DefaultConfig())
|
|
var wg sync.WaitGroup
|
|
for i := range 50 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if b.Allow() {
|
|
if i%2 == 0 {
|
|
b.Success()
|
|
} else {
|
|
b.Failure()
|
|
}
|
|
}
|
|
_ = b.State()
|
|
_ = b.Trips()
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|