CI / test (push) Successful in 3m23s
CI's test job has failed intermittently — two of three runs on some days — while every run here passed, including six repetitions on one CPU under the race detector with eight packages at once. The difference is a shared, loaded Linux runner, and the tests most exposed to it are the ones that sleep a fixed time and then assert: - admission: idle admission bounded at 50ms of wall clock against a 1ms tick, and a queued request checked for after a 50ms sleep; - scrape: a 10ms scrape timeout, which on a slow runner reports the engine unreachable and then the violation a second time, in a test that counts one; - supervisor: log lines, three lines of output and a removed state file each checked after a fixed sleep, and an exited worker bounded at 1.5s; - agent: a heartbeat checked for after four intervals. Each now waits for its condition up to a deadline of seconds, and the two wall-clock bounds are set against what they exist to distinguish — the admission queue's tick raised to ten seconds, the exited worker bounded below the five-second grace it must not wait out — rather than against how fast this laptop is. Without the CI logs this cannot be proven to be the failure; these are the tests that fail when the machine is slow, and they no longer do. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
514 lines
16 KiB
Go
514 lines
16 KiB
Go
package admission
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
func testConfig() Config {
|
|
c := DefaultConfig()
|
|
c.Interval = time.Millisecond
|
|
c.QueueTTL = 2 * time.Second
|
|
return c
|
|
}
|
|
|
|
// fixedSat lets a test drive saturation directly.
|
|
type fixedSat struct{ v atomic.Uint64 }
|
|
|
|
func (f *fixedSat) set(v float64) { f.v.Store(uint64(v * 1000)) }
|
|
func (f *fixedSat) fn(string, int) (float64, bool) {
|
|
return float64(f.v.Load()) / 1000, true
|
|
}
|
|
|
|
func newController(t *testing.T, cfg Config, sat SaturationFunc) *Controller {
|
|
t.Helper()
|
|
c, err := New(cfg, sat)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c.Start()
|
|
t.Cleanup(c.Close)
|
|
return c
|
|
}
|
|
|
|
func TestAdmitsImmediatelyWhenIdle(t *testing.T) {
|
|
s := &fixedSat{}
|
|
// A queue that is looked at every ten seconds, so a request that went
|
|
// through it rather than the fast path would take that long. A bound of
|
|
// 50ms against a 1ms tick measured the machine the test ran on, and a
|
|
// loaded CI runner under the race detector was slower than that.
|
|
cfg := testConfig()
|
|
cfg.Interval = 10 * time.Second
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
start := time.Now()
|
|
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d := time.Since(start); d > 2*time.Second {
|
|
t.Errorf("idle admission took %v; the fast path is not being used", d)
|
|
}
|
|
lease.Release()
|
|
}
|
|
|
|
func TestLowBandHeldWhileHighBandFlows(t *testing.T) {
|
|
// The core of plan §7.2.2: the cluster degrades from the bottom up.
|
|
s := &fixedSat{}
|
|
s.set(0.5) // above bronze (0.30), below gold (0.85)
|
|
c := newController(t, testConfig(), s.fn)
|
|
|
|
ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)
|
|
defer cancel()
|
|
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
|
|
t.Error("bronze was admitted above its ceiling")
|
|
}
|
|
|
|
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"})
|
|
if err != nil {
|
|
t.Fatalf("gold was held below its ceiling: %v", err)
|
|
}
|
|
lease.Release()
|
|
}
|
|
|
|
func TestTopBandFlowsUpToTheHealthyLimit(t *testing.T) {
|
|
// A ceiling of 1.0 means "hold only once the pool is past its healthy
|
|
// limit", not "never hold". At exactly the limit the top band still flows.
|
|
s := &fixedSat{}
|
|
s.set(1.0)
|
|
c := newController(t, testConfig(), s.fn)
|
|
|
|
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"})
|
|
if err != nil {
|
|
t.Fatalf("top band held at exactly its ceiling: %v", err)
|
|
}
|
|
lease.Release()
|
|
}
|
|
|
|
func TestTopBandIsHeldWhenOverloaded(t *testing.T) {
|
|
// Holding even the top band is deliberate: with nobody above it to protect,
|
|
// the benefit is a bounded engine queue. Dumping a flood straight into a
|
|
// FIFO engine queue makes every request in that band slow, whereas queueing
|
|
// at the gateway keeps ordering, TTL and tenant fairness.
|
|
s := &fixedSat{}
|
|
s.set(5)
|
|
cfg := testConfig()
|
|
cfg.QueueTTL = 200 * time.Millisecond
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"}); !errors.Is(err, ErrQueueTimeout) {
|
|
t.Fatalf("expected the top band to queue while overloaded, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCeilingAboveOneNeverHolds(t *testing.T) {
|
|
// The escape hatch for a band that must never wait, whatever the load.
|
|
s := &fixedSat{}
|
|
s.set(50)
|
|
cfg := testConfig()
|
|
cfg.Bands = []Band{{Name: "critical", Priority: 900, SaturationCeiling: 1000}}
|
|
cfg.DefaultBand = "critical"
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "critical"})
|
|
if err != nil {
|
|
t.Fatalf("a band with an effectively unbounded ceiling was held: %v", err)
|
|
}
|
|
lease.Release()
|
|
}
|
|
|
|
func TestQueuedRequestAdmittedWhenSaturationDrops(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(0.9)
|
|
c := newController(t, testConfig(), s.fn)
|
|
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
|
done <- err
|
|
}()
|
|
|
|
// Wait for it to actually queue, then open the gate. Waited for, not
|
|
// slept through: on a loaded runner 50ms was sometimes not long enough
|
|
// for the goroutine to have been scheduled at all.
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for c.Depths()[3].Depth != 1 && time.Now().Before(deadline) {
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
if d := c.Depths(); d[3].Depth != 1 {
|
|
t.Fatalf("expected 1 queued bronze request, got %+v", d)
|
|
}
|
|
s.set(0.1)
|
|
|
|
select {
|
|
case err := <-done:
|
|
if err != nil {
|
|
t.Fatalf("queued request failed: %v", err)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("queued request was not admitted after saturation dropped")
|
|
}
|
|
}
|
|
|
|
func TestQueueTTL(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(1.0)
|
|
cfg := testConfig()
|
|
cfg.QueueTTL = 150 * time.Millisecond
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
start := time.Now()
|
|
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
|
if !errors.Is(err, ErrQueueTimeout) {
|
|
t.Fatalf("expected ErrQueueTimeout, got %v", err)
|
|
}
|
|
if d := time.Since(start); d > time.Second {
|
|
t.Errorf("TTL fired after %v", d)
|
|
}
|
|
}
|
|
|
|
func TestQueueCapacity(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(1.0)
|
|
cfg := testConfig()
|
|
cfg.QueueCapacityPerBand = 3
|
|
cfg.QueueTTL = 5 * time.Second
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
for range 3 {
|
|
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // queued on purpose
|
|
}
|
|
// Wait for the queue to fill.
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) && c.Depths()[3].Depth < 3 {
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
|
|
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
|
if !errors.Is(err, ErrQueueFull) {
|
|
t.Errorf("expected ErrQueueFull, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestClientCancellationLeavesQueue(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(1.0)
|
|
c := newController(t, testConfig(), s.fn)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // cancelled on purpose
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) && c.Depths()[3].Depth == 0 {
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
cancel()
|
|
|
|
deadline = time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if c.Depths()[3].Depth == 0 {
|
|
return
|
|
}
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
t.Fatal("a cancelled request stayed in the queue, occupying capacity")
|
|
}
|
|
|
|
// TestBandQueueRotatesTenants is where the fairness property actually lives.
|
|
//
|
|
// It is tested on the queue rather than through the controller on purpose: at
|
|
// the controller level the observable order is the order goroutines wake up,
|
|
// which the scheduler decides, not the order the dispatcher admitted them. A
|
|
// test asserting on that is measuring the Go runtime.
|
|
func TestBandQueueRotatesTenants(t *testing.T) {
|
|
q := newBandQueue(Band{Name: "bronze"})
|
|
|
|
// One tenant floods, then a quiet one arrives strictly behind it.
|
|
for range 10 {
|
|
q.push(&waiter{tenant: "noisy", model: "m", ready: make(chan struct{})})
|
|
}
|
|
q.push(&waiter{tenant: "quiet", model: "m", ready: make(chan struct{})})
|
|
|
|
var order []string
|
|
for q.len() > 0 {
|
|
order = append(order, q.pop().tenant)
|
|
}
|
|
|
|
if len(order) != 11 {
|
|
t.Fatalf("popped %d waiters, pushed 11", len(order))
|
|
}
|
|
// FIFO would put the quiet tenant last. Round-robin gives it the second
|
|
// turn: one noisy, one quiet, then the rest of the flood.
|
|
if order[0] != "noisy" || order[1] != "quiet" {
|
|
t.Errorf("rotation order starts %v; a flood is taking consecutive turns", order[:3])
|
|
}
|
|
quiet := 0
|
|
for _, v := range order {
|
|
if v == "quiet" {
|
|
quiet++
|
|
}
|
|
}
|
|
if quiet != 1 {
|
|
t.Errorf("the quiet tenant was admitted %d times", quiet)
|
|
}
|
|
}
|
|
|
|
func TestBandQueueRotationIsStable(t *testing.T) {
|
|
// Three tenants with unequal backlogs must still take turns, and a tenant
|
|
// that runs out must drop cleanly out of the rotation.
|
|
q := newBandQueue(Band{Name: "bronze"})
|
|
for _, tenant := range []string{"a", "a", "a", "b", "c"} {
|
|
q.push(&waiter{tenant: tenant, model: "m", ready: make(chan struct{})})
|
|
}
|
|
|
|
var order []string
|
|
for q.len() > 0 {
|
|
order = append(order, q.pop().tenant)
|
|
}
|
|
if strings.Join(order, "") != "abcaa" {
|
|
t.Errorf("order=%v, expected a,b,c then a's remaining two", order)
|
|
}
|
|
}
|
|
|
|
func TestBandQueueRemoveKeepsRotation(t *testing.T) {
|
|
// A cancelled request leaves the queue; the rotation must not skip or
|
|
// repeat a tenant because of it.
|
|
q := newBandQueue(Band{Name: "bronze"})
|
|
a1 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
|
|
b1 := &waiter{tenant: "b", model: "m", ready: make(chan struct{})}
|
|
a2 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
|
|
q.push(a1)
|
|
q.push(b1)
|
|
q.push(a2)
|
|
|
|
if !q.remove(b1) {
|
|
t.Fatal("remove reported the waiter was not queued")
|
|
}
|
|
if q.len() != 2 {
|
|
t.Fatalf("len=%d after removing one of three", q.len())
|
|
}
|
|
|
|
var order []string
|
|
for q.len() > 0 {
|
|
order = append(order, q.pop().tenant)
|
|
}
|
|
if strings.Join(order, "") != "aa" {
|
|
t.Errorf("order=%v after the only b was removed", order)
|
|
}
|
|
}
|
|
|
|
// TestNoTenantIsStarved is the controller-level companion: it asserts that
|
|
// everything queued is eventually admitted, without asserting an order the Go
|
|
// scheduler controls.
|
|
func TestNoTenantIsStarved(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(1.0) // hold everything initially
|
|
cfg := testConfig()
|
|
cfg.QueueTTL = 10 * time.Second
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
const flood = 40
|
|
var noisy, quiet atomic.Int64
|
|
var wg sync.WaitGroup
|
|
|
|
admit := func(tenant string, counter *atomic.Int64) {
|
|
defer wg.Done()
|
|
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze", Tenant: tenant}); err == nil {
|
|
counter.Add(1)
|
|
}
|
|
}
|
|
|
|
for range flood {
|
|
wg.Add(1)
|
|
go admit("noisy", &noisy)
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood {
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
|
|
for range 4 {
|
|
wg.Add(1)
|
|
go admit("quiet", &quiet)
|
|
}
|
|
deadline = time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood+4 {
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
|
|
s.set(0.0) // open the gate
|
|
wg.Wait()
|
|
|
|
if noisy.Load() != flood {
|
|
t.Errorf("the flooding tenant got %d of %d admissions", noisy.Load(), flood)
|
|
}
|
|
if quiet.Load() != 4 {
|
|
t.Errorf("the quiet tenant got %d of 4 admissions; it was starved", quiet.Load())
|
|
}
|
|
}
|
|
|
|
func TestUnknownBandUsesDefault(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(0.5)
|
|
cfg := testConfig()
|
|
cfg.DefaultBand = "bronze"
|
|
c := newController(t, cfg, s.fn)
|
|
|
|
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
|
|
defer cancel()
|
|
// bronze's ceiling is 0.30, so an unknown band must be held here.
|
|
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "no-such-band"}); err == nil {
|
|
t.Error("unknown band did not fall back to the default")
|
|
}
|
|
}
|
|
|
|
func TestUnknownModelIsLetThrough(t *testing.T) {
|
|
// Better a precise 503 from dispatch than a 30-second queue timeout.
|
|
c := newController(t, testConfig(), func(string, int) (float64, bool) { return 0, false })
|
|
lease, err := c.Acquire(t.Context(), Request{Model: "nope", Band: "bronze"})
|
|
if err != nil {
|
|
t.Fatalf("unknown model was queued instead of passed through: %v", err)
|
|
}
|
|
lease.Release()
|
|
}
|
|
|
|
func TestConfigValidation(t *testing.T) {
|
|
if _, err := New(Config{Bands: []Band{{Name: "a"}, {Name: "a"}}}, nil); err == nil {
|
|
t.Error("duplicate band names accepted")
|
|
}
|
|
if _, err := New(Config{Bands: []Band{{Name: "a"}}, DefaultBand: "b"}, nil); err == nil {
|
|
t.Error("default_band outside the band list accepted")
|
|
}
|
|
}
|
|
|
|
func TestBandsSortedByPriority(t *testing.T) {
|
|
c, err := New(Config{Bands: []Band{
|
|
{Name: "low", Priority: 1, SaturationCeiling: 0.2},
|
|
{Name: "high", Priority: 9, SaturationCeiling: 1},
|
|
}, DefaultBand: "low"}, func(string, int) (float64, bool) { return 0, true })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := c.Bands()[0].Name; got != "high" {
|
|
t.Errorf("bands are not ordered by priority: first is %q", got)
|
|
}
|
|
}
|
|
|
|
// --- saturation ---
|
|
|
|
func snap(id string, waiting, running, inflight int, kv float64, open bool) endpoint.Snapshot {
|
|
s := endpoint.Snapshot{
|
|
Info: endpoint.Info{ID: id},
|
|
Inflight: inflight,
|
|
Breaker: breaker.Closed,
|
|
}
|
|
s.Stats.Waiting = waiting
|
|
s.Stats.Running = running
|
|
s.Stats.KVCacheUsage = kv
|
|
if open {
|
|
s.Breaker = breaker.Open
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestSaturationCombinesQueueAndKV(t *testing.T) {
|
|
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
|
|
|
// Empty queue but a nearly full cache still counts as saturated.
|
|
if got := Saturation([]endpoint.Snapshot{snap("a", 0, 1, 1, 0.89, false)}, 0, th); got < 0.9 {
|
|
t.Errorf("KV pressure ignored: %v", got)
|
|
}
|
|
// Full queue with an empty cache likewise.
|
|
if got := Saturation([]endpoint.Snapshot{snap("a", 4, 1, 5, 0, false)}, 0, th); got < 1 {
|
|
t.Errorf("queue depth ignored: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestSaturationCountsUnreportedInflight(t *testing.T) {
|
|
// A burst the engine has not reported yet is real load; missing it would
|
|
// leave admission blind exactly when it needs to bite.
|
|
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
|
cold := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 0, 0, false)}, 0, th)
|
|
burst := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 8, 0, false)}, 0, th)
|
|
if !(burst > cold) {
|
|
t.Errorf("unreported inflight not counted: cold=%v burst=%v", cold, burst)
|
|
}
|
|
}
|
|
|
|
func TestSaturationSkipsOpenBreakers(t *testing.T) {
|
|
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
|
eps := []endpoint.Snapshot{
|
|
snap("busy", 8, 4, 12, 0.5, false),
|
|
snap("dead", 0, 0, 0, 0, true), // would halve the mean if counted
|
|
}
|
|
withDead := Saturation(eps, 0, th)
|
|
alone := Saturation(eps[:1], 0, th)
|
|
if fmt.Sprintf("%.4f", withDead) != fmt.Sprintf("%.4f", alone) {
|
|
t.Errorf("an endpoint with an open breaker was counted as capacity: %v vs %v", withDead, alone)
|
|
}
|
|
}
|
|
|
|
func TestSaturationCountsUnreportedAdmissions(t *testing.T) {
|
|
// The thundering herd guard: a simultaneous burst must register as load
|
|
// before any of it reaches an engine, or the whole burst walks through the
|
|
// fast path while every signal still reads zero.
|
|
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
|
idle := []endpoint.Snapshot{snap("a", 0, 0, 0, 0, false), snap("b", 0, 0, 0, 0, false)}
|
|
|
|
if got := Saturation(idle, 0, th); got != 0 {
|
|
t.Fatalf("idle pool reported %v", got)
|
|
}
|
|
if got := Saturation(idle, 16, th); got <= 1 {
|
|
t.Errorf("16 unreported admissions across 2 replicas should saturate, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestBurstDoesNotAllPassTheFastPath(t *testing.T) {
|
|
// End to end for the same guard: with saturation derived from outstanding
|
|
// leases, a burst of 200 must not be admitted wholesale.
|
|
var admitted atomic.Int64
|
|
c := newController(t, testConfig(), func(_ string, outstanding int) (float64, bool) {
|
|
// Two replicas, healthy buffer of 4 each: 8 in flight is the limit.
|
|
return float64(outstanding) / 8, true
|
|
})
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
|
defer cancel()
|
|
|
|
var wg sync.WaitGroup
|
|
for range 200 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
|
|
admitted.Add(1)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
// bronze's ceiling is 0.30, so roughly 8*0.3 = 2-3 should get in. Allow
|
|
// generous slack for scheduling, but nothing near 200.
|
|
if n := admitted.Load(); n > 20 {
|
|
t.Errorf("%d of 200 requests were admitted at once; the burst guard is not working", n)
|
|
}
|
|
}
|
|
|
|
func TestSaturationWithNoUsableEndpoints(t *testing.T) {
|
|
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
|
if got := Saturation(nil, 0, th); got != 1 {
|
|
t.Errorf("an empty pool should report full saturation, got %v", got)
|
|
}
|
|
}
|