Files
llmgw/internal/router/admission/admission_test.go
T
Ilker ManapandClaude Opus 5.5 b661bfead8
CI / test (push) Successful in 3m23s
Wait for what tests check instead of sleeping and hoping
CI's test job has failed intermittently — two of three runs on some days —
while every run here passed, including six repetitions on one CPU under the
race detector with eight packages at once. The difference is a shared,
loaded Linux runner, and the tests most exposed to it are the ones that
sleep a fixed time and then assert:

- admission: idle admission bounded at 50ms of wall clock against a 1ms
  tick, and a queued request checked for after a 50ms sleep;
- scrape: a 10ms scrape timeout, which on a slow runner reports the engine
  unreachable and then the violation a second time, in a test that counts
  one;
- supervisor: log lines, three lines of output and a removed state file
  each checked after a fixed sleep, and an exited worker bounded at 1.5s;
- agent: a heartbeat checked for after four intervals.

Each now waits for its condition up to a deadline of seconds, and the two
wall-clock bounds are set against what they exist to distinguish — the
admission queue's tick raised to ten seconds, the exited worker bounded
below the five-second grace it must not wait out — rather than against how
fast this laptop is.

Without the CI logs this cannot be proven to be the failure; these are the
tests that fail when the machine is slow, and they no longer do.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 01:36:55 +02:00

514 lines
16 KiB
Go

package admission
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func testConfig() Config {
c := DefaultConfig()
c.Interval = time.Millisecond
c.QueueTTL = 2 * time.Second
return c
}
// fixedSat lets a test drive saturation directly.
type fixedSat struct{ v atomic.Uint64 }
func (f *fixedSat) set(v float64) { f.v.Store(uint64(v * 1000)) }
func (f *fixedSat) fn(string, int) (float64, bool) {
return float64(f.v.Load()) / 1000, true
}
func newController(t *testing.T, cfg Config, sat SaturationFunc) *Controller {
t.Helper()
c, err := New(cfg, sat)
if err != nil {
t.Fatal(err)
}
c.Start()
t.Cleanup(c.Close)
return c
}
func TestAdmitsImmediatelyWhenIdle(t *testing.T) {
s := &fixedSat{}
// A queue that is looked at every ten seconds, so a request that went
// through it rather than the fast path would take that long. A bound of
// 50ms against a 1ms tick measured the machine the test ran on, and a
// loaded CI runner under the race detector was slower than that.
cfg := testConfig()
cfg.Interval = 10 * time.Second
c := newController(t, cfg, s.fn)
start := time.Now()
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if err != nil {
t.Fatal(err)
}
if d := time.Since(start); d > 2*time.Second {
t.Errorf("idle admission took %v; the fast path is not being used", d)
}
lease.Release()
}
func TestLowBandHeldWhileHighBandFlows(t *testing.T) {
// The core of plan §7.2.2: the cluster degrades from the bottom up.
s := &fixedSat{}
s.set(0.5) // above bronze (0.30), below gold (0.85)
c := newController(t, testConfig(), s.fn)
ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)
defer cancel()
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
t.Error("bronze was admitted above its ceiling")
}
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"})
if err != nil {
t.Fatalf("gold was held below its ceiling: %v", err)
}
lease.Release()
}
func TestTopBandFlowsUpToTheHealthyLimit(t *testing.T) {
// A ceiling of 1.0 means "hold only once the pool is past its healthy
// limit", not "never hold". At exactly the limit the top band still flows.
s := &fixedSat{}
s.set(1.0)
c := newController(t, testConfig(), s.fn)
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"})
if err != nil {
t.Fatalf("top band held at exactly its ceiling: %v", err)
}
lease.Release()
}
func TestTopBandIsHeldWhenOverloaded(t *testing.T) {
// Holding even the top band is deliberate: with nobody above it to protect,
// the benefit is a bounded engine queue. Dumping a flood straight into a
// FIFO engine queue makes every request in that band slow, whereas queueing
// at the gateway keeps ordering, TTL and tenant fairness.
s := &fixedSat{}
s.set(5)
cfg := testConfig()
cfg.QueueTTL = 200 * time.Millisecond
c := newController(t, cfg, s.fn)
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"}); !errors.Is(err, ErrQueueTimeout) {
t.Fatalf("expected the top band to queue while overloaded, got %v", err)
}
}
func TestCeilingAboveOneNeverHolds(t *testing.T) {
// The escape hatch for a band that must never wait, whatever the load.
s := &fixedSat{}
s.set(50)
cfg := testConfig()
cfg.Bands = []Band{{Name: "critical", Priority: 900, SaturationCeiling: 1000}}
cfg.DefaultBand = "critical"
c := newController(t, cfg, s.fn)
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "critical"})
if err != nil {
t.Fatalf("a band with an effectively unbounded ceiling was held: %v", err)
}
lease.Release()
}
func TestQueuedRequestAdmittedWhenSaturationDrops(t *testing.T) {
s := &fixedSat{}
s.set(0.9)
c := newController(t, testConfig(), s.fn)
done := make(chan error, 1)
go func() {
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
done <- err
}()
// Wait for it to actually queue, then open the gate. Waited for, not
// slept through: on a loaded runner 50ms was sometimes not long enough
// for the goroutine to have been scheduled at all.
deadline := time.Now().Add(2 * time.Second)
for c.Depths()[3].Depth != 1 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if d := c.Depths(); d[3].Depth != 1 {
t.Fatalf("expected 1 queued bronze request, got %+v", d)
}
s.set(0.1)
select {
case err := <-done:
if err != nil {
t.Fatalf("queued request failed: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("queued request was not admitted after saturation dropped")
}
}
func TestQueueTTL(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
cfg := testConfig()
cfg.QueueTTL = 150 * time.Millisecond
c := newController(t, cfg, s.fn)
start := time.Now()
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if !errors.Is(err, ErrQueueTimeout) {
t.Fatalf("expected ErrQueueTimeout, got %v", err)
}
if d := time.Since(start); d > time.Second {
t.Errorf("TTL fired after %v", d)
}
}
func TestQueueCapacity(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
cfg := testConfig()
cfg.QueueCapacityPerBand = 3
cfg.QueueTTL = 5 * time.Second
c := newController(t, cfg, s.fn)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for range 3 {
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // queued on purpose
}
// Wait for the queue to fill.
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < 3 {
time.Sleep(2 * time.Millisecond)
}
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if !errors.Is(err, ErrQueueFull) {
t.Errorf("expected ErrQueueFull, got %v", err)
}
}
func TestClientCancellationLeavesQueue(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
c := newController(t, testConfig(), s.fn)
ctx, cancel := context.WithCancel(context.Background())
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // cancelled on purpose
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth == 0 {
time.Sleep(2 * time.Millisecond)
}
cancel()
deadline = time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if c.Depths()[3].Depth == 0 {
return
}
time.Sleep(2 * time.Millisecond)
}
t.Fatal("a cancelled request stayed in the queue, occupying capacity")
}
// TestBandQueueRotatesTenants is where the fairness property actually lives.
//
// It is tested on the queue rather than through the controller on purpose: at
// the controller level the observable order is the order goroutines wake up,
// which the scheduler decides, not the order the dispatcher admitted them. A
// test asserting on that is measuring the Go runtime.
func TestBandQueueRotatesTenants(t *testing.T) {
q := newBandQueue(Band{Name: "bronze"})
// One tenant floods, then a quiet one arrives strictly behind it.
for range 10 {
q.push(&waiter{tenant: "noisy", model: "m", ready: make(chan struct{})})
}
q.push(&waiter{tenant: "quiet", model: "m", ready: make(chan struct{})})
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if len(order) != 11 {
t.Fatalf("popped %d waiters, pushed 11", len(order))
}
// FIFO would put the quiet tenant last. Round-robin gives it the second
// turn: one noisy, one quiet, then the rest of the flood.
if order[0] != "noisy" || order[1] != "quiet" {
t.Errorf("rotation order starts %v; a flood is taking consecutive turns", order[:3])
}
quiet := 0
for _, v := range order {
if v == "quiet" {
quiet++
}
}
if quiet != 1 {
t.Errorf("the quiet tenant was admitted %d times", quiet)
}
}
func TestBandQueueRotationIsStable(t *testing.T) {
// Three tenants with unequal backlogs must still take turns, and a tenant
// that runs out must drop cleanly out of the rotation.
q := newBandQueue(Band{Name: "bronze"})
for _, tenant := range []string{"a", "a", "a", "b", "c"} {
q.push(&waiter{tenant: tenant, model: "m", ready: make(chan struct{})})
}
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if strings.Join(order, "") != "abcaa" {
t.Errorf("order=%v, expected a,b,c then a's remaining two", order)
}
}
func TestBandQueueRemoveKeepsRotation(t *testing.T) {
// A cancelled request leaves the queue; the rotation must not skip or
// repeat a tenant because of it.
q := newBandQueue(Band{Name: "bronze"})
a1 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
b1 := &waiter{tenant: "b", model: "m", ready: make(chan struct{})}
a2 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
q.push(a1)
q.push(b1)
q.push(a2)
if !q.remove(b1) {
t.Fatal("remove reported the waiter was not queued")
}
if q.len() != 2 {
t.Fatalf("len=%d after removing one of three", q.len())
}
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if strings.Join(order, "") != "aa" {
t.Errorf("order=%v after the only b was removed", order)
}
}
// TestNoTenantIsStarved is the controller-level companion: it asserts that
// everything queued is eventually admitted, without asserting an order the Go
// scheduler controls.
func TestNoTenantIsStarved(t *testing.T) {
s := &fixedSat{}
s.set(1.0) // hold everything initially
cfg := testConfig()
cfg.QueueTTL = 10 * time.Second
c := newController(t, cfg, s.fn)
const flood = 40
var noisy, quiet atomic.Int64
var wg sync.WaitGroup
admit := func(tenant string, counter *atomic.Int64) {
defer wg.Done()
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze", Tenant: tenant}); err == nil {
counter.Add(1)
}
}
for range flood {
wg.Add(1)
go admit("noisy", &noisy)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood {
time.Sleep(2 * time.Millisecond)
}
for range 4 {
wg.Add(1)
go admit("quiet", &quiet)
}
deadline = time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood+4 {
time.Sleep(2 * time.Millisecond)
}
s.set(0.0) // open the gate
wg.Wait()
if noisy.Load() != flood {
t.Errorf("the flooding tenant got %d of %d admissions", noisy.Load(), flood)
}
if quiet.Load() != 4 {
t.Errorf("the quiet tenant got %d of 4 admissions; it was starved", quiet.Load())
}
}
func TestUnknownBandUsesDefault(t *testing.T) {
s := &fixedSat{}
s.set(0.5)
cfg := testConfig()
cfg.DefaultBand = "bronze"
c := newController(t, cfg, s.fn)
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
defer cancel()
// bronze's ceiling is 0.30, so an unknown band must be held here.
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "no-such-band"}); err == nil {
t.Error("unknown band did not fall back to the default")
}
}
func TestUnknownModelIsLetThrough(t *testing.T) {
// Better a precise 503 from dispatch than a 30-second queue timeout.
c := newController(t, testConfig(), func(string, int) (float64, bool) { return 0, false })
lease, err := c.Acquire(t.Context(), Request{Model: "nope", Band: "bronze"})
if err != nil {
t.Fatalf("unknown model was queued instead of passed through: %v", err)
}
lease.Release()
}
func TestConfigValidation(t *testing.T) {
if _, err := New(Config{Bands: []Band{{Name: "a"}, {Name: "a"}}}, nil); err == nil {
t.Error("duplicate band names accepted")
}
if _, err := New(Config{Bands: []Band{{Name: "a"}}, DefaultBand: "b"}, nil); err == nil {
t.Error("default_band outside the band list accepted")
}
}
func TestBandsSortedByPriority(t *testing.T) {
c, err := New(Config{Bands: []Band{
{Name: "low", Priority: 1, SaturationCeiling: 0.2},
{Name: "high", Priority: 9, SaturationCeiling: 1},
}, DefaultBand: "low"}, func(string, int) (float64, bool) { return 0, true })
if err != nil {
t.Fatal(err)
}
if got := c.Bands()[0].Name; got != "high" {
t.Errorf("bands are not ordered by priority: first is %q", got)
}
}
// --- saturation ---
func snap(id string, waiting, running, inflight int, kv float64, open bool) endpoint.Snapshot {
s := endpoint.Snapshot{
Info: endpoint.Info{ID: id},
Inflight: inflight,
Breaker: breaker.Closed,
}
s.Stats.Waiting = waiting
s.Stats.Running = running
s.Stats.KVCacheUsage = kv
if open {
s.Breaker = breaker.Open
}
return s
}
func TestSaturationCombinesQueueAndKV(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
// Empty queue but a nearly full cache still counts as saturated.
if got := Saturation([]endpoint.Snapshot{snap("a", 0, 1, 1, 0.89, false)}, 0, th); got < 0.9 {
t.Errorf("KV pressure ignored: %v", got)
}
// Full queue with an empty cache likewise.
if got := Saturation([]endpoint.Snapshot{snap("a", 4, 1, 5, 0, false)}, 0, th); got < 1 {
t.Errorf("queue depth ignored: %v", got)
}
}
func TestSaturationCountsUnreportedInflight(t *testing.T) {
// A burst the engine has not reported yet is real load; missing it would
// leave admission blind exactly when it needs to bite.
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
cold := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 0, 0, false)}, 0, th)
burst := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 8, 0, false)}, 0, th)
if !(burst > cold) {
t.Errorf("unreported inflight not counted: cold=%v burst=%v", cold, burst)
}
}
func TestSaturationSkipsOpenBreakers(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
eps := []endpoint.Snapshot{
snap("busy", 8, 4, 12, 0.5, false),
snap("dead", 0, 0, 0, 0, true), // would halve the mean if counted
}
withDead := Saturation(eps, 0, th)
alone := Saturation(eps[:1], 0, th)
if fmt.Sprintf("%.4f", withDead) != fmt.Sprintf("%.4f", alone) {
t.Errorf("an endpoint with an open breaker was counted as capacity: %v vs %v", withDead, alone)
}
}
func TestSaturationCountsUnreportedAdmissions(t *testing.T) {
// The thundering herd guard: a simultaneous burst must register as load
// before any of it reaches an engine, or the whole burst walks through the
// fast path while every signal still reads zero.
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
idle := []endpoint.Snapshot{snap("a", 0, 0, 0, 0, false), snap("b", 0, 0, 0, 0, false)}
if got := Saturation(idle, 0, th); got != 0 {
t.Fatalf("idle pool reported %v", got)
}
if got := Saturation(idle, 16, th); got <= 1 {
t.Errorf("16 unreported admissions across 2 replicas should saturate, got %v", got)
}
}
func TestBurstDoesNotAllPassTheFastPath(t *testing.T) {
// End to end for the same guard: with saturation derived from outstanding
// leases, a burst of 200 must not be admitted wholesale.
var admitted atomic.Int64
c := newController(t, testConfig(), func(_ string, outstanding int) (float64, bool) {
// Two replicas, healthy buffer of 4 each: 8 in flight is the limit.
return float64(outstanding) / 8, true
})
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
var wg sync.WaitGroup
for range 200 {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
admitted.Add(1)
}
}()
}
wg.Wait()
// bronze's ceiling is 0.30, so roughly 8*0.3 = 2-3 should get in. Allow
// generous slack for scheduling, but nothing near 200.
if n := admitted.Load(); n > 20 {
t.Errorf("%d of 200 requests were admitted at once; the burst guard is not working", n)
}
}
func TestSaturationWithNoUsableEndpoints(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
if got := Saturation(nil, 0, th); got != 1 {
t.Errorf("an empty pool should report full saturation, got %v", got)
}
}