Files
llmgw/internal/router/admission/admission_test.go
T
ilkerandClaude Opus 5 e8d90cf375 Treat a zombie as dead, and test fairness where it is deterministic
CI went red on Linux with two failures that never appear on macOS. Both were
worth the trip.

The first is a real bug. processAlive used kill(pid, 0), which succeeds on a
zombie — a process that has exited and is only waiting to be reaped. That
happens for real whenever an engine's parent dies before it does and nothing
reaps the orphan, which in a container is the normal case: PID 1 is usually the
workload rather than an init. The supervisor would then report a dead adopted
engine as running forever, and hold its GPUs against every future placement.
Zombies are now detected through procfs on Linux, where the answer exists, and
the check degrades to its old behaviour elsewhere rather than to something worse.

The second was a bad test. TestTenantFairnessWithinBand asserted on the order
goroutines sent to a channel after being admitted, which is the order the Go
scheduler woke them, not the order the dispatcher chose. Under load it measured
the runtime. The rotation itself lives in bandQueue.pop, which is pure, so it is
tested there — including what happens when a tenant runs out mid-rotation and
when a cancelled waiter is removed. The controller-level test now asserts the
thing that is actually stable: nobody is starved.

Verified with three consecutive `go test -race ./...` runs in a Linux container.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:34:08 +02:00

503 lines
15 KiB
Go

package admission
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func testConfig() Config {
c := DefaultConfig()
c.Interval = time.Millisecond
c.QueueTTL = 2 * time.Second
return c
}
// fixedSat lets a test drive saturation directly.
type fixedSat struct{ v atomic.Uint64 }
func (f *fixedSat) set(v float64) { f.v.Store(uint64(v * 1000)) }
func (f *fixedSat) fn(string, int) (float64, bool) {
return float64(f.v.Load()) / 1000, true
}
func newController(t *testing.T, cfg Config, sat SaturationFunc) *Controller {
t.Helper()
c, err := New(cfg, sat)
if err != nil {
t.Fatal(err)
}
c.Start()
t.Cleanup(c.Close)
return c
}
func TestAdmitsImmediatelyWhenIdle(t *testing.T) {
s := &fixedSat{}
c := newController(t, testConfig(), s.fn)
start := time.Now()
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if err != nil {
t.Fatal(err)
}
if d := time.Since(start); d > 50*time.Millisecond {
t.Errorf("idle admission took %v; the fast path is not being used", d)
}
lease.Release()
}
func TestLowBandHeldWhileHighBandFlows(t *testing.T) {
// The core of plan §7.2.2: the cluster degrades from the bottom up.
s := &fixedSat{}
s.set(0.5) // above bronze (0.30), below gold (0.85)
c := newController(t, testConfig(), s.fn)
ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)
defer cancel()
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
t.Error("bronze was admitted above its ceiling")
}
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"})
if err != nil {
t.Fatalf("gold was held below its ceiling: %v", err)
}
lease.Release()
}
func TestTopBandFlowsUpToTheHealthyLimit(t *testing.T) {
// A ceiling of 1.0 means "hold only once the pool is past its healthy
// limit", not "never hold". At exactly the limit the top band still flows.
s := &fixedSat{}
s.set(1.0)
c := newController(t, testConfig(), s.fn)
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"})
if err != nil {
t.Fatalf("top band held at exactly its ceiling: %v", err)
}
lease.Release()
}
func TestTopBandIsHeldWhenOverloaded(t *testing.T) {
// Holding even the top band is deliberate: with nobody above it to protect,
// the benefit is a bounded engine queue. Dumping a flood straight into a
// FIFO engine queue makes every request in that band slow, whereas queueing
// at the gateway keeps ordering, TTL and tenant fairness.
s := &fixedSat{}
s.set(5)
cfg := testConfig()
cfg.QueueTTL = 200 * time.Millisecond
c := newController(t, cfg, s.fn)
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"}); !errors.Is(err, ErrQueueTimeout) {
t.Fatalf("expected the top band to queue while overloaded, got %v", err)
}
}
func TestCeilingAboveOneNeverHolds(t *testing.T) {
// The escape hatch for a band that must never wait, whatever the load.
s := &fixedSat{}
s.set(50)
cfg := testConfig()
cfg.Bands = []Band{{Name: "critical", Priority: 900, SaturationCeiling: 1000}}
cfg.DefaultBand = "critical"
c := newController(t, cfg, s.fn)
lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "critical"})
if err != nil {
t.Fatalf("a band with an effectively unbounded ceiling was held: %v", err)
}
lease.Release()
}
func TestQueuedRequestAdmittedWhenSaturationDrops(t *testing.T) {
s := &fixedSat{}
s.set(0.9)
c := newController(t, testConfig(), s.fn)
done := make(chan error, 1)
go func() {
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
done <- err
}()
// Give it time to actually queue, then open the gate.
time.Sleep(50 * time.Millisecond)
if d := c.Depths(); d[3].Depth != 1 {
t.Fatalf("expected 1 queued bronze request, got %+v", d)
}
s.set(0.1)
select {
case err := <-done:
if err != nil {
t.Fatalf("queued request failed: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("queued request was not admitted after saturation dropped")
}
}
func TestQueueTTL(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
cfg := testConfig()
cfg.QueueTTL = 150 * time.Millisecond
c := newController(t, cfg, s.fn)
start := time.Now()
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if !errors.Is(err, ErrQueueTimeout) {
t.Fatalf("expected ErrQueueTimeout, got %v", err)
}
if d := time.Since(start); d > time.Second {
t.Errorf("TTL fired after %v", d)
}
}
func TestQueueCapacity(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
cfg := testConfig()
cfg.QueueCapacityPerBand = 3
cfg.QueueTTL = 5 * time.Second
c := newController(t, cfg, s.fn)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
for range 3 {
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // queued on purpose
}
// Wait for the queue to fill.
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < 3 {
time.Sleep(2 * time.Millisecond)
}
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if !errors.Is(err, ErrQueueFull) {
t.Errorf("expected ErrQueueFull, got %v", err)
}
}
func TestClientCancellationLeavesQueue(t *testing.T) {
s := &fixedSat{}
s.set(1.0)
c := newController(t, testConfig(), s.fn)
ctx, cancel := context.WithCancel(context.Background())
go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // cancelled on purpose
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth == 0 {
time.Sleep(2 * time.Millisecond)
}
cancel()
deadline = time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if c.Depths()[3].Depth == 0 {
return
}
time.Sleep(2 * time.Millisecond)
}
t.Fatal("a cancelled request stayed in the queue, occupying capacity")
}
// TestBandQueueRotatesTenants is where the fairness property actually lives.
//
// It is tested on the queue rather than through the controller on purpose: at
// the controller level the observable order is the order goroutines wake up,
// which the scheduler decides, not the order the dispatcher admitted them. A
// test asserting on that is measuring the Go runtime.
func TestBandQueueRotatesTenants(t *testing.T) {
q := newBandQueue(Band{Name: "bronze"})
// One tenant floods, then a quiet one arrives strictly behind it.
for range 10 {
q.push(&waiter{tenant: "noisy", model: "m", ready: make(chan struct{})})
}
q.push(&waiter{tenant: "quiet", model: "m", ready: make(chan struct{})})
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if len(order) != 11 {
t.Fatalf("popped %d waiters, pushed 11", len(order))
}
// FIFO would put the quiet tenant last. Round-robin gives it the second
// turn: one noisy, one quiet, then the rest of the flood.
if order[0] != "noisy" || order[1] != "quiet" {
t.Errorf("rotation order starts %v; a flood is taking consecutive turns", order[:3])
}
quiet := 0
for _, v := range order {
if v == "quiet" {
quiet++
}
}
if quiet != 1 {
t.Errorf("the quiet tenant was admitted %d times", quiet)
}
}
func TestBandQueueRotationIsStable(t *testing.T) {
// Three tenants with unequal backlogs must still take turns, and a tenant
// that runs out must drop cleanly out of the rotation.
q := newBandQueue(Band{Name: "bronze"})
for _, tenant := range []string{"a", "a", "a", "b", "c"} {
q.push(&waiter{tenant: tenant, model: "m", ready: make(chan struct{})})
}
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if strings.Join(order, "") != "abcaa" {
t.Errorf("order=%v, expected a,b,c then a's remaining two", order)
}
}
func TestBandQueueRemoveKeepsRotation(t *testing.T) {
// A cancelled request leaves the queue; the rotation must not skip or
// repeat a tenant because of it.
q := newBandQueue(Band{Name: "bronze"})
a1 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
b1 := &waiter{tenant: "b", model: "m", ready: make(chan struct{})}
a2 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})}
q.push(a1)
q.push(b1)
q.push(a2)
if !q.remove(b1) {
t.Fatal("remove reported the waiter was not queued")
}
if q.len() != 2 {
t.Fatalf("len=%d after removing one of three", q.len())
}
var order []string
for q.len() > 0 {
order = append(order, q.pop().tenant)
}
if strings.Join(order, "") != "aa" {
t.Errorf("order=%v after the only b was removed", order)
}
}
// TestNoTenantIsStarved is the controller-level companion: it asserts that
// everything queued is eventually admitted, without asserting an order the Go
// scheduler controls.
func TestNoTenantIsStarved(t *testing.T) {
s := &fixedSat{}
s.set(1.0) // hold everything initially
cfg := testConfig()
cfg.QueueTTL = 10 * time.Second
c := newController(t, cfg, s.fn)
const flood = 40
var noisy, quiet atomic.Int64
var wg sync.WaitGroup
admit := func(tenant string, counter *atomic.Int64) {
defer wg.Done()
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze", Tenant: tenant}); err == nil {
counter.Add(1)
}
}
for range flood {
wg.Add(1)
go admit("noisy", &noisy)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood {
time.Sleep(2 * time.Millisecond)
}
for range 4 {
wg.Add(1)
go admit("quiet", &quiet)
}
deadline = time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && c.Depths()[3].Depth < flood+4 {
time.Sleep(2 * time.Millisecond)
}
s.set(0.0) // open the gate
wg.Wait()
if noisy.Load() != flood {
t.Errorf("the flooding tenant got %d of %d admissions", noisy.Load(), flood)
}
if quiet.Load() != 4 {
t.Errorf("the quiet tenant got %d of 4 admissions; it was starved", quiet.Load())
}
}
func TestUnknownBandUsesDefault(t *testing.T) {
s := &fixedSat{}
s.set(0.5)
cfg := testConfig()
cfg.DefaultBand = "bronze"
c := newController(t, cfg, s.fn)
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
defer cancel()
// bronze's ceiling is 0.30, so an unknown band must be held here.
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "no-such-band"}); err == nil {
t.Error("unknown band did not fall back to the default")
}
}
func TestUnknownModelIsLetThrough(t *testing.T) {
// Better a precise 503 from dispatch than a 30-second queue timeout.
c := newController(t, testConfig(), func(string, int) (float64, bool) { return 0, false })
lease, err := c.Acquire(t.Context(), Request{Model: "nope", Band: "bronze"})
if err != nil {
t.Fatalf("unknown model was queued instead of passed through: %v", err)
}
lease.Release()
}
func TestConfigValidation(t *testing.T) {
if _, err := New(Config{Bands: []Band{{Name: "a"}, {Name: "a"}}}, nil); err == nil {
t.Error("duplicate band names accepted")
}
if _, err := New(Config{Bands: []Band{{Name: "a"}}, DefaultBand: "b"}, nil); err == nil {
t.Error("default_band outside the band list accepted")
}
}
func TestBandsSortedByPriority(t *testing.T) {
c, err := New(Config{Bands: []Band{
{Name: "low", Priority: 1, SaturationCeiling: 0.2},
{Name: "high", Priority: 9, SaturationCeiling: 1},
}, DefaultBand: "low"}, func(string, int) (float64, bool) { return 0, true })
if err != nil {
t.Fatal(err)
}
if got := c.Bands()[0].Name; got != "high" {
t.Errorf("bands are not ordered by priority: first is %q", got)
}
}
// --- saturation ---
func snap(id string, waiting, running, inflight int, kv float64, open bool) endpoint.Snapshot {
s := endpoint.Snapshot{
Info: endpoint.Info{ID: id},
Inflight: inflight,
Breaker: breaker.Closed,
}
s.Stats.Waiting = waiting
s.Stats.Running = running
s.Stats.KVCacheUsage = kv
if open {
s.Breaker = breaker.Open
}
return s
}
func TestSaturationCombinesQueueAndKV(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
// Empty queue but a nearly full cache still counts as saturated.
if got := Saturation([]endpoint.Snapshot{snap("a", 0, 1, 1, 0.89, false)}, 0, th); got < 0.9 {
t.Errorf("KV pressure ignored: %v", got)
}
// Full queue with an empty cache likewise.
if got := Saturation([]endpoint.Snapshot{snap("a", 4, 1, 5, 0, false)}, 0, th); got < 1 {
t.Errorf("queue depth ignored: %v", got)
}
}
func TestSaturationCountsUnreportedInflight(t *testing.T) {
// A burst the engine has not reported yet is real load; missing it would
// leave admission blind exactly when it needs to bite.
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
cold := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 0, 0, false)}, 0, th)
burst := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 8, 0, false)}, 0, th)
if !(burst > cold) {
t.Errorf("unreported inflight not counted: cold=%v burst=%v", cold, burst)
}
}
func TestSaturationSkipsOpenBreakers(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
eps := []endpoint.Snapshot{
snap("busy", 8, 4, 12, 0.5, false),
snap("dead", 0, 0, 0, 0, true), // would halve the mean if counted
}
withDead := Saturation(eps, 0, th)
alone := Saturation(eps[:1], 0, th)
if fmt.Sprintf("%.4f", withDead) != fmt.Sprintf("%.4f", alone) {
t.Errorf("an endpoint with an open breaker was counted as capacity: %v vs %v", withDead, alone)
}
}
func TestSaturationCountsUnreportedAdmissions(t *testing.T) {
// The thundering herd guard: a simultaneous burst must register as load
// before any of it reaches an engine, or the whole burst walks through the
// fast path while every signal still reads zero.
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
idle := []endpoint.Snapshot{snap("a", 0, 0, 0, 0, false), snap("b", 0, 0, 0, 0, false)}
if got := Saturation(idle, 0, th); got != 0 {
t.Fatalf("idle pool reported %v", got)
}
if got := Saturation(idle, 16, th); got <= 1 {
t.Errorf("16 unreported admissions across 2 replicas should saturate, got %v", got)
}
}
func TestBurstDoesNotAllPassTheFastPath(t *testing.T) {
// End to end for the same guard: with saturation derived from outstanding
// leases, a burst of 200 must not be admitted wholesale.
var admitted atomic.Int64
c := newController(t, testConfig(), func(_ string, outstanding int) (float64, bool) {
// Two replicas, healthy buffer of 4 each: 8 in flight is the limit.
return float64(outstanding) / 8, true
})
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
var wg sync.WaitGroup
for range 200 {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil {
admitted.Add(1)
}
}()
}
wg.Wait()
// bronze's ceiling is 0.30, so roughly 8*0.3 = 2-3 should get in. Allow
// generous slack for scheduling, but nothing near 200.
if n := admitted.Load(); n > 20 {
t.Errorf("%d of 200 requests were admitted at once; the burst guard is not working", n)
}
}
func TestSaturationWithNoUsableEndpoints(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
if got := Saturation(nil, 0, th); got != 1 {
t.Errorf("an empty pool should report full saturation, got %v", got)
}
}