package admission import ( "context" "errors" "fmt" "strings" "sync" "sync/atomic" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) func testConfig() Config { c := DefaultConfig() c.Interval = time.Millisecond c.QueueTTL = 2 * time.Second return c } // fixedSat lets a test drive saturation directly. type fixedSat struct{ v atomic.Uint64 } func (f *fixedSat) set(v float64) { f.v.Store(uint64(v * 1000)) } func (f *fixedSat) fn(string, int) (float64, bool) { return float64(f.v.Load()) / 1000, true } func newController(t *testing.T, cfg Config, sat SaturationFunc) *Controller { t.Helper() c, err := New(cfg, sat) if err != nil { t.Fatal(err) } c.Start() t.Cleanup(c.Close) return c } func TestAdmitsImmediatelyWhenIdle(t *testing.T) { s := &fixedSat{} // A queue that is looked at every ten seconds, so a request that went // through it rather than the fast path would take that long. A bound of // 50ms against a 1ms tick measured the machine the test ran on, and a // loaded CI runner under the race detector was slower than that. cfg := testConfig() cfg.Interval = 10 * time.Second c := newController(t, cfg, s.fn) start := time.Now() lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"}) if err != nil { t.Fatal(err) } if d := time.Since(start); d > 2*time.Second { t.Errorf("idle admission took %v; the fast path is not being used", d) } lease.Release() } func TestLowBandHeldWhileHighBandFlows(t *testing.T) { // The core of plan ยง7.2.2: the cluster degrades from the bottom up. s := &fixedSat{} s.set(0.5) // above bronze (0.30), below gold (0.85) c := newController(t, testConfig(), s.fn) ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) defer cancel() if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil { t.Error("bronze was admitted above its ceiling") } lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"}) if err != nil { t.Fatalf("gold was held below its ceiling: %v", err) } lease.Release() } func TestTopBandFlowsUpToTheHealthyLimit(t *testing.T) { // A ceiling of 1.0 means "hold only once the pool is past its healthy // limit", not "never hold". At exactly the limit the top band still flows. s := &fixedSat{} s.set(1.0) c := newController(t, testConfig(), s.fn) lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"}) if err != nil { t.Fatalf("top band held at exactly its ceiling: %v", err) } lease.Release() } func TestTopBandIsHeldWhenOverloaded(t *testing.T) { // Holding even the top band is deliberate: with nobody above it to protect, // the benefit is a bounded engine queue. Dumping a flood straight into a // FIFO engine queue makes every request in that band slow, whereas queueing // at the gateway keeps ordering, TTL and tenant fairness. s := &fixedSat{} s.set(5) cfg := testConfig() cfg.QueueTTL = 200 * time.Millisecond c := newController(t, cfg, s.fn) if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "platinum"}); !errors.Is(err, ErrQueueTimeout) { t.Fatalf("expected the top band to queue while overloaded, got %v", err) } } func TestCeilingAboveOneNeverHolds(t *testing.T) { // The escape hatch for a band that must never wait, whatever the load. s := &fixedSat{} s.set(50) cfg := testConfig() cfg.Bands = []Band{{Name: "critical", Priority: 900, SaturationCeiling: 1000}} cfg.DefaultBand = "critical" c := newController(t, cfg, s.fn) lease, err := c.Acquire(t.Context(), Request{Model: "m", Band: "critical"}) if err != nil { t.Fatalf("a band with an effectively unbounded ceiling was held: %v", err) } lease.Release() } func TestQueuedRequestAdmittedWhenSaturationDrops(t *testing.T) { s := &fixedSat{} s.set(0.9) c := newController(t, testConfig(), s.fn) done := make(chan error, 1) go func() { _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"}) done <- err }() // Wait for it to actually queue, then open the gate. Waited for, not // slept through: on a loaded runner 50ms was sometimes not long enough // for the goroutine to have been scheduled at all. deadline := time.Now().Add(2 * time.Second) for c.Depths()[3].Depth != 1 && time.Now().Before(deadline) { time.Sleep(5 * time.Millisecond) } if d := c.Depths(); d[3].Depth != 1 { t.Fatalf("expected 1 queued bronze request, got %+v", d) } s.set(0.1) select { case err := <-done: if err != nil { t.Fatalf("queued request failed: %v", err) } case <-time.After(2 * time.Second): t.Fatal("queued request was not admitted after saturation dropped") } } func TestQueueTTL(t *testing.T) { s := &fixedSat{} s.set(1.0) cfg := testConfig() cfg.QueueTTL = 150 * time.Millisecond c := newController(t, cfg, s.fn) start := time.Now() _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"}) if !errors.Is(err, ErrQueueTimeout) { t.Fatalf("expected ErrQueueTimeout, got %v", err) } if d := time.Since(start); d > time.Second { t.Errorf("TTL fired after %v", d) } } func TestQueueCapacity(t *testing.T) { s := &fixedSat{} s.set(1.0) cfg := testConfig() cfg.QueueCapacityPerBand = 3 cfg.QueueTTL = 5 * time.Second c := newController(t, cfg, s.fn) ctx, cancel := context.WithCancel(context.Background()) defer cancel() for range 3 { go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // queued on purpose } // Wait for the queue to fill. deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) && c.Depths()[3].Depth < 3 { time.Sleep(2 * time.Millisecond) } _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"}) if !errors.Is(err, ErrQueueFull) { t.Errorf("expected ErrQueueFull, got %v", err) } } func TestClientCancellationLeavesQueue(t *testing.T) { s := &fixedSat{} s.set(1.0) c := newController(t, testConfig(), s.fn) ctx, cancel := context.WithCancel(context.Background()) go c.Acquire(ctx, Request{Model: "m", Band: "bronze"}) //nolint:errcheck // cancelled on purpose deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) && c.Depths()[3].Depth == 0 { time.Sleep(2 * time.Millisecond) } cancel() deadline = time.Now().Add(time.Second) for time.Now().Before(deadline) { if c.Depths()[3].Depth == 0 { return } time.Sleep(2 * time.Millisecond) } t.Fatal("a cancelled request stayed in the queue, occupying capacity") } // TestBandQueueRotatesTenants is where the fairness property actually lives. // // It is tested on the queue rather than through the controller on purpose: at // the controller level the observable order is the order goroutines wake up, // which the scheduler decides, not the order the dispatcher admitted them. A // test asserting on that is measuring the Go runtime. func TestBandQueueRotatesTenants(t *testing.T) { q := newBandQueue(Band{Name: "bronze"}) // One tenant floods, then a quiet one arrives strictly behind it. for range 10 { q.push(&waiter{tenant: "noisy", model: "m", ready: make(chan struct{})}) } q.push(&waiter{tenant: "quiet", model: "m", ready: make(chan struct{})}) var order []string for q.len() > 0 { order = append(order, q.pop().tenant) } if len(order) != 11 { t.Fatalf("popped %d waiters, pushed 11", len(order)) } // FIFO would put the quiet tenant last. Round-robin gives it the second // turn: one noisy, one quiet, then the rest of the flood. if order[0] != "noisy" || order[1] != "quiet" { t.Errorf("rotation order starts %v; a flood is taking consecutive turns", order[:3]) } quiet := 0 for _, v := range order { if v == "quiet" { quiet++ } } if quiet != 1 { t.Errorf("the quiet tenant was admitted %d times", quiet) } } func TestBandQueueRotationIsStable(t *testing.T) { // Three tenants with unequal backlogs must still take turns, and a tenant // that runs out must drop cleanly out of the rotation. q := newBandQueue(Band{Name: "bronze"}) for _, tenant := range []string{"a", "a", "a", "b", "c"} { q.push(&waiter{tenant: tenant, model: "m", ready: make(chan struct{})}) } var order []string for q.len() > 0 { order = append(order, q.pop().tenant) } if strings.Join(order, "") != "abcaa" { t.Errorf("order=%v, expected a,b,c then a's remaining two", order) } } func TestBandQueueRemoveKeepsRotation(t *testing.T) { // A cancelled request leaves the queue; the rotation must not skip or // repeat a tenant because of it. q := newBandQueue(Band{Name: "bronze"}) a1 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})} b1 := &waiter{tenant: "b", model: "m", ready: make(chan struct{})} a2 := &waiter{tenant: "a", model: "m", ready: make(chan struct{})} q.push(a1) q.push(b1) q.push(a2) if !q.remove(b1) { t.Fatal("remove reported the waiter was not queued") } if q.len() != 2 { t.Fatalf("len=%d after removing one of three", q.len()) } var order []string for q.len() > 0 { order = append(order, q.pop().tenant) } if strings.Join(order, "") != "aa" { t.Errorf("order=%v after the only b was removed", order) } } // TestNoTenantIsStarved is the controller-level companion: it asserts that // everything queued is eventually admitted, without asserting an order the Go // scheduler controls. func TestNoTenantIsStarved(t *testing.T) { s := &fixedSat{} s.set(1.0) // hold everything initially cfg := testConfig() cfg.QueueTTL = 10 * time.Second c := newController(t, cfg, s.fn) const flood = 40 var noisy, quiet atomic.Int64 var wg sync.WaitGroup admit := func(tenant string, counter *atomic.Int64) { defer wg.Done() if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze", Tenant: tenant}); err == nil { counter.Add(1) } } for range flood { wg.Add(1) go admit("noisy", &noisy) } deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) && c.Depths()[3].Depth < flood { time.Sleep(2 * time.Millisecond) } for range 4 { wg.Add(1) go admit("quiet", &quiet) } deadline = time.Now().Add(5 * time.Second) for time.Now().Before(deadline) && c.Depths()[3].Depth < flood+4 { time.Sleep(2 * time.Millisecond) } s.set(0.0) // open the gate wg.Wait() if noisy.Load() != flood { t.Errorf("the flooding tenant got %d of %d admissions", noisy.Load(), flood) } if quiet.Load() != 4 { t.Errorf("the quiet tenant got %d of 4 admissions; it was starved", quiet.Load()) } } func TestUnknownBandUsesDefault(t *testing.T) { s := &fixedSat{} s.set(0.5) cfg := testConfig() cfg.DefaultBand = "bronze" c := newController(t, cfg, s.fn) ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) defer cancel() // bronze's ceiling is 0.30, so an unknown band must be held here. if _, err := c.Acquire(ctx, Request{Model: "m", Band: "no-such-band"}); err == nil { t.Error("unknown band did not fall back to the default") } } func TestUnknownModelIsLetThrough(t *testing.T) { // Better a precise 503 from dispatch than a 30-second queue timeout. c := newController(t, testConfig(), func(string, int) (float64, bool) { return 0, false }) lease, err := c.Acquire(t.Context(), Request{Model: "nope", Band: "bronze"}) if err != nil { t.Fatalf("unknown model was queued instead of passed through: %v", err) } lease.Release() } func TestConfigValidation(t *testing.T) { if _, err := New(Config{Bands: []Band{{Name: "a"}, {Name: "a"}}}, nil); err == nil { t.Error("duplicate band names accepted") } if _, err := New(Config{Bands: []Band{{Name: "a"}}, DefaultBand: "b"}, nil); err == nil { t.Error("default_band outside the band list accepted") } } func TestBandsSortedByPriority(t *testing.T) { c, err := New(Config{Bands: []Band{ {Name: "low", Priority: 1, SaturationCeiling: 0.2}, {Name: "high", Priority: 9, SaturationCeiling: 1}, }, DefaultBand: "low"}, func(string, int) (float64, bool) { return 0, true }) if err != nil { t.Fatal(err) } if got := c.Bands()[0].Name; got != "high" { t.Errorf("bands are not ordered by priority: first is %q", got) } } // --- saturation --- func snap(id string, waiting, running, inflight int, kv float64, open bool) endpoint.Snapshot { s := endpoint.Snapshot{ Info: endpoint.Info{ID: id}, Inflight: inflight, Breaker: breaker.Closed, } s.Stats.Waiting = waiting s.Stats.Running = running s.Stats.KVCacheUsage = kv if open { s.Breaker = breaker.Open } return s } func TestSaturationCombinesQueueAndKV(t *testing.T) { th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} // Empty queue but a nearly full cache still counts as saturated. if got := Saturation([]endpoint.Snapshot{snap("a", 0, 1, 1, 0.89, false)}, 0, th); got < 0.9 { t.Errorf("KV pressure ignored: %v", got) } // Full queue with an empty cache likewise. if got := Saturation([]endpoint.Snapshot{snap("a", 4, 1, 5, 0, false)}, 0, th); got < 1 { t.Errorf("queue depth ignored: %v", got) } } func TestSaturationCountsUnreportedInflight(t *testing.T) { // A burst the engine has not reported yet is real load; missing it would // leave admission blind exactly when it needs to bite. th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} cold := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 0, 0, false)}, 0, th) burst := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 8, 0, false)}, 0, th) if !(burst > cold) { t.Errorf("unreported inflight not counted: cold=%v burst=%v", cold, burst) } } func TestSaturationSkipsOpenBreakers(t *testing.T) { th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} eps := []endpoint.Snapshot{ snap("busy", 8, 4, 12, 0.5, false), snap("dead", 0, 0, 0, 0, true), // would halve the mean if counted } withDead := Saturation(eps, 0, th) alone := Saturation(eps[:1], 0, th) if fmt.Sprintf("%.4f", withDead) != fmt.Sprintf("%.4f", alone) { t.Errorf("an endpoint with an open breaker was counted as capacity: %v vs %v", withDead, alone) } } func TestSaturationCountsUnreportedAdmissions(t *testing.T) { // The thundering herd guard: a simultaneous burst must register as load // before any of it reaches an engine, or the whole burst walks through the // fast path while every signal still reads zero. th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} idle := []endpoint.Snapshot{snap("a", 0, 0, 0, 0, false), snap("b", 0, 0, 0, 0, false)} if got := Saturation(idle, 0, th); got != 0 { t.Fatalf("idle pool reported %v", got) } if got := Saturation(idle, 16, th); got <= 1 { t.Errorf("16 unreported admissions across 2 replicas should saturate, got %v", got) } } func TestBurstDoesNotAllPassTheFastPath(t *testing.T) { // End to end for the same guard: with saturation derived from outstanding // leases, a burst of 200 must not be admitted wholesale. var admitted atomic.Int64 c := newController(t, testConfig(), func(_ string, outstanding int) (float64, bool) { // Two replicas, healthy buffer of 4 each: 8 in flight is the limit. return float64(outstanding) / 8, true }) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() var wg sync.WaitGroup for range 200 { wg.Add(1) go func() { defer wg.Done() if _, err := c.Acquire(ctx, Request{Model: "m", Band: "bronze"}); err == nil { admitted.Add(1) } }() } wg.Wait() // bronze's ceiling is 0.30, so roughly 8*0.3 = 2-3 should get in. Allow // generous slack for scheduling, but nothing near 200. if n := admitted.Load(); n > 20 { t.Errorf("%d of 200 requests were admitted at once; the burst guard is not working", n) } } func TestSaturationWithNoUsableEndpoints(t *testing.T) { th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9} if got := Saturation(nil, 0, th); got != 1 { t.Errorf("an empty pool should report full saturation, got %v", got) } }