Files
llmgw/internal/router/admission/displace_test.go
T
Ilker ManapandClaude Opus 5.5 3d3c96e422 Displace the newest lowest-band request when the gateway queue is full
A total bound across bands (admission.queue_capacity_total, default half of
all bands' capacity together); reached, a request is not refused while a
lower band has requests queued, and the newest of the lowest band gets a 503
with Retry-After instead.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 23:07:53 +02:00

90 lines
2.7 KiB
Go

package admission
import (
"context"
"errors"
"testing"
"time"
)
// queueUntil starts an Acquire in the background and waits until the band
// holds depth requests, returning where its result will arrive.
func queueUntil(t *testing.T, c *Controller, band string, bandIdx, depth int) <-chan error {
t.Helper()
out := make(chan error, 1)
go func() {
l, err := c.Acquire(context.Background(), Request{Model: "m", Band: band})
if err == nil {
l.Release()
}
out <- err
}()
deadline := time.Now().Add(2 * time.Second)
for c.Depths()[bandIdx].Depth < depth && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if got := c.Depths()[bandIdx].Depth; got != depth {
t.Fatalf("band %s holds %d, expected %d", band, got, depth)
}
return out
}
func TestAHigherBandDisplacesTheNewestOfTheLowest(t *testing.T) {
s := &fixedSat{}
s.set(5) // past every ceiling: everything queues
cfg := testConfig()
cfg.QueueCapacityTotal = 3
c := newController(t, cfg, s.fn)
// Bands, highest first: platinum 0, gold 1, silver 2, bronze 3.
oldest := queueUntil(t, c, "bronze", 3, 1)
time.Sleep(2 * time.Millisecond)
silver := queueUntil(t, c, "silver", 2, 1)
time.Sleep(2 * time.Millisecond)
newest := queueUntil(t, c, "bronze", 3, 2)
// Full at three. Gold arrives: the newest bronze goes, not the oldest,
// and not silver, which outranks bronze.
gold := queueUntil(t, c, "gold", 1, 1)
select {
case err := <-newest:
if !errors.Is(err, ErrDisplaced) {
t.Fatalf("the newest bronze got %v", err)
}
case <-time.After(time.Second):
t.Fatal("nothing was displaced")
}
if d := c.Depths(); d[3].Depth != 1 || d[2].Depth != 1 || d[1].Depth != 1 {
t.Errorf("depths %+v: one bronze, the silver and the gold should remain", d)
}
// Bronze cannot displace anything: nothing is below it.
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
if !errors.Is(err, ErrQueueFull) {
t.Errorf("a bronze request at a full queue got %v", err)
}
// Another gold takes the remaining bronze, then silver.
queueUntil(t, c, "gold", 1, 2)
if err := <-oldest; !errors.Is(err, ErrDisplaced) {
t.Errorf("the last bronze got %v", err)
}
queueUntil(t, c, "gold", 1, 3)
if err := <-silver; !errors.Is(err, ErrDisplaced) {
t.Errorf("silver got %v", err)
}
// All gold now: a fourth has nobody lower to displace.
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"}); !errors.Is(err, ErrQueueFull) {
t.Errorf("a gold request with only gold queued got %v", err)
}
// And they are served once there is room.
s.set(0)
select {
case err := <-gold:
if err != nil {
t.Errorf("the first gold was not served: %v", err)
}
case <-time.After(2 * time.Second):
t.Error("the queue did not drain")
}
}