A total bound across bands (admission.queue_capacity_total, default half of all bands' capacity together); reached, a request is not refused while a lower band has requests queued, and the newest of the lowest band gets a 503 with Retry-After instead. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
90 lines
2.7 KiB
Go
90 lines
2.7 KiB
Go
package admission
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// queueUntil starts an Acquire in the background and waits until the band
|
|
// holds depth requests, returning where its result will arrive.
|
|
func queueUntil(t *testing.T, c *Controller, band string, bandIdx, depth int) <-chan error {
|
|
t.Helper()
|
|
out := make(chan error, 1)
|
|
go func() {
|
|
l, err := c.Acquire(context.Background(), Request{Model: "m", Band: band})
|
|
if err == nil {
|
|
l.Release()
|
|
}
|
|
out <- err
|
|
}()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for c.Depths()[bandIdx].Depth < depth && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if got := c.Depths()[bandIdx].Depth; got != depth {
|
|
t.Fatalf("band %s holds %d, expected %d", band, got, depth)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAHigherBandDisplacesTheNewestOfTheLowest(t *testing.T) {
|
|
s := &fixedSat{}
|
|
s.set(5) // past every ceiling: everything queues
|
|
cfg := testConfig()
|
|
cfg.QueueCapacityTotal = 3
|
|
c := newController(t, cfg, s.fn)
|
|
// Bands, highest first: platinum 0, gold 1, silver 2, bronze 3.
|
|
oldest := queueUntil(t, c, "bronze", 3, 1)
|
|
time.Sleep(2 * time.Millisecond)
|
|
silver := queueUntil(t, c, "silver", 2, 1)
|
|
time.Sleep(2 * time.Millisecond)
|
|
newest := queueUntil(t, c, "bronze", 3, 2)
|
|
|
|
// Full at three. Gold arrives: the newest bronze goes, not the oldest,
|
|
// and not silver, which outranks bronze.
|
|
gold := queueUntil(t, c, "gold", 1, 1)
|
|
select {
|
|
case err := <-newest:
|
|
if !errors.Is(err, ErrDisplaced) {
|
|
t.Fatalf("the newest bronze got %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("nothing was displaced")
|
|
}
|
|
if d := c.Depths(); d[3].Depth != 1 || d[2].Depth != 1 || d[1].Depth != 1 {
|
|
t.Errorf("depths %+v: one bronze, the silver and the gold should remain", d)
|
|
}
|
|
|
|
// Bronze cannot displace anything: nothing is below it.
|
|
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
|
if !errors.Is(err, ErrQueueFull) {
|
|
t.Errorf("a bronze request at a full queue got %v", err)
|
|
}
|
|
// Another gold takes the remaining bronze, then silver.
|
|
queueUntil(t, c, "gold", 1, 2)
|
|
if err := <-oldest; !errors.Is(err, ErrDisplaced) {
|
|
t.Errorf("the last bronze got %v", err)
|
|
}
|
|
queueUntil(t, c, "gold", 1, 3)
|
|
if err := <-silver; !errors.Is(err, ErrDisplaced) {
|
|
t.Errorf("silver got %v", err)
|
|
}
|
|
// All gold now: a fourth has nobody lower to displace.
|
|
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"}); !errors.Is(err, ErrQueueFull) {
|
|
t.Errorf("a gold request with only gold queued got %v", err)
|
|
}
|
|
|
|
// And they are served once there is room.
|
|
s.set(0)
|
|
select {
|
|
case err := <-gold:
|
|
if err != nil {
|
|
t.Errorf("the first gold was not served: %v", err)
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Error("the queue did not drain")
|
|
}
|
|
}
|