From 3d3c96e422b713b24c5157c3af336e8930f6fa1e Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 23:07:53 +0200 Subject: [PATCH] Displace the newest lowest-band request when the gateway queue is full A total bound across bands (admission.queue_capacity_total, default half of all bands' capacity together); reached, a request is not refused while a lower band has requests queued, and the newest of the lowest band gets a 503 with Retry-After instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 +++ docs/reference.md | 10 +++ internal/config/config.go | 7 +- internal/router/admission/controller.go | 56 +++++++++++++- internal/router/admission/displace_test.go | 89 ++++++++++++++++++++++ internal/router/admission/queue.go | 17 +++++ internal/router/finish_test.go | 3 + internal/router/handlers.go | 6 ++ internal/router/router.go | 1 + llmgw-plan.md | 3 +- 10 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 internal/router/admission/displace_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5406d63..a887a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,17 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### Displacement: a higher band is not refused while a lower one waits + +Each band's queue was bounded, and nothing bounded them together, so a full +gateway turned an interactive request away while batch requests sat queued +behind it. `admission.queue_capacity_total` now bounds everything queued — +by default half of every band's capacity together. Reached, a request is not +refused while a lower band has any queued: the newest request of the lowest +band is turned away instead, with a 503 and `Retry-After: 5`, so it backs off +and returns. The newest, because the oldest have waited longest and are +nearest their TTL. Counted as `displaced` in the admission rejection metric. + ### Releases can be signed `llmgw release keygen` makes an ed25519 key pair. With the private key in the diff --git a/docs/reference.md b/docs/reference.md index bcef9c7..c76a82d 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -21,6 +21,7 @@ rather than repeated. | `band_header` | `string` | [Admission](#admission) | | `queue_ttl` | `time.Duration` | [Admission](#admission) | | `queue_capacity_per_band` | `int` | [Admission](#admission) | +| `queue_capacity_total` | `int` | [Admission](#admission) | | `interval` | `time.Duration` | [Admission](#admission) | | `enabled` | `bool` | [Auth](#auth) | | `store` | `string` | [Auth](#auth) | @@ -228,6 +229,15 @@ should not be able to promote themselves. `int` +### `queue_capacity_total` + +`int` + +QueueCapacityTotal bounds everything queued across bands. Reached, a +request is not refused while a lower band has any queued: the newest of +the lowest band is turned away instead, with a 503 and Retry-After. +Zero takes half of all the bands' capacities together. + ### `interval` `time.Duration` diff --git a/internal/config/config.go b/internal/config/config.go index 0272dab..3295246 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -378,7 +378,12 @@ type Admission struct { QueueTTL time.Duration `koanf:"queue_ttl"` QueueCapacityPerBand int `koanf:"queue_capacity_per_band"` - Interval time.Duration `koanf:"interval"` + // QueueCapacityTotal bounds everything queued across bands. Reached, a + // request is not refused while a lower band has any queued: the newest of + // the lowest band is turned away instead, with a 503 and Retry-After. + // Zero takes half of all the bands' capacities together. + QueueCapacityTotal int `koanf:"queue_capacity_total"` + Interval time.Duration `koanf:"interval"` } // Prefix configures approximate prefix routing. diff --git a/internal/router/admission/controller.go b/internal/router/admission/controller.go index 77e67aa..6eeb3ed 100644 --- a/internal/router/admission/controller.go +++ b/internal/router/admission/controller.go @@ -34,6 +34,12 @@ type Config struct { // QueueCapacityPerBand bounds each band's queue. Rejecting immediately // beats accumulating silently. QueueCapacityPerBand int `koanf:"queue_capacity_per_band"` + // QueueCapacityTotal bounds everything queued, across bands. When it is + // reached, a request is not refused while a lower band has requests + // queued: the newest of the lowest band is turned away instead + // (displacement), since the oldest have waited longest and are nearest + // their TTL. Zero takes half of every band's capacity together. + QueueCapacityTotal int `koanf:"queue_capacity_total"` // Interval is how often the dispatcher re-checks saturation. Releases also // wake it, so this is only the backstop for saturation falling on its own. Interval time.Duration `koanf:"interval"` @@ -94,6 +100,9 @@ func (c *Config) normalize() error { if c.QueueCapacityPerBand <= 0 { c.QueueCapacityPerBand = 2000 } + if c.QueueCapacityTotal <= 0 { + c.QueueCapacityTotal = max(c.QueueCapacityPerBand*len(c.Bands)/2, 1) + } if c.Interval <= 0 { c.Interval = 10 * time.Millisecond } @@ -123,6 +132,10 @@ var ( ErrQueueTimeout = errors.New("admission queue timeout") // ErrClosed means the controller is shutting down. ErrClosed = errors.New("admission controller closed") + // ErrDisplaced means a queued request was turned away to make room for + // one of a higher band. Worth retrying: the queue is full now, not for + // good. + ErrDisplaced = errors.New("displaced from the admission queue by higher-priority requests") ) // Stats is a snapshot of one band's queue, for metrics. @@ -259,6 +272,12 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) { c.mu.Unlock() return nil, fmt.Errorf("%w: band %q", ErrQueueFull, band.Name) } + if c.queuedLocked() >= c.cfg.QueueCapacityTotal && !c.displaceBelowLocked(idx) { + c.rejected[band.Name]++ + c.mu.Unlock() + return nil, fmt.Errorf("%w: %d queued in all, and nothing of a lower band to make room", + ErrQueueFull, c.cfg.QueueCapacityTotal) + } w := &waiter{ tenant: tenantKey(r), @@ -279,6 +298,10 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) { c.mu.Unlock() return nil, ErrClosed } + if w.err != nil { + c.mu.Unlock() + return nil, w.err + } c.waitTotal[band.Name] += time.Since(w.enqueued) c.mu.Unlock() return &Lease{c: c, band: band.Name, model: r.Model}, nil @@ -297,8 +320,9 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) { func (c *Controller) remove(idx int, w *waiter, reason string) { c.mu.Lock() defer c.mu.Unlock() - if w.admitted { - // It won the race; the caller will see ready closed next time round. + if w.admitted || w.err != nil { + // It won the race, or was displaced; the caller will see ready + // closed next time round. return } c.queues[idx].remove(w) @@ -410,3 +434,31 @@ func tenantKey(r Request) string { } return "anonymous" } + +// queuedLocked is how many requests wait in every band. Called with mu held. +func (c *Controller) queuedLocked() int { + n := 0 + for _, q := range c.queues { + n += q.len() + } + return n +} + +// displaceBelowLocked turns away the newest request of the lowest band below +// idx that has any queued, and says whether there was one. Called with mu held. +func (c *Controller) displaceBelowLocked(idx int) bool { + // Bands are sorted highest priority first, so lower bands have larger + // indices, and the lowest is last. + for j := len(c.queues) - 1; j > idx; j-- { + w := c.queues[j].newest() + if w == nil { + continue + } + c.queues[j].remove(w) + w.err = fmt.Errorf("%w (band %q)", ErrDisplaced, c.cfg.Bands[j].Name) + close(w.ready) + c.rejected[c.cfg.Bands[j].Name+"/displaced"]++ + return true + } + return false +} diff --git a/internal/router/admission/displace_test.go b/internal/router/admission/displace_test.go new file mode 100644 index 0000000..63116ee --- /dev/null +++ b/internal/router/admission/displace_test.go @@ -0,0 +1,89 @@ +package admission + +import ( + "context" + "errors" + "testing" + "time" +) + +// queueUntil starts an Acquire in the background and waits until the band +// holds depth requests, returning where its result will arrive. +func queueUntil(t *testing.T, c *Controller, band string, bandIdx, depth int) <-chan error { + t.Helper() + out := make(chan error, 1) + go func() { + l, err := c.Acquire(context.Background(), Request{Model: "m", Band: band}) + if err == nil { + l.Release() + } + out <- err + }() + deadline := time.Now().Add(2 * time.Second) + for c.Depths()[bandIdx].Depth < depth && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := c.Depths()[bandIdx].Depth; got != depth { + t.Fatalf("band %s holds %d, expected %d", band, got, depth) + } + return out +} + +func TestAHigherBandDisplacesTheNewestOfTheLowest(t *testing.T) { + s := &fixedSat{} + s.set(5) // past every ceiling: everything queues + cfg := testConfig() + cfg.QueueCapacityTotal = 3 + c := newController(t, cfg, s.fn) + // Bands, highest first: platinum 0, gold 1, silver 2, bronze 3. + oldest := queueUntil(t, c, "bronze", 3, 1) + time.Sleep(2 * time.Millisecond) + silver := queueUntil(t, c, "silver", 2, 1) + time.Sleep(2 * time.Millisecond) + newest := queueUntil(t, c, "bronze", 3, 2) + + // Full at three. Gold arrives: the newest bronze goes, not the oldest, + // and not silver, which outranks bronze. + gold := queueUntil(t, c, "gold", 1, 1) + select { + case err := <-newest: + if !errors.Is(err, ErrDisplaced) { + t.Fatalf("the newest bronze got %v", err) + } + case <-time.After(time.Second): + t.Fatal("nothing was displaced") + } + if d := c.Depths(); d[3].Depth != 1 || d[2].Depth != 1 || d[1].Depth != 1 { + t.Errorf("depths %+v: one bronze, the silver and the gold should remain", d) + } + + // Bronze cannot displace anything: nothing is below it. + _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"}) + if !errors.Is(err, ErrQueueFull) { + t.Errorf("a bronze request at a full queue got %v", err) + } + // Another gold takes the remaining bronze, then silver. + queueUntil(t, c, "gold", 1, 2) + if err := <-oldest; !errors.Is(err, ErrDisplaced) { + t.Errorf("the last bronze got %v", err) + } + queueUntil(t, c, "gold", 1, 3) + if err := <-silver; !errors.Is(err, ErrDisplaced) { + t.Errorf("silver got %v", err) + } + // All gold now: a fourth has nobody lower to displace. + if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"}); !errors.Is(err, ErrQueueFull) { + t.Errorf("a gold request with only gold queued got %v", err) + } + + // And they are served once there is room. + s.set(0) + select { + case err := <-gold: + if err != nil { + t.Errorf("the first gold was not served: %v", err) + } + case <-time.After(2 * time.Second): + t.Error("the queue did not drain") + } +} diff --git a/internal/router/admission/queue.go b/internal/router/admission/queue.go index 8aa1e33..3de88da 100644 --- a/internal/router/admission/queue.go +++ b/internal/router/admission/queue.go @@ -12,6 +12,9 @@ type waiter struct { // admitted guards the race between the dispatcher admitting a waiter and // the waiter giving up on its own. admitted bool + // err is why the waiter was turned away while queued — displaced by a + // higher band — set before ready is closed. + err error } // bandQueue holds one band's waiters, grouped by tenant. @@ -166,3 +169,17 @@ func (q *bandQueue) dropExpired(now time.Time, onDrop func()) { q.byTenant[tenant] = kept } } + +// newest is the most recently queued waiter, across every tenant: the one +// displacement takes, since it has invested the least waiting. +func (q *bandQueue) newest() *waiter { + var out *waiter + for _, ws := range q.byTenant { + for _, w := range ws { + if out == nil || w.enqueued.After(out.enqueued) { + out = w + } + } + } + return out +} diff --git a/internal/router/finish_test.go b/internal/router/finish_test.go index 93e47ec..a1a467e 100644 --- a/internal/router/finish_test.go +++ b/internal/router/finish_test.go @@ -88,6 +88,9 @@ func TestARefusalBeforeDispatchSaysWhetherWaitingHelps(t *testing.T) { {"queued too long", func(w http.ResponseWriter) { rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrQueueTimeout, time.Now()) }, 504, ""}, + {"displaced for a higher band", func(w http.ResponseWriter) { + rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrDisplaced, time.Now()) + }, 503, "5"}, {"shutting down", func(w http.ResponseWriter) { rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrClosed, time.Now()) }, 503, ""}, diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 6ff1592..d301557 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -60,9 +60,15 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err return case errors.Is(err, admission.ErrQueueTimeout): status = http.StatusGatewayTimeout + case errors.Is(err, admission.ErrDisplaced): + // Turned away for higher-priority work, not for good: said so the + // client backs off and comes back rather than giving up. + w.Header().Set("Retry-After", "5") } reason := "unavailable" switch { + case errors.Is(err, admission.ErrDisplaced): + reason = "displaced" case errors.Is(err, admission.ErrQueueFull): reason = "queue_full" case errors.Is(err, admission.ErrQueueTimeout): diff --git a/internal/router/router.go b/internal/router/router.go index bba828f..1dba6aa 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -347,6 +347,7 @@ func admissionConfig(rc config.Router) admission.Config { }, QueueTTL: rc.Admission.QueueTTL, QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand, + QueueCapacityTotal: rc.Admission.QueueCapacityTotal, Interval: rc.Admission.Interval, } } diff --git a/llmgw-plan.md b/llmgw-plan.md index efc9b24..c1f9ebd 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1477,7 +1477,8 @@ The launch path is now built too: ### Phase 6 — v2 ideas - [ ] Precise KV index (§7.5 tier 2) — vLLM KV events plus tokenizer integration - [ ] Dynamo-style block cost function, tuning the `softmax` picker -- [ ] Admission displacement (dropping requests from lower bands) +- [x] Admission displacement: a total queue bound, and the newest request of + the lowest band turned away (503, Retry-After) for a higher one - [x] Autoscaling on load: a replica added while requests wait, up to a model's max, and taken back after ten quiet minutes - [x] Priority-based preemption: capacity classes (critical, standard,