Displace the newest lowest-band request when the gateway queue is full
A total bound across bands (admission.queue_capacity_total, default half of all bands' capacity together); reached, a request is not refused while a lower band has requests queued, and the newest of the lowest band gets a 503 with Retry-After instead. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
623cdb5ba3
commit
3d3c96e422
10 files changed
+199
-4
No files matched your search
@@ -42,6 +42,17 @@ What v0.10.5 made settable from the command line is now on the pages too:
|
||||
critical model's card no longer refuses a replica that preemption would make
|
||||
room for.
|
||||
|
||||
### Displacement: a higher band is not refused while a lower one waits
|
||||
|
||||
Each band's queue was bounded, and nothing bounded them together, so a full
|
||||
gateway turned an interactive request away while batch requests sat queued
|
||||
behind it. `admission.queue_capacity_total` now bounds everything queued —
|
||||
by default half of every band's capacity together. Reached, a request is not
|
||||
refused while a lower band has any queued: the newest request of the lowest
|
||||
band is turned away instead, with a 503 and `Retry-After: 5`, so it backs off
|
||||
and returns. The newest, because the oldest have waited longest and are
|
||||
nearest their TTL. Counted as `displaced` in the admission rejection metric.
|
||||
|
||||
### Releases can be signed
|
||||
|
||||
`llmgw release keygen` makes an ed25519 key pair. With the private key in the
|
||||
|
||||
@@ -21,6 +21,7 @@ rather than repeated.
|
||||
| `band_header` | `string` | [Admission](#admission) |
|
||||
| `queue_ttl` | `time.Duration` | [Admission](#admission) |
|
||||
| `queue_capacity_per_band` | `int` | [Admission](#admission) |
|
||||
| `queue_capacity_total` | `int` | [Admission](#admission) |
|
||||
| `interval` | `time.Duration` | [Admission](#admission) |
|
||||
| `enabled` | `bool` | [Auth](#auth) |
|
||||
| `store` | `string` | [Auth](#auth) |
|
||||
@@ -228,6 +229,15 @@ should not be able to promote themselves.
|
||||
|
||||
`int`
|
||||
|
||||
### `queue_capacity_total`
|
||||
|
||||
`int`
|
||||
|
||||
QueueCapacityTotal bounds everything queued across bands. Reached, a
|
||||
request is not refused while a lower band has any queued: the newest of
|
||||
the lowest band is turned away instead, with a 503 and Retry-After.
|
||||
Zero takes half of all the bands' capacities together.
|
||||
|
||||
### `interval`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
@@ -378,7 +378,12 @@ type Admission struct {
|
||||
|
||||
QueueTTL time.Duration `koanf:"queue_ttl"`
|
||||
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
||||
Interval time.Duration `koanf:"interval"`
|
||||
// QueueCapacityTotal bounds everything queued across bands. Reached, a
|
||||
// request is not refused while a lower band has any queued: the newest of
|
||||
// the lowest band is turned away instead, with a 503 and Retry-After.
|
||||
// Zero takes half of all the bands' capacities together.
|
||||
QueueCapacityTotal int `koanf:"queue_capacity_total"`
|
||||
Interval time.Duration `koanf:"interval"`
|
||||
}
|
||||
|
||||
// Prefix configures approximate prefix routing.
|
||||
|
||||
@@ -34,6 +34,12 @@ type Config struct {
|
||||
// QueueCapacityPerBand bounds each band's queue. Rejecting immediately
|
||||
// beats accumulating silently.
|
||||
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
||||
// QueueCapacityTotal bounds everything queued, across bands. When it is
|
||||
// reached, a request is not refused while a lower band has requests
|
||||
// queued: the newest of the lowest band is turned away instead
|
||||
// (displacement), since the oldest have waited longest and are nearest
|
||||
// their TTL. Zero takes half of every band's capacity together.
|
||||
QueueCapacityTotal int `koanf:"queue_capacity_total"`
|
||||
// Interval is how often the dispatcher re-checks saturation. Releases also
|
||||
// wake it, so this is only the backstop for saturation falling on its own.
|
||||
Interval time.Duration `koanf:"interval"`
|
||||
@@ -94,6 +100,9 @@ func (c *Config) normalize() error {
|
||||
if c.QueueCapacityPerBand <= 0 {
|
||||
c.QueueCapacityPerBand = 2000
|
||||
}
|
||||
if c.QueueCapacityTotal <= 0 {
|
||||
c.QueueCapacityTotal = max(c.QueueCapacityPerBand*len(c.Bands)/2, 1)
|
||||
}
|
||||
if c.Interval <= 0 {
|
||||
c.Interval = 10 * time.Millisecond
|
||||
}
|
||||
@@ -123,6 +132,10 @@ var (
|
||||
ErrQueueTimeout = errors.New("admission queue timeout")
|
||||
// ErrClosed means the controller is shutting down.
|
||||
ErrClosed = errors.New("admission controller closed")
|
||||
// ErrDisplaced means a queued request was turned away to make room for
|
||||
// one of a higher band. Worth retrying: the queue is full now, not for
|
||||
// good.
|
||||
ErrDisplaced = errors.New("displaced from the admission queue by higher-priority requests")
|
||||
)
|
||||
|
||||
// Stats is a snapshot of one band's queue, for metrics.
|
||||
@@ -259,6 +272,12 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) {
|
||||
c.mu.Unlock()
|
||||
return nil, fmt.Errorf("%w: band %q", ErrQueueFull, band.Name)
|
||||
}
|
||||
if c.queuedLocked() >= c.cfg.QueueCapacityTotal && !c.displaceBelowLocked(idx) {
|
||||
c.rejected[band.Name]++
|
||||
c.mu.Unlock()
|
||||
return nil, fmt.Errorf("%w: %d queued in all, and nothing of a lower band to make room",
|
||||
ErrQueueFull, c.cfg.QueueCapacityTotal)
|
||||
}
|
||||
|
||||
w := &waiter{
|
||||
tenant: tenantKey(r),
|
||||
@@ -279,6 +298,10 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) {
|
||||
c.mu.Unlock()
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if w.err != nil {
|
||||
c.mu.Unlock()
|
||||
return nil, w.err
|
||||
}
|
||||
c.waitTotal[band.Name] += time.Since(w.enqueued)
|
||||
c.mu.Unlock()
|
||||
return &Lease{c: c, band: band.Name, model: r.Model}, nil
|
||||
@@ -297,8 +320,9 @@ func (c *Controller) Acquire(ctx context.Context, r Request) (*Lease, error) {
|
||||
func (c *Controller) remove(idx int, w *waiter, reason string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if w.admitted {
|
||||
// It won the race; the caller will see ready closed next time round.
|
||||
if w.admitted || w.err != nil {
|
||||
// It won the race, or was displaced; the caller will see ready
|
||||
// closed next time round.
|
||||
return
|
||||
}
|
||||
c.queues[idx].remove(w)
|
||||
@@ -410,3 +434,31 @@ func tenantKey(r Request) string {
|
||||
}
|
||||
return "anonymous"
|
||||
}
|
||||
|
||||
// queuedLocked is how many requests wait in every band. Called with mu held.
|
||||
func (c *Controller) queuedLocked() int {
|
||||
n := 0
|
||||
for _, q := range c.queues {
|
||||
n += q.len()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// displaceBelowLocked turns away the newest request of the lowest band below
|
||||
// idx that has any queued, and says whether there was one. Called with mu held.
|
||||
func (c *Controller) displaceBelowLocked(idx int) bool {
|
||||
// Bands are sorted highest priority first, so lower bands have larger
|
||||
// indices, and the lowest is last.
|
||||
for j := len(c.queues) - 1; j > idx; j-- {
|
||||
w := c.queues[j].newest()
|
||||
if w == nil {
|
||||
continue
|
||||
}
|
||||
c.queues[j].remove(w)
|
||||
w.err = fmt.Errorf("%w (band %q)", ErrDisplaced, c.cfg.Bands[j].Name)
|
||||
close(w.ready)
|
||||
c.rejected[c.cfg.Bands[j].Name+"/displaced"]++
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package admission
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// queueUntil starts an Acquire in the background and waits until the band
|
||||
// holds depth requests, returning where its result will arrive.
|
||||
func queueUntil(t *testing.T, c *Controller, band string, bandIdx, depth int) <-chan error {
|
||||
t.Helper()
|
||||
out := make(chan error, 1)
|
||||
go func() {
|
||||
l, err := c.Acquire(context.Background(), Request{Model: "m", Band: band})
|
||||
if err == nil {
|
||||
l.Release()
|
||||
}
|
||||
out <- err
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for c.Depths()[bandIdx].Depth < depth && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if got := c.Depths()[bandIdx].Depth; got != depth {
|
||||
t.Fatalf("band %s holds %d, expected %d", band, got, depth)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAHigherBandDisplacesTheNewestOfTheLowest(t *testing.T) {
|
||||
s := &fixedSat{}
|
||||
s.set(5) // past every ceiling: everything queues
|
||||
cfg := testConfig()
|
||||
cfg.QueueCapacityTotal = 3
|
||||
c := newController(t, cfg, s.fn)
|
||||
// Bands, highest first: platinum 0, gold 1, silver 2, bronze 3.
|
||||
oldest := queueUntil(t, c, "bronze", 3, 1)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
silver := queueUntil(t, c, "silver", 2, 1)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
newest := queueUntil(t, c, "bronze", 3, 2)
|
||||
|
||||
// Full at three. Gold arrives: the newest bronze goes, not the oldest,
|
||||
// and not silver, which outranks bronze.
|
||||
gold := queueUntil(t, c, "gold", 1, 1)
|
||||
select {
|
||||
case err := <-newest:
|
||||
if !errors.Is(err, ErrDisplaced) {
|
||||
t.Fatalf("the newest bronze got %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("nothing was displaced")
|
||||
}
|
||||
if d := c.Depths(); d[3].Depth != 1 || d[2].Depth != 1 || d[1].Depth != 1 {
|
||||
t.Errorf("depths %+v: one bronze, the silver and the gold should remain", d)
|
||||
}
|
||||
|
||||
// Bronze cannot displace anything: nothing is below it.
|
||||
_, err := c.Acquire(t.Context(), Request{Model: "m", Band: "bronze"})
|
||||
if !errors.Is(err, ErrQueueFull) {
|
||||
t.Errorf("a bronze request at a full queue got %v", err)
|
||||
}
|
||||
// Another gold takes the remaining bronze, then silver.
|
||||
queueUntil(t, c, "gold", 1, 2)
|
||||
if err := <-oldest; !errors.Is(err, ErrDisplaced) {
|
||||
t.Errorf("the last bronze got %v", err)
|
||||
}
|
||||
queueUntil(t, c, "gold", 1, 3)
|
||||
if err := <-silver; !errors.Is(err, ErrDisplaced) {
|
||||
t.Errorf("silver got %v", err)
|
||||
}
|
||||
// All gold now: a fourth has nobody lower to displace.
|
||||
if _, err := c.Acquire(t.Context(), Request{Model: "m", Band: "gold"}); !errors.Is(err, ErrQueueFull) {
|
||||
t.Errorf("a gold request with only gold queued got %v", err)
|
||||
}
|
||||
|
||||
// And they are served once there is room.
|
||||
s.set(0)
|
||||
select {
|
||||
case err := <-gold:
|
||||
if err != nil {
|
||||
t.Errorf("the first gold was not served: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Error("the queue did not drain")
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,9 @@ type waiter struct {
|
||||
// admitted guards the race between the dispatcher admitting a waiter and
|
||||
// the waiter giving up on its own.
|
||||
admitted bool
|
||||
// err is why the waiter was turned away while queued — displaced by a
|
||||
// higher band — set before ready is closed.
|
||||
err error
|
||||
}
|
||||
|
||||
// bandQueue holds one band's waiters, grouped by tenant.
|
||||
@@ -166,3 +169,17 @@ func (q *bandQueue) dropExpired(now time.Time, onDrop func()) {
|
||||
q.byTenant[tenant] = kept
|
||||
}
|
||||
}
|
||||
|
||||
// newest is the most recently queued waiter, across every tenant: the one
|
||||
// displacement takes, since it has invested the least waiting.
|
||||
func (q *bandQueue) newest() *waiter {
|
||||
var out *waiter
|
||||
for _, ws := range q.byTenant {
|
||||
for _, w := range ws {
|
||||
if out == nil || w.enqueued.After(out.enqueued) {
|
||||
out = w
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -88,6 +88,9 @@ func TestARefusalBeforeDispatchSaysWhetherWaitingHelps(t *testing.T) {
|
||||
{"queued too long", func(w http.ResponseWriter) {
|
||||
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrQueueTimeout, time.Now())
|
||||
}, 504, ""},
|
||||
{"displaced for a higher band", func(w http.ResponseWriter) {
|
||||
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrDisplaced, time.Now())
|
||||
}, 503, "5"},
|
||||
{"shutting down", func(w http.ResponseWriter) {
|
||||
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrClosed, time.Now())
|
||||
}, 503, ""},
|
||||
|
||||
@@ -60,9 +60,15 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err
|
||||
return
|
||||
case errors.Is(err, admission.ErrQueueTimeout):
|
||||
status = http.StatusGatewayTimeout
|
||||
case errors.Is(err, admission.ErrDisplaced):
|
||||
// Turned away for higher-priority work, not for good: said so the
|
||||
// client backs off and comes back rather than giving up.
|
||||
w.Header().Set("Retry-After", "5")
|
||||
}
|
||||
reason := "unavailable"
|
||||
switch {
|
||||
case errors.Is(err, admission.ErrDisplaced):
|
||||
reason = "displaced"
|
||||
case errors.Is(err, admission.ErrQueueFull):
|
||||
reason = "queue_full"
|
||||
case errors.Is(err, admission.ErrQueueTimeout):
|
||||
|
||||
@@ -347,6 +347,7 @@ func admissionConfig(rc config.Router) admission.Config {
|
||||
},
|
||||
QueueTTL: rc.Admission.QueueTTL,
|
||||
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
|
||||
QueueCapacityTotal: rc.Admission.QueueCapacityTotal,
|
||||
Interval: rc.Admission.Interval,
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -1477,7 +1477,8 @@ The launch path is now built too:
|
||||
### Phase 6 — v2 ideas
|
||||
- [ ] Precise KV index (§7.5 tier 2) — vLLM KV events plus tokenizer integration
|
||||
- [ ] Dynamo-style block cost function, tuning the `softmax` picker
|
||||
- [ ] Admission displacement (dropping requests from lower bands)
|
||||
- [x] Admission displacement: a total queue bound, and the newest request of
|
||||
the lowest band turned away (503, Retry-After) for a higher one
|
||||
- [x] Autoscaling on load: a replica added while requests wait, up to a
|
||||
model's max, and taken back after ten quiet minutes
|
||||
- [x] Priority-based preemption: capacity classes (critical, standard,
|
||||
|
||||
Reference in new issue
Block a user