Files
llmgw/internal/router/admission/queue.go
T
ilkerandClaude Opus 5 dc38dd6096 Phase 1.5: admission control
Inserted between quota checks and scheduling: admission answers when a request
may proceed, the scheduler still answers where it goes. Keeping them apart is
the point — an engine queue is FIFO and knows nothing about tenants, so once a
batch job is in it, no endpoint selection rescues the request behind it.

- saturation detector: one number per pool, the worse of queue depth and KV
  occupancy, since either alone misleads.
- priority bands with per-band saturation ceilings, so the cluster degrades from
  the bottom up.
- tenant round-robin within a band: one caller sending a thousand requests takes
  one slot in the rotation, not a thousand.
- bounded queues with capacity (503) and TTL (504); cancelled requests leave the
  queue instead of holding capacity.
- metrics for queue depth, wait time, rejections and pool saturation.

Exit criterion, measured in test/e2e: with 500 concurrent bronze requests
running continuously, interactive p99 in the platinum band stays at 1.04-1.24x
idle against a 1.5x budget. With admission off the same load pushes it to
55-60x. Both directions are asserted, so the test fails if admission regresses
and also if the load generator stops being a real load.

Two lags had to be closed, and they needed different answers. Engine metrics are
scraped and lag an interval; router inflight does not, but it counts running
requests, and treating those as queue makes a healthy engine look overloaded
every interval. What is actually queued is inflight minus batch capacity, and
capacity is not in the metric contract, so it is learned as the high-water mark
of observed running. Separately, between admission granting a request and the
router dispatching it the request is invisible everywhere, so a simultaneous
burst passed the check en masse; the controller now counts its own outstanding
leases against dispatched inflight.

One plan correction: a saturation ceiling of 1.0 was documented as "never held
back". It is not — it means "flow up to the healthy limit". A band that must
never wait needs a ceiling above 1. Plan §7.2.2 updated.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:53:38 +02:00

169 lines
3.6 KiB
Go

package admission
import "time"
// waiter is one queued request.
type waiter struct {
tenant string
model string
ready chan struct{}
enqueued time.Time
deadline time.Time
// admitted guards the race between the dispatcher admitting a waiter and
// the waiter giving up on its own.
admitted bool
}
// bandQueue holds one band's waiters, grouped by tenant.
//
// Grouping is what makes fairness possible: a single tenant sending a thousand
// requests occupies one slot in the rotation, not a thousand, so a traffic
// spike from one caller cannot consume the band.
type bandQueue struct {
band Band
byTenant map[string][]*waiter
// rotation is the round-robin order of tenants with pending work.
rotation []string
next int
count int
}
func newBandQueue(b Band) *bandQueue {
return &bandQueue{band: b, byTenant: map[string][]*waiter{}}
}
func (q *bandQueue) len() int { return q.count }
func (q *bandQueue) push(w *waiter) {
if _, ok := q.byTenant[w.tenant]; !ok {
q.byTenant[w.tenant] = nil
q.rotation = append(q.rotation, w.tenant)
}
q.byTenant[w.tenant] = append(q.byTenant[w.tenant], w)
q.count++
}
// peek returns the waiter the next pop would return, without removing it.
func (q *bandQueue) peek() *waiter {
if q.count == 0 {
return nil
}
idx, tenant := q.nextTenant()
if idx < 0 {
return nil
}
_ = tenant
return q.byTenant[q.rotation[idx]][0]
}
// pop removes and returns the next waiter in tenant rotation.
func (q *bandQueue) pop() *waiter {
if q.count == 0 {
return nil
}
idx, _ := q.nextTenant()
if idx < 0 {
return nil
}
tenant := q.rotation[idx]
ws := q.byTenant[tenant]
w := ws[0]
q.byTenant[tenant] = ws[1:]
q.count--
if len(q.byTenant[tenant]) == 0 {
delete(q.byTenant, tenant)
q.rotation = append(q.rotation[:idx], q.rotation[idx+1:]...)
if idx < q.next {
q.next--
}
} else {
q.next = idx + 1
}
if len(q.rotation) > 0 {
q.next %= len(q.rotation)
} else {
q.next = 0
}
return w
}
// nextTenant finds the next rotation entry that still has waiters.
func (q *bandQueue) nextTenant() (int, string) {
n := len(q.rotation)
for i := range n {
idx := (q.next + i) % n
if len(q.byTenant[q.rotation[idx]]) > 0 {
return idx, q.rotation[idx]
}
}
return -1, ""
}
// remove takes a specific waiter out, for cancellation and TTL expiry.
func (q *bandQueue) remove(target *waiter) bool {
ws, ok := q.byTenant[target.tenant]
if !ok {
return false
}
for i, w := range ws {
if w != target {
continue
}
q.byTenant[target.tenant] = append(ws[:i:i], ws[i+1:]...)
q.count--
if len(q.byTenant[target.tenant]) == 0 {
delete(q.byTenant, target.tenant)
q.dropTenant(target.tenant)
}
return true
}
return false
}
func (q *bandQueue) dropTenant(tenant string) {
for i, t := range q.rotation {
if t != tenant {
continue
}
q.rotation = append(q.rotation[:i], q.rotation[i+1:]...)
if i < q.next {
q.next--
}
break
}
if len(q.rotation) > 0 {
q.next %= len(q.rotation)
} else {
q.next = 0
}
}
// dropExpired removes waiters past their deadline, calling onDrop for each.
// Their Acquire call is already unblocking on its own timer; this keeps the
// queue from carrying corpses that would occupy capacity.
func (q *bandQueue) dropExpired(now time.Time, onDrop func()) {
if q.count == 0 {
return
}
for tenant, ws := range q.byTenant {
kept := ws[:0]
for _, w := range ws {
if now.After(w.deadline) && !w.admitted {
q.count--
onDrop()
continue
}
kept = append(kept, w)
}
if len(kept) == 0 {
delete(q.byTenant, tenant)
q.dropTenant(tenant)
continue
}
q.byTenant[tenant] = kept
}
}