CI / test (push) Successful in 48s
Inserted between quota checks and scheduling: admission answers when a request may proceed, the scheduler still answers where it goes. Keeping them apart is the point — an engine queue is FIFO and knows nothing about tenants, so once a batch job is in it, no endpoint selection rescues the request behind it. - saturation detector: one number per pool, the worse of queue depth and KV occupancy, since either alone misleads. - priority bands with per-band saturation ceilings, so the cluster degrades from the bottom up. - tenant round-robin within a band: one caller sending a thousand requests takes one slot in the rotation, not a thousand. - bounded queues with capacity (503) and TTL (504); cancelled requests leave the queue instead of holding capacity. - metrics for queue depth, wait time, rejections and pool saturation. Exit criterion, measured in test/e2e: with 500 concurrent bronze requests running continuously, interactive p99 in the platinum band stays at 1.04-1.24x idle against a 1.5x budget. With admission off the same load pushes it to 55-60x. Both directions are asserted, so the test fails if admission regresses and also if the load generator stops being a real load. Two lags had to be closed, and they needed different answers. Engine metrics are scraped and lag an interval; router inflight does not, but it counts running requests, and treating those as queue makes a healthy engine look overloaded every interval. What is actually queued is inflight minus batch capacity, and capacity is not in the metric contract, so it is learned as the high-water mark of observed running. Separately, between admission granting a request and the router dispatching it the request is invisible everywhere, so a simultaneous burst passed the check en masse; the controller now counts its own outstanding leases against dispatched inflight. One plan correction: a saturation ceiling of 1.0 was documented as "never held back". It is not — it means "flow up to the healthy limit". A band that must never wait needs a ceiling above 1. Plan §7.2.2 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
169 lines
3.6 KiB
Go
169 lines
3.6 KiB
Go
package admission
|
|
|
|
import "time"
|
|
|
|
// waiter is one queued request.
|
|
type waiter struct {
|
|
tenant string
|
|
model string
|
|
ready chan struct{}
|
|
enqueued time.Time
|
|
deadline time.Time
|
|
// admitted guards the race between the dispatcher admitting a waiter and
|
|
// the waiter giving up on its own.
|
|
admitted bool
|
|
}
|
|
|
|
// bandQueue holds one band's waiters, grouped by tenant.
|
|
//
|
|
// Grouping is what makes fairness possible: a single tenant sending a thousand
|
|
// requests occupies one slot in the rotation, not a thousand, so a traffic
|
|
// spike from one caller cannot consume the band.
|
|
type bandQueue struct {
|
|
band Band
|
|
|
|
byTenant map[string][]*waiter
|
|
// rotation is the round-robin order of tenants with pending work.
|
|
rotation []string
|
|
next int
|
|
count int
|
|
}
|
|
|
|
func newBandQueue(b Band) *bandQueue {
|
|
return &bandQueue{band: b, byTenant: map[string][]*waiter{}}
|
|
}
|
|
|
|
func (q *bandQueue) len() int { return q.count }
|
|
|
|
func (q *bandQueue) push(w *waiter) {
|
|
if _, ok := q.byTenant[w.tenant]; !ok {
|
|
q.byTenant[w.tenant] = nil
|
|
q.rotation = append(q.rotation, w.tenant)
|
|
}
|
|
q.byTenant[w.tenant] = append(q.byTenant[w.tenant], w)
|
|
q.count++
|
|
}
|
|
|
|
// peek returns the waiter the next pop would return, without removing it.
|
|
func (q *bandQueue) peek() *waiter {
|
|
if q.count == 0 {
|
|
return nil
|
|
}
|
|
idx, tenant := q.nextTenant()
|
|
if idx < 0 {
|
|
return nil
|
|
}
|
|
_ = tenant
|
|
return q.byTenant[q.rotation[idx]][0]
|
|
}
|
|
|
|
// pop removes and returns the next waiter in tenant rotation.
|
|
func (q *bandQueue) pop() *waiter {
|
|
if q.count == 0 {
|
|
return nil
|
|
}
|
|
idx, _ := q.nextTenant()
|
|
if idx < 0 {
|
|
return nil
|
|
}
|
|
|
|
tenant := q.rotation[idx]
|
|
ws := q.byTenant[tenant]
|
|
w := ws[0]
|
|
q.byTenant[tenant] = ws[1:]
|
|
q.count--
|
|
|
|
if len(q.byTenant[tenant]) == 0 {
|
|
delete(q.byTenant, tenant)
|
|
q.rotation = append(q.rotation[:idx], q.rotation[idx+1:]...)
|
|
if idx < q.next {
|
|
q.next--
|
|
}
|
|
} else {
|
|
q.next = idx + 1
|
|
}
|
|
if len(q.rotation) > 0 {
|
|
q.next %= len(q.rotation)
|
|
} else {
|
|
q.next = 0
|
|
}
|
|
return w
|
|
}
|
|
|
|
// nextTenant finds the next rotation entry that still has waiters.
|
|
func (q *bandQueue) nextTenant() (int, string) {
|
|
n := len(q.rotation)
|
|
for i := range n {
|
|
idx := (q.next + i) % n
|
|
if len(q.byTenant[q.rotation[idx]]) > 0 {
|
|
return idx, q.rotation[idx]
|
|
}
|
|
}
|
|
return -1, ""
|
|
}
|
|
|
|
// remove takes a specific waiter out, for cancellation and TTL expiry.
|
|
func (q *bandQueue) remove(target *waiter) bool {
|
|
ws, ok := q.byTenant[target.tenant]
|
|
if !ok {
|
|
return false
|
|
}
|
|
for i, w := range ws {
|
|
if w != target {
|
|
continue
|
|
}
|
|
q.byTenant[target.tenant] = append(ws[:i:i], ws[i+1:]...)
|
|
q.count--
|
|
if len(q.byTenant[target.tenant]) == 0 {
|
|
delete(q.byTenant, target.tenant)
|
|
q.dropTenant(target.tenant)
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (q *bandQueue) dropTenant(tenant string) {
|
|
for i, t := range q.rotation {
|
|
if t != tenant {
|
|
continue
|
|
}
|
|
q.rotation = append(q.rotation[:i], q.rotation[i+1:]...)
|
|
if i < q.next {
|
|
q.next--
|
|
}
|
|
break
|
|
}
|
|
if len(q.rotation) > 0 {
|
|
q.next %= len(q.rotation)
|
|
} else {
|
|
q.next = 0
|
|
}
|
|
}
|
|
|
|
// dropExpired removes waiters past their deadline, calling onDrop for each.
|
|
// Their Acquire call is already unblocking on its own timer; this keeps the
|
|
// queue from carrying corpses that would occupy capacity.
|
|
func (q *bandQueue) dropExpired(now time.Time, onDrop func()) {
|
|
if q.count == 0 {
|
|
return
|
|
}
|
|
for tenant, ws := range q.byTenant {
|
|
kept := ws[:0]
|
|
for _, w := range ws {
|
|
if now.After(w.deadline) && !w.admitted {
|
|
q.count--
|
|
onDrop()
|
|
continue
|
|
}
|
|
kept = append(kept, w)
|
|
}
|
|
if len(kept) == 0 {
|
|
delete(q.byTenant, tenant)
|
|
q.dropTenant(tenant)
|
|
continue
|
|
}
|
|
q.byTenant[tenant] = kept
|
|
}
|
|
}
|