// Package admission answers "may this request proceed now?" (plan §7.2). // // It is deliberately separate from scheduling, which answers "where should it // go?". Kept apart, the router can hold requests in its own queue and apply // priority and tenant fairness *before* they enter an engine queue. An engine // queue is FIFO and knows nothing about tenants: once a batch job has put 500 // requests into it, no amount of clever endpoint selection will rescue the // interactive request that arrives next. package admission import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // Thresholds define what "saturated" means for a pool. type Thresholds struct { // TargetWaitingPerReplica is the healthy queue depth: enough for continuous // batching to stay efficient, not enough for queueing delay to show up. // Measure it with ctl bench (plan §10); this single number governs how the // whole system behaves under load. TargetWaitingPerReplica float64 // KVCeiling is the KV occupancy treated as full. KVCeiling float64 } func (t *Thresholds) normalize() { if t.TargetWaitingPerReplica <= 0 { t.TargetWaitingPerReplica = 4 } if t.KVCeiling <= 0 || t.KVCeiling > 1 { t.KVCeiling = 0.9 } } // Saturation reduces a pool's state to a single number, where 1 means "at the // healthy limit" and above 1 means overloaded. // // Two signals are combined because either alone misleads: a pool can have an // empty queue while its KV cache is nearly full (long contexts), or plenty of // KV headroom while requests pile up (slow decode). The worse of the two wins. // // unreported is load that has been admitted but that no endpoint has accounted // for yet. Without it a simultaneous burst all passes the check before any of // it becomes visible — which is precisely when admission needs to bite. func Saturation(eps []endpoint.Snapshot, unreported int, t Thresholds) float64 { t.normalize() var sum float64 var n int for _, e := range eps { // A replica the breaker has taken out is not available capacity, so // counting it would make the pool look healthier than it is. if e.Breaker.String() == "open" { continue } // The engine's own queue is the truth, but it is one scrape interval // old. Fill that gap with what the router knows without any lag: how // many requests it has dispatched here, minus what this engine can run // at once. Anything beyond that capacity is queued by definition. // // Subtracting the capacity is the whole point. Counting raw inflight // would make a fully busy but perfectly healthy engine look like it had // a queue as long as its batch, every single scrape interval. waiting := float64(e.Stats.Waiting) if e.RunningCapacity > 0 { if queued := e.Inflight - e.RunningCapacity; float64(queued) > waiting { waiting = float64(queued) } } else if e.Inflight > 0 { // Nothing observed yet, so capacity is unknown. Treat dispatched // work as queued: conservative for the first scrape, and it relaxes // as soon as the engine reports anything. waiting = max(waiting, float64(e.Inflight)) } q := waiting / t.TargetWaitingPerReplica kv := e.Stats.KVCacheUsage / t.KVCeiling sum += max(q, kv) n++ } _ = n if n == 0 { // Every replica is out. Report full saturation so low bands stop // queueing work that has nowhere to go; the dispatch layer will return // a clearer error than a queue timeout. return 1 } // Spread unaccounted admissions evenly; they will land somewhere. if unreported > 0 { sum += float64(unreported) / t.TargetWaitingPerReplica } // The mean, not the minimum: one idle replica does not make a busy pool // idle, and admission is about the pool's capacity as a whole. return sum / float64(n) }