Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
107 lines
4.0 KiB
Go
107 lines
4.0 KiB
Go
// Package admission answers "may this request proceed now?" (plan §7.2).
|
|
//
|
|
// It is deliberately separate from scheduling, which answers "where should it
|
|
// go?". Kept apart, the router can hold requests in its own queue and apply
|
|
// priority and tenant fairness *before* they enter an engine queue. An engine
|
|
// queue is FIFO and knows nothing about tenants: once a batch job has put 500
|
|
// requests into it, no amount of clever endpoint selection will rescue the
|
|
// interactive request that arrives next.
|
|
package admission
|
|
|
|
import (
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Thresholds define what "saturated" means for a pool.
|
|
type Thresholds struct {
|
|
// TargetWaitingPerReplica is the healthy queue depth: enough for continuous
|
|
// batching to stay efficient, not enough for queueing delay to show up.
|
|
// Measure it with ctl bench (plan §10); this single number governs how the
|
|
// whole system behaves under load.
|
|
TargetWaitingPerReplica float64
|
|
// KVCeiling is the KV occupancy treated as full.
|
|
KVCeiling float64
|
|
}
|
|
|
|
func (t *Thresholds) normalize() {
|
|
if t.TargetWaitingPerReplica <= 0 {
|
|
t.TargetWaitingPerReplica = 4
|
|
}
|
|
if t.KVCeiling <= 0 || t.KVCeiling > 1 {
|
|
t.KVCeiling = 0.9
|
|
}
|
|
}
|
|
|
|
// Saturation reduces a pool's state to a single number, where 1 means "at the
|
|
// healthy limit" and above 1 means overloaded.
|
|
//
|
|
// Two signals are combined because either alone misleads: a pool can have an
|
|
// empty queue while its KV cache is nearly full (long contexts), or plenty of
|
|
// KV headroom while requests pile up (slow decode). The worse of the two wins.
|
|
//
|
|
// unreported is load that has been admitted but that no endpoint has accounted
|
|
// for yet. Without it a simultaneous burst all passes the check before any of
|
|
// it becomes visible — which is precisely when admission needs to bite.
|
|
func Saturation(eps []endpoint.Snapshot, unreported int, t Thresholds) float64 {
|
|
t.normalize()
|
|
|
|
var sum float64
|
|
var n int
|
|
for _, e := range eps {
|
|
// A replica the breaker has taken out is not available capacity, so
|
|
// counting it would make the pool look healthier than it is.
|
|
if e.Breaker.String() == "open" {
|
|
continue
|
|
}
|
|
// A provider outside the cluster queues for itself and says 429 when
|
|
// full, which its breaker hears: nothing here measures its queue, and
|
|
// counting every request in flight to it as waiting held an external
|
|
// model to three or four at once, the rest timing out in the queue.
|
|
if e.Engine == endpoint.EngineExternal {
|
|
n++
|
|
continue
|
|
}
|
|
|
|
// The engine's own queue is the truth, but it is one scrape interval
|
|
// old. Fill that gap with what the router knows without any lag: how
|
|
// many requests it has dispatched here, minus what this engine can run
|
|
// at once. Anything beyond that capacity is queued by definition.
|
|
//
|
|
// Subtracting the capacity is the whole point. Counting raw inflight
|
|
// would make a fully busy but perfectly healthy engine look like it had
|
|
// a queue as long as its batch, every single scrape interval.
|
|
waiting := float64(e.Stats.Waiting)
|
|
if e.RunningCapacity > 0 {
|
|
if queued := e.Inflight - e.RunningCapacity; float64(queued) > waiting {
|
|
waiting = float64(queued)
|
|
}
|
|
} else if e.Inflight > 0 {
|
|
// Nothing observed yet, so capacity is unknown. Treat dispatched
|
|
// work as queued: conservative for the first scrape, and it relaxes
|
|
// as soon as the engine reports anything.
|
|
waiting = max(waiting, float64(e.Inflight))
|
|
}
|
|
|
|
q := waiting / t.TargetWaitingPerReplica
|
|
kv := e.Stats.KVCacheUsage / t.KVCeiling
|
|
sum += max(q, kv)
|
|
n++
|
|
}
|
|
_ = n
|
|
|
|
if n == 0 {
|
|
// Every replica is out. Report full saturation so low bands stop
|
|
// queueing work that has nowhere to go; the dispatch layer will return
|
|
// a clearer error than a queue timeout.
|
|
return 1
|
|
}
|
|
// Spread unaccounted admissions evenly; they will land somewhere.
|
|
if unreported > 0 {
|
|
sum += float64(unreported) / t.TargetWaitingPerReplica
|
|
}
|
|
|
|
// The mean, not the minimum: one idle replica does not make a busy pool
|
|
// idle, and admission is about the pool's capacity as a whole.
|
|
return sum / float64(n)
|
|
}
|