From d5f02c0a4b8a61820b612df1667f7cc00cbaaa20 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 23:23:32 +0200 Subject: [PATCH] Spill a saturated pool's requests to the next pool The pool-priority filter kept the highest-priority pool however full it was. It now takes the first pool, by priority, that is not saturated by admission's own measure, and the preferred one when all are. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++++ internal/router/scheduler/filters.go | 46 +++++++++++++-------- internal/router/scheduler/profiles.go | 6 +-- internal/router/scheduler/scheduler_test.go | 38 ++++++++++++++++- llmgw-plan.md | 3 +- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a887a84..d12e1aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,15 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### Pool spillover + +A model served from pools of different priority — H100 before A100, say — was +routed to the preferred pool however full it was. Now a pool whose replicas are +saturated, by the same measure admission uses (queue past its healthy depth, or +KV cache past its ceiling), passes requests to the next pool that is not. When +every pool is saturated the preferred one is kept: a pool that is just as full +gains nothing. + ### Displacement: a higher band is not refused while a lower one waits Each band's queue was bounded, and nothing bounded them together, so a full diff --git a/internal/router/scheduler/filters.go b/internal/router/scheduler/filters.go index 617b8a6..44bca49 100644 --- a/internal/router/scheduler/filters.go +++ b/internal/router/scheduler/filters.go @@ -1,6 +1,9 @@ package scheduler import ( + "sort" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) @@ -51,34 +54,41 @@ func KVSaturated(ceiling float64) Filter { } } -// PoolPriority keeps only the endpoints of the highest-priority pool -// (plan §7.3, the two-stage decision: pool first, then replica). +// PoolPriority keeps the endpoints of one pool: the highest-priority one, +// unless it is saturated and a lower one is not (plan §7.3, the two-stage +// decision — pool first, then replica). // -// Spillover arrives in phase 5, at which point this filter will fall through -// to the next pool once the preferred one is saturated. -func PoolPriority() Filter { - return poolPriorityFilter{} +// Saturated is what admission means by it, the same formula over the same +// thresholds, so the two layers agree on when a pool is full: the H100 pool is +// used until it is, and only then the A100 one — never the A100 because it +// happens to be emptier. When every pool is saturated the preferred one is +// kept: moving to a pool that is just as full buys nothing and loses whatever +// made the first preferred. +func PoolPriority(targetWaiting, kvCeiling float64) Filter { + return poolPriorityFilter{t: admission.Thresholds{TargetWaitingPerReplica: targetWaiting, KVCeiling: kvCeiling}} } -type poolPriorityFilter struct{} +type poolPriorityFilter struct{ t admission.Thresholds } func (poolPriorityFilter) Name() string { return "pool-priority" } -func (poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot { +func (f poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot { if len(eps) == 0 { return eps } - best := eps[0].Priority - for _, e := range eps[1:] { - if e.Priority > best { - best = e.Priority - } - } - out := make([]endpoint.Snapshot, 0, len(eps)) + byPriority := map[int][]endpoint.Snapshot{} + var order []int for _, e := range eps { - if e.Priority == best { - out = append(out, e) + if _, ok := byPriority[e.Priority]; !ok { + order = append(order, e.Priority) + } + byPriority[e.Priority] = append(byPriority[e.Priority], e) + } + sort.Sort(sort.Reverse(sort.IntSlice(order))) + for _, p := range order { + if admission.Saturation(byPriority[p], 0, f.t) < 1 { + return byPriority[p] } } - return out + return byPriority[order[0]] } diff --git a/internal/router/scheduler/profiles.go b/internal/router/scheduler/profiles.go index bcedf4b..0a24a18 100644 --- a/internal/router/scheduler/profiles.go +++ b/internal/router/scheduler/profiles.go @@ -50,7 +50,7 @@ func Chat(c ProfileConfig) *Profile { c.normalize() return &Profile{ Name: "chat", - Filters: []Filter{BreakerOpen(), PoolPriority()}, + Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)}, Scorers: c.withPrefix([]WeightedScorer{ {Session(c.SessionLoadFactor), 3.0}, {Queue(c.TargetWaiting), 1.0}, @@ -81,7 +81,7 @@ func Batch(c ProfileConfig) *Profile { c.normalize() return &Profile{ Name: "batch", - Filters: []Filter{BreakerOpen(), PoolPriority()}, + Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)}, Scorers: []WeightedScorer{ {Queue(c.TargetWaiting), 3.0}, {KV(c.KVCeiling), 1.0}, @@ -99,7 +99,7 @@ func LongContext(c ProfileConfig) *Profile { c.normalize() return &Profile{ Name: "long-context", - Filters: []Filter{BreakerOpen(), PoolPriority()}, + Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)}, // Prefix matching earns more here than anywhere else: a long prompt is // mostly prefill, and prefill is what a cache hit skips. Scorers: c.withPrefix([]WeightedScorer{ diff --git a/internal/router/scheduler/scheduler_test.go b/internal/router/scheduler/scheduler_test.go index 1509b9c..c36c532 100644 --- a/internal/router/scheduler/scheduler_test.go +++ b/internal/router/scheduler/scheduler_test.go @@ -177,7 +177,7 @@ func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) { func TestPoolPriority(t *testing.T) { p := &Profile{ Name: "t", - Filters: []Filter{PoolPriority()}, + Filters: []Filter{PoolPriority(4, 0.9)}, Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore(), } @@ -189,7 +189,8 @@ func TestPoolPriority(t *testing.T) { if err != nil { t.Fatal(err) } - // The lower-priority pool must not win despite being emptier (spillover is phase 5). + // The lower-priority pool must not win for being emptier: the preferred + // one is not saturated (3 waiting of a healthy 4). if d.Endpoint.ID != "h100" { t.Errorf("pool priority was ignored: %s", d.Endpoint.ID) } @@ -383,3 +384,36 @@ func TestBuiltinProfilesSchedule(t *testing.T) { } } } + +func TestASaturatedPoolSpillsToTheNext(t *testing.T) { + p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)}, + Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()} + h100 := func(w int) endpoint.Snapshot { + return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" }) + } + a100 := func(w int) endpoint.Snapshot { + return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" }) + } + pick := func(eps ...endpoint.Snapshot) string { + d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) + if err != nil { + t.Fatal(err) + } + return d.Endpoint.ID + } + if got := pick(h100(8), a100(0)); got != "a100" { + t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got) + } + if got := pick(h100(8), a100(6)); got != "h100" { + t.Errorf("both pools saturated: the preferred one should be kept, got %s", got) + } + if got := pick(h100(4), a100(0)); got != "a100" { + t.Errorf("at exactly its healthy limit a pool is full: %s", got) + } + // KV pressure saturates a pool as queue depth does. + full := h100(0) + full.Stats.KVCacheUsage = 0.95 + if got := pick(full, a100(0)); got != "a100" { + t.Errorf("a pool out of KV cache did not spill: %s", got) + } +} diff --git a/llmgw-plan.md b/llmgw-plan.md index c1f9ebd..e9a8c30 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1459,7 +1459,8 @@ The launch path is now built too: hits. It does nothing when the working set fits in one replica's cache, which is worth knowing rather than assuming. - [x] Fallback models -- [ ] Pool spillover +- [x] Pool spillover: a saturated pool passes requests to the next one that + is not, by admission's own measure of saturation - [x] `ctl bench`: a load ramp that reports capacity as the last concurrency level meeting the SLO, plus the queue depth and KV usage observed there, so `weight`, `target_waiting` and `kv_ceiling` stop being guesses