Spill a saturated pool's requests to the next pool
CI / test (push) Successful in 3m40s

The pool-priority filter kept the highest-priority pool however full it
was. It now takes the first pool, by priority, that is not saturated by
admission's own measure, and the preferred one when all are.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 23:23:32 +02:00
1 parent 3d3c96e422
commit d5f02c0a4b
5 files changed
+78 -24

No files matched your search

+28 -18
View File
@@ -1,6 +1,9 @@
package scheduler
import (
"sort"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
@@ -51,34 +54,41 @@ func KVSaturated(ceiling float64) Filter {
}
}
// PoolPriority keeps only the endpoints of the highest-priority pool
// (plan §7.3, the two-stage decision: pool first, then replica).
// PoolPriority keeps the endpoints of one pool: the highest-priority one,
// unless it is saturated and a lower one is not (plan §7.3, the two-stage
// decision — pool first, then replica).
//
// Spillover arrives in phase 5, at which point this filter will fall through
// to the next pool once the preferred one is saturated.
func PoolPriority() Filter {
return poolPriorityFilter{}
// Saturated is what admission means by it, the same formula over the same
// thresholds, so the two layers agree on when a pool is full: the H100 pool is
// used until it is, and only then the A100 one — never the A100 because it
// happens to be emptier. When every pool is saturated the preferred one is
// kept: moving to a pool that is just as full buys nothing and loses whatever
// made the first preferred.
func PoolPriority(targetWaiting, kvCeiling float64) Filter {
return poolPriorityFilter{t: admission.Thresholds{TargetWaitingPerReplica: targetWaiting, KVCeiling: kvCeiling}}
}
type poolPriorityFilter struct{}
type poolPriorityFilter struct{ t admission.Thresholds }
func (poolPriorityFilter) Name() string { return "pool-priority" }
func (poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
func (f poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
if len(eps) == 0 {
return eps
}
best := eps[0].Priority
for _, e := range eps[1:] {
if e.Priority > best {
best = e.Priority
}
}
out := make([]endpoint.Snapshot, 0, len(eps))
byPriority := map[int][]endpoint.Snapshot{}
var order []int
for _, e := range eps {
if e.Priority == best {
out = append(out, e)
if _, ok := byPriority[e.Priority]; !ok {
order = append(order, e.Priority)
}
byPriority[e.Priority] = append(byPriority[e.Priority], e)
}
sort.Sort(sort.Reverse(sort.IntSlice(order)))
for _, p := range order {
if admission.Saturation(byPriority[p], 0, f.t) < 1 {
return byPriority[p]
}
}
return out
return byPriority[order[0]]
}
+3 -3
View File
@@ -50,7 +50,7 @@ func Chat(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "chat",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
Scorers: c.withPrefix([]WeightedScorer{
{Session(c.SessionLoadFactor), 3.0},
{Queue(c.TargetWaiting), 1.0},
@@ -81,7 +81,7 @@ func Batch(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "batch",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
Scorers: []WeightedScorer{
{Queue(c.TargetWaiting), 3.0},
{KV(c.KVCeiling), 1.0},
@@ -99,7 +99,7 @@ func LongContext(c ProfileConfig) *Profile {
c.normalize()
return &Profile{
Name: "long-context",
Filters: []Filter{BreakerOpen(), PoolPriority()},
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
// Prefix matching earns more here than anywhere else: a long prompt is
// mostly prefill, and prefill is what a cache hit skips.
Scorers: c.withPrefix([]WeightedScorer{
+36 -2
View File
@@ -177,7 +177,7 @@ func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) {
func TestPoolPriority(t *testing.T) {
p := &Profile{
Name: "t",
Filters: []Filter{PoolPriority()},
Filters: []Filter{PoolPriority(4, 0.9)},
Scorers: []WeightedScorer{{Queue(4), 1}},
Picker: MaxScore(),
}
@@ -189,7 +189,8 @@ func TestPoolPriority(t *testing.T) {
if err != nil {
t.Fatal(err)
}
// The lower-priority pool must not win despite being emptier (spillover is phase 5).
// The lower-priority pool must not win for being emptier: the preferred
// one is not saturated (3 waiting of a healthy 4).
if d.Endpoint.ID != "h100" {
t.Errorf("pool priority was ignored: %s", d.Endpoint.ID)
}
@@ -383,3 +384,36 @@ func TestBuiltinProfilesSchedule(t *testing.T) {
}
}
}
func TestASaturatedPoolSpillsToTheNext(t *testing.T) {
p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)},
Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()}
h100 := func(w int) endpoint.Snapshot {
return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" })
}
a100 := func(w int) endpoint.Snapshot {
return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" })
}
pick := func(eps ...endpoint.Snapshot) string {
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
return d.Endpoint.ID
}
if got := pick(h100(8), a100(0)); got != "a100" {
t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got)
}
if got := pick(h100(8), a100(6)); got != "h100" {
t.Errorf("both pools saturated: the preferred one should be kept, got %s", got)
}
if got := pick(h100(4), a100(0)); got != "a100" {
t.Errorf("at exactly its healthy limit a pool is full: %s", got)
}
// KV pressure saturates a pool as queue depth does.
full := h100(0)
full.Stats.KVCacheUsage = 0.95
if got := pick(full, a100(0)); got != "a100" {
t.Errorf("a pool out of KV cache did not spill: %s", got)
}
}