Spill a saturated pool's requests to the next pool

The pool-priority filter kept the highest-priority pool however full it
was. It now takes the first pool, by priority, that is not saturated by
admission's own measure, and the preferred one when all are.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -42,6 +42,15 @@ What v0.10.5 made settable from the command line is now on the pages too:
   critical model's card no longer refuses a replica that preemption would make
   room for.
 
+### Pool spillover
+
+A model served from pools of different priority — H100 before A100, say — was
+routed to the preferred pool however full it was. Now a pool whose replicas are
+saturated, by the same measure admission uses (queue past its healthy depth, or
+KV cache past its ceiling), passes requests to the next pool that is not. When
+every pool is saturated the preferred one is kept: a pool that is just as full
+gains nothing.
+
 ### Displacement: a higher band is not refused while a lower one waits
 
 Each band's queue was bounded, and nothing bounded them together, so a full

internal/router/scheduler/filters.go

@@ -1,6 +1,9 @@
 package scheduler
 
 import (
+	"sort"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
 )
@@ -51,34 +54,41 @@ func KVSaturated(ceiling float64) Filter {
 	}
 }
 
-// PoolPriority keeps only the endpoints of the highest-priority pool
-// (plan §7.3, the two-stage decision: pool first, then replica).
+// PoolPriority keeps the endpoints of one pool: the highest-priority one,
+// unless it is saturated and a lower one is not (plan §7.3, the two-stage
+// decision — pool first, then replica).
 //
-// Spillover arrives in phase 5, at which point this filter will fall through
-// to the next pool once the preferred one is saturated.
-func PoolPriority() Filter {
-	return poolPriorityFilter{}
+// Saturated is what admission means by it, the same formula over the same
+// thresholds, so the two layers agree on when a pool is full: the H100 pool is
+// used until it is, and only then the A100 one — never the A100 because it
+// happens to be emptier. When every pool is saturated the preferred one is
+// kept: moving to a pool that is just as full buys nothing and loses whatever
+// made the first preferred.
+func PoolPriority(targetWaiting, kvCeiling float64) Filter {
+	return poolPriorityFilter{t: admission.Thresholds{TargetWaitingPerReplica: targetWaiting, KVCeiling: kvCeiling}}
 }
 
-type poolPriorityFilter struct{}
+type poolPriorityFilter struct{ t admission.Thresholds }
 
 func (poolPriorityFilter) Name() string { return "pool-priority" }
 
-func (poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
+func (f poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
 	if len(eps) == 0 {
 		return eps
 	}
-	best := eps[0].Priority
-	for _, e := range eps[1:] {
-		if e.Priority > best {
-			best = e.Priority
-		}
-	}
-	out := make([]endpoint.Snapshot, 0, len(eps))
+	byPriority := map[int][]endpoint.Snapshot{}
+	var order []int
 	for _, e := range eps {
-		if e.Priority == best {
-			out = append(out, e)
+		if _, ok := byPriority[e.Priority]; !ok {
+			order = append(order, e.Priority)
+		}
+		byPriority[e.Priority] = append(byPriority[e.Priority], e)
+	}
+	sort.Sort(sort.Reverse(sort.IntSlice(order)))
+	for _, p := range order {
+		if admission.Saturation(byPriority[p], 0, f.t) < 1 {
+			return byPriority[p]
 		}
 	}
-	return out
+	return byPriority[order[0]]
 }

internal/router/scheduler/profiles.go

@@ -50,7 +50,7 @@ func Chat(c ProfileConfig) *Profile {
 	c.normalize()
 	return &Profile{
 		Name:    "chat",
-		Filters: []Filter{BreakerOpen(), PoolPriority()},
+		Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
 		Scorers: c.withPrefix([]WeightedScorer{
 			{Session(c.SessionLoadFactor), 3.0},
 			{Queue(c.TargetWaiting), 1.0},
@@ -81,7 +81,7 @@ func Batch(c ProfileConfig) *Profile {
 	c.normalize()
 	return &Profile{
 		Name:    "batch",
-		Filters: []Filter{BreakerOpen(), PoolPriority()},
+		Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
 		Scorers: []WeightedScorer{
 			{Queue(c.TargetWaiting), 3.0},
 			{KV(c.KVCeiling), 1.0},
@@ -99,7 +99,7 @@ func LongContext(c ProfileConfig) *Profile {
 	c.normalize()
 	return &Profile{
 		Name:    "long-context",
-		Filters: []Filter{BreakerOpen(), PoolPriority()},
+		Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
 		// Prefix matching earns more here than anywhere else: a long prompt is
 		// mostly prefill, and prefill is what a cache hit skips.
 		Scorers: c.withPrefix([]WeightedScorer{

internal/router/scheduler/scheduler_test.go

@@ -177,7 +177,7 @@ func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) {
 func TestPoolPriority(t *testing.T) {
 	p := &Profile{
 		Name:    "t",
-		Filters: []Filter{PoolPriority()},
+		Filters: []Filter{PoolPriority(4, 0.9)},
 		Scorers: []WeightedScorer{{Queue(4), 1}},
 		Picker:  MaxScore(),
 	}
@@ -189,7 +189,8 @@ func TestPoolPriority(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	// The lower-priority pool must not win despite being emptier (spillover is phase 5).
+	// The lower-priority pool must not win for being emptier: the preferred
+	// one is not saturated (3 waiting of a healthy 4).
 	if d.Endpoint.ID != "h100" {
 		t.Errorf("pool priority was ignored: %s", d.Endpoint.ID)
 	}
@@ -383,3 +384,36 @@ func TestBuiltinProfilesSchedule(t *testing.T) {
 		}
 	}
 }
+
+func TestASaturatedPoolSpillsToTheNext(t *testing.T) {
+	p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)},
+		Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()}
+	h100 := func(w int) endpoint.Snapshot {
+		return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" })
+	}
+	a100 := func(w int) endpoint.Snapshot {
+		return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" })
+	}
+	pick := func(eps ...endpoint.Snapshot) string {
+		d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return d.Endpoint.ID
+	}
+	if got := pick(h100(8), a100(0)); got != "a100" {
+		t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got)
+	}
+	if got := pick(h100(8), a100(6)); got != "h100" {
+		t.Errorf("both pools saturated: the preferred one should be kept, got %s", got)
+	}
+	if got := pick(h100(4), a100(0)); got != "a100" {
+		t.Errorf("at exactly its healthy limit a pool is full: %s", got)
+	}
+	// KV pressure saturates a pool as queue depth does.
+	full := h100(0)
+	full.Stats.KVCacheUsage = 0.95
+	if got := pick(full, a100(0)); got != "a100" {
+		t.Errorf("a pool out of KV cache did not spill: %s", got)
+	}
+}

llmgw-plan.md

@@ -1459,7 +1459,8 @@ The launch path is now built too:
       hits. It does nothing when the working set fits in one replica's cache,
       which is worth knowing rather than assuming.
 - [x] Fallback models
-- [ ] Pool spillover
+- [x] Pool spillover: a saturated pool passes requests to the next one that
+      is not, by admission's own measure of saturation
 - [x] `ctl bench`: a load ramp that reports capacity as the last concurrency
       level meeting the SLO, plus the queue depth and KV usage observed there,
       so `weight`, `target_waiting` and `kv_ceiling` stop being guesses