External providers left out of admission's saturation; the scale-out section in the backlog

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 19:13:15 +01:00
1 parent be54f1617b
commit 524f029603
4 files changed
+49

No files matched your search

@@ -543,3 +543,20 @@ func TestABandForBulkWorkWaitsLongerThanTheRest(t *testing.T) {
t.Errorf("bronze gave up after %v, before its own queue_ttl", d)
}
}
// A provider outside the cluster queues for itself: requests in flight to it
// are not a queue here. Counted as one, an external model took three or four
// at once and the rest timed out waiting.
func TestRequestsInFlightToAnExternalProviderAreNotAQueue(t *testing.T) {
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
ext := snap("openai", 0, 0, 200, 0, false)
ext.Engine = endpoint.EngineExternal
if got := Saturation([]endpoint.Snapshot{ext}, 0, th); got >= 0.5 {
t.Errorf("200 in flight to an external provider read as saturation %v", got)
}
// An engine of the cluster with the same in flight and nothing learned
// of its capacity still reads as loaded.
if got := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 200, 0, false)}, 0, th); got < 1 {
t.Errorf("an engine's unreported burst ignored: %v", got)
}
}
+8
View File
@@ -53,6 +53,14 @@ func Saturation(eps []endpoint.Snapshot, unreported int, t Thresholds) float64 {
if e.Breaker.String() == "open" {
continue
}
// A provider outside the cluster queues for itself and says 429 when
// full, which its breaker hears: nothing here measures its queue, and
// counting every request in flight to it as waiting held an external
// model to three or four at once, the rest timing out in the queue.
if e.Engine == endpoint.EngineExternal {
n++
continue
}
// The engine's own queue is the truth, but it is one scrape interval
// old. Fill that gap with what the router knows without any lag: how