External providers left out of admission's saturation; the scale-out section in the backlog
CI / test (push) Successful in 9m25s
CI / test (push) Successful in 9m25s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
be54f1617b
commit
524f029603
4 files changed
+49
No files matched your search
@@ -543,3 +543,20 @@ func TestABandForBulkWorkWaitsLongerThanTheRest(t *testing.T) {
|
||||
t.Errorf("bronze gave up after %v, before its own queue_ttl", d)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider outside the cluster queues for itself: requests in flight to it
|
||||
// are not a queue here. Counted as one, an external model took three or four
|
||||
// at once and the rest timed out waiting.
|
||||
func TestRequestsInFlightToAnExternalProviderAreNotAQueue(t *testing.T) {
|
||||
th := Thresholds{TargetWaitingPerReplica: 4, KVCeiling: 0.9}
|
||||
ext := snap("openai", 0, 0, 200, 0, false)
|
||||
ext.Engine = endpoint.EngineExternal
|
||||
if got := Saturation([]endpoint.Snapshot{ext}, 0, th); got >= 0.5 {
|
||||
t.Errorf("200 in flight to an external provider read as saturation %v", got)
|
||||
}
|
||||
// An engine of the cluster with the same in flight and nothing learned
|
||||
// of its capacity still reads as loaded.
|
||||
if got := Saturation([]endpoint.Snapshot{snap("a", 0, 0, 200, 0, false)}, 0, th); got < 1 {
|
||||
t.Errorf("an engine's unreported burst ignored: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,14 @@ func Saturation(eps []endpoint.Snapshot, unreported int, t Thresholds) float64 {
|
||||
if e.Breaker.String() == "open" {
|
||||
continue
|
||||
}
|
||||
// A provider outside the cluster queues for itself and says 429 when
|
||||
// full, which its breaker hears: nothing here measures its queue, and
|
||||
// counting every request in flight to it as waiting held an external
|
||||
// model to three or four at once, the rest timing out in the queue.
|
||||
if e.Engine == endpoint.EngineExternal {
|
||||
n++
|
||||
continue
|
||||
}
|
||||
|
||||
// The engine's own queue is the truth, but it is one scrape interval
|
||||
// old. Fill that gap with what the router knows without any lag: how
|
||||
|
||||
Reference in new issue
Block a user