A judge replica that hangs is overtaken by another at half the budget
On the three-VM cluster about one answer in twenty-five came back as content_filter: both judge replicas timed out within the same millisecond. The tries shared the policy's 5 s budget, so the first replica, busy on an oversubscribed machine, used it all and the retry on the second had nothing left. A judge request now goes to a second replica too when the first has not answered by half the budget, and the first answer is taken. The first is not cut short. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b09922b29b
commit
bc4c5d5ac9
6 files changed
+124
-1
No files matched your search
@@ -42,6 +42,8 @@ type fakeEngine struct {
|
||||
last atomic.Value
|
||||
// traceparent is the one the last completion came with.
|
||||
traceparent atomic.Value
|
||||
// hang, when set, has a completion wait until its request is given up.
|
||||
hang atomic.Bool
|
||||
}
|
||||
|
||||
func newFakeEngine(t *testing.T, model string) *fakeEngine {
|
||||
@@ -63,6 +65,10 @@ func newFakeEngine(t *testing.T, model string) *fakeEngine {
|
||||
http.Error(w, "unavailable", code)
|
||||
return
|
||||
}
|
||||
if f.hang.Load() {
|
||||
<-r.Context().Done()
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"model":%q,"choices":[{"message":{"content":"hi"}}],"usage":{"prompt_tokens":4,"completion_tokens":2}}`, model)
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user