A judge replica that hangs is overtaken by another at half the budget

On the three-VM cluster about one answer in twenty-five came back as
content_filter: both judge replicas timed out within the same millisecond.
The tries shared the policy's 5 s budget, so the first replica, busy on an
oversubscribed machine, used it all and the retry on the second had nothing
left. A judge request now goes to a second replica too when the first has
not answered by half the budget, and the first answer is taken. The first
is not cut short.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-16 08:35:28 +02:00
1 parent b09922b29b
commit bc4c5d5ac9
6 files changed
+124 -1

No files matched your search

+37
View File
@@ -269,3 +269,40 @@ func TestSignInThroughAProviderNeedsNoAPIKey(t *testing.T) {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
}
// One replica of the judge hangs: the other is asked at half the budget, and
// answers within it, rather than the hung one taking the whole budget and
// the request being refused.
func TestAJudgeThatHangsIsOvertakenByAnother(t *testing.T) {
hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge")
hung.hang.Store(true)
cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}},
config.Endpoint{ID: "quick", URL: quick.srv.URL, Models: []string{"judge"}})
judge := catModel("judge", nil)
judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"}
cfg.Models = []catalog.Model{judge}
rt := newTestRouter(t, cfg)
deadline := time.Now().Add(3 * time.Second)
for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
if n := len(rt.reg.Snapshots("judge")); n != 2 {
t.Fatalf("%d replicas of the judge", n)
}
body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`)
for i := 0; i < 12 && hung.calls.Load() == 0; i++ {
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
start := time.Now()
out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body)
cancel()
if err != nil || !strings.Contains(string(out), `"hi"`) {
t.Fatalf("no verdict in %s: %v", time.Since(start), err)
}
if took := time.Since(start); took > 900*time.Millisecond {
t.Errorf("answered in %s, at the end of the budget", took)
}
}
if hung.calls.Load() == 0 {
t.Fatal("the hung replica was never asked first")
}
}