A judge replica that hangs is overtaken by another at half the budget
On the three-VM cluster about one answer in twenty-five came back as content_filter: both judge replicas timed out within the same millisecond. The tries shared the policy's 5 s budget, so the first replica, busy on an oversubscribed machine, used it all and the retry on the second had nothing left. A judge request now goes to a second replica too when the first has not answered by half the budget, and the first answer is taken. The first is not cut short. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b09922b29b
commit
bc4c5d5ac9
6 files changed
+124
-1
No files matched your search
@@ -269,3 +269,40 @@ func TestSignInThroughAProviderNeedsNoAPIKey(t *testing.T) {
|
||||
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// One replica of the judge hangs: the other is asked at half the budget, and
|
||||
// answers within it, rather than the hung one taking the whole budget and
|
||||
// the request being refused.
|
||||
func TestAJudgeThatHangsIsOvertakenByAnother(t *testing.T) {
|
||||
hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge")
|
||||
hung.hang.Store(true)
|
||||
cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}},
|
||||
config.Endpoint{ID: "quick", URL: quick.srv.URL, Models: []string{"judge"}})
|
||||
judge := catModel("judge", nil)
|
||||
judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"}
|
||||
cfg.Models = []catalog.Model{judge}
|
||||
rt := newTestRouter(t, cfg)
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if n := len(rt.reg.Snapshots("judge")); n != 2 {
|
||||
t.Fatalf("%d replicas of the judge", n)
|
||||
}
|
||||
body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`)
|
||||
for i := 0; i < 12 && hung.calls.Load() == 0; i++ {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
start := time.Now()
|
||||
out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body)
|
||||
cancel()
|
||||
if err != nil || !strings.Contains(string(out), `"hi"`) {
|
||||
t.Fatalf("no verdict in %s: %v", time.Since(start), err)
|
||||
}
|
||||
if took := time.Since(start); took > 900*time.Millisecond {
|
||||
t.Errorf("answered in %s, at the end of the budget", took)
|
||||
}
|
||||
}
|
||||
if hung.calls.Load() == 0 {
|
||||
t.Fatal("the hung replica was never asked first")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user