A judge is asked again only on another machine
CI / test (push) Failing after 30m3s

Measured under CPU load on the three-VM cluster, with both judge replicas on
ha1: 37 of 60 requests failed with the hedge, 38 of 60 without it. Two
replicas on one saturated machine are slow together, and the second
question doubled the work there. The hedge now goes only to a replica whose
address names another host.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-16 10:27:50 +02:00
1 parent bc4c5d5ac9
commit 0b984a92bc
3 files changed
+87 -33

No files matched your search

+5 -2
View File
@@ -101,8 +101,11 @@ APIs change between minors without a deprecation period.
- **A judge replica that hangs no longer costs the request.** All the tries of a judge request shared the policy's
budget, so a replica that hung used it all and the try on another replica had none left: under `fail: closed` the
request was refused while a judge stood idle. When the first replica has not answered by half the budget, the same
question now goes to another replica too, and the first answer is taken. The first replica is not cut short, which on
a slow machine would refuse answers it was about to give.
question now goes to a replica on another machine too, and the first answer is taken. The first replica is not cut
short, which on a slow machine would refuse answers it was about to give. Replicas on one machine whose CPUs are taken
are slow together, so asking both only doubles the work there: measured that way, as many requests failed as before.
On such a machine the remedy is room, not a second question: judges on more than one node, more cores, or the judge
on a GPU.
- **A new leader waits for the judge instead of refusing at once.** The moment a control plane takes the lead it knows no
judge yet. When every judge fails at once, they are asked again every 250 ms while the policy's `latency_budget` lasts,
so `fail: closed` no longer answers every request in those seconds with 503 (about 17 seconds of them in the test). A
+44 -31
View File
@@ -270,39 +270,52 @@ func TestSignInThroughAProviderNeedsNoAPIKey(t *testing.T) {
}
}
// One replica of the judge hangs: the other is asked at half the budget, and
// answers within it, rather than the hung one taking the whole budget and
// the request being refused.
// One replica of the judge hangs: the other, on another machine, is asked
// at half the budget and answers within it, rather than the hung one taking
// the whole budget and the request being refused. Two on one machine are
// not both asked: they are slow together.
func TestAJudgeThatHangsIsOvertakenByAnother(t *testing.T) {
hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge")
hung.hang.Store(true)
cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}},
config.Endpoint{ID: "quick", URL: quick.srv.URL, Models: []string{"judge"}})
judge := catModel("judge", nil)
judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"}
cfg.Models = []catalog.Model{judge}
rt := newTestRouter(t, cfg)
deadline := time.Now().Add(3 * time.Second)
for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
if n := len(rt.reg.Snapshots("judge")); n != 2 {
t.Fatalf("%d replicas of the judge", n)
}
body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`)
for i := 0; i < 12 && hung.calls.Load() == 0; i++ {
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
start := time.Now()
out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body)
cancel()
if err != nil || !strings.Contains(string(out), `"hi"`) {
t.Fatalf("no verdict in %s: %v", time.Since(start), err)
for _, apart := range []bool{true, false} {
hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge")
hung.hang.Store(true)
quickURL := quick.srv.URL
if apart {
// Another host name for the same server: another machine, to
// the router.
quickURL = strings.Replace(quickURL, "127.0.0.1", "localhost", 1)
}
if took := time.Since(start); took > 900*time.Millisecond {
t.Errorf("answered in %s, at the end of the budget", took)
cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}},
config.Endpoint{ID: "quick", URL: quickURL, Models: []string{"judge"}})
judge := catModel("judge", nil)
judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"}
cfg.Models = []catalog.Model{judge}
rt := newTestRouter(t, cfg)
deadline := time.Now().Add(3 * time.Second)
for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
if n := len(rt.reg.Snapshots("judge")); n != 2 {
t.Fatalf("%d replicas of the judge", n)
}
body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`)
for i := 0; i < 12 && hung.calls.Load() == 0; i++ {
before := quick.calls.Load()
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
start := time.Now()
out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body)
cancel()
if hung.calls.Load() == 0 {
continue // the quick one was asked first
}
if apart && (err != nil || !strings.Contains(string(out), `"hi"`) || time.Since(start) > 900*time.Millisecond) {
t.Fatalf("apart: no verdict in time (%s): %v", time.Since(start), err)
}
if !apart && quick.calls.Load() != before {
t.Error("together: the other replica on the same machine was asked as well")
}
}
if hung.calls.Load() == 0 {
t.Fatal("the hung replica was never asked first")
}
}
if hung.calls.Load() == 0 {
t.Fatal("the hung replica was never asked first")
}
}
+38
View File
@@ -5,6 +5,7 @@ import (
"context"
"fmt"
"net/http"
"net/url"
"sync"
"time"
@@ -32,6 +33,10 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]
// on the other found none left: the request was refused (fail: closed)
// though a judge stood idle. The first is not cut short, which on a slow
// machine would refuse what it was about to answer.
//
// Only of a replica on another machine. Two on one machine whose CPUs
// are taken are slow together, and asking both doubled the work there:
// measured, as many requests failed as before, more of them refused.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
type answer struct {
@@ -65,6 +70,9 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]
mu.Lock()
avoid := []string{first}
mu.Unlock()
if !r.elsewhere(canonical, first) {
continue
}
asking++
go func() {
out, err := r.judgeOnce(ctx, path, body, avoid, nil)
@@ -144,3 +152,33 @@ func (b *bufferWriter) Write(p []byte) (int, error) {
return b.buf.Write(p)
}
func (b *bufferWriter) Flush() {}
// elsewhere is whether a model has a replica on another machine than this
// endpoint's, by the host of its address.
func (r *Router) elsewhere(model, endpointID string) bool {
snaps := r.reg.Snapshots(model)
host := ""
for _, e := range snaps {
if e.ID == endpointID {
host = hostOfURL(e.URL)
}
}
if host == "" {
return false
}
for _, e := range snaps {
if e.ID != endpointID && hostOfURL(e.URL) != host {
return true
}
}
return false
}
// hostOfURL is an address's host, without its port.
func hostOfURL(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return u.Hostname()
}