From 0b984a92bcb71306a82073bafedbcf56870f7589 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 16 Oct 2026 10:27:50 +0200 Subject: [PATCH] A judge is asked again only on another machine Measured under CPU load on the three-VM cluster, with both judge replicas on ha1: 37 of 60 requests failed with the hedge, 38 of 60 without it. Two replicas on one saturated machine are slow together, and the second question doubled the work there. The hedge now goes only to a replica whose address names another host. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +++- internal/router/guard_test.go | 75 ++++++++++++++++++++--------------- internal/router/judge.go | 38 ++++++++++++++++++ 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67957f39..28c04050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,8 +101,11 @@ APIs change between minors without a deprecation period. - **A judge replica that hangs no longer costs the request.** All the tries of a judge request shared the policy's budget, so a replica that hung used it all and the try on another replica had none left: under `fail: closed` the request was refused while a judge stood idle. When the first replica has not answered by half the budget, the same - question now goes to another replica too, and the first answer is taken. The first replica is not cut short, which on - a slow machine would refuse answers it was about to give. + question now goes to a replica on another machine too, and the first answer is taken. The first replica is not cut + short, which on a slow machine would refuse answers it was about to give. Replicas on one machine whose CPUs are taken + are slow together, so asking both only doubles the work there: measured that way, as many requests failed as before. + On such a machine the remedy is room, not a second question: judges on more than one node, more cores, or the judge + on a GPU. - **A new leader waits for the judge instead of refusing at once.** The moment a control plane takes the lead it knows no judge yet. When every judge fails at once, they are asked again every 250 ms while the policy's `latency_budget` lasts, so `fail: closed` no longer answers every request in those seconds with 503 (about 17 seconds of them in the test). A diff --git a/internal/router/guard_test.go b/internal/router/guard_test.go index 64b0f513..aca9861d 100644 --- a/internal/router/guard_test.go +++ b/internal/router/guard_test.go @@ -270,39 +270,52 @@ func TestSignInThroughAProviderNeedsNoAPIKey(t *testing.T) { } } -// One replica of the judge hangs: the other is asked at half the budget, and -// answers within it, rather than the hung one taking the whole budget and -// the request being refused. +// One replica of the judge hangs: the other, on another machine, is asked +// at half the budget and answers within it, rather than the hung one taking +// the whole budget and the request being refused. Two on one machine are +// not both asked: they are slow together. func TestAJudgeThatHangsIsOvertakenByAnother(t *testing.T) { - hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge") - hung.hang.Store(true) - cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}}, - config.Endpoint{ID: "quick", URL: quick.srv.URL, Models: []string{"judge"}}) - judge := catModel("judge", nil) - judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"} - cfg.Models = []catalog.Model{judge} - rt := newTestRouter(t, cfg) - deadline := time.Now().Add(3 * time.Second) - for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) { - time.Sleep(20 * time.Millisecond) - } - if n := len(rt.reg.Snapshots("judge")); n != 2 { - t.Fatalf("%d replicas of the judge", n) - } - body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`) - for i := 0; i < 12 && hung.calls.Load() == 0; i++ { - ctx, cancel := context.WithTimeout(t.Context(), time.Second) - start := time.Now() - out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body) - cancel() - if err != nil || !strings.Contains(string(out), `"hi"`) { - t.Fatalf("no verdict in %s: %v", time.Since(start), err) + for _, apart := range []bool{true, false} { + hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge") + hung.hang.Store(true) + quickURL := quick.srv.URL + if apart { + // Another host name for the same server: another machine, to + // the router. + quickURL = strings.Replace(quickURL, "127.0.0.1", "localhost", 1) } - if took := time.Since(start); took > 900*time.Millisecond { - t.Errorf("answered in %s, at the end of the budget", took) + cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}}, + config.Endpoint{ID: "quick", URL: quickURL, Models: []string{"judge"}}) + judge := catModel("judge", nil) + judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"} + cfg.Models = []catalog.Model{judge} + rt := newTestRouter(t, cfg) + deadline := time.Now().Add(3 * time.Second) + for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if n := len(rt.reg.Snapshots("judge")); n != 2 { + t.Fatalf("%d replicas of the judge", n) + } + body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`) + for i := 0; i < 12 && hung.calls.Load() == 0; i++ { + before := quick.calls.Load() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + start := time.Now() + out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body) + cancel() + if hung.calls.Load() == 0 { + continue // the quick one was asked first + } + if apart && (err != nil || !strings.Contains(string(out), `"hi"`) || time.Since(start) > 900*time.Millisecond) { + t.Fatalf("apart: no verdict in time (%s): %v", time.Since(start), err) + } + if !apart && quick.calls.Load() != before { + t.Error("together: the other replica on the same machine was asked as well") + } + } + if hung.calls.Load() == 0 { + t.Fatal("the hung replica was never asked first") } - } - if hung.calls.Load() == 0 { - t.Fatal("the hung replica was never asked first") } } diff --git a/internal/router/judge.go b/internal/router/judge.go index 638ec0d8..360e3a8d 100644 --- a/internal/router/judge.go +++ b/internal/router/judge.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net/http" + "net/url" "sync" "time" @@ -32,6 +33,10 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([] // on the other found none left: the request was refused (fail: closed) // though a judge stood idle. The first is not cut short, which on a slow // machine would refuse what it was about to answer. + // + // Only of a replica on another machine. Two on one machine whose CPUs + // are taken are slow together, and asking both doubled the work there: + // measured, as many requests failed as before, more of them refused. ctx, cancel := context.WithCancel(ctx) defer cancel() type answer struct { @@ -65,6 +70,9 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([] mu.Lock() avoid := []string{first} mu.Unlock() + if !r.elsewhere(canonical, first) { + continue + } asking++ go func() { out, err := r.judgeOnce(ctx, path, body, avoid, nil) @@ -144,3 +152,33 @@ func (b *bufferWriter) Write(p []byte) (int, error) { return b.buf.Write(p) } func (b *bufferWriter) Flush() {} + +// elsewhere is whether a model has a replica on another machine than this +// endpoint's, by the host of its address. +func (r *Router) elsewhere(model, endpointID string) bool { + snaps := r.reg.Snapshots(model) + host := "" + for _, e := range snaps { + if e.ID == endpointID { + host = hostOfURL(e.URL) + } + } + if host == "" { + return false + } + for _, e := range snaps { + if e.ID != endpointID && hostOfURL(e.URL) != host { + return true + } + } + return false +} + +// hostOfURL is an address's host, without its port. +func hostOfURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + return u.Hostname() +}