diff --git a/CHANGELOG.md b/CHANGELOG.md index 67957f39..28c04050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,8 +101,11 @@ APIs change between minors without a deprecation period. - **A judge replica that hangs no longer costs the request.** All the tries of a judge request shared the policy's budget, so a replica that hung used it all and the try on another replica had none left: under `fail: closed` the request was refused while a judge stood idle. When the first replica has not answered by half the budget, the same - question now goes to another replica too, and the first answer is taken. The first replica is not cut short, which on - a slow machine would refuse answers it was about to give. + question now goes to a replica on another machine too, and the first answer is taken. The first replica is not cut + short, which on a slow machine would refuse answers it was about to give. Replicas on one machine whose CPUs are taken + are slow together, so asking both only doubles the work there: measured that way, as many requests failed as before. + On such a machine the remedy is room, not a second question: judges on more than one node, more cores, or the judge + on a GPU. - **A new leader waits for the judge instead of refusing at once.** The moment a control plane takes the lead it knows no judge yet. When every judge fails at once, they are asked again every 250 ms while the policy's `latency_budget` lasts, so `fail: closed` no longer answers every request in those seconds with 503 (about 17 seconds of them in the test). A diff --git a/internal/router/guard_test.go b/internal/router/guard_test.go index 64b0f513..aca9861d 100644 --- a/internal/router/guard_test.go +++ b/internal/router/guard_test.go @@ -270,39 +270,52 @@ func TestSignInThroughAProviderNeedsNoAPIKey(t *testing.T) { } } -// One replica of the judge hangs: the other is asked at half the budget, and -// answers within it, rather than the hung one taking the whole budget and -// the request being refused. +// One replica of the judge hangs: the other, on another machine, is asked +// at half the budget and answers within it, rather than the hung one taking +// the whole budget and the request being refused. Two on one machine are +// not both asked: they are slow together. func TestAJudgeThatHangsIsOvertakenByAnother(t *testing.T) { - hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge") - hung.hang.Store(true) - cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}}, - config.Endpoint{ID: "quick", URL: quick.srv.URL, Models: []string{"judge"}}) - judge := catModel("judge", nil) - judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"} - cfg.Models = []catalog.Model{judge} - rt := newTestRouter(t, cfg) - deadline := time.Now().Add(3 * time.Second) - for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) { - time.Sleep(20 * time.Millisecond) - } - if n := len(rt.reg.Snapshots("judge")); n != 2 { - t.Fatalf("%d replicas of the judge", n) - } - body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`) - for i := 0; i < 12 && hung.calls.Load() == 0; i++ { - ctx, cancel := context.WithTimeout(t.Context(), time.Second) - start := time.Now() - out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body) - cancel() - if err != nil || !strings.Contains(string(out), `"hi"`) { - t.Fatalf("no verdict in %s: %v", time.Since(start), err) + for _, apart := range []bool{true, false} { + hung, quick := newFakeEngine(t, "judge"), newFakeEngine(t, "judge") + hung.hang.Store(true) + quickURL := quick.srv.URL + if apart { + // Another host name for the same server: another machine, to + // the router. + quickURL = strings.Replace(quickURL, "127.0.0.1", "localhost", 1) } - if took := time.Since(start); took > 900*time.Millisecond { - t.Errorf("answered in %s, at the end of the budget", took) + cfg := baseConfig(config.Endpoint{ID: "hung", URL: hung.srv.URL, Models: []string{"judge"}}, + config.Endpoint{ID: "quick", URL: quickURL, Models: []string{"judge"}}) + judge := catModel("judge", nil) + judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "qwen3guard"} + cfg.Models = []catalog.Model{judge} + rt := newTestRouter(t, cfg) + deadline := time.Now().Add(3 * time.Second) + for len(rt.reg.Snapshots("judge")) < 2 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if n := len(rt.reg.Snapshots("judge")); n != 2 { + t.Fatalf("%d replicas of the judge", n) + } + body := []byte(`{"model":"judge","messages":[{"role":"user","content":"x"}]}`) + for i := 0; i < 12 && hung.calls.Load() == 0; i++ { + before := quick.calls.Load() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + start := time.Now() + out, err := rt.Judge(ctx, "judge", "/v1/chat/completions", body) + cancel() + if hung.calls.Load() == 0 { + continue // the quick one was asked first + } + if apart && (err != nil || !strings.Contains(string(out), `"hi"`) || time.Since(start) > 900*time.Millisecond) { + t.Fatalf("apart: no verdict in time (%s): %v", time.Since(start), err) + } + if !apart && quick.calls.Load() != before { + t.Error("together: the other replica on the same machine was asked as well") + } + } + if hung.calls.Load() == 0 { + t.Fatal("the hung replica was never asked first") } - } - if hung.calls.Load() == 0 { - t.Fatal("the hung replica was never asked first") } } diff --git a/internal/router/judge.go b/internal/router/judge.go index 638ec0d8..360e3a8d 100644 --- a/internal/router/judge.go +++ b/internal/router/judge.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net/http" + "net/url" "sync" "time" @@ -32,6 +33,10 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([] // on the other found none left: the request was refused (fail: closed) // though a judge stood idle. The first is not cut short, which on a slow // machine would refuse what it was about to answer. + // + // Only of a replica on another machine. Two on one machine whose CPUs + // are taken are slow together, and asking both doubled the work there: + // measured, as many requests failed as before, more of them refused. ctx, cancel := context.WithCancel(ctx) defer cancel() type answer struct { @@ -65,6 +70,9 @@ func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([] mu.Lock() avoid := []string{first} mu.Unlock() + if !r.elsewhere(canonical, first) { + continue + } asking++ go func() { out, err := r.judgeOnce(ctx, path, body, avoid, nil) @@ -144,3 +152,33 @@ func (b *bufferWriter) Write(p []byte) (int, error) { return b.buf.Write(p) } func (b *bufferWriter) Flush() {} + +// elsewhere is whether a model has a replica on another machine than this +// endpoint's, by the host of its address. +func (r *Router) elsewhere(model, endpointID string) bool { + snaps := r.reg.Snapshots(model) + host := "" + for _, e := range snaps { + if e.ID == endpointID { + host = hostOfURL(e.URL) + } + } + if host == "" { + return false + } + for _, e := range snaps { + if e.ID != endpointID && hostOfURL(e.URL) != host { + return true + } + } + return false +} + +// hostOfURL is an address's host, without its port. +func hostOfURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + return u.Hostname() +}