A lost store machine is given up on in seconds, and a new leader waits for its judge
CI / test (push) Failing after 30m6s

Measured on three VMs: cutting the store primary's power cost about 45 s of
hanging requests. New connections tried the dead member first with no
connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of
both cost a further 17 s of 503s while the new leader knew no judge.

The store dials each member for 3 s and closes connections to a machine that
is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with
ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing
clusters through its API. Judges that all fail at once are asked again
every 250 ms while the policy's budget lasts, and reported once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 20:33:13 +02:00
1 parent eeaffd764d
commit 5a4b022dcd
12 files changed
+313 -23

No files matched your search

+59 -14
View File
@@ -198,25 +198,70 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
}
j.mu.Unlock()
err := errNoJudge
for _, model := range models {
var res judgment
res, err = j.askOne(ctx, jp, model, user, answer)
if err == nil {
j.remember(key, res)
return res, nil
start := time.Now()
for {
err := errNoJudge
var failed []string
var errs []error
// fast is every judge failing at once, as one does that is not
// there yet: the moment a control plane takes the lead, before
// the agents have told it where the judges are.
fast := true
for _, model := range models {
var res judgment
t := time.Now()
res, err = j.askOne(ctx, jp, model, user, answer)
if err == nil {
// The judges before it that failed are said to have, as
// one on a GPU that is down while the CPUs' answers.
j.report(failed, errs)
j.remember(key, res)
return res, nil
}
failed, errs = append(failed, model), append(errs, err)
if time.Since(t) > judgeFastFailure || errors.Is(err, errJudgeFormat) {
fast = false
}
if ctx.Err() != nil {
// The request itself is gone: no judge after this one is wanted.
break
}
}
if j.failed != nil {
j.failed(model, err)
}
if ctx.Err() != nil {
// The request itself is gone: no judge after this one is wanted.
break
// Asked again while the budget lasts, rather than refused at once:
// with fail: closed, every request in those seconds was a 503.
if fast && ctx.Err() == nil && time.Since(start)+judgeRetryEvery < jp.budget() {
select {
case <-ctx.Done():
case <-time.After(judgeRetryEvery):
}
if ctx.Err() == nil {
continue
}
}
// Told once, for the last round: not twenty times in five seconds.
j.report(failed, errs)
return judgment{}, err
}
return judgment{}, err
}
// report tells of each judge that gave no verdict.
func (j *judge) report(models []string, errs []error) {
if j.failed == nil {
return
}
for i, model := range models {
j.failed(model, errs[i])
}
}
// A judge that fails within judgeFastFailure is one that is not there, and
// is asked again every judgeRetryEvery while the policy's budget lasts. One
// that is slow, or answers what cannot be read, is not.
const (
judgeFastFailure = time.Second
judgeRetryEvery = 250 * time.Millisecond
)
// askOne asks one judge, waiting the policy's budget for it.
func (j *judge) askOne(ctx context.Context, jp *JudgePolicy, model, user, answer string) (res judgment, err error) {
start := time.Now()
+42 -1
View File
@@ -222,7 +222,7 @@ func TestJudgesAreAskedInTurn(t *testing.T) {
var asked []string
gpuDown := errors.New("no replica of gpu-judge is serving")
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block"}}}, "p"))
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block", LatencyBudget: 600 * time.Millisecond}}}, "p"))
g.SetJudge(func(_ context.Context, model, _ string, body []byte) ([]byte, error) {
asked = append(asked, model)
if model == "gpu-judge" {
@@ -317,3 +317,44 @@ func TestALongTextIsJudgedByItsEnds(t *testing.T) {
t.Errorf("a short text was cut: %q", got)
}
}
// The moment a control plane takes the lead it knows no judge yet; one that
// arrives within the budget judges the request, instead of fail: closed
// refusing it at once. One that is slow is not asked again.
func TestAJudgeNotThereYetIsWaitedForWithinTheBudget(t *testing.T) {
ctx := context.Background()
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
Judge: &JudgePolicy{Model: "guard", Input: "block", LatencyBudget: 2 * time.Second}}}, "p"))
arrives := time.Now().Add(400 * time.Millisecond)
calls := 0
g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) {
calls++
if time.Now().Before(arrives) {
return nil, errors.New("no replica of guard is serving")
}
return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": "Safety: Safe\nCategories: None"}}})
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
var alarmed []string
g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
if v, err := g.Input(ctx, Subject{}, chat("user", "hello")); err != nil || v.Action != Allow {
t.Fatalf("refused while the judge was on its way: %+v %v", v, err)
}
if calls < 2 || len(alarmed) != 0 {
t.Errorf("asked %d times, alarms %v", calls, alarmed)
}
// Slow, and failing: given up on after one try, not the whole budget again.
calls = 0
g.SetJudge(func(ctx context.Context, _, _ string, _ []byte) ([]byte, error) {
calls++
time.Sleep(1100 * time.Millisecond)
return nil, errors.New("overloaded")
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
start := time.Now()
if _, err := g.Input(ctx, Subject{}, chat("user", "hello again")); err == nil {
t.Error("fail closed served what no judge read")
}
if calls != 1 || time.Since(start) > 1800*time.Millisecond {
t.Errorf("a slow judge was asked %d times over %s", calls, time.Since(start))
}
}