A lost store machine is given up on in seconds, and a new leader waits for its judge
CI / test (push) Failing after 30m6s
CI / test (push) Failing after 30m6s
Measured on three VMs: cutting the store primary's power cost about 45 s of hanging requests. New connections tried the dead member first with no connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of both cost a further 17 s of 503s while the new leader knew no judge. The store dials each member for 3 s and closes connections to a machine that is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing clusters through its API. Judges that all fail at once are asked again every 250 ms while the policy's budget lasts, and reported once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
eeaffd764d
commit
5a4b022dcd
12 files changed
+313
-23
No files matched your search
+59
-14
@@ -198,25 +198,70 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
|
||||
}
|
||||
j.mu.Unlock()
|
||||
|
||||
err := errNoJudge
|
||||
for _, model := range models {
|
||||
var res judgment
|
||||
res, err = j.askOne(ctx, jp, model, user, answer)
|
||||
if err == nil {
|
||||
j.remember(key, res)
|
||||
return res, nil
|
||||
start := time.Now()
|
||||
for {
|
||||
err := errNoJudge
|
||||
var failed []string
|
||||
var errs []error
|
||||
// fast is every judge failing at once, as one does that is not
|
||||
// there yet: the moment a control plane takes the lead, before
|
||||
// the agents have told it where the judges are.
|
||||
fast := true
|
||||
for _, model := range models {
|
||||
var res judgment
|
||||
t := time.Now()
|
||||
res, err = j.askOne(ctx, jp, model, user, answer)
|
||||
if err == nil {
|
||||
// The judges before it that failed are said to have, as
|
||||
// one on a GPU that is down while the CPUs' answers.
|
||||
j.report(failed, errs)
|
||||
j.remember(key, res)
|
||||
return res, nil
|
||||
}
|
||||
failed, errs = append(failed, model), append(errs, err)
|
||||
if time.Since(t) > judgeFastFailure || errors.Is(err, errJudgeFormat) {
|
||||
fast = false
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The request itself is gone: no judge after this one is wanted.
|
||||
break
|
||||
}
|
||||
}
|
||||
if j.failed != nil {
|
||||
j.failed(model, err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The request itself is gone: no judge after this one is wanted.
|
||||
break
|
||||
// Asked again while the budget lasts, rather than refused at once:
|
||||
// with fail: closed, every request in those seconds was a 503.
|
||||
if fast && ctx.Err() == nil && time.Since(start)+judgeRetryEvery < jp.budget() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(judgeRetryEvery):
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Told once, for the last round: not twenty times in five seconds.
|
||||
j.report(failed, errs)
|
||||
return judgment{}, err
|
||||
}
|
||||
return judgment{}, err
|
||||
}
|
||||
|
||||
// report tells of each judge that gave no verdict.
|
||||
func (j *judge) report(models []string, errs []error) {
|
||||
if j.failed == nil {
|
||||
return
|
||||
}
|
||||
for i, model := range models {
|
||||
j.failed(model, errs[i])
|
||||
}
|
||||
}
|
||||
|
||||
// A judge that fails within judgeFastFailure is one that is not there, and
|
||||
// is asked again every judgeRetryEvery while the policy's budget lasts. One
|
||||
// that is slow, or answers what cannot be read, is not.
|
||||
const (
|
||||
judgeFastFailure = time.Second
|
||||
judgeRetryEvery = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// askOne asks one judge, waiting the policy's budget for it.
|
||||
func (j *judge) askOne(ctx context.Context, jp *JudgePolicy, model, user, answer string) (res judgment, err error) {
|
||||
start := time.Now()
|
||||
|
||||
@@ -222,7 +222,7 @@ func TestJudgesAreAskedInTurn(t *testing.T) {
|
||||
var asked []string
|
||||
gpuDown := errors.New("no replica of gpu-judge is serving")
|
||||
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
|
||||
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block"}}}, "p"))
|
||||
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block", LatencyBudget: 600 * time.Millisecond}}}, "p"))
|
||||
g.SetJudge(func(_ context.Context, model, _ string, body []byte) ([]byte, error) {
|
||||
asked = append(asked, model)
|
||||
if model == "gpu-judge" {
|
||||
@@ -317,3 +317,44 @@ func TestALongTextIsJudgedByItsEnds(t *testing.T) {
|
||||
t.Errorf("a short text was cut: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The moment a control plane takes the lead it knows no judge yet; one that
|
||||
// arrives within the budget judges the request, instead of fail: closed
|
||||
// refusing it at once. One that is slow is not asked again.
|
||||
func TestAJudgeNotThereYetIsWaitedForWithinTheBudget(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
|
||||
Judge: &JudgePolicy{Model: "guard", Input: "block", LatencyBudget: 2 * time.Second}}}, "p"))
|
||||
arrives := time.Now().Add(400 * time.Millisecond)
|
||||
calls := 0
|
||||
g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) {
|
||||
calls++
|
||||
if time.Now().Before(arrives) {
|
||||
return nil, errors.New("no replica of guard is serving")
|
||||
}
|
||||
return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": "Safety: Safe\nCategories: None"}}})
|
||||
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
|
||||
var alarmed []string
|
||||
g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
|
||||
if v, err := g.Input(ctx, Subject{}, chat("user", "hello")); err != nil || v.Action != Allow {
|
||||
t.Fatalf("refused while the judge was on its way: %+v %v", v, err)
|
||||
}
|
||||
if calls < 2 || len(alarmed) != 0 {
|
||||
t.Errorf("asked %d times, alarms %v", calls, alarmed)
|
||||
}
|
||||
|
||||
// Slow, and failing: given up on after one try, not the whole budget again.
|
||||
calls = 0
|
||||
g.SetJudge(func(ctx context.Context, _, _ string, _ []byte) ([]byte, error) {
|
||||
calls++
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
return nil, errors.New("overloaded")
|
||||
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
|
||||
start := time.Now()
|
||||
if _, err := g.Input(ctx, Subject{}, chat("user", "hello again")); err == nil {
|
||||
t.Error("fail closed served what no judge read")
|
||||
}
|
||||
if calls != 1 || time.Since(start) > 1800*time.Millisecond {
|
||||
t.Errorf("a slow judge was asked %d times over %s", calls, time.Since(start))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user