Files
llmgw/internal/router/judge.go
T
Ilker ManapandClaude Opus 5.5 bc4c5d5ac9 A judge replica that hangs is overtaken by another at half the budget
On the three-VM cluster about one answer in twenty-five came back as
content_filter: both judge replicas timed out within the same millisecond.
The tries shared the policy's 5 s budget, so the first replica, busy on an
oversubscribed machine, used it all and the retry on the second had nothing
left. A judge request now goes to a second replica too when the first has
not answered by half the budget, and the first answer is taken. The first
is not cut short.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 08:35:28 +02:00

147 lines
4.1 KiB
Go

package router
import (
"bytes"
"context"
"fmt"
"net/http"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// Judge sends a chat completion to a judge model and returns its answer: for
// the guardrails, which ask a model on the cluster about the traffic they
// judge. It goes past admission — a judge call queued behind the tenants it
// is judging would wait on them — and past accounting and the guardrails
// themselves, but through the scheduler and the replicas like any request.
func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]byte, error) {
if path != "/v1/chat/completions" && path != "/v1/completions" {
return nil, fmt.Errorf("a judge is asked at /v1/chat/completions or /v1/completions, not %s", path)
}
deadline, ok := ctx.Deadline()
canonical, _ := r.resolve(model)
if !ok || len(r.reg.Snapshots(canonical)) < 2 {
return r.judgeOnce(ctx, path, body, nil, nil)
}
// Asked of a second replica too when the first has not answered by half
// the budget, and the first answer taken. One replica that hung — busy,
// or on a machine that is going — took the whole budget, and the retry
// on the other found none left: the request was refused (fail: closed)
// though a judge stood idle. The first is not cut short, which on a slow
// machine would refuse what it was about to answer.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
type answer struct {
out []byte
err error
}
answers := make(chan answer, 2)
var mu sync.Mutex
var first string
go func() {
out, err := r.judgeOnce(ctx, path, body, nil, func(id string) {
mu.Lock()
first = id
mu.Unlock()
})
answers <- answer{out, err}
}()
hedge := time.NewTimer(time.Until(deadline) / 2)
defer hedge.Stop()
asking := 1
var lastErr error
for asking > 0 {
select {
case a := <-answers:
asking--
if a.err == nil {
return a.out, nil
}
lastErr = a.err
case <-hedge.C:
mu.Lock()
avoid := []string{first}
mu.Unlock()
asking++
go func() {
out, err := r.judgeOnce(ctx, path, body, avoid, nil)
answers <- answer{out, err}
}()
}
}
return nil, lastErr
}
// judgeOnce asks one judge request of the router, as it would a client's.
func (r *Router) judgeOnce(ctx context.Context, path string, body []byte, avoid []string, picked func(string)) ([]byte, error) {
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
if err != nil {
return nil, err
}
req, err := r.parse(hreq, path)
if err != nil {
return nil, err
}
req.internal = true
req.Stream = false
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
req.avoid, req.picked = avoid, picked
w := &bufferWriter{header: http.Header{}}
res := r.dispatch(ctx, w, req)
if res.err != nil {
return nil, res.err
}
if w.status != 0 && w.status != http.StatusOK {
return nil, fmt.Errorf("the judge %s answered %d", req.Model, w.status)
}
return w.buf.Bytes(), nil
}
// GuardFormat is how a judge model's answers are read, from its catalog
// entry: for guard.Gate.SetJudge.
func (r *Router) GuardFormat(model string) (string, []string, bool) {
_, entry := r.resolve(model)
if !entry.IsGuard() || entry.Guard == nil {
return "", nil, false
}
return entry.Guard.Format, entry.Guard.Categories, true
}
var _ guard.Caller = (*Router)(nil).Judge
// bufferWriter is a ResponseWriter that keeps what is written, for a request
// the gateway makes of itself.
type bufferWriter struct {
header http.Header
status int
buf bytes.Buffer
// max bounds what is kept; 1 MiB when unset, which is a judge's answer
// many times over. A job's result is a file, and sets more.
max int
}
func (b *bufferWriter) Header() http.Header { return b.header }
func (b *bufferWriter) WriteHeader(code int) {
if b.status == 0 {
b.status = code
}
}
func (b *bufferWriter) Write(p []byte) (int, error) {
if b.status == 0 {
b.status = http.StatusOK
}
limit := b.max
if limit <= 0 {
limit = 1 << 20
}
if b.buf.Len()+len(p) > limit {
return 0, fmt.Errorf("an answer over %d bytes", limit)
}
return b.buf.Write(p)
}
func (b *bufferWriter) Flush() {}