On the three-VM cluster about one answer in twenty-five came back as content_filter: both judge replicas timed out within the same millisecond. The tries shared the policy's 5 s budget, so the first replica, busy on an oversubscribed machine, used it all and the retry on the second had nothing left. A judge request now goes to a second replica too when the first has not answered by half the budget, and the first answer is taken. The first is not cut short. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
147 lines
4.1 KiB
Go
147 lines
4.1 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// Judge sends a chat completion to a judge model and returns its answer: for
|
|
// the guardrails, which ask a model on the cluster about the traffic they
|
|
// judge. It goes past admission — a judge call queued behind the tenants it
|
|
// is judging would wait on them — and past accounting and the guardrails
|
|
// themselves, but through the scheduler and the replicas like any request.
|
|
func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]byte, error) {
|
|
if path != "/v1/chat/completions" && path != "/v1/completions" {
|
|
return nil, fmt.Errorf("a judge is asked at /v1/chat/completions or /v1/completions, not %s", path)
|
|
}
|
|
deadline, ok := ctx.Deadline()
|
|
canonical, _ := r.resolve(model)
|
|
if !ok || len(r.reg.Snapshots(canonical)) < 2 {
|
|
return r.judgeOnce(ctx, path, body, nil, nil)
|
|
}
|
|
|
|
// Asked of a second replica too when the first has not answered by half
|
|
// the budget, and the first answer taken. One replica that hung — busy,
|
|
// or on a machine that is going — took the whole budget, and the retry
|
|
// on the other found none left: the request was refused (fail: closed)
|
|
// though a judge stood idle. The first is not cut short, which on a slow
|
|
// machine would refuse what it was about to answer.
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
type answer struct {
|
|
out []byte
|
|
err error
|
|
}
|
|
answers := make(chan answer, 2)
|
|
var mu sync.Mutex
|
|
var first string
|
|
go func() {
|
|
out, err := r.judgeOnce(ctx, path, body, nil, func(id string) {
|
|
mu.Lock()
|
|
first = id
|
|
mu.Unlock()
|
|
})
|
|
answers <- answer{out, err}
|
|
}()
|
|
hedge := time.NewTimer(time.Until(deadline) / 2)
|
|
defer hedge.Stop()
|
|
asking := 1
|
|
var lastErr error
|
|
for asking > 0 {
|
|
select {
|
|
case a := <-answers:
|
|
asking--
|
|
if a.err == nil {
|
|
return a.out, nil
|
|
}
|
|
lastErr = a.err
|
|
case <-hedge.C:
|
|
mu.Lock()
|
|
avoid := []string{first}
|
|
mu.Unlock()
|
|
asking++
|
|
go func() {
|
|
out, err := r.judgeOnce(ctx, path, body, avoid, nil)
|
|
answers <- answer{out, err}
|
|
}()
|
|
}
|
|
}
|
|
return nil, lastErr
|
|
}
|
|
|
|
// judgeOnce asks one judge request of the router, as it would a client's.
|
|
func (r *Router) judgeOnce(ctx context.Context, path string, body []byte, avoid []string, picked func(string)) ([]byte, error) {
|
|
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := r.parse(hreq, path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.internal = true
|
|
req.Stream = false
|
|
req.RequestID = requestID("")
|
|
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
|
|
req.avoid, req.picked = avoid, picked
|
|
w := &bufferWriter{header: http.Header{}}
|
|
res := r.dispatch(ctx, w, req)
|
|
if res.err != nil {
|
|
return nil, res.err
|
|
}
|
|
if w.status != 0 && w.status != http.StatusOK {
|
|
return nil, fmt.Errorf("the judge %s answered %d", req.Model, w.status)
|
|
}
|
|
return w.buf.Bytes(), nil
|
|
}
|
|
|
|
// GuardFormat is how a judge model's answers are read, from its catalog
|
|
// entry: for guard.Gate.SetJudge.
|
|
func (r *Router) GuardFormat(model string) (string, []string, bool) {
|
|
_, entry := r.resolve(model)
|
|
if !entry.IsGuard() || entry.Guard == nil {
|
|
return "", nil, false
|
|
}
|
|
return entry.Guard.Format, entry.Guard.Categories, true
|
|
}
|
|
|
|
var _ guard.Caller = (*Router)(nil).Judge
|
|
|
|
// bufferWriter is a ResponseWriter that keeps what is written, for a request
|
|
// the gateway makes of itself.
|
|
type bufferWriter struct {
|
|
header http.Header
|
|
status int
|
|
buf bytes.Buffer
|
|
// max bounds what is kept; 1 MiB when unset, which is a judge's answer
|
|
// many times over. A job's result is a file, and sets more.
|
|
max int
|
|
}
|
|
|
|
func (b *bufferWriter) Header() http.Header { return b.header }
|
|
func (b *bufferWriter) WriteHeader(code int) {
|
|
if b.status == 0 {
|
|
b.status = code
|
|
}
|
|
}
|
|
func (b *bufferWriter) Write(p []byte) (int, error) {
|
|
if b.status == 0 {
|
|
b.status = http.StatusOK
|
|
}
|
|
limit := b.max
|
|
if limit <= 0 {
|
|
limit = 1 << 20
|
|
}
|
|
if b.buf.Len()+len(p) > limit {
|
|
return 0, fmt.Errorf("an answer over %d bytes", limit)
|
|
}
|
|
return b.buf.Write(p)
|
|
}
|
|
func (b *bufferWriter) Flush() {}
|