Files
llmgw/internal/router/judge.go
T
Ilker ManapandClaude Opus 5.5 0b984a92bc
CI / test (push) Failing after 30m3s
A judge is asked again only on another machine
Measured under CPU load on the three-VM cluster, with both judge replicas on
ha1: 37 of 60 requests failed with the hedge, 38 of 60 without it. Two
replicas on one saturated machine are slow together, and the second
question doubled the work there. The hedge now goes only to a replica whose
address names another host.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 10:27:50 +02:00

185 lines
5.0 KiB
Go

package router
import (
"bytes"
"context"
"fmt"
"net/http"
"net/url"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// Judge sends a chat completion to a judge model and returns its answer: for
// the guardrails, which ask a model on the cluster about the traffic they
// judge. It goes past admission — a judge call queued behind the tenants it
// is judging would wait on them — and past accounting and the guardrails
// themselves, but through the scheduler and the replicas like any request.
func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]byte, error) {
if path != "/v1/chat/completions" && path != "/v1/completions" {
return nil, fmt.Errorf("a judge is asked at /v1/chat/completions or /v1/completions, not %s", path)
}
deadline, ok := ctx.Deadline()
canonical, _ := r.resolve(model)
if !ok || len(r.reg.Snapshots(canonical)) < 2 {
return r.judgeOnce(ctx, path, body, nil, nil)
}
// Asked of a second replica too when the first has not answered by half
// the budget, and the first answer taken. One replica that hung — busy,
// or on a machine that is going — took the whole budget, and the retry
// on the other found none left: the request was refused (fail: closed)
// though a judge stood idle. The first is not cut short, which on a slow
// machine would refuse what it was about to answer.
//
// Only of a replica on another machine. Two on one machine whose CPUs
// are taken are slow together, and asking both doubled the work there:
// measured, as many requests failed as before, more of them refused.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
type answer struct {
out []byte
err error
}
answers := make(chan answer, 2)
var mu sync.Mutex
var first string
go func() {
out, err := r.judgeOnce(ctx, path, body, nil, func(id string) {
mu.Lock()
first = id
mu.Unlock()
})
answers <- answer{out, err}
}()
hedge := time.NewTimer(time.Until(deadline) / 2)
defer hedge.Stop()
asking := 1
var lastErr error
for asking > 0 {
select {
case a := <-answers:
asking--
if a.err == nil {
return a.out, nil
}
lastErr = a.err
case <-hedge.C:
mu.Lock()
avoid := []string{first}
mu.Unlock()
if !r.elsewhere(canonical, first) {
continue
}
asking++
go func() {
out, err := r.judgeOnce(ctx, path, body, avoid, nil)
answers <- answer{out, err}
}()
}
}
return nil, lastErr
}
// judgeOnce asks one judge request of the router, as it would a client's.
func (r *Router) judgeOnce(ctx context.Context, path string, body []byte, avoid []string, picked func(string)) ([]byte, error) {
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
if err != nil {
return nil, err
}
req, err := r.parse(hreq, path)
if err != nil {
return nil, err
}
req.internal = true
req.Stream = false
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
req.avoid, req.picked = avoid, picked
w := &bufferWriter{header: http.Header{}}
res := r.dispatch(ctx, w, req)
if res.err != nil {
return nil, res.err
}
if w.status != 0 && w.status != http.StatusOK {
return nil, fmt.Errorf("the judge %s answered %d", req.Model, w.status)
}
return w.buf.Bytes(), nil
}
// GuardFormat is how a judge model's answers are read, from its catalog
// entry: for guard.Gate.SetJudge.
func (r *Router) GuardFormat(model string) (string, []string, bool) {
_, entry := r.resolve(model)
if !entry.IsGuard() || entry.Guard == nil {
return "", nil, false
}
return entry.Guard.Format, entry.Guard.Categories, true
}
var _ guard.Caller = (*Router)(nil).Judge
// bufferWriter is a ResponseWriter that keeps what is written, for a request
// the gateway makes of itself.
type bufferWriter struct {
header http.Header
status int
buf bytes.Buffer
// max bounds what is kept; 1 MiB when unset, which is a judge's answer
// many times over. A job's result is a file, and sets more.
max int
}
func (b *bufferWriter) Header() http.Header { return b.header }
func (b *bufferWriter) WriteHeader(code int) {
if b.status == 0 {
b.status = code
}
}
func (b *bufferWriter) Write(p []byte) (int, error) {
if b.status == 0 {
b.status = http.StatusOK
}
limit := b.max
if limit <= 0 {
limit = 1 << 20
}
if b.buf.Len()+len(p) > limit {
return 0, fmt.Errorf("an answer over %d bytes", limit)
}
return b.buf.Write(p)
}
func (b *bufferWriter) Flush() {}
// elsewhere is whether a model has a replica on another machine than this
// endpoint's, by the host of its address.
func (r *Router) elsewhere(model, endpointID string) bool {
snaps := r.reg.Snapshots(model)
host := ""
for _, e := range snaps {
if e.ID == endpointID {
host = hostOfURL(e.URL)
}
}
if host == "" {
return false
}
for _, e := range snaps {
if e.ID != endpointID && hostOfURL(e.URL) != host {
return true
}
}
return false
}
// hostOfURL is an address's host, without its port.
func hostOfURL(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return u.Hostname()
}