package router import ( "bytes" "context" "fmt" "net/http" "net/url" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) // Judge sends a chat completion to a judge model and returns its answer: for // the guardrails, which ask a model on the cluster about the traffic they // judge. It goes past admission — a judge call queued behind the tenants it // is judging would wait on them — and past accounting and the guardrails // themselves, but through the scheduler and the replicas like any request. func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]byte, error) { if path != "/v1/chat/completions" && path != "/v1/completions" { return nil, fmt.Errorf("a judge is asked at /v1/chat/completions or /v1/completions, not %s", path) } deadline, ok := ctx.Deadline() canonical, _ := r.resolve(model) if !ok || len(r.reg.Snapshots(canonical)) < 2 { return r.judgeOnce(ctx, path, body, nil, nil) } // Asked of a second replica too when the first has not answered by half // the budget, and the first answer taken. One replica that hung — busy, // or on a machine that is going — took the whole budget, and the retry // on the other found none left: the request was refused (fail: closed) // though a judge stood idle. The first is not cut short, which on a slow // machine would refuse what it was about to answer. // // Only of a replica on another machine. Two on one machine whose CPUs // are taken are slow together, and asking both doubled the work there: // measured, as many requests failed as before, more of them refused. ctx, cancel := context.WithCancel(ctx) defer cancel() type answer struct { out []byte err error } answers := make(chan answer, 2) var mu sync.Mutex var first string go func() { out, err := r.judgeOnce(ctx, path, body, nil, func(id string) { mu.Lock() first = id mu.Unlock() }) answers <- answer{out, err} }() hedge := time.NewTimer(time.Until(deadline) / 2) defer hedge.Stop() asking := 1 var lastErr error for asking > 0 { select { case a := <-answers: asking-- if a.err == nil { return a.out, nil } lastErr = a.err case <-hedge.C: mu.Lock() avoid := []string{first} mu.Unlock() if !r.elsewhere(canonical, first) { continue } asking++ go func() { out, err := r.judgeOnce(ctx, path, body, avoid, nil) answers <- answer{out, err} }() } } return nil, lastErr } // judgeOnce asks one judge request of the router, as it would a client's. func (r *Router) judgeOnce(ctx context.Context, path string, body []byte, avoid []string, picked func(string)) ([]byte, error) { hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body)) if err != nil { return nil, err } req, err := r.parse(hreq, path) if err != nil { return nil, err } req.internal = true req.Stream = false req.RequestID = requestID("") req.Header = http.Header{"X-Request-Id": []string{req.RequestID}} req.avoid, req.picked = avoid, picked w := &bufferWriter{header: http.Header{}} res := r.dispatch(ctx, w, req) if res.err != nil { return nil, res.err } if w.status != 0 && w.status != http.StatusOK { return nil, fmt.Errorf("the judge %s answered %d", req.Model, w.status) } return w.buf.Bytes(), nil } // GuardFormat is how a judge model's answers are read, from its catalog // entry: for guard.Gate.SetJudge. func (r *Router) GuardFormat(model string) (string, []string, bool) { _, entry := r.resolve(model) if !entry.IsGuard() || entry.Guard == nil { return "", nil, false } return entry.Guard.Format, entry.Guard.Categories, true } var _ guard.Caller = (*Router)(nil).Judge // bufferWriter is a ResponseWriter that keeps what is written, for a request // the gateway makes of itself. type bufferWriter struct { header http.Header status int buf bytes.Buffer // max bounds what is kept; 1 MiB when unset, which is a judge's answer // many times over. A job's result is a file, and sets more. max int } func (b *bufferWriter) Header() http.Header { return b.header } func (b *bufferWriter) WriteHeader(code int) { if b.status == 0 { b.status = code } } func (b *bufferWriter) Write(p []byte) (int, error) { if b.status == 0 { b.status = http.StatusOK } limit := b.max if limit <= 0 { limit = 1 << 20 } if b.buf.Len()+len(p) > limit { return 0, fmt.Errorf("an answer over %d bytes", limit) } return b.buf.Write(p) } func (b *bufferWriter) Flush() {} // elsewhere is whether a model has a replica on another machine than this // endpoint's, by the host of its address. func (r *Router) elsewhere(model, endpointID string) bool { snaps := r.reg.Snapshots(model) host := "" for _, e := range snaps { if e.ID == endpointID { host = hostOfURL(e.URL) } } if host == "" { return false } for _, e := range snaps { if e.ID != endpointID && hostOfURL(e.URL) != host { return true } } return false } // hostOfURL is an address's host, without its port. func hostOfURL(raw string) string { u, err := url.Parse(raw) if err != nil { return "" } return u.Hostname() }