Files
llmgw/internal/router/judge.go
T
Ilker ManapandClaude Fable 5.1 b51cc3cf92 A 3D model is a job: queued with an id, asked after, its result a file
POST /v1/3d/generations answers 202 with a job; GET says its status,
progress and result file; cancel stops it in the engine's queue too. The
request is kept as sent and replayed by whichever gateway holds the lease
through the path a client's request takes, so it is charged and judged
like any request (tagged job=<id>), and a model still loading is waited
out for as long as the job has. Migration 0044. Nothing makes 3D models
yet; the door refuses every model until the ComfyUI adapter learns meshes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-10 16:04:11 +02:00

87 lines
2.5 KiB
Go

package router
import (
"bytes"
"context"
"fmt"
"net/http"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// Judge sends a chat completion to a judge model and returns its answer: for
// the guardrails, which ask a model on the cluster about the traffic they
// judge. It goes past admission — a judge call queued behind the tenants it
// is judging would wait on them — and past accounting and the guardrails
// themselves, but through the scheduler and the replicas like any request.
func (r *Router) Judge(ctx context.Context, model, path string, body []byte) ([]byte, error) {
if path != "/v1/chat/completions" && path != "/v1/completions" {
return nil, fmt.Errorf("a judge is asked at /v1/chat/completions or /v1/completions, not %s", path)
}
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
if err != nil {
return nil, err
}
req, err := r.parse(hreq, path)
if err != nil {
return nil, err
}
req.internal = true
req.Stream = false
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
w := &bufferWriter{header: http.Header{}}
res := r.dispatch(ctx, w, req)
if res.err != nil {
return nil, res.err
}
if w.status != 0 && w.status != http.StatusOK {
return nil, fmt.Errorf("the judge %s answered %d", model, w.status)
}
return w.buf.Bytes(), nil
}
// GuardFormat is how a judge model's answers are read, from its catalog
// entry: for guard.Gate.SetJudge.
func (r *Router) GuardFormat(model string) (string, []string, bool) {
_, entry := r.resolve(model)
if !entry.IsGuard() || entry.Guard == nil {
return "", nil, false
}
return entry.Guard.Format, entry.Guard.Categories, true
}
var _ guard.Caller = (*Router)(nil).Judge
// bufferWriter is a ResponseWriter that keeps what is written, for a request
// the gateway makes of itself.
type bufferWriter struct {
header http.Header
status int
buf bytes.Buffer
// max bounds what is kept; 1 MiB when unset, which is a judge's answer
// many times over. A job's result is a file, and sets more.
max int
}
func (b *bufferWriter) Header() http.Header { return b.header }
func (b *bufferWriter) WriteHeader(code int) {
if b.status == 0 {
b.status = code
}
}
func (b *bufferWriter) Write(p []byte) (int, error) {
if b.status == 0 {
b.status = http.StatusOK
}
limit := b.max
if limit <= 0 {
limit = 1 << 20
}
if b.buf.Len()+len(p) > limit {
return 0, fmt.Errorf("an answer over %d bytes", limit)
}
return b.buf.Write(p)
}
func (b *bufferWriter) Flush() {}