Hook a guard into the router: judge requests before dispatch and filter answers as they are relayed
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e701f92806
commit
979aa44804
12 files changed
+448
-12
No files matched your search
@@ -203,6 +203,12 @@ default output cap when the client omits `max_tokens`; `catalog.Capabilities`
|
||||
is advertised but never enforced; tool results are trusted input.
|
||||
|
||||
### B1. Hooks in the proxy and router (S)
|
||||
**Done (branch `guardrails-b`):** `internal/guard` holds the interface;
|
||||
proxy `Filter`/`StreamFilter`; the input hook runs after `loadThread` (the
|
||||
simpler choice); error codes, headers, Responses `incomplete`, usage columns
|
||||
`guard_*`. Moved to B2: holding `delta.tool_calls` until they are whole and
|
||||
reading their arguments — that is the tool policy's filter, not the proxy's.
|
||||
|
||||
- `internal/router/proxy/proxy.go`: add optional `InputRewrite func(body
|
||||
[]byte) ([]byte, Verdict)` on the request and two output hooks: `Filter`
|
||||
for `relayBuffered` (between `ReadAll` and `WriteHeader`) and
|
||||
|
||||
@@ -23,6 +23,21 @@ APIs change between minors without a deprecation period.
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Guardrails
|
||||
|
||||
- **Hooks for guardrails on the request path** (nothing judges yet; B2 adds
|
||||
the policies). A guard, when one is set, judges a request once it is
|
||||
authorised, within its limits and has its stored conversation spliced
|
||||
in, and before it is dispatched; and filters the answer, buffered or
|
||||
streamed, before the client sees it. A blocked request is refused with 400
|
||||
and `code: "content_policy_violation"` — the first error with a code — and
|
||||
costs nothing; a blocked answer ends with `finish_reason: "content_filter"`,
|
||||
and a stopped stream with that chunk and `[DONE]`, the engine cancelled.
|
||||
The Responses API reports either as `incomplete` with
|
||||
`incomplete_details.reason: "content_filter"`. `x-llmgw-guard` and
|
||||
`x-llmgw-guard-policy` say what was done; the usage row records the
|
||||
action, policy, category and time taken — never the text.
|
||||
|
||||
### Security
|
||||
|
||||
- **Built with Go 1.26.6.** Eleven standard-library vulnerabilities reachable
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// The headers a judged request is answered with: what was done, and by
|
||||
// which policy. Set before anything is written, so on a streamed answer they
|
||||
// carry the input's verdict, and the output's only on a buffered one.
|
||||
const (
|
||||
guardHeader = "x-llmgw-guard"
|
||||
guardPolicyHeader = "x-llmgw-guard-policy"
|
||||
)
|
||||
|
||||
// policyViolation is the error code a blocked request is refused with.
|
||||
const policyViolation = "content_policy_violation"
|
||||
|
||||
func (p *parsedRequest) guardSubject(model string) guard.Subject {
|
||||
return guard.Subject{
|
||||
RequestID: p.RequestID, KeyID: p.KeyID, UserID: p.UserID, ProjectID: p.ProjectID,
|
||||
TeamID: p.TeamID, OrgID: p.OrgID, Model: model, Path: p.Path, Stream: p.Stream,
|
||||
}
|
||||
}
|
||||
|
||||
func setGuardHeaders(h http.Header, v guard.Verdict) {
|
||||
if v.Action == guard.Allow {
|
||||
return
|
||||
}
|
||||
h.Set(guardHeader, string(v.Action))
|
||||
if v.Policy != "" {
|
||||
h.Set(guardPolicyHeader, v.Policy)
|
||||
}
|
||||
}
|
||||
|
||||
// guardInput judges the request about to be dispatched. It answers the
|
||||
// client itself and returns false when the request goes no further.
|
||||
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
|
||||
model, _ := r.resolve(req.Model)
|
||||
t := time.Now()
|
||||
v, err := r.guard.Input(ctx, req.guardSubject(model), req.UpstreamBody)
|
||||
if v.Took == 0 {
|
||||
v.Took = time.Since(t)
|
||||
}
|
||||
v.Stage = guard.Input
|
||||
if err != nil {
|
||||
// Not judged is not judged fine: a guard that means to fail open
|
||||
// says Allow instead.
|
||||
r.refuseGuarded(w, req, guard.Verdict{Action: guard.Block, Stage: guard.Input, Took: v.Took},
|
||||
http.StatusServiceUnavailable, "server_error", "guard_unavailable",
|
||||
"the request could not be checked against this gateway's policies; try again", start)
|
||||
return false
|
||||
}
|
||||
req.guardIn = v
|
||||
setGuardHeaders(w.Header(), v)
|
||||
switch v.Action {
|
||||
case guard.Block:
|
||||
msg := v.Message
|
||||
if msg == "" {
|
||||
msg = "this request was refused by policy " + strconv.Quote(v.Policy)
|
||||
}
|
||||
r.refuseGuarded(w, req, v, http.StatusBadRequest, "invalid_request_error", policyViolation, msg, start)
|
||||
return false
|
||||
case guard.Redact:
|
||||
if v.Body != nil {
|
||||
req.UpstreamBody = v.Body
|
||||
// Routed by what is sent, so the prefix tree never holds what
|
||||
// was taken out.
|
||||
var b inferenceBody
|
||||
if json.Unmarshal(v.Body, &b) == nil {
|
||||
req.PromptPrefix = promptPrefix(&b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// refuseGuarded answers a request a guard stopped. Nothing is charged —
|
||||
// nothing was generated — but the row is kept, with the verdict, so a team
|
||||
// running into a policy shows as one.
|
||||
func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guard.Verdict,
|
||||
status int, kind, code, msg string, start time.Time,
|
||||
) {
|
||||
writeErrorCode(w, status, kind, code, msg)
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0)
|
||||
r.m.tenant.observe(req, status, 0, 0, 0)
|
||||
if r.acct != nil && req.KeyID != "" {
|
||||
u := store.Usage{
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: req.Model, Status: status,
|
||||
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, QueueMs: req.QueueMs,
|
||||
}
|
||||
guardFields(&u, v, guard.Verdict{})
|
||||
// Settled at nothing: the estimate the limit check charged is
|
||||
// given back.
|
||||
r.acct.Record(req.reservation, u)
|
||||
}
|
||||
r.accessLog(req, dispatchResult{guard: v}, status, time.Since(start))
|
||||
}
|
||||
|
||||
// outputFilter attaches the guard's filter for one attempt to the proxy
|
||||
// request, and returns it so its verdict can be read afterwards.
|
||||
func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model string, preq *proxy.Request) guard.OutputFilter {
|
||||
if r.guard == nil {
|
||||
return nil
|
||||
}
|
||||
f := r.guard.Output(ctx, req.guardSubject(model))
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
if req.Stream {
|
||||
preq.StreamFilter = f
|
||||
}
|
||||
preq.Filter = func(body []byte, h http.Header) []byte {
|
||||
out := f.Buffered(body)
|
||||
v := f.Verdict()
|
||||
if v.Stricter(req.guardIn) {
|
||||
setGuardHeaders(h, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// guardFields records the stricter of the input's and output's verdicts.
|
||||
func guardFields(u *store.Usage, in, out guard.Verdict) {
|
||||
v := in
|
||||
if out.Stricter(in) {
|
||||
v = out
|
||||
}
|
||||
u.GuardAction, u.GuardPolicy, u.GuardCategory = string(v.Action), v.Policy, v.Category
|
||||
u.GuardMs = (in.Took + out.Took).Milliseconds()
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// wordGuard blocks requests saying "forbidden", rewrites "alice@example.com",
|
||||
// flags "maybe", and stops answers saying "hi".
|
||||
type wordGuard struct {
|
||||
err error
|
||||
subject guard.Subject
|
||||
// answers, when set, stops answers saying "hi".
|
||||
answers bool
|
||||
}
|
||||
|
||||
func (g *wordGuard) Input(_ context.Context, s guard.Subject, body []byte) (guard.Verdict, error) {
|
||||
g.subject = s
|
||||
switch {
|
||||
case g.err != nil:
|
||||
return guard.Verdict{}, g.err
|
||||
case bytes.Contains(body, []byte("forbidden")):
|
||||
return guard.Verdict{Action: guard.Block, Policy: "words", Category: "test"}, nil
|
||||
case bytes.Contains(body, []byte("alice@example.com")):
|
||||
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
|
||||
Body: bytes.ReplaceAll(body, []byte("alice@example.com"), []byte("[EMAIL_1]"))}, nil
|
||||
case bytes.Contains(body, []byte("maybe")):
|
||||
return guard.Verdict{Action: guard.Flag, Policy: "words"}, nil
|
||||
}
|
||||
return guard.Verdict{}, nil
|
||||
}
|
||||
|
||||
func (g *wordGuard) Output(context.Context, guard.Subject) guard.OutputFilter {
|
||||
if !g.answers {
|
||||
return nil
|
||||
}
|
||||
return &stopHi{}
|
||||
}
|
||||
|
||||
type stopHi struct{ v guard.Verdict }
|
||||
|
||||
func (f *stopHi) Buffered(body []byte) []byte {
|
||||
if !bytes.Contains(body, []byte(`"hi"`)) {
|
||||
return body
|
||||
}
|
||||
f.v = guard.Verdict{Action: guard.Block, Stage: guard.Output, Policy: "answers"}
|
||||
return []byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":""},"finish_reason":"content_filter"}]}`)
|
||||
}
|
||||
func (f *stopHi) OnEvent(d string) ([]string, bool) { return []string{d}, false }
|
||||
func (f *stopHi) OnEnd() []string { return nil }
|
||||
func (f *stopHi) Verdict() guard.Verdict { return f.v }
|
||||
|
||||
func guardedRouter(t *testing.T, g guard.Guard) (*Router, *fakeEngine) {
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
|
||||
rt.SetGuard(g)
|
||||
return rt, e
|
||||
}
|
||||
|
||||
func say(text string) map[string]any {
|
||||
return map[string]any{"model": "m", "messages": []any{map[string]any{"role": "user", "content": text}}}
|
||||
}
|
||||
|
||||
func TestABlockedRequestIsRefusedBeforeTheEngine(t *testing.T) {
|
||||
rt, e := guardedRouter(t, &wordGuard{})
|
||||
rec := post(t, rt.Handler(), say("something forbidden"), nil)
|
||||
var body struct {
|
||||
Error struct{ Type, Code, Message string } `json:"error"`
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if rec.Code != http.StatusBadRequest || body.Error.Code != policyViolation ||
|
||||
rec.Header().Get(guardHeader) != "blocked" || rec.Header().Get(guardPolicyHeader) != "words" {
|
||||
t.Errorf("%d %v %s", rec.Code, rec.Header(), rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "forbidden") {
|
||||
t.Error("the refusal quotes what it refused")
|
||||
}
|
||||
if e.calls.Load() != 0 {
|
||||
t.Error("the engine was asked anyway")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARedactedRequestReachesTheEngineRewritten(t *testing.T) {
|
||||
rt, e := guardedRouter(t, &wordGuard{})
|
||||
rec := post(t, rt.Handler(), say("write to alice@example.com"), nil)
|
||||
if rec.Header().Get(guardHeader) != "redacted" {
|
||||
t.Errorf("headers %v", rec.Header())
|
||||
}
|
||||
sent, _ := e.last.Load().([]byte)
|
||||
if bytes.Contains(sent, []byte("alice@example.com")) || !bytes.Contains(sent, []byte("[EMAIL_1]")) {
|
||||
t.Errorf("the engine was sent %s", sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerAGuardStopsEndsAsContentFilter(t *testing.T) {
|
||||
rt, _ := guardedRouter(t, &wordGuard{answers: true})
|
||||
rec := post(t, rt.Handler(), say("hello"), nil)
|
||||
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"finish_reason":"content_filter"`) ||
|
||||
strings.Contains(rec.Body.String(), `"hi"`) || rec.Header().Get(guardHeader) != "blocked" {
|
||||
t.Errorf("%d %v %s", rec.Code, rec.Header(), rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGuardThatCannotJudgeRefuses(t *testing.T) {
|
||||
g := &wordGuard{err: errors.New("judge down")}
|
||||
rt, e := guardedRouter(t, g)
|
||||
rec := post(t, rt.Handler(), say("hello"), nil)
|
||||
if rec.Code != http.StatusServiceUnavailable || e.calls.Load() != 0 || strings.Contains(rec.Body.String(), "judge down") {
|
||||
t.Errorf("%d %s, engine calls %d", rec.Code, rec.Body.String(), e.calls.Load())
|
||||
}
|
||||
if g.subject.Model != "m" || g.subject.Path != "/v1/chat/completions" {
|
||||
t.Errorf("judged as %+v", g.subject)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFlaggedRequestIsServedAndSaysSo(t *testing.T) {
|
||||
rt, e := guardedRouter(t, &wordGuard{})
|
||||
rec := post(t, rt.Handler(), say("maybe"), nil)
|
||||
if rec.Code != http.StatusOK || rec.Header().Get(guardHeader) != "flagged" || e.calls.Load() != 1 {
|
||||
t.Errorf("%d %v", rec.Code, rec.Header())
|
||||
}
|
||||
}
|
||||
|
||||
func TestABlockedRequestIsRecordedWithItsVerdictAndCostsNothing(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
rt.SetGuard(&wordGuard{})
|
||||
rec := post(t, rt.Handler(),
|
||||
map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "forbidden"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "blocked-1"})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
u, ok, err := rt.store.UsageByRequest(t.Context(), "blocked-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
if u.GuardAction != "blocked" || u.GuardPolicy != "words" || u.GuardCategory != "test" ||
|
||||
u.CostMicros != 0 || u.PromptTokens != 0 || u.Status != http.StatusBadRequest {
|
||||
t.Errorf("recorded %+v", u)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the blocked request was not recorded")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
@@ -296,6 +297,8 @@ type parsedRequest struct {
|
||||
// request is settled, so the record counts against the same budget window
|
||||
// the check read.
|
||||
reservation accounting.Reservation
|
||||
// guardIn is what the guard decided about the request.
|
||||
guardIn guard.Verdict
|
||||
}
|
||||
|
||||
// inferenceBody covers the fields the router reads across all inference
|
||||
@@ -422,6 +425,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
w.Header().Set(trimmedHeader, strconv.Itoa(dropped))
|
||||
}
|
||||
|
||||
// Judged as it will be sent: authorised, within its limits, and with
|
||||
// its stored conversation in it, which is text the model reads too.
|
||||
if r.guard != nil && !r.guardInput(req.Context(), w, parsed, start) {
|
||||
return
|
||||
}
|
||||
|
||||
res := r.dispatch(req.Context(), w, parsed)
|
||||
status := r.finish(w, parsed, res, start)
|
||||
span.SetAttributes(attribute.Int("http.response.status_code", status),
|
||||
@@ -521,6 +530,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
RequestID: req.RequestID, Replica: res.endpointID,
|
||||
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
|
||||
}
|
||||
guardFields(&u, req.guardIn, res.guard)
|
||||
if u.ServedModel == "" {
|
||||
u.ServedModel = req.Model
|
||||
}
|
||||
@@ -874,8 +884,18 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
|
||||
// writeError emits an OpenAI-shaped error so clients need no special handling
|
||||
// for gateway failures.
|
||||
func writeError(w http.ResponseWriter, code int, kind, msg string) {
|
||||
writeJSON(w, code, map[string]any{"error": map[string]any{
|
||||
"message": msg, "type": kind, "code": nil,
|
||||
writeErrorCode(w, code, kind, "", msg)
|
||||
}
|
||||
|
||||
// writeErrorCode is writeError with OpenAI's machine-readable code, which
|
||||
// clients branch on: content_policy_violation is the one SDKs know.
|
||||
func writeErrorCode(w http.ResponseWriter, status int, kind, code, msg string) {
|
||||
var c any
|
||||
if code != "" {
|
||||
c = code
|
||||
}
|
||||
writeJSON(w, status, map[string]any{"error": map[string]any{
|
||||
"message": msg, "type": kind, "code": c,
|
||||
}})
|
||||
}
|
||||
|
||||
@@ -1003,6 +1023,15 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t
|
||||
if res.err != nil {
|
||||
attrs = append(attrs, "err", res.err.Error())
|
||||
}
|
||||
// What a guard did, by name: never what it matched.
|
||||
if v := req.guardIn; res.guard.Stricter(v) {
|
||||
attrs = append(attrs, "guard", string(res.guard.Action), "guard_stage", "output", "guard_policy", res.guard.Policy)
|
||||
} else if v.Action != "" {
|
||||
attrs = append(attrs, "guard", string(v.Action), "guard_stage", "input", "guard_policy", v.Policy)
|
||||
}
|
||||
if res.outcome.Stopped {
|
||||
attrs = append(attrs, "stopped_by_guard", true)
|
||||
}
|
||||
r.log.Info("request", attrs...)
|
||||
}
|
||||
|
||||
|
||||
@@ -270,6 +270,9 @@ type respTranslator struct {
|
||||
byIndex map[int]*respCall
|
||||
usage map[string]int
|
||||
finished bool
|
||||
// filtered is a guard having stopped the answer: the response ends
|
||||
// incomplete, saying why, rather than completed.
|
||||
filtered bool
|
||||
}
|
||||
|
||||
type respCall struct {
|
||||
@@ -340,7 +343,10 @@ func (t *respTranslator) event(typ string, fields map[string]any) {
|
||||
// object is the response as it stands.
|
||||
func (t *respTranslator) object(status string) map[string]any {
|
||||
o := map[string]any{"id": t.id, "object": "response", "created_at": t.created, "model": t.model,
|
||||
"status": status, "output": t.output(status == "completed")}
|
||||
"status": status, "output": t.output(status == "completed" || status == "incomplete")}
|
||||
if status == "incomplete" {
|
||||
o["incomplete_details"] = map[string]any{"reason": "content_filter"}
|
||||
}
|
||||
if t.metadata != nil {
|
||||
o["metadata"] = t.metadata
|
||||
}
|
||||
@@ -431,6 +437,7 @@ func (t *respTranslator) chatLine(line string) {
|
||||
} `json:"function"`
|
||||
} `json:"tool_calls"`
|
||||
} `json:"delta"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
Prompt int `json:"prompt_tokens"`
|
||||
@@ -445,6 +452,9 @@ func (t *respTranslator) chatLine(line string) {
|
||||
t.usage = map[string]int{"prompt": chunk.Usage.Prompt, "completion": chunk.Usage.Completion}
|
||||
}
|
||||
for _, ch := range chunk.Choices {
|
||||
if ch.FinishReason == "content_filter" {
|
||||
t.filtered = true
|
||||
}
|
||||
if d := ch.Delta.Content; d != "" {
|
||||
if t.msgID == "" {
|
||||
// The message's place is after whatever calls came first.
|
||||
@@ -522,7 +532,11 @@ func (t *respTranslator) finish() {
|
||||
}
|
||||
t.event("response.output_item.done", map[string]any{"output_index": c.outIndex, "item": t.callItem(c, true)})
|
||||
}
|
||||
t.event("response.completed", map[string]any{"response": t.object("completed")})
|
||||
if t.filtered {
|
||||
t.event("response.incomplete", map[string]any{"response": t.object("incomplete")})
|
||||
} else {
|
||||
t.event("response.completed", map[string]any{"response": t.object("completed")})
|
||||
}
|
||||
t.Flush()
|
||||
}
|
||||
|
||||
@@ -540,6 +554,7 @@ func (t *respTranslator) fromChatJSON(b []byte) {
|
||||
} `json:"function"`
|
||||
} `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
Prompt int `json:"prompt_tokens"`
|
||||
@@ -555,6 +570,9 @@ func (t *respTranslator) fromChatJSON(b []byte) {
|
||||
t.usage = map[string]int{"prompt": c.Usage.Prompt, "completion": c.Usage.Completion}
|
||||
}
|
||||
for _, ch := range c.Choices {
|
||||
if ch.FinishReason == "content_filter" {
|
||||
t.filtered = true
|
||||
}
|
||||
if ch.Message.Content != "" {
|
||||
t.msgID, t.msgIndex = respID("msg"), 0
|
||||
t.text.WriteString(ch.Message.Content)
|
||||
@@ -565,6 +583,10 @@ func (t *respTranslator) fromChatJSON(b []byte) {
|
||||
t.calls = append(t.calls, rc)
|
||||
}
|
||||
}
|
||||
out, _ := json.Marshal(t.object("completed"))
|
||||
status := "completed"
|
||||
if t.filtered {
|
||||
status = "incomplete"
|
||||
}
|
||||
out, _ := json.Marshal(t.object(status))
|
||||
_, _ = t.w.Write(out)
|
||||
}
|
||||
@@ -206,3 +206,25 @@ func TestAResponsesRequestWithoutAStreamGetsAResponseObject(t *testing.T) {
|
||||
t.Errorf("%s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerStoppedByPolicyIsAnIncompleteResponse(t *testing.T) {
|
||||
for _, stream := range []bool{true, false} {
|
||||
w := httptest.NewRecorder()
|
||||
tr := &respTranslator{w: w, model: "m", stream: stream, id: "resp_1", status: 200}
|
||||
if stream {
|
||||
_, _ = tr.Write([]byte("data: {\"choices\":[{\"delta\":{\"content\":\"par\"}}]}\n\n" +
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"content_filter\"}]}\n\ndata: [DONE]\n\n"))
|
||||
} else {
|
||||
_, _ = tr.Write([]byte(`{"choices":[{"message":{"content":""},"finish_reason":"content_filter"}]}`))
|
||||
}
|
||||
tr.finish()
|
||||
body := w.Body.String()
|
||||
if !strings.Contains(body, `"incomplete_details":{"reason":"content_filter"}`) ||
|
||||
!strings.Contains(body, `"status":"incomplete"`) || strings.Contains(body, "response.completed") {
|
||||
t.Errorf("stream=%v: %s", stream, body)
|
||||
}
|
||||
if stream && !strings.Contains(body, "response.incomplete") {
|
||||
t.Errorf("no response.incomplete event: %s", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -88,6 +89,8 @@ type Router struct {
|
||||
// isLeader says whether this process leads; nil is always.
|
||||
isLeader atomic.Pointer[func() bool]
|
||||
acct *accounting.Accountant
|
||||
// guard judges requests and answers; nil judges nothing.
|
||||
guard guard.Guard
|
||||
|
||||
// threads holds conversations and memory notes. It is store when auth is on,
|
||||
// its own file when only the threads block is configured, and nil when
|
||||
@@ -150,6 +153,9 @@ func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
|
||||
// cluster view: the router is built before the things its admin pages show.
|
||||
func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m }
|
||||
|
||||
// SetGuard has every request and answer judged by g.
|
||||
func (r *Router) SetGuard(g guard.Guard) { r.guard = g }
|
||||
|
||||
// SetRegistry gives the mirror page somewhere to look models up.
|
||||
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
|
||||
|
||||
@@ -710,6 +716,8 @@ type dispatchResult struct {
|
||||
servedModel
|
||||
outcome proxy.Outcome
|
||||
err error
|
||||
// guard is what the output filter decided, if there was one.
|
||||
guard guard.Verdict
|
||||
}
|
||||
|
||||
type servedModel struct {
|
||||
@@ -863,7 +871,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
} else {
|
||||
w.Header().Del(hardwareHeader)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
preq := proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
Body: body,
|
||||
@@ -871,13 +879,19 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
Stream: req.Stream,
|
||||
SuppressUsageChunk: req.SuppressUsageChunk,
|
||||
CaptureReply: req.captureBytes(),
|
||||
})
|
||||
}
|
||||
// A filter per attempt, for the model this attempt is served by.
|
||||
filter := r.outputFilter(uctx, req, model, &preq)
|
||||
out, perr := r.px.Do(uctx, w, preq)
|
||||
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
|
||||
endSpan(uspan, perr)
|
||||
// A retryable failure is the endpoint's fault; a client disconnect is not.
|
||||
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
|
||||
|
||||
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
|
||||
if filter != nil {
|
||||
res.guard = filter.Verdict()
|
||||
}
|
||||
r.m.observeScores(dec)
|
||||
|
||||
// Recorded after the decision, not during it: the scorer proposes and
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- What a guard decided about a request: the action (flagged, redacted,
|
||||
-- blocked), the policy and category it was decided by, and how long it took.
|
||||
-- Names only, never the text.
|
||||
ALTER TABLE usage ADD COLUMN guard_action TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_policy TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_category TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_ms BIGINT NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- What a guard decided about a request: the action (flagged, redacted,
|
||||
-- blocked), the policy and category it was decided by, and how long it took.
|
||||
-- Names only, never the text.
|
||||
ALTER TABLE usage ADD COLUMN guard_action TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_policy TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_category TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN guard_ms INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -425,8 +425,9 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -435,7 +436,8 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs); err != nil {
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
|
||||
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardMs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -846,11 +848,12 @@ func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool,
|
||||
var ts int64
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens,
|
||||
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms
|
||||
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_ms
|
||||
FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID).
|
||||
Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens,
|
||||
&u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica,
|
||||
&u.QueueMs, &u.TTFTMs)
|
||||
&u.QueueMs, &u.TTFTMs, &u.GuardAction, &u.GuardPolicy, &u.GuardCategory, &u.GuardMs)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Usage{}, false, nil
|
||||
}
|
||||
|
||||
@@ -253,6 +253,15 @@ type Usage struct {
|
||||
// token, 0 when none came.
|
||||
QueueMs int64
|
||||
TTFTMs int64
|
||||
|
||||
// GuardAction is what a guard did — flagged, redacted, blocked — empty
|
||||
// when it let the request through untouched or there is none;
|
||||
// GuardPolicy and GuardCategory what decided it, GuardMs how long
|
||||
// judging took. Never the text.
|
||||
GuardAction string
|
||||
GuardPolicy string
|
||||
GuardCategory string
|
||||
GuardMs int64
|
||||
}
|
||||
|
||||
// Store is the persistent half of tenancy and accounting.
|
||||
|
||||
Reference in new issue
Block a user