Hook a guard into the router: judge requests before dispatch and filter answers as they are relayed

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 12:20:24 +02:00
1 parent e701f92806
commit 979aa44804
12 files changed
+448 -12

No files matched your search

+6
View File
@@ -203,6 +203,12 @@ default output cap when the client omits `max_tokens`; `catalog.Capabilities`
is advertised but never enforced; tool results are trusted input.
### B1. Hooks in the proxy and router (S)
**Done (branch `guardrails-b`):** `internal/guard` holds the interface;
proxy `Filter`/`StreamFilter`; the input hook runs after `loadThread` (the
simpler choice); error codes, headers, Responses `incomplete`, usage columns
`guard_*`. Moved to B2: holding `delta.tool_calls` until they are whole and
reading their arguments — that is the tool policy's filter, not the proxy's.
- `internal/router/proxy/proxy.go`: add optional `InputRewrite func(body
[]byte) ([]byte, Verdict)` on the request and two output hooks: `Filter`
for `relayBuffered` (between `ReadAll` and `WriteHeader`) and
+15
View File
@@ -23,6 +23,21 @@ APIs change between minors without a deprecation period.
## Unreleased
### Guardrails
- **Hooks for guardrails on the request path** (nothing judges yet; B2 adds
the policies). A guard, when one is set, judges a request once it is
authorised, within its limits and has its stored conversation spliced
in, and before it is dispatched; and filters the answer, buffered or
streamed, before the client sees it. A blocked request is refused with 400
and `code: "content_policy_violation"` — the first error with a code — and
costs nothing; a blocked answer ends with `finish_reason: "content_filter"`,
and a stopped stream with that chunk and `[DONE]`, the engine cancelled.
The Responses API reports either as `incomplete` with
`incomplete_details.reason: "content_filter"`. `x-llmgw-guard` and
`x-llmgw-guard-policy` say what was done; the usage row records the
action, policy, category and time taken — never the text.
### Security
- **Built with Go 1.26.6.** Eleven standard-library vulnerabilities reachable
+140
View File
@@ -0,0 +1,140 @@
package router
import (
"context"
"encoding/json"
"net/http"
"strconv"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// The headers a judged request is answered with: what was done, and by
// which policy. Set before anything is written, so on a streamed answer they
// carry the input's verdict, and the output's only on a buffered one.
const (
guardHeader = "x-llmgw-guard"
guardPolicyHeader = "x-llmgw-guard-policy"
)
// policyViolation is the error code a blocked request is refused with.
const policyViolation = "content_policy_violation"
func (p *parsedRequest) guardSubject(model string) guard.Subject {
return guard.Subject{
RequestID: p.RequestID, KeyID: p.KeyID, UserID: p.UserID, ProjectID: p.ProjectID,
TeamID: p.TeamID, OrgID: p.OrgID, Model: model, Path: p.Path, Stream: p.Stream,
}
}
func setGuardHeaders(h http.Header, v guard.Verdict) {
if v.Action == guard.Allow {
return
}
h.Set(guardHeader, string(v.Action))
if v.Policy != "" {
h.Set(guardPolicyHeader, v.Policy)
}
}
// guardInput judges the request about to be dispatched. It answers the
// client itself and returns false when the request goes no further.
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
model, _ := r.resolve(req.Model)
t := time.Now()
v, err := r.guard.Input(ctx, req.guardSubject(model), req.UpstreamBody)
if v.Took == 0 {
v.Took = time.Since(t)
}
v.Stage = guard.Input
if err != nil {
// Not judged is not judged fine: a guard that means to fail open
// says Allow instead.
r.refuseGuarded(w, req, guard.Verdict{Action: guard.Block, Stage: guard.Input, Took: v.Took},
http.StatusServiceUnavailable, "server_error", "guard_unavailable",
"the request could not be checked against this gateway's policies; try again", start)
return false
}
req.guardIn = v
setGuardHeaders(w.Header(), v)
switch v.Action {
case guard.Block:
msg := v.Message
if msg == "" {
msg = "this request was refused by policy " + strconv.Quote(v.Policy)
}
r.refuseGuarded(w, req, v, http.StatusBadRequest, "invalid_request_error", policyViolation, msg, start)
return false
case guard.Redact:
if v.Body != nil {
req.UpstreamBody = v.Body
// Routed by what is sent, so the prefix tree never holds what
// was taken out.
var b inferenceBody
if json.Unmarshal(v.Body, &b) == nil {
req.PromptPrefix = promptPrefix(&b)
}
}
}
return true
}
// refuseGuarded answers a request a guard stopped. Nothing is charged —
// nothing was generated — but the row is kept, with the verdict, so a team
// running into a policy shows as one.
func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guard.Verdict,
status int, kind, code, msg string, start time.Time,
) {
writeErrorCode(w, status, kind, code, msg)
r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0)
r.m.tenant.observe(req, status, 0, 0, 0)
if r.acct != nil && req.KeyID != "" {
u := store.Usage{
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: req.Model, Status: status,
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, QueueMs: req.QueueMs,
}
guardFields(&u, v, guard.Verdict{})
// Settled at nothing: the estimate the limit check charged is
// given back.
r.acct.Record(req.reservation, u)
}
r.accessLog(req, dispatchResult{guard: v}, status, time.Since(start))
}
// outputFilter attaches the guard's filter for one attempt to the proxy
// request, and returns it so its verdict can be read afterwards.
func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model string, preq *proxy.Request) guard.OutputFilter {
if r.guard == nil {
return nil
}
f := r.guard.Output(ctx, req.guardSubject(model))
if f == nil {
return nil
}
if req.Stream {
preq.StreamFilter = f
}
preq.Filter = func(body []byte, h http.Header) []byte {
out := f.Buffered(body)
v := f.Verdict()
if v.Stricter(req.guardIn) {
setGuardHeaders(h, v)
}
return out
}
return f
}
// guardFields records the stricter of the input's and output's verdicts.
func guardFields(u *store.Usage, in, out guard.Verdict) {
v := in
if out.Stricter(in) {
v = out
}
u.GuardAction, u.GuardPolicy, u.GuardCategory = string(v.Action), v.Policy, v.Category
u.GuardMs = (in.Took + out.Took).Milliseconds()
}
+161
View File
@@ -0,0 +1,161 @@
package router
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// wordGuard blocks requests saying "forbidden", rewrites "alice@example.com",
// flags "maybe", and stops answers saying "hi".
type wordGuard struct {
err error
subject guard.Subject
// answers, when set, stops answers saying "hi".
answers bool
}
func (g *wordGuard) Input(_ context.Context, s guard.Subject, body []byte) (guard.Verdict, error) {
g.subject = s
switch {
case g.err != nil:
return guard.Verdict{}, g.err
case bytes.Contains(body, []byte("forbidden")):
return guard.Verdict{Action: guard.Block, Policy: "words", Category: "test"}, nil
case bytes.Contains(body, []byte("alice@example.com")):
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
Body: bytes.ReplaceAll(body, []byte("alice@example.com"), []byte("[EMAIL_1]"))}, nil
case bytes.Contains(body, []byte("maybe")):
return guard.Verdict{Action: guard.Flag, Policy: "words"}, nil
}
return guard.Verdict{}, nil
}
func (g *wordGuard) Output(context.Context, guard.Subject) guard.OutputFilter {
if !g.answers {
return nil
}
return &stopHi{}
}
type stopHi struct{ v guard.Verdict }
func (f *stopHi) Buffered(body []byte) []byte {
if !bytes.Contains(body, []byte(`"hi"`)) {
return body
}
f.v = guard.Verdict{Action: guard.Block, Stage: guard.Output, Policy: "answers"}
return []byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":""},"finish_reason":"content_filter"}]}`)
}
func (f *stopHi) OnEvent(d string) ([]string, bool) { return []string{d}, false }
func (f *stopHi) OnEnd() []string { return nil }
func (f *stopHi) Verdict() guard.Verdict { return f.v }
func guardedRouter(t *testing.T, g guard.Guard) (*Router, *fakeEngine) {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
rt.SetGuard(g)
return rt, e
}
func say(text string) map[string]any {
return map[string]any{"model": "m", "messages": []any{map[string]any{"role": "user", "content": text}}}
}
func TestABlockedRequestIsRefusedBeforeTheEngine(t *testing.T) {
rt, e := guardedRouter(t, &wordGuard{})
rec := post(t, rt.Handler(), say("something forbidden"), nil)
var body struct {
Error struct{ Type, Code, Message string } `json:"error"`
}
_ = json.Unmarshal(rec.Body.Bytes(), &body)
if rec.Code != http.StatusBadRequest || body.Error.Code != policyViolation ||
rec.Header().Get(guardHeader) != "blocked" || rec.Header().Get(guardPolicyHeader) != "words" {
t.Errorf("%d %v %s", rec.Code, rec.Header(), rec.Body.String())
}
if strings.Contains(rec.Body.String(), "forbidden") {
t.Error("the refusal quotes what it refused")
}
if e.calls.Load() != 0 {
t.Error("the engine was asked anyway")
}
}
func TestARedactedRequestReachesTheEngineRewritten(t *testing.T) {
rt, e := guardedRouter(t, &wordGuard{})
rec := post(t, rt.Handler(), say("write to alice@example.com"), nil)
if rec.Header().Get(guardHeader) != "redacted" {
t.Errorf("headers %v", rec.Header())
}
sent, _ := e.last.Load().([]byte)
if bytes.Contains(sent, []byte("alice@example.com")) || !bytes.Contains(sent, []byte("[EMAIL_1]")) {
t.Errorf("the engine was sent %s", sent)
}
}
func TestAnAnswerAGuardStopsEndsAsContentFilter(t *testing.T) {
rt, _ := guardedRouter(t, &wordGuard{answers: true})
rec := post(t, rt.Handler(), say("hello"), nil)
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"finish_reason":"content_filter"`) ||
strings.Contains(rec.Body.String(), `"hi"`) || rec.Header().Get(guardHeader) != "blocked" {
t.Errorf("%d %v %s", rec.Code, rec.Header(), rec.Body.String())
}
}
func TestAGuardThatCannotJudgeRefuses(t *testing.T) {
g := &wordGuard{err: errors.New("judge down")}
rt, e := guardedRouter(t, g)
rec := post(t, rt.Handler(), say("hello"), nil)
if rec.Code != http.StatusServiceUnavailable || e.calls.Load() != 0 || strings.Contains(rec.Body.String(), "judge down") {
t.Errorf("%d %s, engine calls %d", rec.Code, rec.Body.String(), e.calls.Load())
}
if g.subject.Model != "m" || g.subject.Path != "/v1/chat/completions" {
t.Errorf("judged as %+v", g.subject)
}
}
func TestAFlaggedRequestIsServedAndSaysSo(t *testing.T) {
rt, e := guardedRouter(t, &wordGuard{})
rec := post(t, rt.Handler(), say("maybe"), nil)
if rec.Code != http.StatusOK || rec.Header().Get(guardHeader) != "flagged" || e.calls.Load() != 1 {
t.Errorf("%d %v", rec.Code, rec.Header())
}
}
func TestABlockedRequestIsRecordedWithItsVerdictAndCostsNothing(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
rt.SetGuard(&wordGuard{})
rec := post(t, rt.Handler(),
map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "forbidden"}}},
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "blocked-1"})
if rec.Code != http.StatusBadRequest {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
deadline := time.Now().Add(5 * time.Second)
for {
u, ok, err := rt.store.UsageByRequest(t.Context(), "blocked-1")
if err != nil {
t.Fatal(err)
}
if ok {
if u.GuardAction != "blocked" || u.GuardPolicy != "words" || u.GuardCategory != "test" ||
u.CostMicros != 0 || u.PromptTokens != 0 || u.Status != http.StatusBadRequest {
t.Errorf("recorded %+v", u)
}
return
}
if time.Now().After(deadline) {
t.Fatal("the blocked request was not recorded")
}
time.Sleep(20 * time.Millisecond)
}
}
+31 -2
View File
@@ -19,6 +19,7 @@ import (
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
@@ -296,6 +297,8 @@ type parsedRequest struct {
// request is settled, so the record counts against the same budget window
// the check read.
reservation accounting.Reservation
// guardIn is what the guard decided about the request.
guardIn guard.Verdict
}
// inferenceBody covers the fields the router reads across all inference
@@ -422,6 +425,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
w.Header().Set(trimmedHeader, strconv.Itoa(dropped))
}
// Judged as it will be sent: authorised, within its limits, and with
// its stored conversation in it, which is text the model reads too.
if r.guard != nil && !r.guardInput(req.Context(), w, parsed, start) {
return
}
res := r.dispatch(req.Context(), w, parsed)
status := r.finish(w, parsed, res, start)
span.SetAttributes(attribute.Int("http.response.status_code", status),
@@ -521,6 +530,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
RequestID: req.RequestID, Replica: res.endpointID,
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
}
guardFields(&u, req.guardIn, res.guard)
if u.ServedModel == "" {
u.ServedModel = req.Model
}
@@ -874,8 +884,18 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
// writeError emits an OpenAI-shaped error so clients need no special handling
// for gateway failures.
func writeError(w http.ResponseWriter, code int, kind, msg string) {
writeJSON(w, code, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": nil,
writeErrorCode(w, code, kind, "", msg)
}
// writeErrorCode is writeError with OpenAI's machine-readable code, which
// clients branch on: content_policy_violation is the one SDKs know.
func writeErrorCode(w http.ResponseWriter, status int, kind, code, msg string) {
var c any
if code != "" {
c = code
}
writeJSON(w, status, map[string]any{"error": map[string]any{
"message": msg, "type": kind, "code": c,
}})
}
@@ -1003,6 +1023,15 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t
if res.err != nil {
attrs = append(attrs, "err", res.err.Error())
}
// What a guard did, by name: never what it matched.
if v := req.guardIn; res.guard.Stricter(v) {
attrs = append(attrs, "guard", string(res.guard.Action), "guard_stage", "output", "guard_policy", res.guard.Policy)
} else if v.Action != "" {
attrs = append(attrs, "guard", string(v.Action), "guard_stage", "input", "guard_policy", v.Policy)
}
if res.outcome.Stopped {
attrs = append(attrs, "stopped_by_guard", true)
}
r.log.Info("request", attrs...)
}
+25 -3
View File
@@ -270,6 +270,9 @@ type respTranslator struct {
byIndex map[int]*respCall
usage map[string]int
finished bool
// filtered is a guard having stopped the answer: the response ends
// incomplete, saying why, rather than completed.
filtered bool
}
type respCall struct {
@@ -340,7 +343,10 @@ func (t *respTranslator) event(typ string, fields map[string]any) {
// object is the response as it stands.
func (t *respTranslator) object(status string) map[string]any {
o := map[string]any{"id": t.id, "object": "response", "created_at": t.created, "model": t.model,
"status": status, "output": t.output(status == "completed")}
"status": status, "output": t.output(status == "completed" || status == "incomplete")}
if status == "incomplete" {
o["incomplete_details"] = map[string]any{"reason": "content_filter"}
}
if t.metadata != nil {
o["metadata"] = t.metadata
}
@@ -431,6 +437,7 @@ func (t *respTranslator) chatLine(line string) {
} `json:"function"`
} `json:"tool_calls"`
} `json:"delta"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *struct {
Prompt int `json:"prompt_tokens"`
@@ -445,6 +452,9 @@ func (t *respTranslator) chatLine(line string) {
t.usage = map[string]int{"prompt": chunk.Usage.Prompt, "completion": chunk.Usage.Completion}
}
for _, ch := range chunk.Choices {
if ch.FinishReason == "content_filter" {
t.filtered = true
}
if d := ch.Delta.Content; d != "" {
if t.msgID == "" {
// The message's place is after whatever calls came first.
@@ -522,7 +532,11 @@ func (t *respTranslator) finish() {
}
t.event("response.output_item.done", map[string]any{"output_index": c.outIndex, "item": t.callItem(c, true)})
}
t.event("response.completed", map[string]any{"response": t.object("completed")})
if t.filtered {
t.event("response.incomplete", map[string]any{"response": t.object("incomplete")})
} else {
t.event("response.completed", map[string]any{"response": t.object("completed")})
}
t.Flush()
}
@@ -540,6 +554,7 @@ func (t *respTranslator) fromChatJSON(b []byte) {
} `json:"function"`
} `json:"tool_calls"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *struct {
Prompt int `json:"prompt_tokens"`
@@ -555,6 +570,9 @@ func (t *respTranslator) fromChatJSON(b []byte) {
t.usage = map[string]int{"prompt": c.Usage.Prompt, "completion": c.Usage.Completion}
}
for _, ch := range c.Choices {
if ch.FinishReason == "content_filter" {
t.filtered = true
}
if ch.Message.Content != "" {
t.msgID, t.msgIndex = respID("msg"), 0
t.text.WriteString(ch.Message.Content)
@@ -565,6 +583,10 @@ func (t *respTranslator) fromChatJSON(b []byte) {
t.calls = append(t.calls, rc)
}
}
out, _ := json.Marshal(t.object("completed"))
status := "completed"
if t.filtered {
status = "incomplete"
}
out, _ := json.Marshal(t.object(status))
_, _ = t.w.Write(out)
}
+22
View File
@@ -206,3 +206,25 @@ func TestAResponsesRequestWithoutAStreamGetsAResponseObject(t *testing.T) {
t.Errorf("%s", w.Body.String())
}
}
func TestAnAnswerStoppedByPolicyIsAnIncompleteResponse(t *testing.T) {
for _, stream := range []bool{true, false} {
w := httptest.NewRecorder()
tr := &respTranslator{w: w, model: "m", stream: stream, id: "resp_1", status: 200}
if stream {
_, _ = tr.Write([]byte("data: {\"choices\":[{\"delta\":{\"content\":\"par\"}}]}\n\n" +
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"content_filter\"}]}\n\ndata: [DONE]\n\n"))
} else {
_, _ = tr.Write([]byte(`{"choices":[{"message":{"content":""},"finish_reason":"content_filter"}]}`))
}
tr.finish()
body := w.Body.String()
if !strings.Contains(body, `"incomplete_details":{"reason":"content_filter"}`) ||
!strings.Contains(body, `"status":"incomplete"`) || strings.Contains(body, "response.completed") {
t.Errorf("stream=%v: %s", stream, body)
}
if stream && !strings.Contains(body, "response.incomplete") {
t.Errorf("no response.incomplete event: %s", body)
}
}
}
+16 -2
View File
@@ -25,6 +25,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -88,6 +89,8 @@ type Router struct {
// isLeader says whether this process leads; nil is always.
isLeader atomic.Pointer[func() bool]
acct *accounting.Accountant
// guard judges requests and answers; nil judges nothing.
guard guard.Guard
// threads holds conversations and memory notes. It is store when auth is on,
// its own file when only the threads block is configured, and nil when
@@ -150,6 +153,9 @@ func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
// cluster view: the router is built before the things its admin pages show.
func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m }
// SetGuard has every request and answer judged by g.
func (r *Router) SetGuard(g guard.Guard) { r.guard = g }
// SetRegistry gives the mirror page somewhere to look models up.
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
@@ -710,6 +716,8 @@ type dispatchResult struct {
servedModel
outcome proxy.Outcome
err error
// guard is what the output filter decided, if there was one.
guard guard.Verdict
}
type servedModel struct {
@@ -863,7 +871,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
} else {
w.Header().Del(hardwareHeader)
}
out, perr := r.px.Do(uctx, w, proxy.Request{
preq := proxy.Request{
URL: dec.Endpoint.URL + req.Path,
APIKey: dec.Endpoint.APIKey,
Body: body,
@@ -871,13 +879,19 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
Stream: req.Stream,
SuppressUsageChunk: req.SuppressUsageChunk,
CaptureReply: req.captureBytes(),
})
}
// A filter per attempt, for the model this attempt is served by.
filter := r.outputFilter(uctx, req, model, &preq)
out, perr := r.px.Do(uctx, w, preq)
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
endSpan(uspan, perr)
// A retryable failure is the endpoint's fault; a client disconnect is not.
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
if filter != nil {
res.guard = filter.Verdict()
}
r.m.observeScores(dec)
// Recorded after the decision, not during it: the scorer proposes and
@@ -0,0 +1,8 @@
-- Ported from the SQLite migration of the same number.
-- What a guard decided about a request: the action (flagged, redacted,
-- blocked), the policy and category it was decided by, and how long it took.
-- Names only, never the text.
ALTER TABLE usage ADD COLUMN guard_action TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_policy TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_category TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_ms BIGINT NOT NULL DEFAULT 0;
@@ -0,0 +1,7 @@
-- What a guard decided about a request: the action (flagged, redacted,
-- blocked), the policy and category it was decided by, and how long it took.
-- Names only, never the text.
ALTER TABLE usage ADD COLUMN guard_action TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_policy TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_category TEXT NOT NULL DEFAULT '';
ALTER TABLE usage ADD COLUMN guard_ms INTEGER NOT NULL DEFAULT 0;
+8 -5
View File
@@ -425,8 +425,9 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
stmt, err := tx.PrepareContext(ctx,
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
@@ -435,7 +436,8 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs); err != nil {
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardMs); err != nil {
return err
}
}
@@ -846,11 +848,12 @@ func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool,
var ts int64
err := s.db.QueryRowContext(ctx,
`SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens,
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_ms
FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID).
Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens,
&u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica,
&u.QueueMs, &u.TTFTMs)
&u.QueueMs, &u.TTFTMs, &u.GuardAction, &u.GuardPolicy, &u.GuardCategory, &u.GuardMs)
if errors.Is(err, sql.ErrNoRows) {
return Usage{}, false, nil
}
+9
View File
@@ -253,6 +253,15 @@ type Usage struct {
// token, 0 when none came.
QueueMs int64
TTFTMs int64
// GuardAction is what a guard did — flagged, redacted, blocked — empty
// when it let the request through untouched or there is none;
// GuardPolicy and GuardCategory what decided it, GuardMs how long
// judging took. Never the text.
GuardAction string
GuardPolicy string
GuardCategory string
GuardMs int64
}
// Store is the persistent half of tenancy and accounting.