From e701f9280673080b53f6636851341da2906a70f3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 12:16:00 +0200 Subject: [PATCH] Add the guard interface, and filter hooks in the proxy for buffered and streamed answers Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/guard/guard.go | 116 ++++++++++++++++++++++++++ internal/router/proxy/filter_test.go | 118 +++++++++++++++++++++++++++ internal/router/proxy/proxy.go | 93 ++++++++++++++++++++- 3 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 internal/guard/guard.go create mode 100644 internal/router/proxy/filter_test.go diff --git a/internal/guard/guard.go b/internal/guard/guard.go new file mode 100644 index 00000000..8f7779bc --- /dev/null +++ b/internal/guard/guard.go @@ -0,0 +1,116 @@ +// Package guard is what the gateway asks about a request's text and its +// answer's: whether to let it through, flag it, rewrite it or stop it. +// +// The router calls a Guard at two points. Input, once the request is +// authorised, limited and has its stored conversation spliced in, and before +// it is dispatched: the whole upstream body. Output, for the answer: a filter +// over the response as it is relayed, buffered or streamed. A router without +// a Guard calls nothing and pays nothing. +// +// Nothing here may log, record or return the text it judges. Verdicts carry +// policy, rule and category names, never what matched. +package guard + +import ( + "context" + "time" +) + +// Action is what a verdict does to a request or a response. +type Action string + +const ( + // Allow lets it through untouched. + Allow Action = "" + // Flag lets it through, and says so: in a header and in the usage row. + Flag Action = "flagged" + // Redact lets it through rewritten. + Redact Action = "redacted" + // Block stops it. + Block Action = "blocked" +) + +// Stage is where a verdict was reached. +type Stage string + +const ( + Input Stage = "input" + Output Stage = "output" +) + +// Subject is who is asking, for which model: what a policy is chosen by. +type Subject struct { + RequestID string + KeyID string + UserID string + ProjectID string + TeamID string + OrgID string + // Model is the canonical model that will serve. + Model string + // Path is the API path asked: /v1/chat/completions, /v1/completions, …. + Path string + // Stream says whether the answer is streamed. + Stream bool +} + +// Verdict is a judgement on one stage of one request. +type Verdict struct { + Action Action + Stage Stage + Policy string + Rule string + Category string + // Message is what a blocked caller is told. It names the policy, never + // the text that tripped it. + Message string + // Body replaces the request's upstream body when the input was + // rewritten; nil leaves it as it was. + Body []byte + // Took is how long the judgement took. + Took time.Duration +} + +// Stricter reports whether v outranks w: block over redact over flag over +// allow. A response judged at several points carries its strictest verdict. +func (v Verdict) Stricter(w Verdict) bool { return rank(v.Action) > rank(w.Action) } + +func rank(a Action) int { + switch a { + case Block: + return 3 + case Redact: + return 2 + case Flag: + return 1 + } + return 0 +} + +// Guard judges requests and their answers. +type Guard interface { + // Input judges the body about to be sent upstream. An error means the + // guard could not judge; the router refuses the request rather than + // serve it unjudged — a guard that wants to fail open says Allow. + Input(ctx context.Context, s Subject, body []byte) (Verdict, error) + // Output is the filter for one attempt's answer, or nil when nothing + // watches it. + Output(ctx context.Context, s Subject) OutputFilter +} + +// OutputFilter watches one answer as it is relayed. +type OutputFilter interface { + // Buffered sees a whole non-streamed response body before anything is + // sent, and returns the body to send. + Buffered(body []byte) []byte + // OnEvent sees each streamed event's data — a chunk's JSON — and + // returns the events to send in its place: itself to pass it on, + // nothing to hold it back, more than one to release what it held. stop + // ends the stream after those are sent. + OnEvent(data string) (emit []string, stop bool) + // OnEnd is called when the upstream stream ends, and returns what is + // still held. + OnEnd() []string + // Verdict is the strictest judgement so far. + Verdict() Verdict +} diff --git a/internal/router/proxy/filter_test.go b/internal/router/proxy/filter_test.go new file mode 100644 index 00000000..cc9b29f2 --- /dev/null +++ b/internal/router/proxy/filter_test.go @@ -0,0 +1,118 @@ +package proxy + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// holdTwo releases events two at a time, and stops at one saying "stop". +type holdTwo struct { + held []string + ended bool +} + +func (h *holdTwo) OnEvent(data string) ([]string, bool) { + if strings.Contains(data, "stop") { + return []string{`{"choices":[{"index":0,"delta":{},"finish_reason":"content_filter"}]}`}, true + } + h.held = append(h.held, data) + if len(h.held) < 2 { + return nil, false + } + out := h.held + h.held = nil + return out, false +} + +func (h *holdTwo) OnEnd() []string { + h.ended = true + out := h.held + h.held = nil + return out +} + +func TestAStreamFilterHoldsReleasesAndEnds(t *testing.T) { + up, _ := sseServer(t, []string{ + dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"b"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"c"}}]}`), + dataChunk(`{"choices":[],"usage":{"prompt_tokens":1,"completion_tokens":3,"total_tokens":4}}`), + "data: [DONE]\n\n", + }, time.Millisecond) + f := &holdTwo{} + rec := httptest.NewRecorder() + out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true, + StreamFilter: f, CaptureReply: 100}) + if err != nil { + t.Fatal(err) + } + body := rec.Body.String() + for _, want := range []string{`"content":"a"`, `"content":"b"`, `"content":"c"`, `"usage"`} { + if !strings.Contains(body, want) { + t.Errorf("no %s in %q", want, body) + } + } + if !f.ended || !strings.HasSuffix(body, "data: [DONE]\n\n") || strings.Count(body, "[DONE]") != 1 { + t.Errorf("the held event did not go out before one [DONE]: %q", body) + } + if out.Usage == nil || out.Usage.TotalTokens != 4 || out.Reply != "abc" || out.Stopped { + t.Errorf("outcome %+v", out) + } +} + +func TestAStreamFilterStopsTheStreamAndTheEngine(t *testing.T) { + up, cancelled := sseServer(t, []string{ + dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"b"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"stop"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"never"}}]}`), + dataChunk(`{"choices":[{"delta":{"content":"seen"}}]}`), + "data: [DONE]\n\n", + }, 20*time.Millisecond) + rec := httptest.NewRecorder() + out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true, + StreamFilter: &holdTwo{}}) + if err != nil { + t.Fatal(err) + } + body := rec.Body.String() + if !out.Stopped || !strings.Contains(body, `"finish_reason":"content_filter"`) || + !strings.HasSuffix(body, "data: [DONE]\n\n") || strings.Contains(body, "never") { + t.Errorf("stopped=%v body %q", out.Stopped, body) + } + deadline := time.Now().Add(2 * time.Second) + for !cancelled.Load() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !cancelled.Load() { + t.Error("the engine went on generating after the stream was stopped") + } +} + +func TestABufferedFilterRewritesTheBodyNotTheCharge(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"choices":[{"message":{"content":"secret"}}],"usage":{"prompt_tokens":7,"completion_tokens":3}}`) + })) + defer up.Close() + rec := httptest.NewRecorder() + out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), + Filter: func(body []byte, h http.Header) []byte { + h.Set("x-llmgw-guard", "blocked") + return []byte(`{"choices":[{"message":{"content":""},"finish_reason":"content_filter"}]}`) + }}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(rec.Body.String(), "secret") || rec.Header().Get("x-llmgw-guard") != "blocked" || + rec.Header().Get("Content-Length") != "" { + t.Errorf("headers %v body %q", rec.Header(), rec.Body.String()) + } + if out.Usage == nil || out.Usage.TotalTokens != 10 { + t.Errorf("the engine's work was not charged: %+v", out.Usage) + } +} diff --git a/internal/router/proxy/proxy.go b/internal/router/proxy/proxy.go index a7eb2860..0be6249e 100644 --- a/internal/router/proxy/proxy.go +++ b/internal/router/proxy/proxy.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -164,6 +165,25 @@ type Request struct { // and a gateway that accumulated every reply in memory would be one long // answer away from trouble; a request that stores nothing must pay nothing. CaptureReply int + + // Filter, when set, sees a whole non-streamed answer and the headers + // about to be sent with it, and returns the body to send: a guard's + // last word before anything reaches the client. + Filter func(body []byte, h http.Header) []byte + // StreamFilter, when set, sees a streamed answer event by event and + // decides what is sent in its place. Nil costs nothing. + StreamFilter StreamFilter +} + +// StreamFilter watches a streamed answer. OnEvent gets each event's data — +// not the usage-only chunk, nor [DONE] — and returns the data of the events +// to send instead: itself to pass it on, none to hold it back, several to +// release what it held. stop ends the stream once those are sent: [DONE] +// follows and the engine is cancelled. OnEnd returns what is still held +// when the engine ends the stream. +type StreamFilter interface { + OnEvent(data string) (emit []string, stop bool) + OnEnd() []string } // Usage holds the token counts reported by the engine. @@ -196,6 +216,8 @@ type Outcome struct { // ReplyTruncated says the cap was reached, so nothing downstream records a // half answer as a whole one. ReplyTruncated bool + // Stopped says a StreamFilter ended the stream before the engine did. + Stopped bool } var ( @@ -264,6 +286,12 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response, r Requ } copyHeaders(w.Header(), resp.Header) + // Charged for what the engine did, whatever the client is shown. + usage := usageFromJSON(body) + if r.Filter != nil && resp.StatusCode == http.StatusOK { + body = r.Filter(body, w.Header()) + w.Header().Del("Content-Length") + } w.WriteHeader(resp.StatusCode) n, werr := w.Write(body) @@ -271,7 +299,7 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response, r Requ Wrote: n > 0, Status: resp.StatusCode, Bytes: int64(n), - Usage: usageFromJSON(body), + Usage: usage, } if r.CaptureReply > 0 && resp.StatusCode == http.StatusOK { out.Reply, out.ReplyTruncated = capReply(replyFromJSON(body), r.CaptureReply) @@ -328,7 +356,7 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt out.TTFT = time.Since(start) wd.setInterval(p.cfg.IdleTimeout) } - if r.CaptureReply > 0 { + if r.CaptureReply > 0 && r.StreamFilter == nil { reply.add(replyDelta(ev.data.String())) } if u := ev.usage(); u != nil { @@ -340,6 +368,29 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt continue } } + if r.StreamFilter != nil && !usageOnly(ev.data.String()) { + data := ev.data.String() + ev.reset() + var emit []string + stop := false + if data == "[DONE]" { + emit = append(r.StreamFilter.OnEnd(), "[DONE]") + } else if data != "" { + emit, stop = r.StreamFilter.OnEvent(data) + if stop { + emit = append(emit, "[DONE]") + } + } + if err := p.emit(w, flusher, &out, &reply, r, emit); err != nil { + return out, err + } + if stop { + out.Stopped = true + abort() + return out, nil + } + continue + } n, werr := io.WriteString(w, ev.string()) out.Bytes += int64(n) ev.reset() @@ -352,6 +403,12 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt if err != nil { if errors.Is(err, io.EOF) { + if r.StreamFilter != nil { + // The engine ended without [DONE]: what was held goes out. + if err := p.emit(w, flusher, &out, &reply, r, r.StreamFilter.OnEnd()); err != nil { + return out, err + } + } // Flush a trailing event that never got its blank line. if s := ev.string(); s != "" { n, _ := io.WriteString(w, s) @@ -371,6 +428,38 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt } } +// usageOnly reports whether an event is the usage chunk, which carries no +// choices and so nothing a filter judges. +func usageOnly(data string) bool { + if !strings.Contains(data, "\"usage\"") { + return false + } + var c struct { + Choices []json.RawMessage `json:"choices"` + } + return json.Unmarshal([]byte(data), &c) == nil && len(c.Choices) == 0 +} + +// emit writes events a StreamFilter chose, as data events, and captures the +// reply from what the client is actually sent. +func (p *Proxy) emit(w io.Writer, f http.Flusher, out *Outcome, reply *replyBuilder, r Request, data []string) error { + if len(data) == 0 { + return nil + } + for _, d := range data { + if r.CaptureReply > 0 && d != "[DONE]" { + reply.add(replyDelta(d)) + } + n, err := io.WriteString(w, "data: "+d+"\n\n") + out.Bytes += int64(n) + if err != nil { + return ErrClientGone + } + } + f.Flush() + return nil +} + // --- SSE event assembly --- // eventBuffer accumulates the lines of one SSE event. Events are relayed whole