Add the guard interface, and filter hooks in the proxy for buffered and streamed answers
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
379074b6a5
commit
e701f92806
3 files changed
+325
-2
No files matched your search
@@ -0,0 +1,116 @@
|
||||
// Package guard is what the gateway asks about a request's text and its
|
||||
// answer's: whether to let it through, flag it, rewrite it or stop it.
|
||||
//
|
||||
// The router calls a Guard at two points. Input, once the request is
|
||||
// authorised, limited and has its stored conversation spliced in, and before
|
||||
// it is dispatched: the whole upstream body. Output, for the answer: a filter
|
||||
// over the response as it is relayed, buffered or streamed. A router without
|
||||
// a Guard calls nothing and pays nothing.
|
||||
//
|
||||
// Nothing here may log, record or return the text it judges. Verdicts carry
|
||||
// policy, rule and category names, never what matched.
|
||||
package guard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Action is what a verdict does to a request or a response.
|
||||
type Action string
|
||||
|
||||
const (
|
||||
// Allow lets it through untouched.
|
||||
Allow Action = ""
|
||||
// Flag lets it through, and says so: in a header and in the usage row.
|
||||
Flag Action = "flagged"
|
||||
// Redact lets it through rewritten.
|
||||
Redact Action = "redacted"
|
||||
// Block stops it.
|
||||
Block Action = "blocked"
|
||||
)
|
||||
|
||||
// Stage is where a verdict was reached.
|
||||
type Stage string
|
||||
|
||||
const (
|
||||
Input Stage = "input"
|
||||
Output Stage = "output"
|
||||
)
|
||||
|
||||
// Subject is who is asking, for which model: what a policy is chosen by.
|
||||
type Subject struct {
|
||||
RequestID string
|
||||
KeyID string
|
||||
UserID string
|
||||
ProjectID string
|
||||
TeamID string
|
||||
OrgID string
|
||||
// Model is the canonical model that will serve.
|
||||
Model string
|
||||
// Path is the API path asked: /v1/chat/completions, /v1/completions, ….
|
||||
Path string
|
||||
// Stream says whether the answer is streamed.
|
||||
Stream bool
|
||||
}
|
||||
|
||||
// Verdict is a judgement on one stage of one request.
|
||||
type Verdict struct {
|
||||
Action Action
|
||||
Stage Stage
|
||||
Policy string
|
||||
Rule string
|
||||
Category string
|
||||
// Message is what a blocked caller is told. It names the policy, never
|
||||
// the text that tripped it.
|
||||
Message string
|
||||
// Body replaces the request's upstream body when the input was
|
||||
// rewritten; nil leaves it as it was.
|
||||
Body []byte
|
||||
// Took is how long the judgement took.
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// Stricter reports whether v outranks w: block over redact over flag over
|
||||
// allow. A response judged at several points carries its strictest verdict.
|
||||
func (v Verdict) Stricter(w Verdict) bool { return rank(v.Action) > rank(w.Action) }
|
||||
|
||||
func rank(a Action) int {
|
||||
switch a {
|
||||
case Block:
|
||||
return 3
|
||||
case Redact:
|
||||
return 2
|
||||
case Flag:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Guard judges requests and their answers.
|
||||
type Guard interface {
|
||||
// Input judges the body about to be sent upstream. An error means the
|
||||
// guard could not judge; the router refuses the request rather than
|
||||
// serve it unjudged — a guard that wants to fail open says Allow.
|
||||
Input(ctx context.Context, s Subject, body []byte) (Verdict, error)
|
||||
// Output is the filter for one attempt's answer, or nil when nothing
|
||||
// watches it.
|
||||
Output(ctx context.Context, s Subject) OutputFilter
|
||||
}
|
||||
|
||||
// OutputFilter watches one answer as it is relayed.
|
||||
type OutputFilter interface {
|
||||
// Buffered sees a whole non-streamed response body before anything is
|
||||
// sent, and returns the body to send.
|
||||
Buffered(body []byte) []byte
|
||||
// OnEvent sees each streamed event's data — a chunk's JSON — and
|
||||
// returns the events to send in its place: itself to pass it on,
|
||||
// nothing to hold it back, more than one to release what it held. stop
|
||||
// ends the stream after those are sent.
|
||||
OnEvent(data string) (emit []string, stop bool)
|
||||
// OnEnd is called when the upstream stream ends, and returns what is
|
||||
// still held.
|
||||
OnEnd() []string
|
||||
// Verdict is the strictest judgement so far.
|
||||
Verdict() Verdict
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// holdTwo releases events two at a time, and stops at one saying "stop".
|
||||
type holdTwo struct {
|
||||
held []string
|
||||
ended bool
|
||||
}
|
||||
|
||||
func (h *holdTwo) OnEvent(data string) ([]string, bool) {
|
||||
if strings.Contains(data, "stop") {
|
||||
return []string{`{"choices":[{"index":0,"delta":{},"finish_reason":"content_filter"}]}`}, true
|
||||
}
|
||||
h.held = append(h.held, data)
|
||||
if len(h.held) < 2 {
|
||||
return nil, false
|
||||
}
|
||||
out := h.held
|
||||
h.held = nil
|
||||
return out, false
|
||||
}
|
||||
|
||||
func (h *holdTwo) OnEnd() []string {
|
||||
h.ended = true
|
||||
out := h.held
|
||||
h.held = nil
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAStreamFilterHoldsReleasesAndEnds(t *testing.T) {
|
||||
up, _ := sseServer(t, []string{
|
||||
dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"b"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"c"}}]}`),
|
||||
dataChunk(`{"choices":[],"usage":{"prompt_tokens":1,"completion_tokens":3,"total_tokens":4}}`),
|
||||
"data: [DONE]\n\n",
|
||||
}, time.Millisecond)
|
||||
f := &holdTwo{}
|
||||
rec := httptest.NewRecorder()
|
||||
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true,
|
||||
StreamFilter: f, CaptureReply: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{`"content":"a"`, `"content":"b"`, `"content":"c"`, `"usage"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("no %s in %q", want, body)
|
||||
}
|
||||
}
|
||||
if !f.ended || !strings.HasSuffix(body, "data: [DONE]\n\n") || strings.Count(body, "[DONE]") != 1 {
|
||||
t.Errorf("the held event did not go out before one [DONE]: %q", body)
|
||||
}
|
||||
if out.Usage == nil || out.Usage.TotalTokens != 4 || out.Reply != "abc" || out.Stopped {
|
||||
t.Errorf("outcome %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStreamFilterStopsTheStreamAndTheEngine(t *testing.T) {
|
||||
up, cancelled := sseServer(t, []string{
|
||||
dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"b"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"stop"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"never"}}]}`),
|
||||
dataChunk(`{"choices":[{"delta":{"content":"seen"}}]}`),
|
||||
"data: [DONE]\n\n",
|
||||
}, 20*time.Millisecond)
|
||||
rec := httptest.NewRecorder()
|
||||
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true,
|
||||
StreamFilter: &holdTwo{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !out.Stopped || !strings.Contains(body, `"finish_reason":"content_filter"`) ||
|
||||
!strings.HasSuffix(body, "data: [DONE]\n\n") || strings.Contains(body, "never") {
|
||||
t.Errorf("stopped=%v body %q", out.Stopped, body)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !cancelled.Load() && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !cancelled.Load() {
|
||||
t.Error("the engine went on generating after the stream was stopped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABufferedFilterRewritesTheBodyNotTheCharge(t *testing.T) {
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.WriteString(w, `{"choices":[{"message":{"content":"secret"}}],"usage":{"prompt_tokens":7,"completion_tokens":3}}`)
|
||||
}))
|
||||
defer up.Close()
|
||||
rec := httptest.NewRecorder()
|
||||
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`),
|
||||
Filter: func(body []byte, h http.Header) []byte {
|
||||
h.Set("x-llmgw-guard", "blocked")
|
||||
return []byte(`{"choices":[{"message":{"content":""},"finish_reason":"content_filter"}]}`)
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "secret") || rec.Header().Get("x-llmgw-guard") != "blocked" ||
|
||||
rec.Header().Get("Content-Length") != "" {
|
||||
t.Errorf("headers %v body %q", rec.Header(), rec.Body.String())
|
||||
}
|
||||
if out.Usage == nil || out.Usage.TotalTokens != 10 {
|
||||
t.Errorf("the engine's work was not charged: %+v", out.Usage)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -164,6 +165,25 @@ type Request struct {
|
||||
// and a gateway that accumulated every reply in memory would be one long
|
||||
// answer away from trouble; a request that stores nothing must pay nothing.
|
||||
CaptureReply int
|
||||
|
||||
// Filter, when set, sees a whole non-streamed answer and the headers
|
||||
// about to be sent with it, and returns the body to send: a guard's
|
||||
// last word before anything reaches the client.
|
||||
Filter func(body []byte, h http.Header) []byte
|
||||
// StreamFilter, when set, sees a streamed answer event by event and
|
||||
// decides what is sent in its place. Nil costs nothing.
|
||||
StreamFilter StreamFilter
|
||||
}
|
||||
|
||||
// StreamFilter watches a streamed answer. OnEvent gets each event's data —
|
||||
// not the usage-only chunk, nor [DONE] — and returns the data of the events
|
||||
// to send instead: itself to pass it on, none to hold it back, several to
|
||||
// release what it held. stop ends the stream once those are sent: [DONE]
|
||||
// follows and the engine is cancelled. OnEnd returns what is still held
|
||||
// when the engine ends the stream.
|
||||
type StreamFilter interface {
|
||||
OnEvent(data string) (emit []string, stop bool)
|
||||
OnEnd() []string
|
||||
}
|
||||
|
||||
// Usage holds the token counts reported by the engine.
|
||||
@@ -196,6 +216,8 @@ type Outcome struct {
|
||||
// ReplyTruncated says the cap was reached, so nothing downstream records a
|
||||
// half answer as a whole one.
|
||||
ReplyTruncated bool
|
||||
// Stopped says a StreamFilter ended the stream before the engine did.
|
||||
Stopped bool
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -264,6 +286,12 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response, r Requ
|
||||
}
|
||||
|
||||
copyHeaders(w.Header(), resp.Header)
|
||||
// Charged for what the engine did, whatever the client is shown.
|
||||
usage := usageFromJSON(body)
|
||||
if r.Filter != nil && resp.StatusCode == http.StatusOK {
|
||||
body = r.Filter(body, w.Header())
|
||||
w.Header().Del("Content-Length")
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
n, werr := w.Write(body)
|
||||
|
||||
@@ -271,7 +299,7 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response, r Requ
|
||||
Wrote: n > 0,
|
||||
Status: resp.StatusCode,
|
||||
Bytes: int64(n),
|
||||
Usage: usageFromJSON(body),
|
||||
Usage: usage,
|
||||
}
|
||||
if r.CaptureReply > 0 && resp.StatusCode == http.StatusOK {
|
||||
out.Reply, out.ReplyTruncated = capReply(replyFromJSON(body), r.CaptureReply)
|
||||
@@ -328,7 +356,7 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
out.TTFT = time.Since(start)
|
||||
wd.setInterval(p.cfg.IdleTimeout)
|
||||
}
|
||||
if r.CaptureReply > 0 {
|
||||
if r.CaptureReply > 0 && r.StreamFilter == nil {
|
||||
reply.add(replyDelta(ev.data.String()))
|
||||
}
|
||||
if u := ev.usage(); u != nil {
|
||||
@@ -340,6 +368,29 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.StreamFilter != nil && !usageOnly(ev.data.String()) {
|
||||
data := ev.data.String()
|
||||
ev.reset()
|
||||
var emit []string
|
||||
stop := false
|
||||
if data == "[DONE]" {
|
||||
emit = append(r.StreamFilter.OnEnd(), "[DONE]")
|
||||
} else if data != "" {
|
||||
emit, stop = r.StreamFilter.OnEvent(data)
|
||||
if stop {
|
||||
emit = append(emit, "[DONE]")
|
||||
}
|
||||
}
|
||||
if err := p.emit(w, flusher, &out, &reply, r, emit); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if stop {
|
||||
out.Stopped = true
|
||||
abort()
|
||||
return out, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
n, werr := io.WriteString(w, ev.string())
|
||||
out.Bytes += int64(n)
|
||||
ev.reset()
|
||||
@@ -352,6 +403,12 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
if r.StreamFilter != nil {
|
||||
// The engine ended without [DONE]: what was held goes out.
|
||||
if err := p.emit(w, flusher, &out, &reply, r, r.StreamFilter.OnEnd()); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
// Flush a trailing event that never got its blank line.
|
||||
if s := ev.string(); s != "" {
|
||||
n, _ := io.WriteString(w, s)
|
||||
@@ -371,6 +428,38 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
}
|
||||
}
|
||||
|
||||
// usageOnly reports whether an event is the usage chunk, which carries no
|
||||
// choices and so nothing a filter judges.
|
||||
func usageOnly(data string) bool {
|
||||
if !strings.Contains(data, "\"usage\"") {
|
||||
return false
|
||||
}
|
||||
var c struct {
|
||||
Choices []json.RawMessage `json:"choices"`
|
||||
}
|
||||
return json.Unmarshal([]byte(data), &c) == nil && len(c.Choices) == 0
|
||||
}
|
||||
|
||||
// emit writes events a StreamFilter chose, as data events, and captures the
|
||||
// reply from what the client is actually sent.
|
||||
func (p *Proxy) emit(w io.Writer, f http.Flusher, out *Outcome, reply *replyBuilder, r Request, data []string) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, d := range data {
|
||||
if r.CaptureReply > 0 && d != "[DONE]" {
|
||||
reply.add(replyDelta(d))
|
||||
}
|
||||
n, err := io.WriteString(w, "data: "+d+"\n\n")
|
||||
out.Bytes += int64(n)
|
||||
if err != nil {
|
||||
return ErrClientGone
|
||||
}
|
||||
}
|
||||
f.Flush()
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- SSE event assembly ---
|
||||
|
||||
// eventBuffer accumulates the lines of one SSE event. Events are relayed whole
|
||||
|
||||
Reference in new issue
Block a user