Files
llmgw/internal/guard/output.go
T
Ilker ManapandClaude Opus 5.5 54c6b66cb9
CI / test (push) Successful in 9m48s
Fixes from review: the similar cache, restore, outside services, latency
- The similar-question cache embeds a question as its project would: a
  project kept in the cluster has external fallbacks refused, and an
  embedding model outside the cluster is not asked at all. The vectors are
  compared outside the cache's lock. The question's bound is in characters.
- The restorer puts values back in streamed tool calls' arguments (escaped
  for the JSON they are), completions' text and reasoning, each held across
  events on its own, and releases a held tail with the finish event. A
  whole answer is walked as JSON, so a value with a quote no longer breaks
  tool-call arguments.
- An outside guardrail service reads the user's turn as redacted, and a
  streamed answer once rather than at the finish chunk and again at the
  end. guardrails.md says it can stop a stream only under buffer_all.
- A replica's smoothed time to first token is forgotten after five minutes
  without an answer: one found slow and then given nothing was never tried
  again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:15:43 +01:00

586 lines
16 KiB
Go

package guard
import (
"context"
"encoding/json"
"slices"
"strings"
"sync"
"time"
)
// The tools a request offered, kept from Input until its answer is judged:
// a call's arguments are checked against the parameters declared for it.
type toolMemo struct {
mu sync.Mutex
byID map[string]toolSet
}
type toolSet struct {
schemas map[string]json.RawMessage
// answer is the schema response_format asks the answer to fit.
answer json.RawMessage
// user is the request's last user turn, for a judge reading the answer.
user string
at time.Time
}
func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, user string, now time.Time) {
if id == "" || len(tools) == 0 && answer == nil && user == "" {
return
}
m.mu.Lock()
defer m.mu.Unlock()
if m.byID == nil {
m.byID = map[string]toolSet{}
}
for k, v := range m.byID {
if now.Sub(v.at) > 10*time.Minute {
delete(m.byID, k)
}
}
set := toolSet{schemas: map[string]json.RawMessage{}, answer: answer, user: user, at: now}
for _, t := range tools {
set.schemas[t.name] = t.parameters
}
m.byID[id] = set
}
func (m *toolMemo) get(id string) toolSet {
m.mu.Lock()
defer m.mu.Unlock()
return m.byID[id]
}
func outputWatched(p Policy) bool {
o, t := p.Output, p.Tools
return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 || p.ValidateStructured ||
p.Judge != nil && p.Judge.Output != "" ||
slices.ContainsFunc(p.Services, Service.onOutput) ||
len(t.Allow) > 0 || len(t.Deny) > 0 || t.MaxCallsPerTurn > 0 || t.ValidateArguments
}
// Output is the filter for one answer, or nil when the policy in force
// watches no answers.
func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter {
p, names := g.Policy(s)
if !p.active() || !outputWatched(p) {
return nil
}
memo := g.tools.get(s.RequestID)
return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow",
schemas: memo.schemas, answer: memo.answer, user: memo.user, choices: map[int]*choiceState{}}
}
// outFilter judges one answer.
type outFilter struct {
// servicesAsked says the outside services have read the answer.
servicesAsked bool
ctx context.Context
g *Gate
p Policy
names string
shadow bool
schemas map[string]json.RawMessage
answer json.RawMessage
full strings.Builder // the whole answer, when it must fit a schema
user string // the request's last user turn, for the judge
sofar strings.Builder // the answer so far, for the judge
served strings.Builder // the answer so far, for outside services, asked once at its end
judged int // how much of it the judge has read
v Verdict
took time.Duration
stopped bool
held []string // events not yet released
sent bool // whether a chunk this filter made has gone out
template map[string]json.RawMessage
choices map[int]*choiceState
total int
}
type choiceState struct {
pending strings.Builder // content held, not yet judged
// carried says pending begins with text whose events were already
// replaced: it reaches the client only in a chunk this filter makes.
carried bool
calls map[int]*call
}
type call struct {
name string
args strings.Builder
}
func (f *outFilter) Verdict() Verdict {
v := f.v
v.Took = f.took
return v
}
func (f *outFilter) record(fs []finding) finding {
best, found := strictest(fs)
if found && best.action != Allow && rank(best.action) >= rank(f.v.Action) {
f.v = Verdict{Action: best.action, Stage: Output, Policy: f.names, Category: best.category,
Rule: best.rule, Shadow: f.shadow}
}
return best
}
// judgeText judges answer text, returning the decisive finding and the text
// to send in its place.
func (f *outFilter) judgeText(text string) (finding, string) {
start := time.Now()
defer func() { f.took += time.Since(start) }()
f.total += len(text)
fs, red := f.g.judge(text, f.p.Output)
if f.p.Output.MaxChars > 0 && f.total > f.p.Output.MaxChars {
fs = append(fs, finding{action: Block, category: "size", rule: "max_chars"})
}
best := f.record(fs)
if f.shadow || len(red) == 0 {
return best, text
}
f.g.countRedactions(red)
return best, newPlaceholders().rewrite(text, red)
}
// judgeCalls checks an answer's tool calls: allowed names, how many, and
// arguments that fit what the request declared.
func (f *outFilter) judgeCalls(calls []call) finding {
t := f.p.Tools
var fs []finding
if t.MaxCallsPerTurn > 0 && len(calls) > t.MaxCallsPerTurn {
fs = append(fs, finding{action: Block, category: "tool", rule: "max_calls_per_turn"})
}
for _, c := range calls {
if !toolAllowed(t, c.name) {
fs = append(fs, finding{action: Block, category: "tool", rule: "tool:" + c.name})
continue
}
if t.ValidateArguments {
var v any
if err := json.Unmarshal([]byte(c.args.String()), &v); err != nil {
fs = append(fs, finding{action: Block, category: "tool", rule: "arguments:" + c.name})
continue
}
if schema, ok := f.schemas[c.name]; ok {
if err := validate(schema, v); err != nil {
fs = append(fs, finding{action: Block, category: "tool", rule: "arguments:" + c.name})
}
}
}
// What a tool is told to do is answer text too.
if fd, _ := f.judgeText(c.args.String()); fd.action == Block {
fs = append(fs, fd)
}
}
return f.record(fs)
}
// judgeEvery is how much new answer text a streamed answer's judge waits
// for before reading it again.
const judgeEvery = 600
// askJudge has the judge read the answer so far, when the policy asks and
// enough has arrived since it last did, or at the end.
func (f *outFilter) askJudge(text string, final bool) finding {
jp := f.p.Judge
if jp == nil || jp.Output == "" {
return finding{}
}
f.sofar.WriteString(text)
if !final && f.sofar.Len()-f.judged < judgeEvery || f.sofar.Len() == f.judged {
return finding{}
}
f.judged = f.sofar.Len()
start := time.Now()
res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String())
f.took += time.Since(start)
if err != nil {
f.g.log.Warn("the judge gave no verdict on an answer", "judge", jp.Name(), "err", err)
f.g.noJudger(jp, err)
}
jf := judgeFinding(f.p, jp.Output, res, err)
if jf.failOpen {
f.g.m.failOpen.Inc()
}
return f.record([]finding{jf})
}
// askServices has the outside services read the whole answer, once, at its
// end. One that cannot be asked under fail: closed stops it.
func (f *outFilter) askServices(text string) finding {
if !slices.ContainsFunc(f.p.Services, Service.onOutput) || text == "" {
return finding{}
}
start := time.Now()
fs, err := f.g.askServices(f.ctx, f.p, false, text)
f.took += time.Since(start)
if err != nil {
fs = append(fs, finding{action: Block, category: "service", rule: "service:unavailable"})
}
return f.record(fs)
}
// judgeStructured checks a whole answer against the schema the request
// asked it to fit.
func (f *outFilter) judgeStructured(text string, enforce bool) finding {
if !f.p.ValidateStructured || f.answer == nil {
return finding{}
}
var v any
err := json.Unmarshal([]byte(text), &v)
if err == nil {
err = validate(f.answer, v)
}
if err == nil {
return finding{}
}
a := Block
if !enforce {
a = Flag
}
return f.record([]finding{{action: a, category: "structured", rule: "response_format"}})
}
// --- buffered ---
func (f *outFilter) Buffered(body []byte) []byte {
var top map[string]json.RawMessage
if json.Unmarshal(body, &top) != nil {
return body
}
var choices []map[string]json.RawMessage
if json.Unmarshal(top["choices"], &choices) != nil {
return body
}
blocked := false
changed := false
for _, ch := range choices {
var msg map[string]json.RawMessage
if json.Unmarshal(ch["message"], &msg) != nil {
if t, ok := str(ch["text"]); ok { // a completion
fd, out := f.judgeText(t)
blocked = blocked || fd.action == Block
if out != t {
ch["text"] = rawString(out)
changed = true
}
}
continue
}
if t, ok := str(msg["content"]); ok {
fd, out := f.judgeText(t)
blocked = blocked || fd.action == Block
if f.judgeStructured(t, true).action == Block {
blocked = true
}
if f.askJudge(t, true).action == Block {
blocked = true
}
if f.askServices(t).action == Block {
blocked = true
}
if out != t {
msg["content"] = rawString(out)
ch["message"], _ = json.Marshal(msg)
changed = true
}
}
var tcs []struct {
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
}
if json.Unmarshal(msg["tool_calls"], &tcs) == nil && len(tcs) > 0 {
calls := make([]call, len(tcs))
for i, tc := range tcs {
calls[i].name = tc.Function.Name
calls[i].args.WriteString(tc.Function.Arguments)
}
if f.judgeCalls(calls).action == Block {
blocked = true
}
}
}
if f.shadow {
return body
}
if blocked {
top["choices"] = json.RawMessage(`[{"index":0,"message":{"role":"assistant","content":""},"finish_reason":"content_filter"}]`)
out, _ := json.Marshal(top)
return out
}
if changed {
top["choices"], _ = json.Marshal(choices)
out, _ := json.Marshal(top)
return out
}
return body
}
// --- streamed ---
type chunk struct {
Choices []struct {
Index int `json:"index"`
Delta struct {
Content *string `json:"content"`
ToolCalls []struct {
Index int `json:"index"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
} `json:"tool_calls"`
} `json:"delta"`
Text *string `json:"text"`
FinishReason *string `json:"finish_reason"`
} `json:"choices"`
}
// releaseAt is how much held text is judged at once when no sentence ends.
const releaseAt = 200
func (f *outFilter) OnEvent(data string) ([]string, bool) {
if f.stopped {
return nil, true
}
var c chunk
if json.Unmarshal([]byte(data), &c) != nil {
return []string{data}, false
}
if f.template == nil {
_ = json.Unmarshal([]byte(data), &f.template)
}
finished := false
for _, ch := range c.Choices {
st := f.choices[ch.Index]
if st == nil {
st = &choiceState{calls: map[int]*call{}}
f.choices[ch.Index] = st
}
if ch.Delta.Content != nil {
st.pending.WriteString(*ch.Delta.Content)
}
if ch.Text != nil {
st.pending.WriteString(*ch.Text)
}
for _, tc := range ch.Delta.ToolCalls {
cl := st.calls[tc.Index]
if cl == nil {
cl = &call{}
st.calls[tc.Index] = cl
}
if tc.Function.Name != "" {
cl.name = tc.Function.Name
}
cl.args.WriteString(tc.Function.Arguments)
}
if ch.FinishReason != nil && *ch.FinishReason != "" {
finished = true
}
}
if f.shadow {
// Passed on at once; judged as it goes, to be recorded.
if finished || f.ready() {
f.flush(finished)
}
return []string{data}, false
}
f.held = append(f.held, data)
if finished || f.ready() {
return f.release(finished)
}
return nil, false
}
// ready reports whether held text has reached a place to judge it: a
// finished sentence, a line, or releaseAt characters — never while a tool
// call is still arriving, nor when the whole answer is to be held.
func (f *outFilter) ready() bool {
if f.p.Stream == "buffer_all" {
return false
}
for _, st := range f.choices {
if len(st.calls) > 0 {
return false
}
}
for _, st := range f.choices {
p := st.pending.String()
if len(p) >= releaseAt || strings.Contains(p, "\n") ||
strings.Contains(p, ". ") || strings.Contains(p, "! ") || strings.Contains(p, "? ") {
return true
}
}
return false
}
// flush judges what each choice holds — up to its last whitespace, keeping
// a word that may not be whole, unless final — and with final its tool calls.
// It returns whether anything blocks, each choice's text to send, and
// whether that differs from what the held events carry.
func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
blocked, differs := false, false
out := map[int]string{}
for i, st := range f.choices {
t := st.pending.String()
head, tail := t, ""
if !final {
cut := strings.LastIndexAny(t, " \n\t") + 1
head, tail = t[:cut], t[cut:]
}
st.pending.Reset()
st.pending.WriteString(tail)
if tail != "" || st.carried {
differs = true
}
st.carried = tail != ""
if f.answer != nil {
f.full.WriteString(head)
}
if i == 0 && f.askJudge(head, final).action == Block {
blocked = true
}
if i == 0 {
f.served.WriteString(head)
// Once: the finish chunk is final, and so is the end of the
// stream after it, which would ask again of the same answer.
if final && !f.servicesAsked {
f.servicesAsked = true
if f.askServices(f.served.String()).action == Block {
blocked = true
}
}
}
if head != "" {
fd, r := f.judgeText(head)
blocked = blocked || fd.action == Block
if r != head {
differs = true
}
out[i] = r
}
if final && i == 0 && f.answer != nil {
// Held whole, it can still be stopped; released a sentence at
// a time, it is flagged.
if f.judgeStructured(f.full.String(), f.p.Stream == "buffer_all").action == Block {
blocked = true
}
}
if final && len(st.calls) > 0 {
calls := make([]call, 0, len(st.calls))
for j := 0; j < len(st.calls); j++ {
if c := st.calls[j]; c != nil {
calls = append(calls, *c)
}
}
if f.judgeCalls(calls).action == Block {
blocked = true
}
st.calls = map[int]*call{}
}
}
return blocked, out, differs
}
func (f *outFilter) release(final bool) ([]string, bool) {
blocked, texts, differs := f.flush(final)
held := f.held
f.held = nil
if blocked {
f.stopped = true
return []string{f.synth(map[int]string{0: ""}, "content_filter")}, true
}
if !differs {
return held, false
}
// Rewritten, or cut short of a word still arriving: sent as one chunk
// per choice in place of the held ones, and what else they carried —
// tool calls, a finish — after.
var out []string
if len(texts) > 0 {
out = append(out, f.synth(texts, ""))
}
for _, h := range held {
var c chunk
if json.Unmarshal([]byte(h), &c) != nil {
continue
}
for _, ch := range c.Choices {
if len(ch.Delta.ToolCalls) > 0 || ch.FinishReason != nil && *ch.FinishReason != "" {
out = append(out, withoutContent(h))
break
}
}
}
return out, false
}
// synth is a chunk like the stream's own, carrying text per choice, or a
// finish reason.
func (f *outFilter) synth(texts map[int]string, finish string) string {
top := map[string]json.RawMessage{}
for k, v := range f.template {
if k != "choices" && k != "usage" {
top[k] = v
}
}
var choices []map[string]any
for i, t := range texts {
delta := map[string]any{"content": t}
if !f.sent {
delta["role"] = "assistant"
}
c := map[string]any{"index": i, "delta": delta, "finish_reason": nil}
if finish != "" {
c["delta"] = map[string]any{}
c["finish_reason"] = finish
}
choices = append(choices, c)
}
f.sent = true
top["choices"], _ = json.Marshal(choices)
b, _ := json.Marshal(top)
return string(b)
}
// withoutContent is a chunk with its text taken out, sent after a chunk
// that already carried it rewritten.
func withoutContent(data string) string {
var top map[string]json.RawMessage
if json.Unmarshal([]byte(data), &top) != nil {
return data
}
var choices []map[string]json.RawMessage
if json.Unmarshal(top["choices"], &choices) != nil {
return data
}
for _, c := range choices {
var d map[string]json.RawMessage
if json.Unmarshal(c["delta"], &d) == nil {
delete(d, "content")
c["delta"], _ = json.Marshal(d)
}
delete(c, "text")
}
top["choices"], _ = json.Marshal(choices)
b, _ := json.Marshal(top)
return string(b)
}
func (f *outFilter) OnEnd() []string {
switch {
case f.stopped:
return nil
case f.shadow:
f.flush(true)
return nil
}
out, _ := f.release(true)
return out
}