Opt-in capability checks and structured-answer validation in guard policies

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 12:39:40 +02:00
1 parent 2f07b2ba0b
commit dc788e5cda
9 files changed
+169 -11

No files matched your search

+12
View File
@@ -240,6 +240,18 @@ reading their arguments — that is the tool policy's filter, not the proxy's.
gains `GuardAction`, `GuardPolicy`, `GuardCategory`, `GuardMs`.
### B2. Package `internal/guard` (M)
**Mostly done (branch `guardrails-b`):** policies in server.yaml with
attachments in the store (`ctl guard attach`), narrowed by the limits' rule;
`guard/detect` (validated PII, secrets, injection through normalisation and
decoding); redaction with consistent placeholders before prefix routing;
tool allow/deny/strip, call count and argument schemas; capabilities and
structured output (opt-in); default `max_tokens`; pinned system prompt and
dropping the client's; `no_store` (threads, notes, prefix tree, external
models); shadow mode. **Left:** a `guard_events` table — the usage row's
`guard_*` columns carry action/policy/category/time, not rule and stage;
language identification; the re-identification map that restores
redacted values in answers.
- `Policy`, `Effective` resolution over the tenancy tree with the same
narrowing rule as limits (`internal/store/sqlite.go:383-416` `tighten`):
deny sets union, allow sets intersect, `mode` takes the strictest of
+10
View File
@@ -29,6 +29,8 @@ guard:
system_prompt: "You answer questions about Acme's products only."
drop_client_system: true
stream: sentence # sentence (default) or buffer_all
require_capabilities: true # no tools/images to models whose catalog entry lacks them
validate_structured: true # answers must fit response_format's json_schema
input:
pii: {email: redact, phone: redact, iban: block, tckn: redact}
injection: block
@@ -81,6 +83,14 @@ guard:
- In the answer, a call to a tool that is not allowed, more calls than
`max_calls_per_turn`, or arguments that do not fit the parameters the
request declared (`validate_arguments`) stop it.
- `require_capabilities` refuses tools offered to a model whose catalog
entry does not say `capabilities.tools`, and images to one without
`capabilities.vision`. It is opt-in because an entry that says nothing is
read as "cannot".
- `validate_structured` checks an answer against the `json_schema` the
request asked for in `response_format`. Buffered, or streamed with
`buffer_all`, an answer that does not fit is stopped; streamed a sentence
at a time it has gone out already, and is flagged.
## Where a policy applies
+32
View File
@@ -203,3 +203,35 @@ func (d *doc) capOutput(limit int) bool {
}
return changed
}
// hasImages reports whether any message carries an image part.
func (d *doc) hasImages() bool {
for _, m := range d.msgs {
var parts []struct {
Type string `json:"type"`
}
if json.Unmarshal(m["content"], &parts) == nil {
for _, p := range parts {
if p.Type == "image_url" || p.Type == "input_image" || p.Type == "image" {
return true
}
}
}
}
return false
}
// responseSchema is the JSON schema the request asks the answer to fit, if
// it asks for one.
func (d *doc) responseSchema() json.RawMessage {
var rf struct {
Type string `json:"type"`
JSONSchema struct {
Schema json.RawMessage `json:"schema"`
} `json:"json_schema"`
}
if raw, ok := d.top["response_format"]; !ok || json.Unmarshal(raw, &rf) != nil || rf.Type != "json_schema" {
return nil
}
return rf.JSONSchema.Schema
}
+16 -1
View File
@@ -2,6 +2,7 @@ package guard
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"regexp"
@@ -305,8 +306,22 @@ func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error)
}
}
var answer json.RawMessage
if p.ValidateStructured {
answer = d.responseSchema()
}
var offered []tool
if p.Tools.ValidateArguments {
g.tools.put(s.RequestID, d.tools(), start)
offered = d.tools()
}
g.tools.put(s.RequestID, offered, answer, start)
if p.RequireCapabilities {
if len(d.tools()) > 0 && !s.Tools {
fs = append(fs, finding{action: Block, category: "capability", rule: "tools"})
}
if d.hasImages() && !s.Vision {
fs = append(fs, finding{action: Block, category: "capability", rule: "vision"})
}
}
segs := d.segments()
+28
View File
@@ -250,3 +250,31 @@ func TestNoStoreIsDemandedOnlyWhenEnforced(t *testing.T) {
t.Error("no_store")
}
}
func TestCapabilitiesAndStructuredAnswersWhenAskedFor(t *testing.T) {
g := gate(t, lib(map[string]Policy{"p": {RequireCapabilities: true, ValidateStructured: true}}, "p"))
ctx := context.Background()
tools := []byte(`{"messages":[{"role":"user","content":"x"}],"tools":[{"type":"function","function":{"name":"f"}}]}`)
if v, _ := g.Input(ctx, Subject{}, tools); v.Action != Block || v.Rule != "tools" {
t.Errorf("tools to a model without them: %+v", v)
}
if v, _ := g.Input(ctx, Subject{Tools: true}, tools); v.Action == Block {
t.Errorf("tools to a model with them: %+v", v)
}
img := []byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"what"},{"type":"image_url","image_url":{"url":"data:"}}]}]}`)
if v, _ := g.Input(ctx, Subject{}, img); v.Action != Block || v.Rule != "vision" {
t.Errorf("an image to a model without vision: %+v", v)
}
s := Subject{RequestID: "r9", Tools: true}
_, _ = g.Input(ctx, s, []byte(`{"messages":[],"response_format":{"type":"json_schema","json_schema":{"name":"x",
"schema":{"type":"object","required":["n"],"properties":{"n":{"type":"integer"}}}}}}`))
for content, block := range map[string]bool{`{"n": 3}`: false, `{"m": 3}`: true, `not json`: true} {
f := g.Output(ctx, s)
b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
f.Buffered(b)
if (f.Verdict().Action == Block) != block {
t.Errorf("%s: %+v", content, f.Verdict())
}
}
}
+2
View File
@@ -52,6 +52,8 @@ type Subject struct {
Path string
// Stream says whether the answer is streamed.
Stream bool
// Tools and Vision are what the model's catalog entry says it can do.
Tools, Vision bool
}
// Verdict is a judgement on one stage of one request.
+47 -8
View File
@@ -17,11 +17,13 @@ type toolMemo struct {
type toolSet struct {
schemas map[string]json.RawMessage
at time.Time
// answer is the schema response_format asks the answer to fit.
answer json.RawMessage
at time.Time
}
func (m *toolMemo) put(id string, tools []tool, now time.Time) {
if id == "" || len(tools) == 0 {
func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, now time.Time) {
if id == "" || len(tools) == 0 && answer == nil {
return
}
m.mu.Lock()
@@ -34,22 +36,22 @@ func (m *toolMemo) put(id string, tools []tool, now time.Time) {
delete(m.byID, k)
}
}
set := toolSet{schemas: map[string]json.RawMessage{}, at: now}
set := toolSet{schemas: map[string]json.RawMessage{}, answer: answer, at: now}
for _, t := range tools {
set.schemas[t.name] = t.parameters
}
m.byID[id] = set
}
func (m *toolMemo) get(id string) map[string]json.RawMessage {
func (m *toolMemo) get(id string) toolSet {
m.mu.Lock()
defer m.mu.Unlock()
return m.byID[id].schemas
return m.byID[id]
}
func outputWatched(p Policy) bool {
o, t := p.Output, p.Tools
return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 ||
return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 || p.ValidateStructured ||
len(t.Allow) > 0 || len(t.Deny) > 0 || t.MaxCallsPerTurn > 0 || t.ValidateArguments
}
@@ -60,8 +62,9 @@ func (g *Gate) Output(_ context.Context, s Subject) OutputFilter {
if !p.active() || !outputWatched(p) {
return nil
}
memo := g.tools.get(s.RequestID)
return &outFilter{g: g, p: p, names: names, shadow: p.Mode == "shadow",
schemas: g.tools.get(s.RequestID), choices: map[int]*choiceState{}}
schemas: memo.schemas, answer: memo.answer, choices: map[int]*choiceState{}}
}
// outFilter judges one answer.
@@ -71,6 +74,8 @@ type outFilter struct {
names string
shadow bool
schemas map[string]json.RawMessage
answer json.RawMessage
full strings.Builder // the whole answer, when it must fit a schema
v Verdict
took time.Duration
stopped bool
@@ -157,6 +162,27 @@ func (f *outFilter) judgeCalls(calls []call) finding {
return f.record(fs)
}
// judgeStructured checks a whole answer against the schema the request
// asked it to fit.
func (f *outFilter) judgeStructured(text string, enforce bool) finding {
if !f.p.ValidateStructured || f.answer == nil {
return finding{}
}
var v any
err := json.Unmarshal([]byte(text), &v)
if err == nil {
err = validate(f.answer, v)
}
if err == nil {
return finding{}
}
a := Block
if !enforce {
a = Flag
}
return f.record([]finding{{action: a, category: "structured", rule: "response_format"}})
}
// --- buffered ---
func (f *outFilter) Buffered(body []byte) []byte {
@@ -186,6 +212,9 @@ func (f *outFilter) Buffered(body []byte) []byte {
if t, ok := str(msg["content"]); ok {
fd, out := f.judgeText(t)
blocked = blocked || fd.action == Block
if f.judgeStructured(t, true).action == Block {
blocked = true
}
if out != t {
msg["content"] = rawString(out)
ch["message"], _ = json.Marshal(msg)
@@ -342,6 +371,9 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
if tail != "" {
differs = true
}
if f.answer != nil {
f.full.WriteString(head)
}
if head != "" {
fd, r := f.judgeText(head)
blocked = blocked || fd.action == Block
@@ -350,6 +382,13 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
}
out[i] = r
}
if final && i == 0 && f.answer != nil {
// Held whole, it can still be stopped; released a sentence at
// a time, it is flagged.
if f.judgeStructured(f.full.String(), f.p.Stream == "buffer_all").action == Block {
blocked = true
}
}
if final && len(st.calls) > 0 {
calls := make([]call, 0, len(st.calls))
for j := 0; j < len(st.calls); j++ {
+10
View File
@@ -53,6 +53,14 @@ type Policy struct {
// DropClientSystem removes the caller's own system and developer
// messages, so only the pinned prompt instructs the model.
DropClientSystem bool `koanf:"drop_client_system"`
// RequireCapabilities refuses a request offering tools, or sending
// images, to a model whose catalog entry does not say it takes them.
RequireCapabilities bool `koanf:"require_capabilities"`
// ValidateStructured checks an answer against the JSON schema the
// request asked for in response_format: one that does not fit is
// stopped — or, streamed a sentence at a time, flagged, having been
// sent already.
ValidateStructured bool `koanf:"validate_structured"`
// Stream: sentence scans a streamed answer a sentence at a time,
// releasing each once it passes; buffer_all holds the whole answer
// until it is judged. Empty is sentence.
@@ -179,6 +187,8 @@ func Merge(names []string, ps []Policy) Policy {
out.Stream = stricterOf(out.Stream, p.Stream, []string{"", "sentence", "buffer_all"})
out.NoStore = out.NoStore || p.NoStore
out.DropClientSystem = out.DropClientSystem || p.DropClientSystem
out.RequireCapabilities = out.RequireCapabilities || p.RequireCapabilities
out.ValidateStructured = out.ValidateStructured || p.ValidateStructured
out.MaxOutputTokens = minSet(out.MaxOutputTokens, p.MaxOutputTokens)
if p.SystemPrompt != "" {
prompts = append(prompts, p.SystemPrompt)
+12 -2
View File
@@ -31,6 +31,16 @@ func (p *parsedRequest) guardSubject(model string) guard.Subject {
}
}
// subject is a request as a guard sees it, with what the model serving it
// can do.
func (r *Router) subject(req *parsedRequest, model string) guard.Subject {
s := req.guardSubject(model)
if _, entry := r.resolve(model); entry.Name != "" {
s.Tools, s.Vision = entry.Capabilities.Tools, entry.Capabilities.Vision
}
return s
}
func setGuardHeaders(h http.Header, v guard.Verdict) {
if v.Action == guard.Allow || v.Shadow {
return
@@ -63,7 +73,7 @@ const errNoStoreMsg = "the gateway's policy for this caller keeps nothing: no st
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
model, _ := r.resolve(req.Model)
t := time.Now()
v, err := r.guard.Input(ctx, req.guardSubject(model), req.UpstreamBody)
v, err := r.guard.Input(ctx, r.subject(req, model), req.UpstreamBody)
if v.Took == 0 {
v.Took = time.Since(t)
}
@@ -139,7 +149,7 @@ func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model str
if r.guard == nil {
return nil
}
f := r.guard.Output(ctx, req.guardSubject(model))
f := r.guard.Output(ctx, r.subject(req, model))
if f == nil {
return nil
}