Opt-in capability checks and structured-answer validation in guard policies
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2f07b2ba0b
commit
dc788e5cda
9 files changed
+169
-11
No files matched your search
+12
@@ -240,6 +240,18 @@ reading their arguments — that is the tool policy's filter, not the proxy's.
|
||||
gains `GuardAction`, `GuardPolicy`, `GuardCategory`, `GuardMs`.
|
||||
|
||||
### B2. Package `internal/guard` (M)
|
||||
**Mostly done (branch `guardrails-b`):** policies in server.yaml with
|
||||
attachments in the store (`ctl guard attach`), narrowed by the limits' rule;
|
||||
`guard/detect` (validated PII, secrets, injection through normalisation and
|
||||
decoding); redaction with consistent placeholders before prefix routing;
|
||||
tool allow/deny/strip, call count and argument schemas; capabilities and
|
||||
structured output (opt-in); default `max_tokens`; pinned system prompt and
|
||||
dropping the client's; `no_store` (threads, notes, prefix tree, external
|
||||
models); shadow mode. **Left:** a `guard_events` table — the usage row's
|
||||
`guard_*` columns carry action/policy/category/time, not rule and stage;
|
||||
language identification; the re-identification map that restores
|
||||
redacted values in answers.
|
||||
|
||||
- `Policy`, `Effective` resolution over the tenancy tree with the same
|
||||
narrowing rule as limits (`internal/store/sqlite.go:383-416` `tighten`):
|
||||
deny sets union, allow sets intersect, `mode` takes the strictest of
|
||||
|
||||
@@ -29,6 +29,8 @@ guard:
|
||||
system_prompt: "You answer questions about Acme's products only."
|
||||
drop_client_system: true
|
||||
stream: sentence # sentence (default) or buffer_all
|
||||
require_capabilities: true # no tools/images to models whose catalog entry lacks them
|
||||
validate_structured: true # answers must fit response_format's json_schema
|
||||
input:
|
||||
pii: {email: redact, phone: redact, iban: block, tckn: redact}
|
||||
injection: block
|
||||
@@ -81,6 +83,14 @@ guard:
|
||||
- In the answer, a call to a tool that is not allowed, more calls than
|
||||
`max_calls_per_turn`, or arguments that do not fit the parameters the
|
||||
request declared (`validate_arguments`) stop it.
|
||||
- `require_capabilities` refuses tools offered to a model whose catalog
|
||||
entry does not say `capabilities.tools`, and images to one without
|
||||
`capabilities.vision`. It is opt-in because an entry that says nothing is
|
||||
read as "cannot".
|
||||
- `validate_structured` checks an answer against the `json_schema` the
|
||||
request asked for in `response_format`. Buffered, or streamed with
|
||||
`buffer_all`, an answer that does not fit is stopped; streamed a sentence
|
||||
at a time it has gone out already, and is flagged.
|
||||
|
||||
## Where a policy applies
|
||||
|
||||
|
||||
@@ -203,3 +203,35 @@ func (d *doc) capOutput(limit int) bool {
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// hasImages reports whether any message carries an image part.
|
||||
func (d *doc) hasImages() bool {
|
||||
for _, m := range d.msgs {
|
||||
var parts []struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
if json.Unmarshal(m["content"], &parts) == nil {
|
||||
for _, p := range parts {
|
||||
if p.Type == "image_url" || p.Type == "input_image" || p.Type == "image" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// responseSchema is the JSON schema the request asks the answer to fit, if
|
||||
// it asks for one.
|
||||
func (d *doc) responseSchema() json.RawMessage {
|
||||
var rf struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema struct {
|
||||
Schema json.RawMessage `json:"schema"`
|
||||
} `json:"json_schema"`
|
||||
}
|
||||
if raw, ok := d.top["response_format"]; !ok || json.Unmarshal(raw, &rf) != nil || rf.Type != "json_schema" {
|
||||
return nil
|
||||
}
|
||||
return rf.JSONSchema.Schema
|
||||
}
|
||||
+16
-1
@@ -2,6 +2,7 @@ package guard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
@@ -305,8 +306,22 @@ func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error)
|
||||
}
|
||||
}
|
||||
|
||||
var answer json.RawMessage
|
||||
if p.ValidateStructured {
|
||||
answer = d.responseSchema()
|
||||
}
|
||||
var offered []tool
|
||||
if p.Tools.ValidateArguments {
|
||||
g.tools.put(s.RequestID, d.tools(), start)
|
||||
offered = d.tools()
|
||||
}
|
||||
g.tools.put(s.RequestID, offered, answer, start)
|
||||
if p.RequireCapabilities {
|
||||
if len(d.tools()) > 0 && !s.Tools {
|
||||
fs = append(fs, finding{action: Block, category: "capability", rule: "tools"})
|
||||
}
|
||||
if d.hasImages() && !s.Vision {
|
||||
fs = append(fs, finding{action: Block, category: "capability", rule: "vision"})
|
||||
}
|
||||
}
|
||||
|
||||
segs := d.segments()
|
||||
|
||||
@@ -250,3 +250,31 @@ func TestNoStoreIsDemandedOnlyWhenEnforced(t *testing.T) {
|
||||
t.Error("no_store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapabilitiesAndStructuredAnswersWhenAskedFor(t *testing.T) {
|
||||
g := gate(t, lib(map[string]Policy{"p": {RequireCapabilities: true, ValidateStructured: true}}, "p"))
|
||||
ctx := context.Background()
|
||||
tools := []byte(`{"messages":[{"role":"user","content":"x"}],"tools":[{"type":"function","function":{"name":"f"}}]}`)
|
||||
if v, _ := g.Input(ctx, Subject{}, tools); v.Action != Block || v.Rule != "tools" {
|
||||
t.Errorf("tools to a model without them: %+v", v)
|
||||
}
|
||||
if v, _ := g.Input(ctx, Subject{Tools: true}, tools); v.Action == Block {
|
||||
t.Errorf("tools to a model with them: %+v", v)
|
||||
}
|
||||
img := []byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"what"},{"type":"image_url","image_url":{"url":"data:"}}]}]}`)
|
||||
if v, _ := g.Input(ctx, Subject{}, img); v.Action != Block || v.Rule != "vision" {
|
||||
t.Errorf("an image to a model without vision: %+v", v)
|
||||
}
|
||||
|
||||
s := Subject{RequestID: "r9", Tools: true}
|
||||
_, _ = g.Input(ctx, s, []byte(`{"messages":[],"response_format":{"type":"json_schema","json_schema":{"name":"x",
|
||||
"schema":{"type":"object","required":["n"],"properties":{"n":{"type":"integer"}}}}}}`))
|
||||
for content, block := range map[string]bool{`{"n": 3}`: false, `{"m": 3}`: true, `not json`: true} {
|
||||
f := g.Output(ctx, s)
|
||||
b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||
f.Buffered(b)
|
||||
if (f.Verdict().Action == Block) != block {
|
||||
t.Errorf("%s: %+v", content, f.Verdict())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,8 @@ type Subject struct {
|
||||
Path string
|
||||
// Stream says whether the answer is streamed.
|
||||
Stream bool
|
||||
// Tools and Vision are what the model's catalog entry says it can do.
|
||||
Tools, Vision bool
|
||||
}
|
||||
|
||||
// Verdict is a judgement on one stage of one request.
|
||||
|
||||
@@ -17,11 +17,13 @@ type toolMemo struct {
|
||||
|
||||
type toolSet struct {
|
||||
schemas map[string]json.RawMessage
|
||||
at time.Time
|
||||
// answer is the schema response_format asks the answer to fit.
|
||||
answer json.RawMessage
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (m *toolMemo) put(id string, tools []tool, now time.Time) {
|
||||
if id == "" || len(tools) == 0 {
|
||||
func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, now time.Time) {
|
||||
if id == "" || len(tools) == 0 && answer == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
@@ -34,22 +36,22 @@ func (m *toolMemo) put(id string, tools []tool, now time.Time) {
|
||||
delete(m.byID, k)
|
||||
}
|
||||
}
|
||||
set := toolSet{schemas: map[string]json.RawMessage{}, at: now}
|
||||
set := toolSet{schemas: map[string]json.RawMessage{}, answer: answer, at: now}
|
||||
for _, t := range tools {
|
||||
set.schemas[t.name] = t.parameters
|
||||
}
|
||||
m.byID[id] = set
|
||||
}
|
||||
|
||||
func (m *toolMemo) get(id string) map[string]json.RawMessage {
|
||||
func (m *toolMemo) get(id string) toolSet {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.byID[id].schemas
|
||||
return m.byID[id]
|
||||
}
|
||||
|
||||
func outputWatched(p Policy) bool {
|
||||
o, t := p.Output, p.Tools
|
||||
return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 ||
|
||||
return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 || p.ValidateStructured ||
|
||||
len(t.Allow) > 0 || len(t.Deny) > 0 || t.MaxCallsPerTurn > 0 || t.ValidateArguments
|
||||
}
|
||||
|
||||
@@ -60,8 +62,9 @@ func (g *Gate) Output(_ context.Context, s Subject) OutputFilter {
|
||||
if !p.active() || !outputWatched(p) {
|
||||
return nil
|
||||
}
|
||||
memo := g.tools.get(s.RequestID)
|
||||
return &outFilter{g: g, p: p, names: names, shadow: p.Mode == "shadow",
|
||||
schemas: g.tools.get(s.RequestID), choices: map[int]*choiceState{}}
|
||||
schemas: memo.schemas, answer: memo.answer, choices: map[int]*choiceState{}}
|
||||
}
|
||||
|
||||
// outFilter judges one answer.
|
||||
@@ -71,6 +74,8 @@ type outFilter struct {
|
||||
names string
|
||||
shadow bool
|
||||
schemas map[string]json.RawMessage
|
||||
answer json.RawMessage
|
||||
full strings.Builder // the whole answer, when it must fit a schema
|
||||
v Verdict
|
||||
took time.Duration
|
||||
stopped bool
|
||||
@@ -157,6 +162,27 @@ func (f *outFilter) judgeCalls(calls []call) finding {
|
||||
return f.record(fs)
|
||||
}
|
||||
|
||||
// judgeStructured checks a whole answer against the schema the request
|
||||
// asked it to fit.
|
||||
func (f *outFilter) judgeStructured(text string, enforce bool) finding {
|
||||
if !f.p.ValidateStructured || f.answer == nil {
|
||||
return finding{}
|
||||
}
|
||||
var v any
|
||||
err := json.Unmarshal([]byte(text), &v)
|
||||
if err == nil {
|
||||
err = validate(f.answer, v)
|
||||
}
|
||||
if err == nil {
|
||||
return finding{}
|
||||
}
|
||||
a := Block
|
||||
if !enforce {
|
||||
a = Flag
|
||||
}
|
||||
return f.record([]finding{{action: a, category: "structured", rule: "response_format"}})
|
||||
}
|
||||
|
||||
// --- buffered ---
|
||||
|
||||
func (f *outFilter) Buffered(body []byte) []byte {
|
||||
@@ -186,6 +212,9 @@ func (f *outFilter) Buffered(body []byte) []byte {
|
||||
if t, ok := str(msg["content"]); ok {
|
||||
fd, out := f.judgeText(t)
|
||||
blocked = blocked || fd.action == Block
|
||||
if f.judgeStructured(t, true).action == Block {
|
||||
blocked = true
|
||||
}
|
||||
if out != t {
|
||||
msg["content"] = rawString(out)
|
||||
ch["message"], _ = json.Marshal(msg)
|
||||
@@ -342,6 +371,9 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
|
||||
if tail != "" {
|
||||
differs = true
|
||||
}
|
||||
if f.answer != nil {
|
||||
f.full.WriteString(head)
|
||||
}
|
||||
if head != "" {
|
||||
fd, r := f.judgeText(head)
|
||||
blocked = blocked || fd.action == Block
|
||||
@@ -350,6 +382,13 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
|
||||
}
|
||||
out[i] = r
|
||||
}
|
||||
if final && i == 0 && f.answer != nil {
|
||||
// Held whole, it can still be stopped; released a sentence at
|
||||
// a time, it is flagged.
|
||||
if f.judgeStructured(f.full.String(), f.p.Stream == "buffer_all").action == Block {
|
||||
blocked = true
|
||||
}
|
||||
}
|
||||
if final && len(st.calls) > 0 {
|
||||
calls := make([]call, 0, len(st.calls))
|
||||
for j := 0; j < len(st.calls); j++ {
|
||||
|
||||
@@ -53,6 +53,14 @@ type Policy struct {
|
||||
// DropClientSystem removes the caller's own system and developer
|
||||
// messages, so only the pinned prompt instructs the model.
|
||||
DropClientSystem bool `koanf:"drop_client_system"`
|
||||
// RequireCapabilities refuses a request offering tools, or sending
|
||||
// images, to a model whose catalog entry does not say it takes them.
|
||||
RequireCapabilities bool `koanf:"require_capabilities"`
|
||||
// ValidateStructured checks an answer against the JSON schema the
|
||||
// request asked for in response_format: one that does not fit is
|
||||
// stopped — or, streamed a sentence at a time, flagged, having been
|
||||
// sent already.
|
||||
ValidateStructured bool `koanf:"validate_structured"`
|
||||
// Stream: sentence scans a streamed answer a sentence at a time,
|
||||
// releasing each once it passes; buffer_all holds the whole answer
|
||||
// until it is judged. Empty is sentence.
|
||||
@@ -179,6 +187,8 @@ func Merge(names []string, ps []Policy) Policy {
|
||||
out.Stream = stricterOf(out.Stream, p.Stream, []string{"", "sentence", "buffer_all"})
|
||||
out.NoStore = out.NoStore || p.NoStore
|
||||
out.DropClientSystem = out.DropClientSystem || p.DropClientSystem
|
||||
out.RequireCapabilities = out.RequireCapabilities || p.RequireCapabilities
|
||||
out.ValidateStructured = out.ValidateStructured || p.ValidateStructured
|
||||
out.MaxOutputTokens = minSet(out.MaxOutputTokens, p.MaxOutputTokens)
|
||||
if p.SystemPrompt != "" {
|
||||
prompts = append(prompts, p.SystemPrompt)
|
||||
|
||||
@@ -31,6 +31,16 @@ func (p *parsedRequest) guardSubject(model string) guard.Subject {
|
||||
}
|
||||
}
|
||||
|
||||
// subject is a request as a guard sees it, with what the model serving it
|
||||
// can do.
|
||||
func (r *Router) subject(req *parsedRequest, model string) guard.Subject {
|
||||
s := req.guardSubject(model)
|
||||
if _, entry := r.resolve(model); entry.Name != "" {
|
||||
s.Tools, s.Vision = entry.Capabilities.Tools, entry.Capabilities.Vision
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func setGuardHeaders(h http.Header, v guard.Verdict) {
|
||||
if v.Action == guard.Allow || v.Shadow {
|
||||
return
|
||||
@@ -63,7 +73,7 @@ const errNoStoreMsg = "the gateway's policy for this caller keeps nothing: no st
|
||||
func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *parsedRequest, start time.Time) bool {
|
||||
model, _ := r.resolve(req.Model)
|
||||
t := time.Now()
|
||||
v, err := r.guard.Input(ctx, req.guardSubject(model), req.UpstreamBody)
|
||||
v, err := r.guard.Input(ctx, r.subject(req, model), req.UpstreamBody)
|
||||
if v.Took == 0 {
|
||||
v.Took = time.Since(t)
|
||||
}
|
||||
@@ -139,7 +149,7 @@ func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model str
|
||||
if r.guard == nil {
|
||||
return nil
|
||||
}
|
||||
f := r.guard.Output(ctx, req.guardSubject(model))
|
||||
f := r.guard.Output(ctx, r.subject(req, model))
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user