Guardrails in the console
The console never mentioned them. A Guardrails view shows each policy with its mode, fail behaviour and checks, whether its judge models have replicas ready, where policies are attached (attach and detach from it), and what they did in a window. The admin API gains GET /api/v1/guard/events, and GET /api/v1/guard says what each policy does. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
77fb48ba32
commit
c5b34978f5
9 files changed
+644
-1
No files matched your search
@@ -4,11 +4,14 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// GuardStore is where policy attachments are kept.
|
||||
@@ -34,6 +37,114 @@ type GuardView struct {
|
||||
Policies []string `json:"policies"`
|
||||
Default []string `json:"default"`
|
||||
Attachments []GuardAttachmentView `json:"attachments"`
|
||||
// Details says what each policy does, in the order of Policies.
|
||||
Details []GuardPolicyView `json:"details"`
|
||||
}
|
||||
|
||||
// GuardPolicyView is what one policy does, in brief.
|
||||
type GuardPolicyView struct {
|
||||
Name string `json:"name"`
|
||||
// Mode is enforce, shadow or off; Fail is open or closed.
|
||||
Mode string `json:"mode"`
|
||||
Fail string `json:"fail"`
|
||||
NoStore bool `json:"no_store,omitempty"`
|
||||
// Judge is the judge model, when one reads the text.
|
||||
Judge string `json:"judge,omitempty"`
|
||||
// Checks are what it looks for, one line each: "input: pii email → redact".
|
||||
Checks []string `json:"checks"`
|
||||
}
|
||||
|
||||
// GuardHistory is what the guardrails did, from the usage record; the store
|
||||
// has it, and a store that does not leaves the events endpoint unanswered.
|
||||
type GuardHistory interface {
|
||||
GuardEvents(ctx context.Context, since time.Time, limit int) ([]store.GuardEvent, error)
|
||||
GuardCounts(ctx context.Context, since time.Time) ([]store.GuardCount, error)
|
||||
}
|
||||
|
||||
// GuardEventsView is what the guardrails did since a moment: totals, and the
|
||||
// newest requests.
|
||||
type GuardEventsView struct {
|
||||
Since time.Time `json:"since"`
|
||||
Counts []GuardCountView `json:"counts"`
|
||||
Events []GuardEventView `json:"events"`
|
||||
}
|
||||
|
||||
// GuardCountView is how many requests one policy did one thing to.
|
||||
type GuardCountView struct {
|
||||
Project string `json:"project,omitempty"`
|
||||
Action string `json:"action"`
|
||||
Policy string `json:"policy"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// GuardEventView is one request the guardrails did something to.
|
||||
type GuardEventView struct {
|
||||
At time.Time `json:"at"`
|
||||
RequestID string `json:"request_id"`
|
||||
Project string `json:"project,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Status int `json:"status"`
|
||||
Action string `json:"action"`
|
||||
Policy string `json:"policy"`
|
||||
Category string `json:"category,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms"`
|
||||
}
|
||||
|
||||
// policyView sums one policy up.
|
||||
func policyView(name string, p guard.Policy) GuardPolicyView {
|
||||
v := GuardPolicyView{Name: name, Mode: or(p.Mode, "enforce"), Fail: or(p.Fail, "open"), NoStore: p.NoStore, Checks: []string{}}
|
||||
rules := func(dir string, r guard.Rules) {
|
||||
for _, kind := range slices.Sorted(maps.Keys(r.PII)) {
|
||||
v.Checks = append(v.Checks, fmt.Sprintf("%s: pii %s → %s", dir, kind, r.PII[kind]))
|
||||
}
|
||||
if r.Injection != "" {
|
||||
v.Checks = append(v.Checks, fmt.Sprintf("%s: prompt injection → %s", dir, r.Injection))
|
||||
}
|
||||
for _, d := range r.Deny {
|
||||
v.Checks = append(v.Checks, fmt.Sprintf("%s: pattern %s → %s", dir, d.Name, or(d.Action, "block")))
|
||||
}
|
||||
if r.MaxChars > 0 {
|
||||
v.Checks = append(v.Checks, fmt.Sprintf("%s: at most %d characters", dir, r.MaxChars))
|
||||
}
|
||||
}
|
||||
rules("input", p.Input)
|
||||
rules("output", p.Output)
|
||||
if j := p.Judge; j != nil {
|
||||
v.Judge = j.Model
|
||||
var dirs []string
|
||||
if j.Input != "" {
|
||||
dirs = append(dirs, "input → "+j.Input)
|
||||
}
|
||||
if j.Output != "" {
|
||||
dirs = append(dirs, "output → "+j.Output)
|
||||
}
|
||||
line := "judge " + j.Model + ": " + strings.Join(dirs, ", ")
|
||||
if len(j.Categories) > 0 {
|
||||
line += " (" + strings.Join(j.Categories, " ") + ")"
|
||||
}
|
||||
v.Checks = append(v.Checks, line)
|
||||
}
|
||||
if len(p.Tools.Allow) > 0 {
|
||||
v.Checks = append(v.Checks, "tools: only "+strings.Join(p.Tools.Allow, ", "))
|
||||
}
|
||||
if len(p.Tools.Deny) > 0 {
|
||||
v.Checks = append(v.Checks, "tools: never "+strings.Join(p.Tools.Deny, ", "))
|
||||
}
|
||||
if p.MaxOutputTokens > 0 {
|
||||
v.Checks = append(v.Checks, fmt.Sprintf("answers of at most %d tokens", p.MaxOutputTokens))
|
||||
}
|
||||
if p.SystemPrompt != "" {
|
||||
v.Checks = append(v.Checks, "a system prompt of its own")
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func or(s, dflt string) string {
|
||||
if s == "" {
|
||||
return dflt
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// GuardAttachmentView is one attachment, with the name people know its
|
||||
@@ -43,6 +154,15 @@ type GuardAttachmentView struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// guardHistory is where what the guardrails did is read, when there is one.
|
||||
func (a *API) guardHistory() (GuardHistory, bool) {
|
||||
if a.guard == nil {
|
||||
return nil, false
|
||||
}
|
||||
h, ok := a.guard.Store.(GuardHistory)
|
||||
return h, ok
|
||||
}
|
||||
|
||||
// guardTarget reads where a policy is to be attached.
|
||||
func (a *API) guardTarget(ctx context.Context, target string) (guard.Attachment, error) {
|
||||
kind, what, ok := strings.Cut(strings.TrimSpace(target), ":")
|
||||
@@ -99,6 +219,57 @@ func (a *API) registerGuard(mux Mux) {
|
||||
v.Policies = append(v.Policies, name)
|
||||
}
|
||||
slices.Sort(v.Policies)
|
||||
for _, name := range v.Policies {
|
||||
v.Details = append(v.Details, policyView(name, a.guard.Library.Policies[name]))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/guard/events", func(w http.ResponseWriter, r *http.Request) {
|
||||
h, ok := a.guardHistory()
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies, or no store to record what they did")
|
||||
return
|
||||
}
|
||||
since := 24 * time.Hour
|
||||
if q := r.URL.Query().Get("since"); q != "" {
|
||||
d, err := parseSince(q)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
since = d
|
||||
}
|
||||
from := time.Now().Add(-since)
|
||||
counts, err := h.GuardCounts(r.Context(), from)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
events, err := h.GuardEvents(r.Context(), from, 100)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
var names map[string]string
|
||||
if a.quotas != nil {
|
||||
names, _ = a.scopeNames(r.Context())
|
||||
}
|
||||
project := func(id string) string {
|
||||
if n := names["project:"+id]; n != "" {
|
||||
return n
|
||||
}
|
||||
return id
|
||||
}
|
||||
v := GuardEventsView{Since: from, Counts: []GuardCountView{}, Events: []GuardEventView{}}
|
||||
for _, c := range counts {
|
||||
v.Counts = append(v.Counts, GuardCountView{Project: project(c.ProjectID), Action: c.Action,
|
||||
Policy: c.Policy, Category: c.Category, Count: c.Count})
|
||||
}
|
||||
slices.SortStableFunc(v.Counts, func(a, b GuardCountView) int { return int(b.Count - a.Count) })
|
||||
for _, e := range events {
|
||||
v.Events = append(v.Events, GuardEventView{At: e.At, RequestID: e.RequestID, Project: project(e.ProjectID),
|
||||
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, GuardMs: e.GuardMs})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
change := func(attach bool) http.HandlerFunc {
|
||||
|
||||
@@ -116,3 +116,54 @@ func TestUsageIsExportedAsCSVThatASpreadsheetCannotRun(t *testing.T) {
|
||||
t.Errorf("%d %s %+v", rec.Code, body, m)
|
||||
}
|
||||
}
|
||||
|
||||
// memGuardHistory is a guard store that also remembers what was done.
|
||||
type memGuardHistory struct {
|
||||
memGuard
|
||||
since time.Time
|
||||
}
|
||||
|
||||
func (m *memGuardHistory) GuardEvents(_ context.Context, since time.Time, _ int) ([]store.GuardEvent, error) {
|
||||
m.since = since
|
||||
return []store.GuardEvent{{RequestID: "r1", Model: "m", Status: 400, Action: "block", Policy: "base", Category: "S1", GuardMs: 410}}, nil
|
||||
}
|
||||
|
||||
func (m *memGuardHistory) GuardCounts(context.Context, time.Time) ([]store.GuardCount, error) {
|
||||
return []store.GuardCount{
|
||||
{Action: "flag", Policy: "base", Category: "email", Count: 2},
|
||||
{Action: "block", Policy: "base", Category: "S1", Count: 7},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestTheGuardrailsSayWhatEachPolicyDoesAndWhatTheyDid(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
h := &memGuardHistory{}
|
||||
a.SetGuard(&GuardAdmin{Store: h, Library: guard.Config{Policies: map[string]guard.Policy{
|
||||
"base": {Fail: "closed", Input: guard.Rules{PII: map[string]string{"email": "redact"}, Injection: "block"},
|
||||
Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block", Categories: []string{"S1", "S9"}}},
|
||||
}}})
|
||||
|
||||
var v GuardView
|
||||
if code := call(t, a, "GET", "/api/v1/guard", "", &v); code != 200 || len(v.Details) != 1 {
|
||||
t.Fatalf("%d %+v", code, v)
|
||||
}
|
||||
d := v.Details[0]
|
||||
want := []string{"input: pii email → redact", "input: prompt injection → block", "judge llama-guard-3-1b: input → block (S1 S9)"}
|
||||
if d.Mode != "enforce" || d.Fail != "closed" || d.Judge != "llama-guard-3-1b" || !slices.Equal(d.Checks, want) {
|
||||
t.Errorf("%+v", d)
|
||||
}
|
||||
|
||||
var ev GuardEventsView
|
||||
if code := call(t, a, "GET", "/api/v1/guard/events?since=1h", "", &ev); code != 200 {
|
||||
t.Fatalf("events: %d", code)
|
||||
}
|
||||
if time.Since(h.since) > time.Hour+time.Minute || time.Since(h.since) < time.Hour-time.Minute {
|
||||
t.Errorf("asked since %s", h.since)
|
||||
}
|
||||
if len(ev.Counts) != 2 || ev.Counts[0].Count != 7 || len(ev.Events) != 1 || ev.Events[0].GuardMs != 410 {
|
||||
t.Errorf("most first, and the events: %+v", ev)
|
||||
}
|
||||
if code := call(t, a, "GET", "/api/v1/guard/events?since=soon", "", nil); code != 400 {
|
||||
t.Errorf("a bad window: %d", code)
|
||||
}
|
||||
}
|
||||
@@ -249,6 +249,11 @@ var Routes = []Route{
|
||||
Response: Removed{}},
|
||||
{Method: "GET", Path: "/api/v1/guard", Summary: "The guardrail policies and where they are attached",
|
||||
Response: GuardView{}},
|
||||
{Method: "GET", Path: "/api/v1/guard/events", Summary: "What the guardrails did: totals and the newest requests",
|
||||
Query: []Param{
|
||||
{Name: "since", Description: "How far back: 90m, 24h, 7d; 24h when absent."},
|
||||
},
|
||||
Response: GuardEventsView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
|
||||
|
||||
Reference in new issue
Block a user