Guardrails turn off and on; a judge's refusal says which hazard; ctl guard judge ask

guard.enabled (unset is on) makes every policy check nothing while false,
live: llmgw ctl guard off/on set it in the cluster's settings, and ctl
status, ctl guard list and the guardrails page say when they are off.

A refusal by a judge names the hazard it saw - judge: S6,Specialized
Advice - and the verdict is kept with the request in a new usage column,
guard_rule, shown by ctl request, the request log and the guardrails page.

llmgw ctl guard judge ask has a judge read a text, past the cache, and shows
its raw answer and how it is read; the endpoint is kept out of the audit so
no text is recorded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-06 12:41:08 +02:00
1 parent 3f5a809373
commit 249a91aeec
34 files changed
+505 -33

No files matched your search

+2 -1
View File
@@ -1545,6 +1545,7 @@ type RequestView struct {
GuardAction string `json:"guard_action,omitempty"`
GuardPolicy string `json:"guard_policy,omitempty"`
GuardCategory string `json:"guard_category,omitempty"`
GuardRule string `json:"guard_rule,omitempty"`
GuardMs int64 `json:"guard_ms,omitempty"`
// What the client said about it: OpenAI's user field, and its tags.
EndUser string `json:"end_user,omitempty"`
@@ -1573,7 +1574,7 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardMs: u.GuardMs,
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardRule: u.GuardRule, GuardMs: u.GuardMs,
EndUser: u.EndUser, Tags: u.Tags, CacheHit: u.CacheHit,
})
}
+11 -3
View File
@@ -30,6 +30,8 @@ type GuardAdmin struct {
Invalidate func()
// JudgeErrors are each judge's last failure to give a verdict.
JudgeErrors func() map[string]guard.JudgeError
// Ask has a judge read a text, past the cache.
Ask func(ctx context.Context, model, text, answer string) (guard.JudgeAnswer, error)
}
// SetGuard enables the guard endpoints.
@@ -37,6 +39,8 @@ func (a *API) SetGuard(g *GuardAdmin) { a.guard = g }
// GuardView is the library and where its policies are attached.
type GuardView struct {
// Enabled is whether the guardrails check anything (guard.enabled).
Enabled bool `json:"enabled"`
Policies []string `json:"policies"`
Default []string `json:"default"`
Attachments []GuardAttachmentView `json:"attachments"`
@@ -91,7 +95,10 @@ type GuardEventView struct {
Action string `json:"action"`
Policy string `json:"policy"`
Category string `json:"category,omitempty"`
GuardMs int64 `json:"guard_ms"`
// Rule is what decided it within the category: a pattern's name, or a
// judge's verdict, "S6,Specialized Advice".
Rule string `json:"rule,omitempty"`
GuardMs int64 `json:"guard_ms"`
}
// policyView sums one policy up.
@@ -201,6 +208,7 @@ type GuardAttachmentRequest struct {
func (a *API) registerGuard(mux Mux) {
mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge)
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
if a.guard == nil {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
@@ -211,7 +219,7 @@ func (a *API) registerGuard(mux Mux) {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
v := GuardView{Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
v := GuardView{Enabled: a.guard.Library().On(), Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
var names map[string]string
if a.quotas != nil {
names, _ = a.scopeNames(r.Context())
@@ -272,7 +280,7 @@ func (a *API) registerGuard(mux Mux) {
slices.SortStableFunc(v.Counts, func(a, b GuardCountView) int { return int(b.Count - a.Count) })
for _, e := range events {
v.Events = append(v.Events, GuardEventView{At: e.At, RequestID: e.RequestID, Project: project(e.ProjectID),
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, GuardMs: e.GuardMs})
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, Rule: e.Rule, GuardMs: e.GuardMs})
}
writeJSON(w, http.StatusOK, v)
})
+57
View File
@@ -1,10 +1,12 @@
package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -44,6 +46,10 @@ func (a *API) Judges() []JudgeView {
return nil
}
lib := a.guard.Library()
if !lib.On() {
// Off: no judge is asked, so none is missing.
return nil
}
byModel := map[string]*JudgeView{}
for name, p := range lib.Policies {
if p.Judge == nil || p.Judge.Model == "" {
@@ -122,3 +128,54 @@ func (a *API) listJudges(w http.ResponseWriter, _ *http.Request) {
}
writeJSON(w, http.StatusOK, a.Judges())
}
// JudgeAskRequest is a text to have a judge read.
type JudgeAskRequest struct {
// Model is the judge; empty is the one the policies ask, when they ask one.
Model string `json:"model,omitempty"`
Text string `json:"text"`
// Answer, when given, is read as the assistant's reply to Text.
Answer string `json:"answer,omitempty"`
}
// JudgeAskView is what the judge answered.
type JudgeAskView struct {
Model string `json:"model"`
// Verdict is safe or unsafe, as the gateway reads the answer.
Verdict string `json:"verdict"`
Categories []string `json:"categories,omitempty"`
// Raw is the judge's answer as it gave it.
Raw string `json:"raw"`
TookMs int64 `json:"took_ms"`
}
func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
if a.guard == nil || a.guard.Ask == nil {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
return
}
var req JudgeAskRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || strings.TrimSpace(req.Text) == "" {
writeError(w, http.StatusBadRequest, "give the text for the judge to read")
return
}
if req.Model == "" {
judges := a.Judges()
if len(judges) == 0 {
writeError(w, http.StatusBadRequest, "no policy asks a judge: give the model")
return
}
req.Model = judges[0].Model
}
ans, err := a.guard.Ask(r.Context(), req.Model, req.Text, req.Answer)
v := JudgeAskView{Model: req.Model, Raw: ans.Raw, Categories: ans.Categories, TookMs: ans.Took.Milliseconds()}
if err != nil {
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v (it answered %q)", req.Model, err, ans.Raw))
return
}
v.Verdict = "safe"
if ans.Unsafe {
v.Verdict = "unsafe"
}
writeJSON(w, http.StatusOK, v)
}
+25
View File
@@ -1,6 +1,7 @@
package adminapi
import (
"context"
"strings"
"testing"
"time"
@@ -64,3 +65,27 @@ func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
t.Errorf("an old failure still counts: %q", p)
}
}
func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) {
a, _ := setup(t, testModels())
lib := guard.Config{Policies: map[string]guard.Policy{
"baseline": {Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}}}
var asked string
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
Ask: func(_ context.Context, model, text, _ string) (guard.JudgeAnswer, error) {
asked = model + "|" + text
return guard.JudgeAnswer{Model: model, Raw: "unsafe\nS6", Unsafe: true,
Categories: []string{"S6", "Specialized Advice"}}, nil
}})
var v JudgeAskView
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"how do I sort a list"}`, &v); code != 200 {
t.Fatalf("%d", code)
}
if asked != "llama-guard-3-1b|how do I sort a list" || v.Verdict != "unsafe" || v.Raw != "unsafe\nS6" ||
len(v.Categories) != 2 {
t.Errorf("asked %q, got %+v", asked, v)
}
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":" "}`, nil); code != 400 {
t.Errorf("no text: %d", code)
}
}
+2
View File
@@ -256,6 +256,8 @@ var Routes = []Route{
Response: GuardEventsView{}},
{Method: "GET", Path: "/api/v1/guard/judges", Summary: "The judge models the policies ask, and what is wrong with each",
Response: []JudgeView{}},
{Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept",
Request: JudgeAskRequest{}, Response: JudgeAskView{}},
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
+6
View File
@@ -44,6 +44,9 @@ type Summary struct {
ControlPlanes ControlPlanes `json:"control_planes,omitzero"`
// Judges are the guardrails' judge models with something wrong.
Judges []JudgeView `json:"judges,omitempty"`
// GuardOff says the guardrails are turned off (guard.enabled: false):
// no request is checked, whatever the policies say.
GuardOff bool `json:"guard_off,omitempty"`
}
// StoreMember is one member of a replicated store, as its Patroni reports it.
@@ -151,6 +154,9 @@ const idleThreshold = 10 * time.Minute
// Summary builds the fleet summary.
func (a *API) Summary() Summary {
s := Summary{Version: version.Version}
if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() {
s.GuardOff = true
}
for _, j := range a.Judges() {
if j.Problem != "" {
s.Judges = append(s.Judges, j)