Guardrails turn off and on; a judge's refusal says which hazard; ctl guard judge ask
guard.enabled (unset is on) makes every policy check nothing while false, live: llmgw ctl guard off/on set it in the cluster's settings, and ctl status, ctl guard list and the guardrails page say when they are off. A refusal by a judge names the hazard it saw - judge: S6,Specialized Advice - and the verdict is kept with the request in a new usage column, guard_rule, shown by ctl request, the request log and the guardrails page. llmgw ctl guard judge ask has a judge read a text, past the cache, and shows its raw answer and how it is read; the endpoint is kept out of the audit so no text is recorded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3f5a809373
commit
249a91aeec
34 files changed
+505
-33
No files matched your search
@@ -1545,6 +1545,7 @@ type RequestView struct {
|
||||
GuardAction string `json:"guard_action,omitempty"`
|
||||
GuardPolicy string `json:"guard_policy,omitempty"`
|
||||
GuardCategory string `json:"guard_category,omitempty"`
|
||||
GuardRule string `json:"guard_rule,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms,omitempty"`
|
||||
// What the client said about it: OpenAI's user field, and its tags.
|
||||
EndUser string `json:"end_user,omitempty"`
|
||||
@@ -1573,7 +1574,7 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardMs: u.GuardMs,
|
||||
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardRule: u.GuardRule, GuardMs: u.GuardMs,
|
||||
EndUser: u.EndUser, Tags: u.Tags, CacheHit: u.CacheHit,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -30,6 +30,8 @@ type GuardAdmin struct {
|
||||
Invalidate func()
|
||||
// JudgeErrors are each judge's last failure to give a verdict.
|
||||
JudgeErrors func() map[string]guard.JudgeError
|
||||
// Ask has a judge read a text, past the cache.
|
||||
Ask func(ctx context.Context, model, text, answer string) (guard.JudgeAnswer, error)
|
||||
}
|
||||
|
||||
// SetGuard enables the guard endpoints.
|
||||
@@ -37,6 +39,8 @@ func (a *API) SetGuard(g *GuardAdmin) { a.guard = g }
|
||||
|
||||
// GuardView is the library and where its policies are attached.
|
||||
type GuardView struct {
|
||||
// Enabled is whether the guardrails check anything (guard.enabled).
|
||||
Enabled bool `json:"enabled"`
|
||||
Policies []string `json:"policies"`
|
||||
Default []string `json:"default"`
|
||||
Attachments []GuardAttachmentView `json:"attachments"`
|
||||
@@ -91,7 +95,10 @@ type GuardEventView struct {
|
||||
Action string `json:"action"`
|
||||
Policy string `json:"policy"`
|
||||
Category string `json:"category,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms"`
|
||||
// Rule is what decided it within the category: a pattern's name, or a
|
||||
// judge's verdict, "S6,Specialized Advice".
|
||||
Rule string `json:"rule,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms"`
|
||||
}
|
||||
|
||||
// policyView sums one policy up.
|
||||
@@ -201,6 +208,7 @@ type GuardAttachmentRequest struct {
|
||||
|
||||
func (a *API) registerGuard(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
|
||||
mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge)
|
||||
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
@@ -211,7 +219,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
v := GuardView{Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
|
||||
v := GuardView{Enabled: a.guard.Library().On(), Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
|
||||
var names map[string]string
|
||||
if a.quotas != nil {
|
||||
names, _ = a.scopeNames(r.Context())
|
||||
@@ -272,7 +280,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
slices.SortStableFunc(v.Counts, func(a, b GuardCountView) int { return int(b.Count - a.Count) })
|
||||
for _, e := range events {
|
||||
v.Events = append(v.Events, GuardEventView{At: e.At, RequestID: e.RequestID, Project: project(e.ProjectID),
|
||||
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, GuardMs: e.GuardMs})
|
||||
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, Rule: e.Rule, GuardMs: e.GuardMs})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -44,6 +46,10 @@ func (a *API) Judges() []JudgeView {
|
||||
return nil
|
||||
}
|
||||
lib := a.guard.Library()
|
||||
if !lib.On() {
|
||||
// Off: no judge is asked, so none is missing.
|
||||
return nil
|
||||
}
|
||||
byModel := map[string]*JudgeView{}
|
||||
for name, p := range lib.Policies {
|
||||
if p.Judge == nil || p.Judge.Model == "" {
|
||||
@@ -122,3 +128,54 @@ func (a *API) listJudges(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, a.Judges())
|
||||
}
|
||||
|
||||
// JudgeAskRequest is a text to have a judge read.
|
||||
type JudgeAskRequest struct {
|
||||
// Model is the judge; empty is the one the policies ask, when they ask one.
|
||||
Model string `json:"model,omitempty"`
|
||||
Text string `json:"text"`
|
||||
// Answer, when given, is read as the assistant's reply to Text.
|
||||
Answer string `json:"answer,omitempty"`
|
||||
}
|
||||
|
||||
// JudgeAskView is what the judge answered.
|
||||
type JudgeAskView struct {
|
||||
Model string `json:"model"`
|
||||
// Verdict is safe or unsafe, as the gateway reads the answer.
|
||||
Verdict string `json:"verdict"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
// Raw is the judge's answer as it gave it.
|
||||
Raw string `json:"raw"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil || a.guard.Ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
var req JudgeAskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || strings.TrimSpace(req.Text) == "" {
|
||||
writeError(w, http.StatusBadRequest, "give the text for the judge to read")
|
||||
return
|
||||
}
|
||||
if req.Model == "" {
|
||||
judges := a.Judges()
|
||||
if len(judges) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "no policy asks a judge: give the model")
|
||||
return
|
||||
}
|
||||
req.Model = judges[0].Model
|
||||
}
|
||||
ans, err := a.guard.Ask(r.Context(), req.Model, req.Text, req.Answer)
|
||||
v := JudgeAskView{Model: req.Model, Raw: ans.Raw, Categories: ans.Categories, TookMs: ans.Took.Milliseconds()}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v (it answered %q)", req.Model, err, ans.Raw))
|
||||
return
|
||||
}
|
||||
v.Verdict = "safe"
|
||||
if ans.Unsafe {
|
||||
v.Verdict = "unsafe"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -64,3 +65,27 @@ func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
|
||||
t.Errorf("an old failure still counts: %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}}}
|
||||
var asked string
|
||||
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
|
||||
Ask: func(_ context.Context, model, text, _ string) (guard.JudgeAnswer, error) {
|
||||
asked = model + "|" + text
|
||||
return guard.JudgeAnswer{Model: model, Raw: "unsafe\nS6", Unsafe: true,
|
||||
Categories: []string{"S6", "Specialized Advice"}}, nil
|
||||
}})
|
||||
var v JudgeAskView
|
||||
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"how do I sort a list"}`, &v); code != 200 {
|
||||
t.Fatalf("%d", code)
|
||||
}
|
||||
if asked != "llama-guard-3-1b|how do I sort a list" || v.Verdict != "unsafe" || v.Raw != "unsafe\nS6" ||
|
||||
len(v.Categories) != 2 {
|
||||
t.Errorf("asked %q, got %+v", asked, v)
|
||||
}
|
||||
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":" "}`, nil); code != 400 {
|
||||
t.Errorf("no text: %d", code)
|
||||
}
|
||||
}
|
||||
@@ -256,6 +256,8 @@ var Routes = []Route{
|
||||
Response: GuardEventsView{}},
|
||||
{Method: "GET", Path: "/api/v1/guard/judges", Summary: "The judge models the policies ask, and what is wrong with each",
|
||||
Response: []JudgeView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept",
|
||||
Request: JudgeAskRequest{}, Response: JudgeAskView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
|
||||
|
||||
@@ -44,6 +44,9 @@ type Summary struct {
|
||||
ControlPlanes ControlPlanes `json:"control_planes,omitzero"`
|
||||
// Judges are the guardrails' judge models with something wrong.
|
||||
Judges []JudgeView `json:"judges,omitempty"`
|
||||
// GuardOff says the guardrails are turned off (guard.enabled: false):
|
||||
// no request is checked, whatever the policies say.
|
||||
GuardOff bool `json:"guard_off,omitempty"`
|
||||
}
|
||||
|
||||
// StoreMember is one member of a replicated store, as its Patroni reports it.
|
||||
@@ -151,6 +154,9 @@ const idleThreshold = 10 * time.Minute
|
||||
// Summary builds the fleet summary.
|
||||
func (a *API) Summary() Summary {
|
||||
s := Summary{Version: version.Version}
|
||||
if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() {
|
||||
s.GuardOff = true
|
||||
}
|
||||
for _, j := range a.Judges() {
|
||||
if j.Problem != "" {
|
||||
s.Judges = append(s.Judges, j)
|
||||
|
||||
Reference in new issue
Block a user