Guardrail metrics and alerts: checks, latency, judge failures, fail-open, redactions, stopped streams, tenant blocks

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 15:11:50 +02:00
1 parent 43f6a585a6
commit 47e847423d
13 files changed
+300 -17

No files matched your search

+66
View File
@@ -489,3 +489,69 @@ groups:
description: >-
Patroni will not promote a standby more than 1 MB behind, so a
failover now would find none fit. -1 is a lag Patroni cannot read.
- name: llmgw-guard
rules:
- alert: GuardFailingOpen
expr: >-
sum(rate(llmgw_guard_fail_open_total[5m]))
/ clamp_min(sum(rate(llmgw_guard_checks_total[5m])), 1e-9) > 0.01
for: 5m
labels: { severity: warning }
annotations:
summary: "{{ $value | humanizePercentage }} of judged traffic is served unjudged"
description: >-
A guard policy's judge model is not answering within its latency
budget, and the policy fails open: requests go through flagged
"judge unavailable" instead of being judged.
llmgw_guard_judge_errors_total says whether it is timing out or
not there; `llmgw ctl replicas` whether the judge has replicas.
- alert: GuardJudgeSlow
expr: >-
histogram_quantile(0.95, sum by (le, judge_model) (rate(llmgw_guard_judge_latency_seconds_bucket[5m]))) > 0.25
for: 10m
labels: { severity: warning }
annotations:
summary: "Judge {{ $labels.judge_model }} answers in {{ $value | humanizeDuration }} at p95"
description: >-
Over the default latency budget: every judged request waits this
long, and those over their policy's budget are served unjudged or
refused. Give the judge more replicas, or faster ones.
- alert: GuardJudgeUnavailable
expr: sum by (reason) (rate(llmgw_guard_judge_errors_total{reason!="timeout"}[5m])) > 0
for: 5m
labels: { severity: critical }
annotations:
summary: "The guardrails cannot ask their judge ({{ $labels.reason }})"
description: >-
no_judge: a policy names a model that is not in the catalog with
role: guard. unavailable: it has no replica answering. format: it
answers in a form its guard.format does not read.
- alert: GuardBlocksSpiking
expr: >-
sum by (tenant) (rate(llmgw_tenant_guard_blocks_total[1h]))
> 3 * sum by (tenant) (rate(llmgw_tenant_guard_blocks_total[7d]))
and sum by (tenant) (increase(llmgw_tenant_guard_blocks_total[1h])) > 10
for: 15m
labels: { severity: warning }
annotations:
summary: "{{ $labels.tenant }} is being blocked three times as often as usual"
description: >-
A tenant's requests or answers are blocked by the guardrails
more than three times their weekly rate. Someone probing, a
client sending data it should not, or a policy that changed.
The /admin/guard page lists the recent ones by request.
- alert: GuardSecretInAnswer
expr: sum(increase(llmgw_guard_checks_total{stage="output",category="secret",action!="allowed"}[5m])) > 0
labels: { severity: critical }
annotations:
summary: "A model put a credential in an answer"
description: >-
An answer contained something shaped like a key or a private key.
It was stopped or rewritten if the policy enforces; either way a
model has seen it, in its prompt or its training. The usage rows
with guard_category secret say which request.
+3
View File
@@ -132,6 +132,9 @@ func newServerCmd() *cobra.Command {
}
gate.SetJudge(rt.Judge, rt.GuardFormat)
rt.SetGuard(gate)
if err := rt.RegisterMetrics(gate); err != nil {
return fmt.Errorf("guard metrics: %w", err)
}
log.Info("guardrails on", "policies", len(cfg.Guard.Policies), "default", cfg.Guard.Default)
}
+22 -6
View File
@@ -39,6 +39,7 @@ type Gate struct {
deny map[string]*regexp.Regexp // by pattern regex
tools toolMemo
judger *judge
m *gateMetrics
mu sync.Mutex
attached map[string][]string // "scope/id" → policy names
@@ -56,7 +57,7 @@ func NewGate(cfg Config, store Attachments, log *slog.Logger) (*Gate, error) {
log = slog.Default()
}
g := &Gate{cfg: cfg, store: store, log: log, now: time.Now,
deny: map[string]*regexp.Regexp{}, merged: map[string]Policy{}}
deny: map[string]*regexp.Regexp{}, merged: map[string]Policy{}, m: newGateMetrics()}
for _, p := range cfg.Policies {
for _, r := range []Rules{p.Input, p.Output} {
for _, d := range r.Deny {
@@ -155,6 +156,9 @@ type finding struct {
action Action
category string
rule string
// failOpen is a judge that did not answer, under a policy that serves
// anyway.
failOpen bool
}
// judge scans one piece of text under one direction's rules, returning
@@ -334,8 +338,9 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
total := 0
ph := newPlaceholders()
type rewrite struct {
seg segment
text string
seg segment
text string
spans []detect.Span
}
var rewrites []rewrite
for _, sg := range segs {
@@ -343,7 +348,7 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
f, red := g.judge(sg.text, p.Input)
fs = append(fs, f...)
if len(red) > 0 {
rewrites = append(rewrites, rewrite{sg, ph.rewrite(sg.text, red)})
rewrites = append(rewrites, rewrite{sg, ph.rewrite(sg.text, red), red})
}
}
if p.Input.MaxChars > 0 && total > p.Input.MaxChars {
@@ -360,13 +365,18 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
v.Took = g.now().Sub(start)
return v, fmt.Errorf("the judge could not be asked: %w", err)
}
fs = append(fs, judgeFinding(p, jp.Input, res, err))
jf := judgeFinding(p, jp.Input, res, err)
if jf.failOpen {
g.m.failOpen.Inc()
}
fs = append(fs, jf)
if best, found = strictest(fs); found && best.action == Block {
return done(best), nil
}
}
if !v.Shadow {
for _, rw := range rewrites {
g.countRedactions(rw.spans)
rw.seg.set(rw.text)
changed = true
}
@@ -395,7 +405,7 @@ func judgeFinding(p Policy, action string, res judgment, err error) finding {
if p.Fail == "closed" {
return finding{action: Block, category: "judge", rule: "unavailable"}
}
return finding{action: Flag, category: "judge", rule: "unavailable"}
return finding{action: Flag, category: "judge", rule: "unavailable", failOpen: true}
}
if ok, cat := p.Judge.counts(res); ok {
return finding{action: actions[action], category: "judge", rule: cat}
@@ -405,6 +415,12 @@ func judgeFinding(p Policy, action string, res judgment, err error) finding {
func segsOf(d *doc) []segment { return d.segments() }
func (g *Gate) countRedactions(spans []detect.Span) {
for _, s := range spans {
g.m.redactions.WithLabelValues(s.Kind).Inc()
}
}
// lastUser is the last thing the user said: what a judge reads.
func lastUser(segs []segment) string {
for i := len(segs) - 1; i >= 0; i-- {
+20 -7
View File
@@ -98,6 +98,7 @@ type judgment struct {
type judge struct {
call Caller
format ModelFormat
m *gateMetrics
mu sync.Mutex
cache map[[32]byte]cached
@@ -115,18 +116,29 @@ const (
// SetJudge lets policies with a judge: section ask a model on the cluster.
func (g *Gate) SetJudge(call Caller, format ModelFormat) {
g.judger = &judge{call: call, format: format, cache: map[[32]byte]cached{}}
g.judger = &judge{call: call, format: format, m: g.m, cache: map[[32]byte]cached{}}
}
// errNoJudge is a policy naming a judge the gateway cannot ask.
var errNoJudge = errors.New("no judge model to ask")
// errNoJudge is a policy naming a judge the gateway cannot ask, and
// errJudgeFormat a judge whose answer could not be read.
var (
errNoJudge = errors.New("no judge model to ask")
errJudgeFormat = errors.New("the judge's answer could not be read")
)
// ask has the judge read conversation turns — the user's, and the
// assistant's answer when judging output.
func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) {
func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (res judgment, err error) {
if j == nil {
return judgment{}, errNoJudge
}
start := time.Now()
cachedHit := false
defer func() {
if !cachedHit && j.m != nil {
j.m.judged(jp.Model, time.Since(start), err)
}
}()
format, names, ok := j.format(jp.Model)
if !ok {
return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, jp.Model)
@@ -135,6 +147,7 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
j.mu.Lock()
if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL {
j.mu.Unlock()
cachedHit = true
return c.j, nil
}
j.mu.Unlock()
@@ -146,7 +159,7 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
if err != nil {
return judgment{}, err
}
res, err := parseJudgment(format, names, out)
res, err = parseJudgment(format, names, out)
if err != nil {
return judgment{}, err
}
@@ -227,7 +240,7 @@ func parseJudgment(format string, names []string, body []byte) (judgment, error)
} `json:"choices"`
}
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 {
return judgment{}, errors.New("the judge's answer is not a chat completion")
return judgment{}, fmt.Errorf("%w: not a chat completion", errJudgeFormat)
}
text := strings.TrimSpace(resp.Choices[0].Message.Content)
switch format {
@@ -274,5 +287,5 @@ func parseJudgment(format string, names []string, body []byte) (judgment, error)
}
}
}
return judgment{}, fmt.Errorf("the judge answered in a form %s does not have", format)
return judgment{}, fmt.Errorf("%w: not in the form %s has", errJudgeFormat, format)
}
+14
View File
@@ -8,6 +8,8 @@ import (
"sync/atomic"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
)
// fakeJudge answers as a judge model would, saying unsafe to text with
@@ -139,3 +141,15 @@ func TestAStreamedAnswerIsJudgedAsItGrows(t *testing.T) {
t.Errorf("buffer_all: stopped=%v got %q", stopped, got)
}
}
func TestTheGateCountsJudgeFailuresAndRedactions(t *testing.T) {
g := judged(t, Policy{Input: Rules{PII: map[string]string{"email": "redact"}},
Judge: &JudgePolicy{Model: "guard", Input: "flag", LatencyBudget: 10 * time.Millisecond}}, &fakeJudge{delay: time.Second})
_, _ = g.Input(context.Background(), Subject{}, chat("user", "mail a@example.com and b@example.com"))
if n := testutil.ToFloat64(g.m.redactions.WithLabelValues("email")); n != 2 {
t.Errorf("redactions %v", n)
}
if testutil.ToFloat64(g.m.failOpen) != 1 || testutil.ToFloat64(g.m.judgeErrors.WithLabelValues("timeout")) != 1 {
t.Error("a judge that timed out under fail open was not counted")
}
}
+74
View File
@@ -0,0 +1,74 @@
package guard
import (
"context"
"errors"
"time"
"github.com/prometheus/client_golang/prometheus"
)
// gateMetrics are what only the gate sees: the judge, and redactions.
type gateMetrics struct {
judgeLatency *prometheus.HistogramVec
judgeErrors *prometheus.CounterVec
failOpen prometheus.Counter
redactions *prometheus.CounterVec
}
func newGateMetrics() *gateMetrics {
return &gateMetrics{
judgeLatency: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_guard_judge_latency_seconds",
Help: "Time a judge model took to answer the guardrails, cached answers excluded.",
Buckets: []float64{.01, .025, .05, .1, .25, .5, 1, 2.5, 5},
}, []string{"judge_model"}),
judgeErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_guard_judge_errors_total",
Help: "Judge calls that gave no verdict, by reason: timeout, unavailable, format, no_judge.",
}, []string{"reason"}),
failOpen: prometheus.NewCounter(prometheus.CounterOpts{
Name: "llmgw_guard_fail_open_total",
Help: "Requests and answers served unjudged because a judge did not answer and the policy fails open.",
}),
redactions: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_guard_redactions_total",
Help: "Values the guardrails rewrote as placeholders, by kind.",
}, []string{"kind"}),
}
}
// Describe and Collect make the gate a Prometheus collector.
func (g *Gate) Describe(ch chan<- *prometheus.Desc) {
for _, c := range g.collectors() {
c.Describe(ch)
}
}
func (g *Gate) Collect(ch chan<- prometheus.Metric) {
for _, c := range g.collectors() {
c.Collect(ch)
}
}
func (g *Gate) collectors() []prometheus.Collector {
m := g.m
return []prometheus.Collector{m.judgeLatency, m.judgeErrors, m.failOpen, m.redactions}
}
func (m *gateMetrics) judged(model string, took time.Duration, err error) {
if err == nil {
m.judgeLatency.WithLabelValues(model).Observe(took.Seconds())
return
}
reason := "unavailable"
switch {
case errors.Is(err, context.DeadlineExceeded):
reason = "timeout"
case errors.Is(err, errNoJudge):
reason = "no_judge"
case errors.Is(err, errJudgeFormat):
reason = "format"
}
m.judgeErrors.WithLabelValues(reason).Inc()
}
+6 -1
View File
@@ -133,6 +133,7 @@ func (f *outFilter) judgeText(text string) (finding, string) {
if f.shadow || len(red) == 0 {
return best, text
}
f.g.countRedactions(red)
return best, newPlaceholders().rewrite(text, red)
}
@@ -188,7 +189,11 @@ func (f *outFilter) askJudge(text string, final bool) finding {
start := time.Now()
res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String())
f.took += time.Since(start)
return f.record([]finding{judgeFinding(f.p, jp.Output, res, err)})
jf := judgeFinding(f.p, jp.Output, res, err)
if jf.failOpen {
f.g.m.failOpen.Inc()
}
return f.record([]finding{jf})
}
// judgeStructured checks a whole answer against the schema the request
+13
View File
@@ -87,6 +87,7 @@ func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *par
return false
}
req.guardIn = v
r.m.guard.observe(v, guard.Input, r.m.tenant.tenantOf(req))
if v.Shadow {
// Recorded, not done: the client sees nothing of it.
return true
@@ -143,6 +144,18 @@ func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guar
r.accessLog(req, dispatchResult{guard: v}, status, time.Since(start))
}
// observeOutput counts what the output filter decided for an answer that
// was judged, and a stream it stopped.
func (r *Router) observeOutput(req *parsedRequest, res dispatchResult, judged bool) {
if !judged {
return
}
r.m.guard.observe(res.guard, guard.Output, r.m.tenant.tenantOf(req))
if res.outcome.Stopped {
r.m.guard.stops.Inc()
}
}
// outputFilter attaches the guard's filter for one attempt to the proxy
// request, and returns it so its verdict can be read afterwards.
func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model string, preq *proxy.Request) guard.OutputFilter {
+14
View File
@@ -11,6 +11,8 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
@@ -227,3 +229,15 @@ func TestAJudgeModelIsNobodysToAskButTheGuardrails(t *testing.T) {
t.Errorf("format %q %v", f, ok)
}
}
func TestWhatTheGuardrailsDidIsCounted(t *testing.T) {
rt, _ := guardedRouter(t, &wordGuard{answers: true})
post(t, rt.Handler(), say("something forbidden"), nil)
post(t, rt.Handler(), say("hello"), nil)
if n := testutil.ToFloat64(rt.m.guard.checks.WithLabelValues("input", "words", "blocked", "test", "enforce")); n != 1 {
t.Errorf("input blocks counted %v", n)
}
if n := testutil.ToFloat64(rt.m.guard.checks.WithLabelValues("output", "answers", "blocked", "", "enforce")); n != 1 {
t.Errorf("output blocks counted %v", n)
}
}
+59
View File
@@ -0,0 +1,59 @@
package router
import (
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)
// guardMetrics count what the guardrails did, by name: never what they read.
type guardMetrics struct {
checks *prometheus.CounterVec
latency *prometheus.HistogramVec
stops prometheus.Counter
blocks *prometheus.CounterVec
}
func newGuardMetrics() *guardMetrics {
return &guardMetrics{
checks: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_guard_checks_total",
Help: "Requests and answers judged by the guardrails, by stage (input, output), policy, action " +
"(allowed, flagged, redacted, blocked), category, and mode (enforce, shadow).",
}, []string{"stage", "policy", "action", "category", "mode"}),
latency: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_guard_latency_seconds",
Help: "Time the guardrails took to judge a request or an answer, judge model included.",
Buckets: []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5},
}, []string{"stage"}),
stops: prometheus.NewCounter(prometheus.CounterOpts{
Name: "llmgw_guard_stream_stops_total",
Help: "Streamed answers the guardrails stopped part way.",
}),
blocks: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tenant_guard_blocks_total",
Help: "Requests and answers the guardrails blocked, by tenant (the level router.tenant_label names).",
}, []string{"tenant"}),
}
}
func (g *guardMetrics) collectors() []prometheus.Collector {
return []prometheus.Collector{g.checks, g.latency, g.stops, g.blocks}
}
// observe counts one verdict; tenant is "" where tenants are not labelled.
func (g *guardMetrics) observe(v guard.Verdict, stage guard.Stage, tenant string) {
action := string(v.Action)
if action == "" {
action = "allowed"
}
mode := "enforce"
if v.Shadow {
mode = "shadow"
}
g.checks.WithLabelValues(string(stage), v.Policy, action, v.Category, mode).Inc()
g.latency.WithLabelValues(string(stage)).Observe(v.Took.Seconds())
if v.Action == guard.Block && !v.Shadow && tenant != "" {
g.blocks.WithLabelValues(tenant).Inc()
}
}
+1
View File
@@ -457,6 +457,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
res := r.dispatch(req.Context(), w, parsed)
status := r.finish(w, parsed, res, start)
r.observeOutput(parsed, res, res.judged)
span.SetAttributes(attribute.Int("http.response.status_code", status),
attribute.String("llmgw.served_model", res.model),
attribute.String("llmgw.endpoint", res.endpointID))
+3 -1
View File
@@ -31,6 +31,7 @@ type metrics struct {
prefixDecisions *prometheus.CounterVec
tenant *tenantMetrics
guard *guardMetrics
}
func newMetrics() *metrics {
@@ -89,6 +90,7 @@ func newMetrics() *metrics {
Help: "Requests refused by flow control, by reason.",
}, []string{"band", "reason"}),
tenant: newTenantMetrics(),
guard: newGuardMetrics(),
}
}
@@ -98,7 +100,7 @@ func (m *metrics) collectors() []prometheus.Collector {
m.requests, m.duration, m.ttft, m.tokens,
m.scorerValue, m.filtered, m.considered,
m.admissionWait, m.admissionRejected, m.prefixDecisions,
}, m.tenant.collectors()...)
}, append(m.tenant.collectors(), m.guard.collectors()...)...)
}
// observePrefix records which branch a prefix decision took.
+5 -2
View File
@@ -716,8 +716,10 @@ type dispatchResult struct {
servedModel
outcome proxy.Outcome
err error
// guard is what the output filter decided, if there was one.
guard guard.Verdict
// guard is what the output filter decided, if there was one, and
// judged whether there was.
guard guard.Verdict
judged bool
}
type servedModel struct {
@@ -895,6 +897,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
if filter != nil {
res.guard = filter.Verdict()
res.judged = true
}
r.m.observeScores(dec)