diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index 3940ecb..1055462 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -489,3 +489,69 @@ groups: description: >- Patroni will not promote a standby more than 1 MB behind, so a failover now would find none fit. -1 is a lag Patroni cannot read. + + - name: llmgw-guard + rules: + - alert: GuardFailingOpen + expr: >- + sum(rate(llmgw_guard_fail_open_total[5m])) + / clamp_min(sum(rate(llmgw_guard_checks_total[5m])), 1e-9) > 0.01 + for: 5m + labels: { severity: warning } + annotations: + summary: "{{ $value | humanizePercentage }} of judged traffic is served unjudged" + description: >- + A guard policy's judge model is not answering within its latency + budget, and the policy fails open: requests go through flagged + "judge unavailable" instead of being judged. + llmgw_guard_judge_errors_total says whether it is timing out or + not there; `llmgw ctl replicas` whether the judge has replicas. + + - alert: GuardJudgeSlow + expr: >- + histogram_quantile(0.95, sum by (le, judge_model) (rate(llmgw_guard_judge_latency_seconds_bucket[5m]))) > 0.25 + for: 10m + labels: { severity: warning } + annotations: + summary: "Judge {{ $labels.judge_model }} answers in {{ $value | humanizeDuration }} at p95" + description: >- + Over the default latency budget: every judged request waits this + long, and those over their policy's budget are served unjudged or + refused. Give the judge more replicas, or faster ones. + + - alert: GuardJudgeUnavailable + expr: sum by (reason) (rate(llmgw_guard_judge_errors_total{reason!="timeout"}[5m])) > 0 + for: 5m + labels: { severity: critical } + annotations: + summary: "The guardrails cannot ask their judge ({{ $labels.reason }})" + description: >- + no_judge: a policy names a model that is not in the catalog with + role: guard. unavailable: it has no replica answering. format: it + answers in a form its guard.format does not read. + + - alert: GuardBlocksSpiking + expr: >- + sum by (tenant) (rate(llmgw_tenant_guard_blocks_total[1h])) + > 3 * sum by (tenant) (rate(llmgw_tenant_guard_blocks_total[7d])) + and sum by (tenant) (increase(llmgw_tenant_guard_blocks_total[1h])) > 10 + for: 15m + labels: { severity: warning } + annotations: + summary: "{{ $labels.tenant }} is being blocked three times as often as usual" + description: >- + A tenant's requests or answers are blocked by the guardrails + more than three times their weekly rate. Someone probing, a + client sending data it should not, or a policy that changed. + The /admin/guard page lists the recent ones by request. + + - alert: GuardSecretInAnswer + expr: sum(increase(llmgw_guard_checks_total{stage="output",category="secret",action!="allowed"}[5m])) > 0 + labels: { severity: critical } + annotations: + summary: "A model put a credential in an answer" + description: >- + An answer contained something shaped like a key or a private key. + It was stopped or rewritten if the policy enforces; either way a + model has seen it, in its prompt or its training. The usage rows + with guard_category secret say which request. diff --git a/internal/cli/server.go b/internal/cli/server.go index b1695e4..fb9b736 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -132,6 +132,9 @@ func newServerCmd() *cobra.Command { } gate.SetJudge(rt.Judge, rt.GuardFormat) rt.SetGuard(gate) + if err := rt.RegisterMetrics(gate); err != nil { + return fmt.Errorf("guard metrics: %w", err) + } log.Info("guardrails on", "policies", len(cfg.Guard.Policies), "default", cfg.Guard.Default) } diff --git a/internal/guard/gate.go b/internal/guard/gate.go index 2edf02e..c062e13 100644 --- a/internal/guard/gate.go +++ b/internal/guard/gate.go @@ -39,6 +39,7 @@ type Gate struct { deny map[string]*regexp.Regexp // by pattern regex tools toolMemo judger *judge + m *gateMetrics mu sync.Mutex attached map[string][]string // "scope/id" → policy names @@ -56,7 +57,7 @@ func NewGate(cfg Config, store Attachments, log *slog.Logger) (*Gate, error) { log = slog.Default() } g := &Gate{cfg: cfg, store: store, log: log, now: time.Now, - deny: map[string]*regexp.Regexp{}, merged: map[string]Policy{}} + deny: map[string]*regexp.Regexp{}, merged: map[string]Policy{}, m: newGateMetrics()} for _, p := range cfg.Policies { for _, r := range []Rules{p.Input, p.Output} { for _, d := range r.Deny { @@ -155,6 +156,9 @@ type finding struct { action Action category string rule string + // failOpen is a judge that did not answer, under a policy that serves + // anyway. + failOpen bool } // judge scans one piece of text under one direction's rules, returning @@ -334,8 +338,9 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro total := 0 ph := newPlaceholders() type rewrite struct { - seg segment - text string + seg segment + text string + spans []detect.Span } var rewrites []rewrite for _, sg := range segs { @@ -343,7 +348,7 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro f, red := g.judge(sg.text, p.Input) fs = append(fs, f...) if len(red) > 0 { - rewrites = append(rewrites, rewrite{sg, ph.rewrite(sg.text, red)}) + rewrites = append(rewrites, rewrite{sg, ph.rewrite(sg.text, red), red}) } } if p.Input.MaxChars > 0 && total > p.Input.MaxChars { @@ -360,13 +365,18 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro v.Took = g.now().Sub(start) return v, fmt.Errorf("the judge could not be asked: %w", err) } - fs = append(fs, judgeFinding(p, jp.Input, res, err)) + jf := judgeFinding(p, jp.Input, res, err) + if jf.failOpen { + g.m.failOpen.Inc() + } + fs = append(fs, jf) if best, found = strictest(fs); found && best.action == Block { return done(best), nil } } if !v.Shadow { for _, rw := range rewrites { + g.countRedactions(rw.spans) rw.seg.set(rw.text) changed = true } @@ -395,7 +405,7 @@ func judgeFinding(p Policy, action string, res judgment, err error) finding { if p.Fail == "closed" { return finding{action: Block, category: "judge", rule: "unavailable"} } - return finding{action: Flag, category: "judge", rule: "unavailable"} + return finding{action: Flag, category: "judge", rule: "unavailable", failOpen: true} } if ok, cat := p.Judge.counts(res); ok { return finding{action: actions[action], category: "judge", rule: cat} @@ -405,6 +415,12 @@ func judgeFinding(p Policy, action string, res judgment, err error) finding { func segsOf(d *doc) []segment { return d.segments() } +func (g *Gate) countRedactions(spans []detect.Span) { + for _, s := range spans { + g.m.redactions.WithLabelValues(s.Kind).Inc() + } +} + // lastUser is the last thing the user said: what a judge reads. func lastUser(segs []segment) string { for i := len(segs) - 1; i >= 0; i-- { diff --git a/internal/guard/judge.go b/internal/guard/judge.go index 441b7c4..237b984 100644 --- a/internal/guard/judge.go +++ b/internal/guard/judge.go @@ -98,6 +98,7 @@ type judgment struct { type judge struct { call Caller format ModelFormat + m *gateMetrics mu sync.Mutex cache map[[32]byte]cached @@ -115,18 +116,29 @@ const ( // SetJudge lets policies with a judge: section ask a model on the cluster. func (g *Gate) SetJudge(call Caller, format ModelFormat) { - g.judger = &judge{call: call, format: format, cache: map[[32]byte]cached{}} + g.judger = &judge{call: call, format: format, m: g.m, cache: map[[32]byte]cached{}} } -// errNoJudge is a policy naming a judge the gateway cannot ask. -var errNoJudge = errors.New("no judge model to ask") +// errNoJudge is a policy naming a judge the gateway cannot ask, and +// errJudgeFormat a judge whose answer could not be read. +var ( + errNoJudge = errors.New("no judge model to ask") + errJudgeFormat = errors.New("the judge's answer could not be read") +) // ask has the judge read conversation turns — the user's, and the // assistant's answer when judging output. -func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) { +func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (res judgment, err error) { if j == nil { return judgment{}, errNoJudge } + start := time.Now() + cachedHit := false + defer func() { + if !cachedHit && j.m != nil { + j.m.judged(jp.Model, time.Since(start), err) + } + }() format, names, ok := j.format(jp.Model) if !ok { return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, jp.Model) @@ -135,6 +147,7 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) ( j.mu.Lock() if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL { j.mu.Unlock() + cachedHit = true return c.j, nil } j.mu.Unlock() @@ -146,7 +159,7 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) ( if err != nil { return judgment{}, err } - res, err := parseJudgment(format, names, out) + res, err = parseJudgment(format, names, out) if err != nil { return judgment{}, err } @@ -227,7 +240,7 @@ func parseJudgment(format string, names []string, body []byte) (judgment, error) } `json:"choices"` } if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { - return judgment{}, errors.New("the judge's answer is not a chat completion") + return judgment{}, fmt.Errorf("%w: not a chat completion", errJudgeFormat) } text := strings.TrimSpace(resp.Choices[0].Message.Content) switch format { @@ -274,5 +287,5 @@ func parseJudgment(format string, names []string, body []byte) (judgment, error) } } } - return judgment{}, fmt.Errorf("the judge answered in a form %s does not have", format) + return judgment{}, fmt.Errorf("%w: not in the form %s has", errJudgeFormat, format) } diff --git a/internal/guard/judge_test.go b/internal/guard/judge_test.go index 74d1bb2..119e6dd 100644 --- a/internal/guard/judge_test.go +++ b/internal/guard/judge_test.go @@ -8,6 +8,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/prometheus/client_golang/prometheus/testutil" ) // fakeJudge answers as a judge model would, saying unsafe to text with @@ -139,3 +141,15 @@ func TestAStreamedAnswerIsJudgedAsItGrows(t *testing.T) { t.Errorf("buffer_all: stopped=%v got %q", stopped, got) } } + +func TestTheGateCountsJudgeFailuresAndRedactions(t *testing.T) { + g := judged(t, Policy{Input: Rules{PII: map[string]string{"email": "redact"}}, + Judge: &JudgePolicy{Model: "guard", Input: "flag", LatencyBudget: 10 * time.Millisecond}}, &fakeJudge{delay: time.Second}) + _, _ = g.Input(context.Background(), Subject{}, chat("user", "mail a@example.com and b@example.com")) + if n := testutil.ToFloat64(g.m.redactions.WithLabelValues("email")); n != 2 { + t.Errorf("redactions %v", n) + } + if testutil.ToFloat64(g.m.failOpen) != 1 || testutil.ToFloat64(g.m.judgeErrors.WithLabelValues("timeout")) != 1 { + t.Error("a judge that timed out under fail open was not counted") + } +} diff --git a/internal/guard/metrics.go b/internal/guard/metrics.go new file mode 100644 index 0000000..2723225 --- /dev/null +++ b/internal/guard/metrics.go @@ -0,0 +1,74 @@ +package guard + +import ( + "context" + "errors" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// gateMetrics are what only the gate sees: the judge, and redactions. +type gateMetrics struct { + judgeLatency *prometheus.HistogramVec + judgeErrors *prometheus.CounterVec + failOpen prometheus.Counter + redactions *prometheus.CounterVec +} + +func newGateMetrics() *gateMetrics { + return &gateMetrics{ + judgeLatency: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "llmgw_guard_judge_latency_seconds", + Help: "Time a judge model took to answer the guardrails, cached answers excluded.", + Buckets: []float64{.01, .025, .05, .1, .25, .5, 1, 2.5, 5}, + }, []string{"judge_model"}), + judgeErrors: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_guard_judge_errors_total", + Help: "Judge calls that gave no verdict, by reason: timeout, unavailable, format, no_judge.", + }, []string{"reason"}), + failOpen: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "llmgw_guard_fail_open_total", + Help: "Requests and answers served unjudged because a judge did not answer and the policy fails open.", + }), + redactions: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_guard_redactions_total", + Help: "Values the guardrails rewrote as placeholders, by kind.", + }, []string{"kind"}), + } +} + +// Describe and Collect make the gate a Prometheus collector. +func (g *Gate) Describe(ch chan<- *prometheus.Desc) { + for _, c := range g.collectors() { + c.Describe(ch) + } +} + +func (g *Gate) Collect(ch chan<- prometheus.Metric) { + for _, c := range g.collectors() { + c.Collect(ch) + } +} + +func (g *Gate) collectors() []prometheus.Collector { + m := g.m + return []prometheus.Collector{m.judgeLatency, m.judgeErrors, m.failOpen, m.redactions} +} + +func (m *gateMetrics) judged(model string, took time.Duration, err error) { + if err == nil { + m.judgeLatency.WithLabelValues(model).Observe(took.Seconds()) + return + } + reason := "unavailable" + switch { + case errors.Is(err, context.DeadlineExceeded): + reason = "timeout" + case errors.Is(err, errNoJudge): + reason = "no_judge" + case errors.Is(err, errJudgeFormat): + reason = "format" + } + m.judgeErrors.WithLabelValues(reason).Inc() +} diff --git a/internal/guard/output.go b/internal/guard/output.go index 9b58395..0b8e28a 100644 --- a/internal/guard/output.go +++ b/internal/guard/output.go @@ -133,6 +133,7 @@ func (f *outFilter) judgeText(text string) (finding, string) { if f.shadow || len(red) == 0 { return best, text } + f.g.countRedactions(red) return best, newPlaceholders().rewrite(text, red) } @@ -188,7 +189,11 @@ func (f *outFilter) askJudge(text string, final bool) finding { start := time.Now() res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String()) f.took += time.Since(start) - return f.record([]finding{judgeFinding(f.p, jp.Output, res, err)}) + jf := judgeFinding(f.p, jp.Output, res, err) + if jf.failOpen { + f.g.m.failOpen.Inc() + } + return f.record([]finding{jf}) } // judgeStructured checks a whole answer against the schema the request diff --git a/internal/router/guard.go b/internal/router/guard.go index f937856..114e467 100644 --- a/internal/router/guard.go +++ b/internal/router/guard.go @@ -87,6 +87,7 @@ func (r *Router) guardInput(ctx context.Context, w http.ResponseWriter, req *par return false } req.guardIn = v + r.m.guard.observe(v, guard.Input, r.m.tenant.tenantOf(req)) if v.Shadow { // Recorded, not done: the client sees nothing of it. return true @@ -143,6 +144,18 @@ func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guar r.accessLog(req, dispatchResult{guard: v}, status, time.Since(start)) } +// observeOutput counts what the output filter decided for an answer that +// was judged, and a stream it stopped. +func (r *Router) observeOutput(req *parsedRequest, res dispatchResult, judged bool) { + if !judged { + return + } + r.m.guard.observe(res.guard, guard.Output, r.m.tenant.tenantOf(req)) + if res.outcome.Stopped { + r.m.guard.stops.Inc() + } +} + // outputFilter attaches the guard's filter for one attempt to the proxy // request, and returns it so its verdict can be read afterwards. func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model string, preq *proxy.Request) guard.OutputFilter { diff --git a/internal/router/guard_test.go b/internal/router/guard_test.go index a19630f..70829db 100644 --- a/internal/router/guard_test.go +++ b/internal/router/guard_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" @@ -227,3 +229,15 @@ func TestAJudgeModelIsNobodysToAskButTheGuardrails(t *testing.T) { t.Errorf("format %q %v", f, ok) } } + +func TestWhatTheGuardrailsDidIsCounted(t *testing.T) { + rt, _ := guardedRouter(t, &wordGuard{answers: true}) + post(t, rt.Handler(), say("something forbidden"), nil) + post(t, rt.Handler(), say("hello"), nil) + if n := testutil.ToFloat64(rt.m.guard.checks.WithLabelValues("input", "words", "blocked", "test", "enforce")); n != 1 { + t.Errorf("input blocks counted %v", n) + } + if n := testutil.ToFloat64(rt.m.guard.checks.WithLabelValues("output", "answers", "blocked", "", "enforce")); n != 1 { + t.Errorf("output blocks counted %v", n) + } +} diff --git a/internal/router/guardmetrics.go b/internal/router/guardmetrics.go new file mode 100644 index 0000000..4620571 --- /dev/null +++ b/internal/router/guardmetrics.go @@ -0,0 +1,59 @@ +package router + +import ( + "github.com/prometheus/client_golang/prometheus" + + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +// guardMetrics count what the guardrails did, by name: never what they read. +type guardMetrics struct { + checks *prometheus.CounterVec + latency *prometheus.HistogramVec + stops prometheus.Counter + blocks *prometheus.CounterVec +} + +func newGuardMetrics() *guardMetrics { + return &guardMetrics{ + checks: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_guard_checks_total", + Help: "Requests and answers judged by the guardrails, by stage (input, output), policy, action " + + "(allowed, flagged, redacted, blocked), category, and mode (enforce, shadow).", + }, []string{"stage", "policy", "action", "category", "mode"}), + latency: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "llmgw_guard_latency_seconds", + Help: "Time the guardrails took to judge a request or an answer, judge model included.", + Buckets: []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5}, + }, []string{"stage"}), + stops: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "llmgw_guard_stream_stops_total", + Help: "Streamed answers the guardrails stopped part way.", + }), + blocks: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_tenant_guard_blocks_total", + Help: "Requests and answers the guardrails blocked, by tenant (the level router.tenant_label names).", + }, []string{"tenant"}), + } +} + +func (g *guardMetrics) collectors() []prometheus.Collector { + return []prometheus.Collector{g.checks, g.latency, g.stops, g.blocks} +} + +// observe counts one verdict; tenant is "" where tenants are not labelled. +func (g *guardMetrics) observe(v guard.Verdict, stage guard.Stage, tenant string) { + action := string(v.Action) + if action == "" { + action = "allowed" + } + mode := "enforce" + if v.Shadow { + mode = "shadow" + } + g.checks.WithLabelValues(string(stage), v.Policy, action, v.Category, mode).Inc() + g.latency.WithLabelValues(string(stage)).Observe(v.Took.Seconds()) + if v.Action == guard.Block && !v.Shadow && tenant != "" { + g.blocks.WithLabelValues(tenant).Inc() + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index c8f5b95..68d34c7 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -457,6 +457,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { res := r.dispatch(req.Context(), w, parsed) status := r.finish(w, parsed, res, start) + r.observeOutput(parsed, res, res.judged) span.SetAttributes(attribute.Int("http.response.status_code", status), attribute.String("llmgw.served_model", res.model), attribute.String("llmgw.endpoint", res.endpointID)) diff --git a/internal/router/metrics.go b/internal/router/metrics.go index ddc5d38..cdea858 100644 --- a/internal/router/metrics.go +++ b/internal/router/metrics.go @@ -31,6 +31,7 @@ type metrics struct { prefixDecisions *prometheus.CounterVec tenant *tenantMetrics + guard *guardMetrics } func newMetrics() *metrics { @@ -89,6 +90,7 @@ func newMetrics() *metrics { Help: "Requests refused by flow control, by reason.", }, []string{"band", "reason"}), tenant: newTenantMetrics(), + guard: newGuardMetrics(), } } @@ -98,7 +100,7 @@ func (m *metrics) collectors() []prometheus.Collector { m.requests, m.duration, m.ttft, m.tokens, m.scorerValue, m.filtered, m.considered, m.admissionWait, m.admissionRejected, m.prefixDecisions, - }, m.tenant.collectors()...) + }, append(m.tenant.collectors(), m.guard.collectors()...)...) } // observePrefix records which branch a prefix decision took. diff --git a/internal/router/router.go b/internal/router/router.go index 2139992..122c7c4 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -716,8 +716,10 @@ type dispatchResult struct { servedModel outcome proxy.Outcome err error - // guard is what the output filter decided, if there was one. - guard guard.Verdict + // guard is what the output filter decided, if there was one, and + // judged whether there was. + guard guard.Verdict + judged bool } type servedModel struct { @@ -895,6 +897,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr} if filter != nil { res.guard = filter.Verdict() + res.judged = true } r.m.observeScores(dec)