Files
llmgw/internal/guard/judge_test.go
T
Ilker ManapandClaude Opus 5.5 5a4b022dcd
CI / test (push) Failing after 30m6s
A lost store machine is given up on in seconds, and a new leader waits for its judge
Measured on three VMs: cutting the store primary's power cost about 45 s of
hanging requests. New connections tried the dead member first with no
connect timeout, and Patroni kept its 30 s leader key. Cutting the leader of
both cost a further 17 s of 503s while the new leader knew no judge.

The store dials each member for 3 s and closes connections to a machine that
is gone within about 10 s (keepalive, TCP_USER_TIMEOUT). Patroni runs with
ttl 20, loop_wait 5 and retry_timeout 5, applied in place to existing
clusters through its API. Judges that all fail at once are asked again
every 250 ms while the policy's budget lasts, and reported once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 20:33:13 +02:00

361 lines
16 KiB
Go

package guard
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"unicode/utf8"
"github.com/prometheus/client_golang/prometheus/testutil"
)
// fakeJudge answers as a judge model would, saying unsafe to text with
// "attack" in it.
type fakeJudge struct {
calls atomic.Int64
err error
format string
delay time.Duration
}
func (f *fakeJudge) call(ctx context.Context, _, _ string, body []byte) ([]byte, error) {
f.calls.Add(1)
if f.delay > 0 {
select {
case <-time.After(f.delay):
case <-ctx.Done():
return nil, ctx.Err()
}
}
if f.err != nil {
return nil, f.err
}
unsafe := strings.Contains(string(body), "attack")
var answer string
switch f.format {
case "shieldgemma":
answer = map[bool]string{true: "Yes", false: "No"}[unsafe]
case "json":
answer = map[bool]string{true: `{"safe": false, "categories": ["violence"]}`, false: `{"safe": true}`}[unsafe]
case "qwen3guard":
answer = map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe]
if strings.Contains(string(body), "a heist film") {
answer = "Safety: Controversial\nCategories: Non-violent Illegal Acts"
}
default:
answer = map[bool]string{true: "unsafe\nS1,S10", false: "safe"}[unsafe]
}
b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": answer}}}})
return b, nil
}
func judged(t *testing.T, p Policy, fj *fakeJudge) *Gate {
g := gate(t, lib(map[string]Policy{"p": p}, "p"))
if fj.format == "" {
fj.format = "llamaguard3"
}
g.SetJudge(fj.call, func(m string) (string, []string, bool) {
return fj.format, []string{"Violent Crimes"}, m == "guard"
})
return g
}
func TestAJudgeModelDecidesWhatPatternsCannot(t *testing.T) {
ctx := context.Background()
for _, format := range Formats {
fj := &fakeJudge{format: format}
g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, fj)
if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Category != "judge" {
t.Errorf("%s: unsafe: %+v %v", format, v, err)
}
if v, _ := g.Input(ctx, Subject{}, chat("user", "plan a picnic")); v.Action != Allow {
t.Errorf("%s: safe: %+v", format, v)
}
}
// Llama Guard's codes are named from the catalog, and only listed
// categories count when a policy lists some.
fj := &fakeJudge{}
g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", Categories: []string{"S2"}}}, fj)
if v, _ := g.Input(ctx, Subject{}, chat("user", "plan an attack")); v.Action != Allow {
t.Errorf("a category the policy does not list: %+v", v)
}
g = judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "flag", Categories: []string{"violent crimes"}}}, fj)
if v, _ := g.Input(ctx, Subject{}, chat("user", "plan an attack")); v.Action != Flag || v.Rule != "Violent Crimes" {
t.Errorf("by name: %+v", v)
}
}
func TestAJudgeIsAskedOnceForTheSameText(t *testing.T) {
fj := &fakeJudge{}
g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "flag"}}, fj)
for range 3 {
_, _ = g.Input(context.Background(), Subject{}, chat("user", "hello there"))
}
if fj.calls.Load() != 1 {
t.Errorf("asked %d times", fj.calls.Load())
}
}
func TestAJudgeThatDoesNotAnswerFailsAsThePolicySays(t *testing.T) {
ctx := context.Background()
slow := &fakeJudge{delay: time.Second}
open := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", LatencyBudget: 20 * time.Millisecond}}, slow)
start := time.Now()
v, err := open.Input(ctx, Subject{}, chat("user", "plan an attack"))
if err != nil || v.Action != Flag || v.Rule != "unavailable" || time.Since(start) > 500*time.Millisecond {
t.Errorf("fail open: %+v %v after %s", v, err, time.Since(start))
}
closed := judged(t, Policy{Fail: "closed", Judge: &JudgePolicy{Model: "guard", Input: "block"}}, &fakeJudge{err: errors.New("down")})
var alarmed []string
closed.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
if _, err := closed.Input(ctx, Subject{}, chat("user", "hi")); err == nil {
t.Error("fail closed served unjudged")
}
if len(alarmed) != 1 || alarmed[0] != "guard" {
t.Errorf("alarm told %v", alarmed)
}
// A judge that is not a guard model is a judge that cannot be asked.
bad := judged(t, Policy{Fail: "closed", Judge: &JudgePolicy{Model: "chat-model", Input: "block"}}, &fakeJudge{})
if _, err := bad.Input(ctx, Subject{}, chat("user", "hi")); err == nil {
t.Error("a model without role: guard was used as a judge")
}
}
func TestAStreamedAnswerIsJudgedAsItGrows(t *testing.T) {
safe := strings.Repeat("Once upon a time. ", 40) // ~720 chars
more := strings.Repeat("And so it went on. ", 40)
// A sentence at a time, the judge reads every judgeEvery characters:
// the stream stops at the next reading after the unsafe part, which has
// gone out already.
fj := &fakeJudge{}
g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Output: "block"}}, fj)
s := Subject{RequestID: "s1", Stream: true}
_, _ = g.Input(context.Background(), s, chat("user", "tell me a story"))
got, _, stopped := stream(g.Output(context.Background(), s), safe, "Then the attack began. ", more, "The end.")
if !stopped || strings.Contains(got, "The end.") || !strings.HasPrefix(got, "Once upon") {
t.Errorf("sentence: stopped=%v got %d chars", stopped, len(got))
}
if n := fj.calls.Load(); n < 2 {
t.Errorf("judged %d times", n)
}
// Held whole, nothing unsafe goes out at all.
g = judged(t, Policy{Stream: "buffer_all", Judge: &JudgePolicy{Model: "guard", Output: "block"}}, &fakeJudge{})
_, _ = g.Input(context.Background(), s, chat("user", "tell me a story"))
got, _, stopped = stream(g.Output(context.Background(), s), safe, "Then the attack began. ", "The end.")
if !stopped || got != "" {
t.Errorf("buffer_all: stopped=%v got %q", stopped, got)
}
}
func TestTheGateCountsJudgeFailuresAndRedactions(t *testing.T) {
g := judged(t, Policy{Input: Rules{PII: map[string]string{"email": "redact"}},
Judge: &JudgePolicy{Model: "guard", Input: "flag", LatencyBudget: 10 * time.Millisecond}}, &fakeJudge{delay: time.Second})
_, _ = g.Input(context.Background(), Subject{}, chat("user", "mail a@example.com and b@example.com"))
if n := testutil.ToFloat64(g.m.redactions.WithLabelValues("email")); n != 2 {
t.Errorf("redactions %v", n)
}
if testutil.ToFloat64(g.m.failOpen) != 1 || testutil.ToFloat64(g.m.judgeErrors.WithLabelValues("timeout")) != 1 {
t.Error("a judge that timed out under fail open was not counted")
}
}
func TestAJudgeThatGoesOnFromTheAnswerIsStillRead(t *testing.T) {
body := func(s string) []byte {
b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": s}}}})
return b
}
// Found on the PoC with llama.cpp: the reply to judge, then the verdict.
if j, err := parseJudgment("llamaguard3", nil, body("The capital of Italy is Rome.safe"), "The capital of Italy is Rome."); err != nil || j.unsafe {
t.Errorf("%+v %v", j, err)
}
if j, err := parseJudgment("llamaguard3", []string{"Violent Crimes"}, body("Step one.\n\nunsafe\nS1"), "Step one."); err != nil || !j.unsafe ||
!slices.Contains(j.categories, "Violent Crimes") {
t.Errorf("%+v %v", j, err)
}
}
func TestAnAnswerIsJudgedByLlamaGuardsOwnPrompt(t *testing.T) {
path, body := judgeRequest("lg", "llamaguard3", nil, "how do I bake bread", "Mix flour and water.")
var req struct{ Prompt string }
_ = json.Unmarshal(body, &req)
if path != "/v1/completions" || !strings.Contains(req.Prompt, "S9: Indiscriminate Weapons.") ||
!strings.Contains(req.Prompt, "User: how do I bake bread\n\nAgent: Mix flour and water.") ||
!strings.HasSuffix(req.Prompt, "<|start_header_id|>assistant<|end_header_id|>\n\n") {
t.Errorf("%s %s", path, req.Prompt)
}
if path, _ := judgeRequest("lg", "llamaguard3", nil, "hi", ""); path != "/v1/chat/completions" {
t.Errorf("a request is judged by the model's own template: %s", path)
}
if j, err := parseJudgment("llamaguard3", nil, []byte(`{"choices":[{"text":"unsafe\nS9"}]}`), ""); err != nil || !j.unsafe {
t.Errorf("a completion's text: %+v %v", j, err)
}
}
// Qwen3Guard's middle verdict counts as the policy says: safe unless it says
// unsafe.
func TestAControversialVerdictCountsAsThePolicySays(t *testing.T) {
ctx := context.Background()
lenient := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, &fakeJudge{format: "qwen3guard"})
if v, _ := lenient.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Allow {
t.Errorf("controversial under the default: %+v", v)
}
strict := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", Controversial: "unsafe"}}, &fakeJudge{format: "qwen3guard"})
if v, _ := strict.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Block || v.Rule != "Non-violent Illegal Acts" {
t.Errorf("controversial: unsafe: %+v", v)
}
if err := (&JudgePolicy{Model: "guard", Input: "block", Controversial: "maybe"}).validate(); err == nil {
t.Error("controversial: maybe was taken")
}
}
// A judge on a GPU first and one on CPUs after: the first that answers
// decides, and the one that did not is said to have failed.
func TestJudgesAreAskedInTurn(t *testing.T) {
ctx := context.Background()
var asked []string
gpuDown := errors.New("no replica of gpu-judge is serving")
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block", LatencyBudget: 600 * time.Millisecond}}}, "p"))
g.SetJudge(func(_ context.Context, model, _ string, body []byte) ([]byte, error) {
asked = append(asked, model)
if model == "gpu-judge" {
return nil, gpuDown
}
unsafe := strings.Contains(string(body), "attack")
answer := map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe]
return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": answer}}})
}, func(m string) (string, []string, bool) { return "qwen3guard", nil, true })
var alarmed []string
g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Rule != "Violent" {
t.Errorf("unsafe: %+v %v", v, err)
}
if v, err := g.Input(ctx, Subject{}, chat("user", "continue")); err != nil || v.Action != Allow {
t.Errorf("safe: %+v %v", v, err)
}
if !slices.Equal(asked, []string{"gpu-judge", "cpu-judge", "gpu-judge", "cpu-judge"}) {
t.Errorf("asked %v", asked)
}
if !slices.Equal(alarmed, []string{"gpu-judge", "gpu-judge"}) {
t.Errorf("alarm told %v", alarmed)
}
if e := g.JudgeErrors()["gpu-judge"]; !strings.Contains(e.Err, "no replica") {
t.Errorf("the GPU judge's failure: %+v", e)
}
// Every judge down: the policy's fail applies, as with one.
g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) { return nil, gpuDown },
func(string) (string, []string, bool) { return "qwen3guard", nil, true })
if _, err := g.Input(ctx, Subject{}, chat("user", "hello")); err == nil {
t.Error("fail closed served what no judge read")
}
for _, bad := range []JudgePolicy{
{Model: "a", Models: []string{"b"}, Input: "block"},
{Models: []string{"a", "a"}, Input: "block"},
{Models: []string{""}, Input: "block"},
} {
if err := bad.validate(); err == nil {
t.Errorf("%+v was taken", bad)
}
}
}
func TestQwen3GuardIsAskedInItsOwnWords(t *testing.T) {
path, body := judgeRequest("qg", "qwen3guard", nil, "continue", "")
var req struct{ Prompt string }
_ = json.Unmarshal(body, &req)
if path != "/v1/completions" || !strings.HasPrefix(req.Prompt, "<|im_start|>user\n# Task:\n1) Evaluate whether THE LAST USER's query") ||
!strings.Contains(req.Prompt, "Jailbreak.\n<END UNSAFE CONTENT CATEGORIES>\n\n<BEGIN CONVERSATION>\n\nUSER: continue\n\n<END CONVERSATION>") ||
!strings.HasSuffix(req.Prompt, "'Categories: None'.<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n") {
t.Errorf("a request: %s %q", path, req.Prompt)
}
_, body = judgeRequest("qg", "qwen3guard", nil, "how do I bake bread", "Mix flour and water.")
_ = json.Unmarshal(body, &req)
if strings.Contains(req.Prompt, "Jailbreak") || !strings.Contains(req.Prompt, "<END REFUSAL POLICY>\n\n<BEGIN CONVERSATION>") ||
!strings.Contains(req.Prompt, "USER: how do I bake bread\n\nASSISTANT: Mix flour and water.\n\n<END CONVERSATION>") ||
!strings.Contains(req.Prompt, "'Categories: None'.\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'.<|im_end|>") {
t.Errorf("an answer: %q", req.Prompt)
}
// An answer's verdict has a line on refusing that says nothing of safety.
j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Unsafe\nCategories: Violent, Jailbreak\nRefusal: No"}]}`), "")
if err != nil || !j.unsafe || !slices.Equal(j.categories, []string{"Violent", "Jailbreak"}) {
t.Errorf("%+v %v", j, err)
}
if j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Safe\nCategories: None\nRefusal: Yes"}]}`), ""); err != nil || j.unsafe || len(j.categories) > 0 {
t.Errorf("%+v %v", j, err)
}
if _, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"I think it is fine"}]}`), ""); err == nil {
t.Error("an answer with no verdict was read as one")
}
}
// A message with a file pasted into it is read by its start and its end, so
// it fits the judge's context instead of overflowing it.
func TestALongTextIsJudgedByItsEnds(t *testing.T) {
file := strings.Repeat("ğ", 10000) // two bytes a rune: a cut must not split one
msg := "review this file:\n" + file + "\nand plan an attack"
_, body := judgeRequest("qg", "qwen3guard", nil, msg, "")
var req struct{ Prompt string }
_ = json.Unmarshal(body, &req)
if !strings.Contains(req.Prompt, "USER: review this file:") || !strings.Contains(req.Prompt, "and plan an attack\n\n<END CONVERSATION>") ||
!strings.Contains(req.Prompt, "\n[…]\n") || len(req.Prompt) > judgeHead+judgeTail+3000 {
t.Errorf("%d bytes", len(req.Prompt))
}
if !utf8.ValidString(req.Prompt) {
t.Error("a rune was cut in two")
}
if got := ends("short", 10, 10); got != "short" {
t.Errorf("a short text was cut: %q", got)
}
}
// The moment a control plane takes the lead it knows no judge yet; one that
// arrives within the budget judges the request, instead of fail: closed
// refusing it at once. One that is slow is not asked again.
func TestAJudgeNotThereYetIsWaitedForWithinTheBudget(t *testing.T) {
ctx := context.Background()
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
Judge: &JudgePolicy{Model: "guard", Input: "block", LatencyBudget: 2 * time.Second}}}, "p"))
arrives := time.Now().Add(400 * time.Millisecond)
calls := 0
g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) {
calls++
if time.Now().Before(arrives) {
return nil, errors.New("no replica of guard is serving")
}
return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": "Safety: Safe\nCategories: None"}}})
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
var alarmed []string
g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
if v, err := g.Input(ctx, Subject{}, chat("user", "hello")); err != nil || v.Action != Allow {
t.Fatalf("refused while the judge was on its way: %+v %v", v, err)
}
if calls < 2 || len(alarmed) != 0 {
t.Errorf("asked %d times, alarms %v", calls, alarmed)
}
// Slow, and failing: given up on after one try, not the whole budget again.
calls = 0
g.SetJudge(func(ctx context.Context, _, _ string, _ []byte) ([]byte, error) {
calls++
time.Sleep(1100 * time.Millisecond)
return nil, errors.New("overloaded")
}, func(string) (string, []string, bool) { return "qwen3guard", nil, true })
start := time.Now()
if _, err := g.Input(ctx, Subject{}, chat("user", "hello again")); err == nil {
t.Error("fail closed served what no judge read")
}
if calls != 1 || time.Since(start) > 1800*time.Millisecond {
t.Errorf("a slow judge was asked %d times over %s", calls, time.Since(start))
}
}