From b0273ffb5faec4bebd27ce29b74234382652c7f6 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 8 Oct 2026 10:22:47 +0200 Subject: [PATCH] Qwen3Guard is the judge, and judges on a GPU and on CPUs are asked in turn Llama Guard 3 1B could not tell ordinary coding requests from harmful ones: on the PoC it scored "continue" 0.51, "continue the code above" 0.89 and "how do I kill all child processes in bash" 0.92 unsafe, against 0.97 to 1.00 for harmful requests, at Q4_K_M and Q8_0 alike, with its own template. A site had "continue" refused. Qwen3Guard-Gen 0.6B gave the ordinary ones 0.15 at the most and the harmful ones 1.00, in 0.8 GB on CPUs. - format qwen3guard: its prompt written out as its template writes it, for requests and answers, and its Safe / Unsafe / Controversial verdict read; controversial counts as the policy's judge.controversial says, safe by default. - judge.models: judges asked in turn, the first that answers deciding; a failure is recorded against the judge that failed. guard judge use sets the order, replacing categories none of the judges names. - known judges (Qwen3Guard 0.6B/4B/8B, Llama Guard 1B/8B): guard judge enable serves any of them, --gpu the 4B on the roomiest GPU class as a standard model; the leader fetches and serves one a policy names that the catalog lacks. - server init installs qwen3guard-0.6b with the baseline's categories. - a judge reads a long text by its start and end, which fit its context, rather than overflowing it and refusing the request under fail: closed. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 28 ++ docs/admin-api.openapi.json | 19 +- docs/guardrails.md | 130 ++++++--- docs/install.md | 5 +- docs/quick-install.md | 2 +- docs/reference.md | 30 +- internal/cli/guard_judge.go | 259 ++++++++++++++---- internal/cli/judgekeeper.go | 116 ++++++++ internal/cli/judgekeeper_test.go | 126 +++++++++ internal/cli/judges_known.go | 158 +++++++++++ internal/cli/server.go | 15 + internal/cli/serverinit.go | 21 +- internal/cli/serverinit_cluster.go | 20 +- internal/cli/serverinit_cluster_test.go | 16 +- internal/control/adminapi/api.go | 11 +- internal/control/adminapi/judges.go | 37 ++- internal/control/catalog/catalog.go | 17 +- internal/control/modelserve/modelserve.go | 17 +- internal/docgen/reference_gen.go | 6 +- internal/guard/gate.go | 16 +- internal/guard/judge.go | 317 +++++++++++++++++++--- internal/guard/judge_test.go | 128 ++++++++- internal/guard/output.go | 4 +- internal/web/admin_serve.go | 23 +- 24 files changed, 1313 insertions(+), 208 deletions(-) create mode 100644 internal/cli/judgekeeper.go create mode 100644 internal/cli/judgekeeper_test.go create mode 100644 internal/cli/judges_known.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5960dc8..4769e87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,20 @@ What was left of the field report from v0.10.6. ### Added +- **A judge on a GPU and one on CPUs together**: a policy's `judge.models` + is a list asked in turn, the first that answers giving the verdict; the + next is asked when one has no replica serving, fails, or misses its + `latency_budget`. `llmgw ctl guard judge use qwen3guard-4b + qwen3guard-0.6b` sets the order for every policy with a judge, and + `guard judge status` says which judge stands in for which. A judge on a + GPU is served as a `standard` model, so it takes a card only from what + may give one up. +- **A judge a policy names is fetched if it is missing**: the leader + fetches the weights of a judge llmgw knows (`qwen3guard-0.6b`, `-4b`, + `-8b`, `llama-guard-3-1b`, `-8b`) into the mirror and serves it, when a + policy names it and the catalog does not have it — and again if it is + removed. `guard judge enable --gpu` serves Qwen3Guard-Gen 4B on the class + with the most free cards. - **The whole system stops, starts and restarts from one command**: `llmgw stop` takes every model down and answers every request for one with 503 (`system_stopped`), keeping the control planes, agents and mirror running @@ -65,6 +79,20 @@ What was left of the field report from v0.10.6. ### Fixed +- **The judge no longer refuses ordinary coding requests**: Llama Guard 3 + 1B, the judge `server init` installed, could not tell "continue", "devam + et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe) + from harmful requests (0.97 to 1.00), at any quantisation. The judge is + now Qwen3Guard-Gen 0.6B (format `qwen3guard`), which gave the same + ordinary requests 0.15 at the most and the harmful ones 1.00, on CPUs, + in 0.8 GB. Its third verdict, controversial, counts as safe unless the + policy says `controversial: unsafe`. `guard judge use qwen3guard-0.6b` + moves an installed cluster to it, and replaces Llama Guard's category + codes, which Qwen3Guard never names, with the baseline's. +- **A long message is judged rather than refused**: a judge read the whole + last user message, and one with a file pasted into it overflowed the + judge's context, so it gave no verdict and `fail: closed` refused the + request. It now reads the start and the end, about 6000 characters. - **A replica no longer stays "starting" while it serves after a restart**: an agent that registered before its adopted engines had answered their health check never said they had, so the control plane kept them starting diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index f253b42..bb1acfe 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -4573,6 +4573,13 @@ "type": "string" } }, + "then": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Then are the judges its policies ask after it, when it gives no verdict: a CPU judge behind a GPU one." + }, "in_catalog": { "type": "boolean", "description": "InCatalog is whether a model of that name is in the catalog, and Judge whether it is there as a judge (role: guard)." @@ -4830,6 +4837,10 @@ "source": { "type": "string" }, + "role": { + "type": "string", + "description": "Role is \"guard\" for a guardrails judge, empty for a model clients use." + }, "min_replicas": { "type": "integer" }, @@ -6331,7 +6342,7 @@ "properties": { "format": { "type": "string", - "description": "Format is llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and the categories), shieldgemma (Google's ShieldGemma: Yes or No), or json ({\"safe\": false, \"categories\": [\"...\"]})." + "description": "Format is qwen3guard (Qwen3Guard-Gen: \"Safety: Safe\", \"Unsafe\" or \"Controversial\", and the categories), llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and the categories), shieldgemma (Google's ShieldGemma: Yes or No), or json ({\"safe\": false, \"categories\": [\"...\"]})." }, "categories": { "type": "array", @@ -7511,7 +7522,11 @@ }, "guard_format": { "type": "string", - "description": "GuardFormat makes it a judge model for the guardrails (role: guard): llamaguard3, shieldgemma or json. Empty serves clients." + "description": "GuardFormat makes it a judge model for the guardrails (role: guard): llamaguard3, qwen3guard, shieldgemma or json. Empty serves clients." + }, + "class": { + "type": "string", + "description": "Class is what it may take from others and they from it when there is not room for both: critical, standard or opportunistic. Empty is the catalog's default for its role." }, "ctx_pool": { "type": "integer", diff --git a/docs/guardrails.md b/docs/guardrails.md index 4d1f309..73f7a59 100644 --- a/docs/guardrails.md +++ b/docs/guardrails.md @@ -4,7 +4,7 @@ Written for the operator deciding what the gateway lets through: what a policy can check, what it can do, and how one is attached to a team. `llmgw server init` writes a `baseline` policy that applies to every -request, with Llama Guard 3 1B as its judge on a CPU; see +request, with Qwen3Guard-Gen 0.6B as its judge on CPUs; see [install.md](install.md#2-the-control-plane) for what it checks. The policies are part of the cluster's settings, kept in the store: `llmgw ctl config edit` (or the console's Settings) changes them, and the change is in @@ -114,79 +114,136 @@ guard: ## A judge model -Patterns find what has a shape. For what does not — violence, self-harm, -abuse, whatever a moderation model was trained on — a policy can have a -**judge model on the cluster** read the text. +Patterns find what has a shape. For what does not — violence, crime, +sexual content, self-harm, jailbreaks, whatever a moderation model was +trained on — a policy can have a **judge model on the cluster** read the +text. The judge must be in the catalog under the name the policy gives, as a judge (`role: guard`), with a replica ready. When it is not, what the policy judges is refused with `guard_unavailable` (`fail: closed`) or served unjudged (`fail: open`), and `llmgw ctl status` and the console's Overview -say which judge and why. To serve it: +say which judge and why. + +### Which judge + +| judge | runs on | size | | +|---|---|---|---| +| `qwen3guard-0.6b` | CPUs | 0.8 GB | the default: about 2 s a check on two threads | +| `qwen3guard-4b` | a GPU | 2.5 GB | for a cluster with a card to spare | +| `qwen3guard-8b` | a GPU | 4.7 GB | | +| `llama-guard-3-1b` | CPUs | 0.9 GB | the earlier default: refuses ordinary coding requests | +| `llama-guard-3-8b` | a GPU | 4.6 GB | | + +Measured on the reference cluster, as the judge's probability of "unsafe" +for ordinary coding requests against harmful ones, Llama Guard 3 1B could +not tell them apart: "continue" 0.51, "continue the code above" 0.89, "how +do I kill all child processes in bash" 0.92, against 0.97 to 1.00 for +harmful requests, at any quantisation. Qwen3Guard-Gen 0.6B gave the +ordinary ones 0.15 at the most and the harmful ones 1.00. Its verdicts are +safe, unsafe and **controversial**; controversial counts as safe unless the +policy says `controversial: unsafe`. + +A policy that names one of these is enough: the leader fetches its weights +into the mirror and serves it as a judge, and serves it again if it is +removed from the catalog. Another judge is served by hand with `--repo`. ``` -llmgw ctl guard judge status # each judge the policies ask, and what is wrong with it -llmgw ctl guard judge enable # Llama Guard 3 1B on CPUs, under the name the policies ask -llmgw ctl guard judge enable --replace # when a model of that name is there, but not as a judge +llmgw ctl guard judge status # each judge the policies ask, and what is wrong with it +llmgw ctl guard judge enable # Qwen3Guard-Gen 0.6B on CPUs, or the judge the policies ask that is missing +llmgw ctl guard judge enable --gpu # Qwen3Guard-Gen 4B on a GPU +llmgw ctl guard judge use qwen3guard-4b qwen3guard-0.6b # the order every policy asks them in +llmgw ctl guard judge enable --replace # when a model of that name is there, but not as a judge ``` or **Do → Serve the guardrails' judge** in the console. -**When it refuses ordinary requests.** A refusal by a judge names the hazard -it saw — the client is told `refused by the gateway's policy (judge: -S6,Specialized Advice)` — and the same is kept with the request: `llmgw ctl -request `, the server's log (`guard_rule`) and the guardrails page show -it. To see what the judge says of any text, without anything being kept: +### A GPU and CPUs together + +`judge.models` is a list asked in turn: the first judge that answers gives +the verdict. The next is asked when one has no replica serving, fails, or +does not answer within `latency_budget`; a judge with no replica at all is +passed over at once. A judge on a GPU first and one on CPUs after it is +the GPU's speed while it has its card, and the CPUs' when it does not. + +```yaml +judge: + models: [qwen3guard-4b, qwen3guard-0.6b] + input: block + output: block + latency_budget: 5s # each judge's wait +``` + +A judge on a GPU is served as a `standard` model, on one card: it takes a +card only from what may give one up, and waits while every card is busy, +the judge on CPUs standing in. A judge on CPUs is `critical`, with two +replicas, because every judged request waits on one. `llmgw ctl guard judge +status` says, for each judge, which is asked when it gives no verdict. + +### When it refuses ordinary requests + +A refusal by a judge names the hazard it saw — the client is told `refused +by the gateway's policy (judge: Violent)` — and the same is kept with the +request: `llmgw ctl request `, the server's log (`guard_rule`) and the +guardrails page show it. To see what the judge says of any text, without +anything being kept: ``` llmgw ctl guard judge ask 'How do I sort a list in Python?' -# llama-guard-3-1b: safe, 140 ms -# it answered: "safe" +# qwen3guard-0.6b: safe, 1900 ms +# it answered: "Safety: Safe\nCategories: None" ``` -A judge that says `unsafe` to ordinary text is usually the wrong weights -(another repository's or quantisation's, without Llama Guard's template: -`judge enable --replace` serves the right ones), or a policy counting every -hazard: give its judge `categories`, leaving out S5, S6, S8 and S13 as the -baseline does. While finding out, `judge.input: flag` records instead of -refusing. A Llama Guard file -served from the mirror page is served as a judge too, under the name the -policies ask. By hand, the catalog entry is: +A judge that says `unsafe` to ordinary text is usually Llama Guard 3 1B +(`llmgw ctl guard judge use qwen3guard-0.6b`), the wrong weights (another +repository's, without the judge's template: `judge enable --replace` serves +the right ones), or a policy counting every category: give its judge +`categories`. The baseline counts Violent, Non-violent Illegal Acts, Sexual +Content or Sexual Acts, Suicide & Self-Harm and Jailbreak; PII is the +patterns', and copyright, unethical acts and politically sensitive topics +refuse too much ordinary work. While finding out, `judge.input: flag` +records instead of refusing. + +A judge's file served from the mirror page — Llama Guard, Qwen3Guard, +ShieldGemma — is served as a judge too, under the name the policies ask. +By hand, the catalog entry is: ```yaml # a catalog model (server.yaml models:, or ctl model add) -- name: llama-guard-3-1b - source: http://mirror:8081/llama-guard-3-1b/…/Llama-Guard-3-1B-Q4_K_M.gguf +- name: qwen3guard-0.6b + source: http://mirror:8081/qwen3guard-0.6b/…/Qwen3Guard-Gen-0.6B.Q8_0.gguf role: guard guard: - format: llamaguard3 # llamaguard3, shieldgemma or json - categories: [Violent Crimes, Non-Violent Crimes, Sex-Related Crimes, …] # names for S1, S2, … + format: qwen3guard # qwen3guard, llamaguard3, shieldgemma or json scaling: {min: 2, max: 2} profiles: [{name: cpu, engine: llamacpp}] # in a guard policy judge: - model: llama-guard-3-1b + models: [qwen3guard-0.6b] # or model: one judge input: block # flag or block; empty does not ask about input output: flag - categories: [S1, S10] # only these count; empty: any unsafe verdict - latency_budget: 250ms # then the policy's fail: open serves (flagged), closed refuses + categories: [Violent, Jailbreak] # only these count; empty: any unsafe verdict + controversial: safe # or unsafe + latency_budget: 250ms # each judge's wait; after the last, fail: open serves (flagged), closed refuses ``` A model with `role: guard` is the guardrails' alone: - it is not in `/v1/models` or the chat page, and is refused (404) to every key, as a name and as a fallback; -- it is `critical` by default and must always run — on demand is refused. - Give it two replicas on different nodes, because every judged request - waits on it. +- it must always run — on demand is refused. One on CPUs is `critical` + and should have two replicas on different nodes, because every judged + request waits on it. The gateway's calls to it go past admission, accounting and the guardrails themselves. - **Formats:** - - `llamaguard3` reads Meta's "safe" / "unsafe -S1,S10". + - `qwen3guard` asks in Qwen3Guard's own words and reads "Safety: Safe | + Unsafe | Controversial" and its "Categories:". + - `llamaguard3` reads Meta's "safe" / "unsafe\nS1,S10"; give the catalog + entry `categories` to have S1 recorded as its name. - `shieldgemma` asks Google's model whether the text breaks the listed principles (`categories`, each a sentence) and reads Yes or No. - `json` expects `{"safe": false, "categories": [...]}`. @@ -196,6 +253,9 @@ S1,S10". is read every 600 characters and at the end, so text between two readings has gone out already; `stream: buffer_all` holds it until it is read. + - **A long text** is read by its start and its end (about 6000 + characters), so a message with a file pasted into it fits the judge's + context rather than overflowing it and going unjudged. - **Cache:** the same text is not asked about twice within five minutes. ## Where a policy applies diff --git a/docs/install.md b/docs/install.md index 5151ae5..f148818 100644 --- a/docs/install.md +++ b/docs/install.md @@ -129,8 +129,9 @@ That is the whole step. In order, it: `nvidia-smi` finds any, on CPU otherwise — and waits for it to install its engine 9. closes llmgw's internal ports to everything but the cluster's networks -10. fetches Llama Guard 3 1B, the guardrails' judge, and runs two replicas - on the CPUs of nodes with room — this one, or the first that join +10. fetches Qwen3Guard-Gen 0.6B, the guardrails' judge, and runs two + replicas on the CPUs of nodes with room — this one, or the first that + join ([guardrails.md](guardrails.md#a-judge-model) has one on a GPU too) 11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the first thing anyone opens `/chat` to find is a model that answers. A machine without room for it is told so within two minutes, and why. diff --git a/docs/quick-install.md b/docs/quick-install.md index c97037e..0a1b754 100644 --- a/docs/quick-install.md +++ b/docs/quick-install.md @@ -52,7 +52,7 @@ install: - your administrator account. **The password is shown once. Copy it now.** - a model mirror, which downloads model files once for the whole cluster; - this machine as a first node, and a small model (Qwen2.5 0.5B) on it; -- guardrails on every request, with a judge model (Llama Guard 3 1B, two +- guardrails on every request, with a judge model (Qwen3Guard-Gen 0.6B, two replicas) on CPUs — a request it cannot check is refused, not let through — and API keys that expire within 90 days. What is hardened, and how to loosen it: [install.md](install.md#2-the-control-plane). diff --git a/docs/reference.md b/docs/reference.md index 2d5698f..44ac033 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -102,9 +102,11 @@ rather than repeated. | `format` | `string` | [GuardModel](#guardmodel) | | `categories` | `[]string` | [GuardModel](#guardmodel) | | `model` | `string` | [JudgePolicy](#judgepolicy) | +| `models` | `[]string` | [JudgePolicy](#judgepolicy) | | `input` | `string` | [JudgePolicy](#judgepolicy) | | `output` | `string` | [JudgePolicy](#judgepolicy) | | `categories` | `[]string` | [JudgePolicy](#judgepolicy) | +| `controversial` | `string` | [JudgePolicy](#judgepolicy) | | `latency_budget` | `time.Duration` | [JudgePolicy](#judgepolicy) | | `tokenize_timeout` | `time.Duration` | [KVIndex](#kvindex) | | `min_blocks` | `int` | [KVIndex](#kvindex) | @@ -994,9 +996,11 @@ every node and control plane within seconds. `string` -Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and -the categories), shieldgemma (Google's ShieldGemma: Yes or No), or -json ({"safe": false, "categories": ["..."]}). +Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or +"Controversial", and the categories), llamaguard3 (Meta's Llama Guard +3: "safe", or "unsafe" and the categories), shieldgemma (Google's +ShieldGemma: Yes or No), or json ({"safe": false, "categories": +["..."]}). ### `categories` @@ -1013,6 +1017,15 @@ sentence — and the names reported for llamaguard3's S1, S2, …. Model is a catalog model with role: guard. +### `models` + +`[]string` + +Models are judges asked in turn, instead of Model: the first that +answers gives the verdict. A judge on a GPU first and one on CPUs +after it is the GPU's speed while it is there, and the CPUs' when it +is not — busy, failed, or the node down. + ### `input` `string` @@ -1031,12 +1044,19 @@ Empty does not ask the judge about that direction. Categories, when set, are the only ones that count: Llama Guard's codes (S1, S10) or the names the model reports. +### `controversial` + +`string` + +Controversial is what Qwen3Guard's third verdict counts as: safe +(the default) or unsafe. + ### `latency_budget` `time.Duration` -LatencyBudget is how long the judge is waited for before the policy's -fail applies. Zero is 250ms. +LatencyBudget is how long each judge is waited for before the next is +asked, or, after the last, the policy's fail applies. Zero is 250ms. ## KVIndex diff --git a/internal/cli/guard_judge.go b/internal/cli/guard_judge.go index 4ca6f2d..4034231 100644 --- a/internal/cli/guard_judge.go +++ b/internal/cli/guard_judge.go @@ -4,20 +4,17 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" -) - -// The judge the guardrails are installed with: Meta's Llama Guard 3 1B, -// from the repository server init fetches it from. -const ( - guardModelRepo = "QuantFactory/Llama-Guard-3-1B-GGUF" - guardModelFile = "Llama-Guard-3-1B.Q4_K_M.gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" ) func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { @@ -27,9 +24,15 @@ func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { Long: "A policy with a judge has a model read what patterns cannot. The judge must\n" + "be in the catalog under the name the policy gives, as a judge (role:\n" + "guard), with a replica ready; otherwise what the policy judges is refused\n" + - "(fail: closed, guard_unavailable) or served unjudged (fail: open).", + "(fail: closed, guard_unavailable) or served unjudged (fail: open).\n\n" + + "A policy may name several, asked in turn (judge.models): a judge on a GPU\n" + + "first and one on CPUs after it is the GPU's speed while it is there, and\n" + + "the CPUs' when it is not.\n\n" + + "Judges llmgw knows where to fetch, which a policy may name and the leader\n" + + "then fetches and serves by itself: " + knownJudgeNames() + ".", } - c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client), newGuardJudgeAskCmd(client, asJSON)) + c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client), + newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON)) return c } @@ -61,6 +64,11 @@ func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Comma fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d/%d\t%s\n", j.Model, strings.Join(j.Policies, ","), yesNo(j.InCatalog), yesNo(j.Judge), j.Ready, j.Wanted, state) } + for _, j := range js { + if len(j.Then) > 0 { + fmt.Fprintf(w, "\n%s gives no verdict: %s is asked instead", j.Model, strings.Join(j.Then, ", else ")) + } + } if err := w.Flush(); err != nil { return err } @@ -88,7 +96,7 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud } var out []string for _, m := range models { - if !asked[m.Name] && strings.Contains(strings.ToLower(m.Source), "llama-guard") { + if _, judge := guard.FormatOfWeights(m.Source); !asked[m.Name] && judge { out = append(out, m.Name) } } @@ -97,22 +105,29 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command { var ( - model, repo, file string - replicas, cores int - replace, detach bool + model, repo, file, format, gpuClass string + replicas, cores int + gpu, replace, detach bool ) c := &cobra.Command{ Use: "enable", - Short: "Serve the judge the policies ask, as a judge: Llama Guard 3 1B on CPUs", - Long: "Fetches Llama Guard 3 1B into the mirror (once), adds it to the catalog under\n" + - "the name the policies give its judge, as a judge (role: guard,\n" + - "format llamaguard3), on CPUs so it takes no card, and waits for a replica\n" + - "to answer. With no --model, the judge the policies ask that is missing.\n\n" + - "A model of that name already in the catalog as a model clients use is\n" + - "refused; --replace removes it first. An offline cluster gets its judge\n" + - "from its bundle with server init.", + Short: "Serve a judge: Qwen3Guard-Gen 0.6B on CPUs, or with --gpu the 4B on a GPU", + Long: "Fetches a judge's weights into the mirror (once; what is there already, its\n" + + "checksums matching, is not fetched again), adds it to the catalog as a judge\n" + + "(role: guard) and waits for a replica to answer. Which judge: --model, or\n" + + "with --gpu " + defaultGPUJudge + ", or else the first the policies ask that is\n" + + "missing, or else " + defaultJudge + ".\n\n" + + "A judge on CPUs takes no card. One on a GPU takes one card, as a standard\n" + + "model: from nothing more important, so it waits while the cards are busy;\n" + + "put a judge on CPUs after it in the policies (llmgw ctl guard judge use) so\n" + + "there is always one to ask.\n\n" + + "Known judges: " + knownJudgeNames() + ". Another is served with --repo,\n" + + "--file and --format. A model of that name already in the catalog as a\n" + + "model clients use is refused; --replace removes it first. An offline\n" + + "cluster gets its judge from its bundle with server init.", Example: " llmgw ctl guard judge enable\n" + - " llmgw ctl guard judge enable --replace", + " llmgw ctl guard judge enable --gpu\n" + + " llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() @@ -121,45 +136,72 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command { if err := client().get(ctx, "/api/v1/guard/judges", &js); err != nil { return err } - var target *adminapi.JudgeView - for i, j := range js { - if (model != "" && j.Model == model) || (model == "" && (!j.InCatalog || !j.Judge)) { - target = &js[i] - break + name := model + if name == "" && gpu { + name = defaultGPUJudge + } + if name == "" { + for _, j := range js { + if !j.InCatalog || !j.Judge { + name = j.Model + break + } } } + if name == "" { + name = defaultJudge + } + + k, known := knownJudgeNamed(name) switch { - case target == nil && model != "": - target = &adminapi.JudgeView{Model: model} - fmt.Fprintf(out, "no policy asks %s yet; it is served, and a policy's judge.model names it\n", model) - case target == nil && len(js) == 0: - return errors.New("no policy asks a judge: give one a judge (llmgw ctl config edit, guard.policies..judge.model)") - case target == nil: - fmt.Fprintln(out, "every judge the policies ask is in the catalog, as a judge") - return nil + case repo != "": + f := format + if f == "" { + f, _ = guard.FormatOfWeights(repo + "/" + file) + } + if f == "" || !guard.KnownFormat(f) { + return fmt.Errorf("say how its answers are read: --format %s", strings.Join(guard.Formats, "|")) + } + k = knownJudge{Name: name, Repo: repo, File: file, Format: f, GPU: gpu} + case !known: + return fmt.Errorf("%s is not a judge llmgw knows where to fetch (%s): give --repo, --file and --format", name, knownJudgeNames()) } - if target.InCatalog && target.Judge { - fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", target.Model) - return nil + if gpu { + k.GPU = true } - if target.InCatalog && !target.Judge { + + var models []adminapi.ModelView + if err := client().get(ctx, "/api/v1/models", &models); err != nil { + return err + } + if i := slices.IndexFunc(models, func(m adminapi.ModelView) bool { return m.Name == name }); i >= 0 { + if models[i].Role == catalog.RoleGuard { + fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", name) + return nil + } if !replace { return fmt.Errorf("%s is in the catalog as a model clients use, not as a judge: "+ - "--replace removes it and serves it again as a judge", target.Model) + "--replace removes it and serves it again as a judge", name) } var res map[string]string - if err := client().delete(ctx, "/api/v1/models/"+target.Model, &res); err != nil { - return fmt.Errorf("removing %s: %w", target.Model, err) + if err := client().delete(ctx, "/api/v1/models/"+name, &res); err != nil { + return fmt.Errorf("removing %s: %w", name, err) } - fmt.Fprintf(out, "%s removed\n", target.Model) + fmt.Fprintf(out, "%s removed\n", name) } - req := modelserve.Request{ - Repo: repo, File: file, Name: target.Model, GuardFormat: "llamaguard3", - Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", - CPU: true, Replicas: replicas, CPUCores: cores, - CtxPool: 8192, ContextLength: 4096, + + opts := judgeServeOptions{Replicas: replicas, Cores: cores, GPUClass: gpuClass} + if k.GPU && opts.GPUClass == "" { + var s adminapi.Summary + if err := client().get(ctx, "/api/v1/summary", &s); err != nil { + return err + } + if opts.GPUClass = roomiestGPUClass(s.GPUs); opts.GPUClass == "" { + return fmt.Errorf("this cluster has no GPU for %s: serve %s on CPUs instead", name, defaultJudge) + } } - body, _ := json.Marshal(req) + fmt.Fprintf(out, "serving %s\n", k) + body, _ := json.Marshal(k.serveRequest(opts)) var job modelserve.Job if err := client().patient(30*time.Second).post(ctx, "/api/v1/models/serve", body, &job); err != nil { return err @@ -170,6 +212,13 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command { if err := followJob(cmd, client, job, "is the judge"); err != nil { return err } + if !slices.ContainsFunc(js, func(j adminapi.JudgeView) bool { return j.Model == name }) { + fmt.Fprintf(out, "\nno policy asks %s yet: llmgw ctl guard judge use %s", name, name) + if k.GPU { + fmt.Fprintf(out, " %s", defaultJudge) + } + fmt.Fprintln(out) + } if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 { fmt.Fprintf(out, "\nno policy asks %s, served from a judge's weights as a model clients use: "+ "llmgw ctl model remove %s\n", strings.Join(strays, ", "), strings.Join(strays, " ")) @@ -178,16 +227,118 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command { }, } f := c.Flags() - f.StringVar(&model, "model", "", "the judge to serve (default: the one the policies ask that is missing)") - f.StringVar(&repo, "repo", guardModelRepo, "the registry repository its weights come from") - f.StringVar(&file, "file", guardModelFile, "the file in it") - f.IntVar(&replicas, "replicas", 2, "replicas: every judged request waits on one") - f.IntVar(&cores, "cores", 4, "CPU cores a replica computes on") + f.StringVar(&model, "model", "", "the judge to serve: "+knownJudgeNames()+", or a name for --repo's") + f.BoolVar(&gpu, "gpu", false, "serve it on a GPU ("+defaultGPUJudge+" when no --model)") + f.StringVar(&gpuClass, "gpu-class", "", "the GPUs it runs on (default: the class with the most free cards)") + f.StringVar(&repo, "repo", "", "the registry repository of a judge llmgw does not know") + f.StringVar(&file, "file", "", "the GGUF file in --repo") + f.StringVar(&format, "format", "", "how --repo's judge answers: "+strings.Join(guard.Formats, ", ")) + f.IntVar(&replicas, "replicas", 0, "replicas (default: 2 on CPUs, as every judged request waits on one; 1 on a GPU)") + f.IntVar(&cores, "cores", 4, "CPU cores a replica on CPUs computes on") f.BoolVar(&replace, "replace", false, "remove a model of that name that is not a judge first") f.BoolVar(&detach, "detach", false, "start it and return; llmgw ctl jobs follows it") return c } +// roomiestGPUClass is the class with the most free cards, else the most +// cards; empty when the cluster has none. +func roomiestGPUClass(classes []adminapi.ClassUse) string { + best := -1 + for i, c := range classes { + if c.Total == 0 { + continue + } + if best < 0 || c.Free() > classes[best].Free() || (c.Free() == classes[best].Free() && c.Total > classes[best].Total) { + best = i + } + } + if best < 0 { + return "" + } + return classes[best].Class +} + +func newGuardJudgeUseCmd(client func() *ctlClient) *cobra.Command { + var policies []string + var note string + c := &cobra.Command{ + Use: "use JUDGE...", + Short: "Set the judges the policies ask, in the order they are asked", + Long: "Sets judge.models of every policy that has a judge, or of those given with\n" + + "--policy: the first judge that answers gives the verdict, and the next is\n" + + "asked when one has no replica serving, fails, or does not answer within\n" + + "the policy's latency_budget. A judge on a GPU first and one on CPUs after\n" + + "it is the GPU's speed while it is there, and the CPUs' when it is not.\n\n" + + "A judge llmgw knows (" + knownJudgeNames() + ")\n" + + "that is not in the catalog is fetched and served by the leader; another\n" + + "is served first with llmgw ctl guard judge enable --repo. In force on\n" + + "every control plane within seconds.", + Example: " llmgw ctl guard judge use " + defaultJudge + "\n" + + " llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + "\n" + + " llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + " --policy strict", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + for i, a := range args { + if slices.Contains(args[:i], a) { + return fmt.Errorf("%s is named twice", a) + } + } + if note == "" { + note = "judges: " + strings.Join(args, ", else ") + } + list := "[" + strings.Join(args, ", ") + "]" + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + names := policies + if len(names) == 0 { + for _, p := range doc.MapKeys([]string{"guard", "policies"}) { + if doc.Node([]string{"guard", "policies", p, "judge"}) != nil { + names = append(names, p) + } + } + if len(names) == 0 { + return errors.New("no policy has a judge: name one with --policy, and it is given one " + + "that blocks what the judge finds unsafe in requests (judge.input: block)") + } + } + for _, p := range names { + base := []string{"guard", "policies", p} + if doc.Node(base) == nil { + return fmt.Errorf("there is no policy %s (llmgw ctl guard list)", p) + } + judge := append(slices.Clone(base), "judge") + if doc.Node(judge) == nil { + if err := doc.SetYAML(append(slices.Clone(judge), "input"), "block"); err != nil { + return err + } + } + doc.Unset(append(slices.Clone(judge), "model")) + if err := doc.SetYAML(append(slices.Clone(judge), "models"), list); err != nil { + return err + } + cats := append(slices.Clone(judge), "categories") + var have []string + if n := doc.Node(cats); n != nil { + for _, c := range n.Content { + have = append(have, c.Value) + } + } + if next := categoriesFor(args, have); next != nil { + if err := doc.SetYAML(cats, "["+strings.Join(next, ", ")+"]"); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: categories [%s] are not what %s name; it counts [%s] instead\n", + p, strings.Join(have, ", "), strings.Join(args, " or "), strings.Join(next, ", ")) + } + } + return nil + }) + }, + } + c.Flags().StringSliceVar(&policies, "policy", nil, "the policies to change (default: every one with a judge)") + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + func newGuardJudgeAskCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var model, answer string c := &cobra.Command{ diff --git a/internal/cli/judgekeeper.go b/internal/cli/judgekeeper.go new file mode 100644 index 0000000..de72f81 --- /dev/null +++ b/internal/cli/judgekeeper.go @@ -0,0 +1,116 @@ +package cli + +import ( + "context" + "log/slog" + "slices" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +// judgeKeeper fetches and serves, on the leader, each judge llmgw knows that +// a policy names and the catalog does not have: naming qwen3guard-4b in a +// policy is all it takes to have one, and a judge removed by mistake comes +// back. A judge it does not know is left to llmgw ctl guard judge enable +// --repo, said in guard judge status. +type judgeKeeper struct { + library func() guard.Config + models func() []catalog.Model + serve interface { + Start(modelserve.Request) (modelserve.Job, error) + Jobs() []modelserve.Job + } + leader func() bool + gpus func() []adminapi.ClassUse + log *slog.Logger + now func() time.Time + + // tried is when each judge was last asked for, so one that cannot be + // fetched is not asked for again on every look. + tried map[string]time.Time +} + +// judgeRetry is how long a judge whose serving failed waits before it is +// asked for again. +const judgeRetry = 10 * time.Minute + +func (k *judgeKeeper) run(ctx context.Context, every time.Duration) { + t := time.NewTicker(every) + defer t.Stop() + for { + k.check() + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +func (k *judgeKeeper) check() { + if k.leader != nil && !k.leader() { + return + } + lib := k.library() + if !lib.On() { + return + } + if k.tried == nil { + k.tried = map[string]time.Time{} + } + now := time.Now + if k.now != nil { + now = k.now + } + have := map[string]bool{} + for _, m := range k.models() { + have[m.Name] = true + } + busy := map[string]bool{} + for _, j := range k.serve.Jobs() { + if j.State == modelserve.Running { + busy[j.Model] = true + busy[j.Request.Name] = true + } + } + var wanted []string + for _, p := range lib.Policies { + for _, j := range p.Judge.Judges() { + if !slices.Contains(wanted, j) { + wanted = append(wanted, j) + } + } + } + slices.Sort(wanted) + for _, name := range wanted { + kj, known := knownJudgeNamed(name) + if have[name] || !known || busy[name] { + continue + } + if at, ok := k.tried[name]; ok && now().Sub(at) < judgeRetry { + continue + } + k.tried[name] = now() + var opts judgeServeOptions + if kj.GPU { + if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" { + k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none", + "judge", name, "instead", defaultJudge) + continue + } + } + req := kj.serveRequest(opts) + req.By = "llmgw (a guard policy names it)" + job, err := k.serve.Start(req) + if err != nil { + k.log.Error("could not start serving a judge a policy names", "judge", name, "err", err) + continue + } + k.log.Info("serving a judge a policy names, which the catalog did not have", + "judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID) + } +} diff --git a/internal/cli/judgekeeper_test.go b/internal/cli/judgekeeper_test.go new file mode 100644 index 0000000..accd62e --- /dev/null +++ b/internal/cli/judgekeeper_test.go @@ -0,0 +1,126 @@ +package cli + +import ( + "errors" + "log/slog" + "slices" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +type fakeServe struct { + started []modelserve.Request + running []modelserve.Job + err error +} + +func (f *fakeServe) Start(r modelserve.Request) (modelserve.Job, error) { + if f.err != nil { + return modelserve.Job{}, f.err + } + f.started = append(f.started, r) + return modelserve.Job{ID: "j", Model: r.Name, Request: r, State: modelserve.Running}, nil +} +func (f *fakeServe) Jobs() []modelserve.Job { return f.running } + +func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) { + lib := guard.Config{Policies: map[string]guard.Policy{ + "baseline": {Judge: &guard.JudgePolicy{Models: []string{"qwen3guard-4b", "qwen3guard-0.6b"}, Input: "block"}}, + "custom": {Judge: &guard.JudgePolicy{Model: "our-own-judge", Input: "block"}}, + }} + inCatalog := []catalog.Model{{Name: "qwen3guard-0.6b"}} + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + fs := &fakeServe{} + leader := true + k := &judgeKeeper{ + library: func() guard.Config { return lib }, + models: func() []catalog.Model { return inCatalog }, + serve: fs, + leader: func() bool { return leader }, + gpus: func() []adminapi.ClassUse { return []adminapi.ClassUse{{Class: "6000-24g", Total: 1, Used: 1}} }, + log: slog.New(slog.DiscardHandler), + now: func() time.Time { return now }, + } + + k.check() + // The GPU judge alone: the CPU one is there, and one llmgw does not know + // is not guessed at. + if len(fs.started) != 1 || fs.started[0].Name != "qwen3guard-4b" || fs.started[0].GPUClass != "6000-24g" || + fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU { + t.Fatalf("started %+v", fs.started) + } + + // Being fetched: not asked for again. + fs.running = []modelserve.Job{{Model: "qwen3guard-4b", State: modelserve.Running}} + k.check() + if len(fs.started) != 1 { + t.Errorf("asked for again while it is being fetched: %d", len(fs.started)) + } + // Failed: asked for again only after a while. + fs.running = nil + now = now.Add(time.Minute) + k.check() + if len(fs.started) != 1 { + t.Error("asked for again at once after it failed") + } + now = now.Add(judgeRetry) + k.check() + if len(fs.started) != 2 { + t.Error("never asked for again") + } + + // Only the leader fetches, and nothing while the guardrails are off. + leader = false + now = now.Add(judgeRetry) + k.check() + leader = true + off := false + lib.Enabled = &off + k.check() + if len(fs.started) != 2 { + t.Errorf("a standby or the guardrails off fetched: %d", len(fs.started)) + } + + // A CPU judge on a cluster without GPUs is served on CPUs, two of it. + lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}} + inCatalog = nil + k.gpus = func() []adminapi.ClassUse { return nil } + k.check() + if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 { + t.Errorf("CPU judge: %+v", last) + } + fs.err = errors.New("the mirror is down") + lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-8b", Input: "block"}}}} + k.check() // no GPU, so not started, and no panic on the error path either +} + +// A policy moved from Llama Guard's codes to Qwen3Guard would count nothing: +// its categories become the judge's own. +func TestCategoriesFollowTheJudges(t *testing.T) { + lg := []string{"S1", "S2", "S10"} + if got := categoriesFor([]string{"qwen3guard-0.6b"}, lg); !slices.Equal(got, baselineCategories["qwen3guard"]) { + t.Errorf("Llama Guard's codes for Qwen3Guard: %v", got) + } + if got := categoriesFor([]string{"qwen3guard-4b", "qwen3guard-0.6b"}, []string{"Violent", "Jailbreak"}); got != nil { + t.Errorf("its own names were replaced: %v", got) + } + if got := categoriesFor([]string{"llama-guard-3-1b"}, lg); got != nil { + t.Errorf("Llama Guard's own codes were replaced: %v", got) + } + // Both kinds asked in turn: each must have something to count. + got := categoriesFor([]string{"qwen3guard-4b", "llama-guard-3-1b"}, []string{"Violent"}) + if !slices.Contains(got, "Violent") || !slices.Contains(got, "S1") { + t.Errorf("mixed: %v", got) + } + if got := categoriesFor([]string{"our-own-judge"}, lg); got != nil { + t.Errorf("a judge llmgw does not know: %v", got) + } + if got := categoriesFor([]string{"qwen3guard-0.6b"}, nil); got != nil { + t.Errorf("no categories (every one counts) became %v", got) + } +} diff --git a/internal/cli/judges_known.go b/internal/cli/judges_known.go new file mode 100644 index 0000000..d8354c5 --- /dev/null +++ b/internal/cli/judges_known.go @@ -0,0 +1,158 @@ +package cli + +import ( + "fmt" + "slices" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +// knownJudge is a judge model llmgw knows where to fetch and how to serve: +// what a policy may name without anyone saying where its weights are. +type knownJudge struct { + Name, Repo, File, Format string + // GPU is a judge too slow on CPUs to wait for: served on one card. + GPU bool + About string +} + +// knownJudges, the default first. Measured on the reference cluster against +// ordinary coding requests ("continue", "kill the process on port 8080") and +// harmful ones: Llama Guard 3 1B scored both alike, about one in two +// ordinary ones refused; Qwen3Guard-Gen 0.6B kept them apart, 0.15 against +// 1.00 at the worst, at about two seconds a check on two CPU threads. +var knownJudges = []knownJudge{ + {Name: "qwen3guard-0.6b", Repo: "mradermacher/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B.Q8_0.gguf", + Format: "qwen3guard", About: "Qwen3Guard-Gen 0.6B on CPUs, 0.8 GB: the default"}, + {Name: "qwen3guard-4b", Repo: "mradermacher/Qwen3Guard-Gen-4B-GGUF", File: "Qwen3Guard-Gen-4B.Q4_K_M.gguf", + Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 4B on a GPU, 2.5 GB"}, + {Name: "qwen3guard-8b", Repo: "mradermacher/Qwen3Guard-Gen-8B-GGUF", File: "Qwen3Guard-Gen-8B.Q4_K_M.gguf", + Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 8B on a GPU, 4.7 GB"}, + {Name: "llama-guard-3-1b", Repo: "QuantFactory/Llama-Guard-3-1B-GGUF", File: "Llama-Guard-3-1B.Q4_K_M.gguf", + Format: "llamaguard3", About: "Llama Guard 3 1B on CPUs, 0.9 GB: refuses ordinary coding requests"}, + {Name: "llama-guard-3-8b", Repo: "QuantFactory/Llama-Guard-3-8B-GGUF", File: "Llama-Guard-3-8B.Q4_K_M.gguf", + Format: "llamaguard3", GPU: true, About: "Llama Guard 3 8B on a GPU, 4.6 GB"}, +} + +const ( + defaultJudge = "qwen3guard-0.6b" + defaultGPUJudge = "qwen3guard-4b" +) + +// baselineCategories are what the baseline policy counts of each format's +// verdicts: not PII, which is the patterns', nor what refuses ordinary work +// — copyright, unethical acts, politically sensitive topics; Llama Guard's +// defamation, specialized advice, intellectual property and elections. +var baselineCategories = map[string][]string{ + "qwen3guard": {"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "Suicide & Self-Harm", "Jailbreak"}, + "llamaguard3": {"S1", "S2", "S3", "S4", "S7", "S9", "S10", "S11", "S12"}, +} + +// reported is whether a judge of that format ever names category c: its own +// names, or for Llama Guard its codes. +func reported(format, c string) bool { + if slices.ContainsFunc(guard.DefaultCategories(format), func(n string) bool { return strings.EqualFold(n, c) }) { + return true + } + var n int + _, err := fmt.Sscanf(c, "S%d", &n) + return format == "llamaguard3" && err == nil && n >= 1 && n <= 14 +} + +// categoriesFor is what a policy asking these judges should count, when the +// categories it has are ones none of them names: a policy moved from Llama +// Guard's S1 to Qwen3Guard, which says Violent, would otherwise count +// nothing. Nil keeps what it has; so does a judge llmgw does not know. +func categoriesFor(judges, have []string) []string { + var formats []string + for _, j := range judges { + k, ok := knownJudgeNamed(j) + if !ok { + return nil + } + if !slices.Contains(formats, k.Format) { + formats = append(formats, k.Format) + } + } + if len(have) == 0 { + return nil + } + stray := false + for _, f := range formats { + for _, c := range have { + if !reported(f, c) && !slices.ContainsFunc(formats, func(o string) bool { return reported(o, c) }) { + stray = true + } + } + // Every judge must have something it counts. + if !slices.ContainsFunc(have, func(c string) bool { return reported(f, c) }) { + stray = true + } + } + if !stray { + return nil + } + var out []string + for _, f := range formats { + out = append(out, baselineCategories[f]...) + } + return out +} + +func knownJudgeNamed(name string) (knownJudge, bool) { + i := slices.IndexFunc(knownJudges, func(k knownJudge) bool { return k.Name == name }) + if i < 0 { + return knownJudge{}, false + } + return knownJudges[i], true +} + +func knownJudgeNames() string { + var n []string + for _, k := range knownJudges { + n = append(n, k.Name) + } + return strings.Join(n, ", ") +} + +// judgeServeOptions are how a judge is served beyond which weights. +type judgeServeOptions struct { + Replicas, Cores int + // GPUClass is the cards a GPU judge runs on; empty lets the plan choose. + GPUClass string +} + +// serveRequest is the serve job for a judge. A judge on CPUs is critical, +// as every judged request waits on it; one on a GPU is standard, so it takes +// a card only from what may give one up, and the judge on CPUs after it in +// the policies stands in while it has none. +func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request { + req := modelserve.Request{ + Repo: k.Repo, File: k.File, Name: k.Name, GuardFormat: k.Format, + Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", + Replicas: o.Replicas, CtxPool: 8192, ContextLength: 4096, + } + if k.GPU { + req.GPUClass, req.GPUsPerNode, req.Class = o.GPUClass, 1, catalog.ClassStandard + if req.Replicas == 0 { + req.Replicas = 1 + } + return req + } + req.CPU, req.CPUCores = true, o.Cores + if req.Replicas == 0 { + req.Replicas = 2 + } + return req +} + +func (k knownJudge) String() string { + where := "CPUs" + if k.GPU { + where = "a GPU" + } + return fmt.Sprintf("%s (%s, %s/%s, on %s)", k.Name, k.Format, k.Repo, k.File, where) +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 80259a7..2c5fa45 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -518,6 +518,21 @@ func newServerCmd() *cobra.Command { }) go serve.WatchCanaries(ctx, 30*time.Second) adminAPI.SetServe(serve) + if gate != nil { + // A judge a policy names and the catalog lacks is + // fetched and served, on the leader. + keeper := &judgeKeeper{ + library: gate.Library, + models: func() []catalog.Model { return admin.Catalog().Models() }, + serve: serve, + leader: func() bool { + return admin.Leadership() == nil || admin.Leadership().IsLeader() + }, + gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs }, + log: log, + } + go keeper.run(ctx, 30*time.Second) + } updateMetrics.Set(serve) go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck) } diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 7658397..ec24cf4 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -293,7 +293,7 @@ func (in *serverInit) configJudgesWith(model string) bool { return false } for _, p := range cfg.Guard.Policies { - if p.Judge != nil && p.Judge.Model == model { + if slices.Contains(p.Judge.Judges(), model) { return true } } @@ -308,18 +308,21 @@ func guardSection(judge bool) string { # judge: {model: ` + guardModelName + `, input: block, output: block, ...} ` if judge { - j = ` # A judge model reads what patterns cannot: violence, weapons, abuse, - # self-harm. Two replicas run on CPUs with AVX2, about half a second - # for a short text; an answer is read every 600 characters as it - # streams. With fail: closed, a request the judge cannot read within + j = ` # A judge model reads what patterns cannot: violence, crime, sexual + # content, self-harm, jailbreaks. Two replicas run on CPUs, about two + # seconds for a short text; an answer is read every 600 characters as + # it streams. With fail: closed, a request no judge can read within # latency_budget is refused (503); open serves it, flagged, unjudged. - # Defamation (S5), specialized advice (S6), intellectual property (S8) - # and elections (S13) are left out: they refuse too much ordinary work. + # PII is the patterns' (above), and copyright, unethical acts and + # politically sensitive topics are left out: they refuse too much + # ordinary work. Controversial counts as safe; controversial: unsafe + # makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then + # llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `. judge: - model: ` + guardModelName + ` + models: [` + guardModelName + `] input: block output: block - categories: [S1, S2, S3, S4, S7, S9, S10, S11, S12] + categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak] latency_budget: 5s ` } diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 27cf3c5..e3a88fa 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -41,11 +41,13 @@ const ( starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf" starterVersion = "starter" - // The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt - // or an answer and says whether it is unsafe and why. Q4_K_M is about - // 900 MB and judges a short text in about half a second on a CPU. - guardModelName = "llama-guard-3-1b" - guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf" + // The guardrails' judge: Qwen3Guard-Gen 0.6B, which reads a prompt or an + // answer and says whether it is safe, controversial or unsafe, and why. + // Q8_0 is 0.8 GB and judges a short text in about two seconds on two CPU + // threads. It replaced Llama Guard 3 1B, which refused about one in two + // ordinary coding requests ("continue"). + guardModelName = defaultJudge + guardModelSource = "hf:mradermacher/Qwen3Guard-Gen-0.6B-GGUF/Qwen3Guard-Gen-0.6B.Q8_0.gguf" guardModelVersion = "guard" // localLabel marks the control plane's own node, so the starter model is @@ -443,12 +445,6 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model } } -// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is -// recorded as "Violent Crimes" rather than "S1". -var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes", - "Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property", - "Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"} - // guardModel is the judge's catalog entry: CPU only, so it takes no card from // the models people use, on whichever node has room. func guardModel(source string) catalog.Model { @@ -457,7 +453,7 @@ func guardModel(source string) catalog.Model { Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", Source: source, Role: catalog.RoleGuard, - Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories}, + Guard: &catalog.GuardModel{Format: "qwen3guard"}, // Two: every judged request waits on it, and with fail: closed a // judge that is not there refuses everything. Scaling: catalog.Scaling{Min: 2, Max: 2}, diff --git a/internal/cli/serverinit_cluster_test.go b/internal/cli/serverinit_cluster_test.go index 837a4fe..5a605fd 100644 --- a/internal/cli/serverinit_cluster_test.go +++ b/internal/cli/serverinit_cluster_test.go @@ -7,12 +7,14 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) { @@ -150,9 +152,17 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) { if (p.Judge != nil) != judge { t.Errorf("judge=%v: the policy's judge is %+v", judge, p.Judge) } - if judge && (p.Judge.Model != guardModelName || p.Judge.Input != "block") { + if judge && (!slices.Equal(p.Judge.Judges(), []string{guardModelName}) || p.Judge.Input != "block") { t.Errorf("judge %+v", p.Judge) } + // A category the judge never names would never count. + if judge { + for _, c := range p.Judge.Categories { + if !slices.Contains(guard.DefaultCategories("qwen3guard"), c) { + t.Errorf("the baseline counts %q, which Qwen3Guard does not name", c) + } + } + } if cfg.Auth.MaxKeyTTL != 90*24*time.Hour || cfg.Auth.SignupOpen || cfg.DebugPprof || cfg.AdminRemote { t.Errorf("not hardened: ttl %v signup %v pprof %v remote %v", cfg.Auth.MaxKeyTTL, cfg.Auth.SignupOpen, cfg.DebugPprof, cfg.AdminRemote) @@ -161,13 +171,13 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) { } func TestTheGuardModelIsAValidJudge(t *testing.T) { - m := guardModel("http://10.0.0.5:8081/llama-guard-3-1b/guard/g.gguf") + m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf") c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("not a valid catalog entry: %v", err) } got, _ := c.Resolve(guardModelName) - if !got.IsGuard() || got.Guard.Format != "llamaguard3" || len(got.Guard.Categories) != 13 || got.Scaling.Min != 2 { + if !got.IsGuard() || got.Guard.Format != "qwen3guard" || got.Scaling.Min != 2 { t.Errorf("%+v", got) } for _, p := range got.Profiles { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a574db9..e8570d5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -431,10 +431,12 @@ type ModelView struct { Description string `json:"description,omitempty"` Aliases []string `json:"aliases,omitempty"` Source string `json:"source"` - Min int `json:"min_replicas"` - Max int `json:"max_replicas"` - Ready int `json:"ready_replicas"` - Total int `json:"total_replicas"` + // Role is "guard" for a guardrails judge, empty for a model clients use. + Role string `json:"role,omitempty"` + Min int `json:"min_replicas"` + Max int `json:"max_replicas"` + Ready int `json:"ready_replicas"` + Total int `json:"total_replicas"` // Desired is how many it should have now: Min, or for an on-demand // model one while it is being asked for and none otherwise. Desired int `json:"desired_replicas"` @@ -810,6 +812,7 @@ func (a *API) Models() []ModelView { Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor, External: externalLabel(m), Adapters: m.Adapters, Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval, + Role: m.Role, } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) diff --git a/internal/control/adminapi/judges.go b/internal/control/adminapi/judges.go index 930511c..b36adc4 100644 --- a/internal/control/adminapi/judges.go +++ b/internal/control/adminapi/judges.go @@ -21,6 +21,9 @@ type JudgeView struct { // cannot read (fail: closed). Policies []string `json:"policies"` Closed []string `json:"closed,omitempty"` + // Then are the judges its policies ask after it, when it gives no + // verdict: a CPU judge behind a GPU one. + Then []string `json:"then,omitempty"` // InCatalog is whether a model of that name is in the catalog, and // Judge whether it is there as a judge (role: guard). InCatalog bool `json:"in_catalog"` @@ -52,17 +55,24 @@ func (a *API) Judges() []JudgeView { } byModel := map[string]*JudgeView{} for name, p := range lib.Policies { - if p.Judge == nil || p.Judge.Model == "" { - continue - } - v := byModel[p.Judge.Model] - if v == nil { - v = &JudgeView{Model: p.Judge.Model} - byModel[p.Judge.Model] = v - } - v.Policies = append(v.Policies, name) - if p.Fail == "closed" && p.Mode != "shadow" { - v.Closed = append(v.Closed, name) + judges := p.Judge.Judges() + for i, model := range judges { + v := byModel[model] + if v == nil { + v = &JudgeView{Model: model} + byModel[model] = v + } + if !slices.Contains(v.Policies, name) { + v.Policies = append(v.Policies, name) + } + if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) { + v.Closed = append(v.Closed, name) + } + for _, after := range judges[i+1:] { + if !slices.Contains(v.Then, after) { + v.Then = append(v.Then, after) + } + } } } ready := map[string]int{} @@ -104,7 +114,10 @@ func (a *API) Judges() []JudgeView { func judgeProblem(v JudgeView) string { consequence := "requests its policies judge are served unjudged" - if len(v.Closed) > 0 { + switch { + case len(v.Then) > 0: + consequence = "its policies ask " + strings.Join(v.Then, ", else ") + " instead" + case len(v.Closed) > 0: consequence = "requests its policies judge are refused (guard_unavailable)" } switch { diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index e2dab0f..2170afd 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) // Scaling is how many replicas a model should have. @@ -386,9 +387,11 @@ func (m Model) IsGuard() bool { return m.Role == RoleGuard } // GuardModel is how a judge model's answers are read. type GuardModel struct { - // Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and - // the categories), shieldgemma (Google's ShieldGemma: Yes or No), or - // json ({"safe": false, "categories": ["..."]}). + // Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or + // "Controversial", and the categories), llamaguard3 (Meta's Llama Guard + // 3: "safe", or "unsafe" and the categories), shieldgemma (Google's + // ShieldGemma: Yes or No), or json ({"safe": false, "categories": + // ["..."]}). Format string `json:"format" koanf:"format"` // Categories the model is asked about, for shieldgemma — each a policy // sentence — and the names reported for llamaguard3's S1, S2, …. @@ -708,12 +711,10 @@ func (m Model) validate() error { } case RoleGuard: if m.Guard == nil { - return fmt.Errorf("model %q: a guard model needs guard.format: llamaguard3, shieldgemma or json", m.Name) + return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", ")) } - switch m.Guard.Format { - case "llamaguard3", "shieldgemma", "json": - default: - return fmt.Errorf("model %q: guard.format %q is not llamaguard3, shieldgemma or json", m.Name, m.Guard.Format) + if !guard.KnownFormat(m.Guard.Format) { + return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", ")) } if m.Scaling.OnDemand() { // A judge loaded on demand makes the first request of every diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index fe0a05d..5310594 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -29,6 +29,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" @@ -137,8 +138,12 @@ type Request struct { // Replicas to run; 0 makes it on demand, loaded when first asked for. Replicas int `json:"replicas"` // GuardFormat makes it a judge model for the guardrails (role: guard): - // llamaguard3, shieldgemma or json. Empty serves clients. + // llamaguard3, qwen3guard, shieldgemma or json. Empty serves clients. GuardFormat string `json:"guard_format,omitempty"` + // Class is what it may take from others and they from it when there is + // not room for both: critical, standard or opportunistic. Empty is the + // catalog's default for its role. + Class string `json:"class,omitempty"` // CtxPool is llama.cpp's --ctx-size; ContextLength what one request may // use. They default to 16384 and 4096, what four parallel slots divide; // llama.cpp is given as many slots as the pool holds requests. @@ -252,10 +257,13 @@ func (s *Server) Start(req Request) (Job, error) { default: return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) } - switch req.GuardFormat { - case "", "llamaguard3", "shieldgemma", "json": + if req.GuardFormat != "" && !guard.KnownFormat(req.GuardFormat) { + return Job{}, fmt.Errorf("the guard format is one of %s, not %q", strings.Join(guard.Formats, ", "), req.GuardFormat) + } + switch req.Class { + case "", catalog.ClassCritical, catalog.ClassStandard, catalog.ClassOpportunistic: default: - return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat) + return Job{}, fmt.Errorf("the class is critical, standard or opportunistic, not %q", req.Class) } if req.GuardFormat != "" && req.Replicas == 0 { return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand") @@ -618,6 +626,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F if req.GuardFormat != "" { m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, "" } + m.Class = req.Class s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { s.fail(j, "catalog", err) diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 8cb6e52..259b830 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -121,15 +121,17 @@ var Reference = []Block{ {Key: "enabled", Type: "bool", Doc: "Enabled keeps llmgw's internal ports — engines, node exporter, the\nagents' stream, the mirror, the store — to the cluster's networks on\nevery machine. False removes llmgw's nftables table everywhere, for a\nsite whose own firewall does that job; the rules are kept, and true\nputs them back. Changed in the cluster's settings, it is in force on\nevery node and control plane within seconds.", Nested: ""}, }}, {Package: "catalog", Name: "GuardModel", Title: "GuardModel", Doc: "", Fields: []Field{ - {Key: "format", Type: "string", Doc: "Format is llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and\nthe categories), shieldgemma (Google's ShieldGemma: Yes or No), or\njson ({\"safe\": false, \"categories\": [\"...\"]}).", Nested: ""}, + {Key: "format", Type: "string", Doc: "Format is qwen3guard (Qwen3Guard-Gen: \"Safety: Safe\", \"Unsafe\" or\n\"Controversial\", and the categories), llamaguard3 (Meta's Llama Guard\n3: \"safe\", or \"unsafe\" and the categories), shieldgemma (Google's\nShieldGemma: Yes or No), or json ({\"safe\": false, \"categories\":\n[\"...\"]}).", Nested: ""}, {Key: "categories", Type: "[]string", Doc: "Categories the model is asked about, for shieldgemma — each a policy\nsentence — and the names reported for llamaguard3's S1, S2, ….", Nested: ""}, }}, {Package: "guard", Name: "JudgePolicy", Title: "JudgePolicy", Doc: "", Fields: []Field{ {Key: "model", Type: "string", Doc: "Model is a catalog model with role: guard.", Nested: ""}, + {Key: "models", Type: "[]string", Doc: "Models are judges asked in turn, instead of Model: the first that\nanswers gives the verdict. A judge on a GPU first and one on CPUs\nafter it is the GPU's speed while it is there, and the CPUs' when it\nis not — busy, failed, or the node down.", Nested: ""}, {Key: "input", Type: "string", Doc: "Input and Output are what an unsafe verdict does: flag or block.\nEmpty does not ask the judge about that direction.", Nested: ""}, {Key: "output", Type: "string", Doc: "", Nested: ""}, {Key: "categories", Type: "[]string", Doc: "Categories, when set, are the only ones that count: Llama Guard's\ncodes (S1, S10) or the names the model reports.", Nested: ""}, - {Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long the judge is waited for before the policy's\nfail applies. Zero is 250ms.", Nested: ""}, + {Key: "controversial", Type: "string", Doc: "Controversial is what Qwen3Guard's third verdict counts as: safe\n(the default) or unsafe.", Nested: ""}, + {Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long each judge is waited for before the next is\nasked, or, after the last, the policy's fail applies. Zero is 250ms.", Nested: ""}, }}, {Package: "config", Name: "KVIndex", Title: "KVIndex", Doc: "", Fields: []Field{ {Key: "tokenize_timeout", Type: "time.Duration", Doc: "TokenizeTimeout bounds the call that turns a request into token ids,\nmade to one of the model's replicas before the request is placed. Past\nit the request is routed as tier 1 would. Zero takes 250ms.", Nested: ""}, diff --git a/internal/guard/gate.go b/internal/guard/gate.go index bea639e..a2c7959 100644 --- a/internal/guard/gate.go +++ b/internal/guard/gate.go @@ -410,8 +410,8 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro if jp := p.Judge; jp != nil && jp.Input != "" && user != "" { res, err := g.judger.ask(ctx, jp, user, "") if err != nil { - g.log.Warn("the judge gave no verdict on a request", "judge", jp.Model, "err", err) - g.judgeFailed(jp.Model, err) + g.log.Warn("the judge gave no verdict on a request", "judge", jp.Name(), "err", err) + g.noJudger(jp, err) } if err != nil && p.Fail == "closed" && !v.Shadow { v.Took = g.now().Sub(start) @@ -455,6 +455,18 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro // the gate serves. func (g *Gate) SetJudgeAlarm(f func(model string, err error)) { g.alarm = f } +// noJudger records, against each judge a policy asks, that there was no way +// to ask any: a gate built without SetJudge. With one, each judge's failure +// is recorded as it is asked. +func (g *Gate) noJudger(jp *JudgePolicy, err error) { + if g.judger != nil { + return + } + for _, m := range jp.Judges() { + g.judgeFailed(m, err) + } +} + func (g *Gate) judgeFailed(model string, err error) { g.errMu.Lock() if g.lastErr == nil { diff --git a/internal/guard/judge.go b/internal/guard/judge.go index 5e375f2..3bd9abc 100644 --- a/internal/guard/judge.go +++ b/internal/guard/judge.go @@ -10,6 +10,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "gitea.mynodes.xyz/ilker/llmgw/internal/guard/detect" ) @@ -19,6 +20,11 @@ import ( type JudgePolicy struct { // Model is a catalog model with role: guard. Model string `koanf:"model"` + // Models are judges asked in turn, instead of Model: the first that + // answers gives the verdict. A judge on a GPU first and one on CPUs + // after it is the GPU's speed while it is there, and the CPUs' when it + // is not — busy, failed, or the node down. + Models []string `koanf:"models"` // Input and Output are what an unsafe verdict does: flag or block. // Empty does not ask the judge about that direction. Input string `koanf:"input"` @@ -26,11 +32,31 @@ type JudgePolicy struct { // Categories, when set, are the only ones that count: Llama Guard's // codes (S1, S10) or the names the model reports. Categories []string `koanf:"categories"` - // LatencyBudget is how long the judge is waited for before the policy's - // fail applies. Zero is 250ms. + // Controversial is what Qwen3Guard's third verdict counts as: safe + // (the default) or unsafe. + Controversial string `koanf:"controversial"` + // LatencyBudget is how long each judge is waited for before the next is + // asked, or, after the last, the policy's fail applies. Zero is 250ms. LatencyBudget time.Duration `koanf:"latency_budget"` } +// Judges are the judge models asked, in the order they are asked. +func (j *JudgePolicy) Judges() []string { + if j == nil { + return nil + } + if len(j.Models) > 0 { + return j.Models + } + if j.Model != "" { + return []string{j.Model} + } + return nil +} + +// Name is the judges as one name, for logs and errors: "a, else b". +func (j *JudgePolicy) Name() string { return strings.Join(j.Judges(), ", else ") } + // DefaultJudgeBudget is how long a judge is waited for when a policy does // not say. const DefaultJudgeBudget = 250 * time.Millisecond @@ -43,8 +69,26 @@ func (j *JudgePolicy) budget() time.Duration { } func (j *JudgePolicy) validate() error { - if j.Model == "" { - return errors.New("judge.model is required") + switch { + case j.Model == "" && len(j.Models) == 0: + return errors.New("judge.model (or judge.models, judges asked in turn) is required") + case j.Model != "" && len(j.Models) > 0: + return errors.New("judge: give model or models, not both") + } + seen := map[string]bool{} + for _, m := range j.Models { + if strings.TrimSpace(m) == "" { + return errors.New("judge.models has an empty name") + } + if seen[m] { + return fmt.Errorf("judge.models names %s twice", m) + } + seen[m] = true + } + switch j.Controversial { + case "", "safe", "unsafe": + default: + return fmt.Errorf("judge.controversial is %q: safe or unsafe", j.Controversial) } for k, a := range map[string]string{"input": j.Input, "output": j.Output} { switch a { @@ -59,8 +103,8 @@ func (j *JudgePolicy) validate() error { return nil } -// mergeJudge keeps the first judge model asked for, the strictest actions, -// every category and the shortest wait. +// mergeJudge keeps the first judges asked for, the strictest actions, every +// category and the shortest wait. func mergeJudge(a, b *JudgePolicy) *JudgePolicy { if b == nil { return a @@ -68,11 +112,14 @@ func mergeJudge(a, b *JudgePolicy) *JudgePolicy { if a == nil { c := *b c.Categories = slices.Clone(b.Categories) + c.Models = slices.Clone(b.Models) return &c } out := *a + out.Models = slices.Clone(a.Models) out.Input = stricterOf(a.Input, b.Input, []string{"", "flag", "block"}) out.Output = stricterOf(a.Output, b.Output, []string{"", "flag", "block"}) + out.Controversial = stricterOf(a.Controversial, b.Controversial, []string{"", "safe", "unsafe"}) out.Categories = union(slices.Clone(a.Categories), b.Categories) if b.LatencyBudget > 0 && (a.LatencyBudget == 0 || b.LatencyBudget < a.LatencyBudget) { out.LatencyBudget = b.LatencyBudget @@ -91,8 +138,13 @@ type ModelFormat func(model string) (format string, categories []string, ok bool // judgment is what a judge said. type judgment struct { - unsafe bool - categories []string + unsafe bool + // controversial is Qwen3Guard's middle verdict, which the policy says + // how to count. + controversial bool + categories []string + // by is the judge that said it. + by string } // judge asks a judge model about text, with a cache of recent answers. @@ -100,6 +152,9 @@ type judge struct { call Caller format ModelFormat m *gateMetrics + // failed is told of each judge that gave no verdict, whether or not + // another after it did. + failed func(model string, err error) mu sync.Mutex cache map[[32]byte]cached @@ -117,7 +172,7 @@ const ( // SetJudge lets policies with a judge: section ask a model on the cluster. func (g *Gate) SetJudge(call Caller, format ModelFormat) { - g.judger = &judge{call: call, format: format, m: g.m, cache: map[[32]byte]cached{}} + g.judger = &judge{call: call, format: format, m: g.m, failed: g.judgeFailed, cache: map[[32]byte]cached{}} } // errNoJudge is a policy naming a judge the gateway cannot ask, and @@ -127,36 +182,57 @@ var ( errJudgeFormat = errors.New("the judge's answer could not be read") ) -// ask has the judge read conversation turns — the user's, and the -// assistant's answer when judging output. -func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (res judgment, err error) { +// ask has the judges read conversation turns — the user's, and the +// assistant's answer when judging output — each in turn until one answers. +// The error is the last judge's, when none did. +func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) { if j == nil { return judgment{}, errNoJudge } - start := time.Now() - cachedHit := false - defer func() { - if !cachedHit && j.m != nil { - j.m.judged(jp.Model, time.Since(start), err) - } - }() - format, names, ok := j.format(jp.Model) - if !ok { - return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, jp.Model) - } - key := sha256.Sum256([]byte(jp.Model + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer))) + models := jp.Judges() + key := sha256.Sum256([]byte(strings.Join(models, "\x01") + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer))) j.mu.Lock() if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL { j.mu.Unlock() - cachedHit = true return c.j, nil } j.mu.Unlock() + err := errNoJudge + for _, model := range models { + var res judgment + res, err = j.askOne(ctx, jp, model, user, answer) + if err == nil { + j.remember(key, res) + return res, nil + } + if j.failed != nil { + j.failed(model, err) + } + if ctx.Err() != nil { + // The request itself is gone: no judge after this one is wanted. + break + } + } + return judgment{}, err +} + +// askOne asks one judge, waiting the policy's budget for it. +func (j *judge) askOne(ctx context.Context, jp *JudgePolicy, model, user, answer string) (res judgment, err error) { + start := time.Now() + defer func() { + if j.m != nil { + j.m.judged(model, time.Since(start), err) + } + }() + format, names, ok := j.format(model) + if !ok { + return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, model) + } ctx, cancel := context.WithTimeout(ctx, jp.budget()) defer cancel() - path, body := judgeRequest(jp.Model, format, names, user, answer) - out, err := j.call(ctx, jp.Model, path, body) + path, body := judgeRequest(model, format, names, user, answer) + out, err := j.call(ctx, model, path, body) if err != nil { return judgment{}, err } @@ -164,6 +240,11 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) ( if err != nil { return judgment{}, err } + res.by = model + return res, nil +} + +func (j *judge) remember(key [32]byte, res judgment) { j.mu.Lock() if len(j.cache) >= judgeCacheSize { for k, c := range j.cache { @@ -174,12 +255,11 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) ( } j.cache[key] = cached{j: res, at: time.Now()} j.mu.Unlock() - return res, nil } // counts reports whether a judgment is one the policy acts on. func (jp *JudgePolicy) counts(res judgment) (bool, string) { - if !res.unsafe { + if !res.unsafe && !(res.controversial && jp.Controversial == "unsafe") { return false, "" } if len(jp.Categories) == 0 { @@ -228,13 +308,116 @@ func llamaGuard3Answer(categories []string, user, answer string) string { return b.String() } +// qwen3GuardCategories are Qwen3Guard's own, as it names them in a verdict. +// Jailbreak is judged in a user's message only. +var qwen3GuardCategories = []string{"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "PII", + "Suicide & Self-Harm", "Unethical Acts", "Politically Sensitive Topics", "Copyright Violation", "Jailbreak"} + +// qwen3Guard is Qwen3Guard-Gen's prompt, as its chat template writes it, +// for a user's message or, with answer, the assistant's reply to it. Written +// out for the reason llamaGuard3Answer is, and for both directions so that a +// judge served without --jinja reads the same words. +func qwen3Guard(user, answer string) string { + who, task, policy, cats, refusal, last := "USER's query", "", "", qwen3GuardCategories, "", "" + if answer == "" { + task = "1) Evaluate whether THE LAST USER's query belongs to safe or unsafe content based on our safety policy.\n" + + "2) Identify the category of unsafe content found.\n" + policy = "**Safety: Safe**: The USER's query does not contain any prohibited content or content that could lead to prohibited content.\n" + + "**Safety: Unsafe**: The USER's query contains prohibited content.\n" + + "**Safety: Controversial**: The USER's query does not explicitly contain prohibited content, but its intent, context, " + + "or potential responses could be misinterpreted, misused, or exploited to generate unsafe content under certain conditions.\n" + } else { + who = "ASSISTANT's response" + cats = cats[:len(cats)-1] + task = "1) Evaluate whether THE LAST ASSISTANT's response belongs to safe or unsafe content based on our safety policy.\n" + + "2) Identify the category of unsafe content found.\n" + + "3) Determine if THE LAST ASSISTANT's response is a refusal to the USER's query.\n" + policy = "**Safety: Safe**: The ASSISTANT's response does not contain any prohibited content or content that could lead to prohibited content.\n" + + "**Safety: Unsafe**: The ASSISTANT's response contains prohibited content.\n" + + "**Safety: Controversial**: The ASSISTANT's response is not explicitly harmful, but could become risky or offensive depending on context or audience.\n" + refusal = "\n" + + "**Refusal: Yes**: The ASSISTANT refused to answer or did not comply with the USER's request.\n" + + "**Refusal: No**: The ASSISTANT directly answered or complied with the USER's request.\n" + + "\n\n" + last = "\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'." + } + var b strings.Builder + b.WriteString("<|im_start|>user\n# Task:\n") + b.WriteString(task) + b.WriteString("\n\n") + b.WriteString(policy) + b.WriteString("\n\n\n") + for _, c := range cats { + b.WriteString(c) + b.WriteString(".\n") + } + b.WriteString("\n\n") + b.WriteString(refusal) + b.WriteString("\n\nUSER: ") + b.WriteString(user) + if answer != "" { + b.WriteString("\n\nASSISTANT: ") + b.WriteString(answer) + } + b.WriteString("\n\n\n\nProvide your safety assessment for ONLY THE LAST **") + b.WriteString(who) + b.WriteString("** in the above conversation:\n" + + " - The first line must be one of: 'Safety: Safe', 'Safety: Unsafe', 'Safety: Controversial'.\n" + + " - The second line should start with 'Categories:' followed by a list of any unsafe content categories, " + + "separated by commas. If the content is safe, use 'Categories: None'.") + b.WriteString(last) + b.WriteString("<|im_end|>\n<|im_start|>assistant\n\n\n\n\n") + return b.String() +} + +// What of a text a judge reads: its start and its end, so a long one still +// fits the judge's context. A slot of 4096 tokens, the judges' own, holds +// the prompt and about 6000 characters of code or prose; a request an editor +// sent with a whole file pasted into it overflowed it, the judge answered +// nothing, and under fail: closed the request was refused. +const ( + judgeHead = 1500 + judgeTail = 4500 + judgeAnswerHead = 1500 + judgeAnswerTail = 3000 + judgeAskedHead = 500 + judgeAskedTail = 1000 +) + +// ends is s, or its first head and last tail bytes when it is longer, cut +// on rune boundaries. +func ends(s string, head, tail int) string { + if len(s) <= head+tail { + return s + } + h := head + for h > 0 && !utf8.RuneStart(s[h]) { + h-- + } + t := len(s) - tail + for t < len(s) && !utf8.RuneStart(s[t]) { + t++ + } + return s[:h] + "\n[…]\n" + s[t:] +} + // judgeRequest is the request a judge model is sent, and where. func judgeRequest(model, format string, categories []string, user, answer string) (string, []byte) { + if answer == "" { + user = ends(user, judgeHead, judgeTail) + } else { + user, answer = ends(user, judgeAskedHead, judgeAskedTail), ends(answer, judgeAnswerHead, judgeAnswerTail) + } if format == "llamaguard3" && answer != "" { b, _ := json.Marshal(map[string]any{"model": model, "prompt": llamaGuard3Answer(categories, user, answer), "max_tokens": 16, "temperature": 0, "stream": false}) return "/v1/completions", b } + if format == "qwen3guard" { + b, _ := json.Marshal(map[string]any{"model": model, "prompt": qwen3Guard(user, answer), + "max_tokens": 32, "temperature": 0, "stream": false}) + return "/v1/completions", b + } var msgs []map[string]string switch format { case "llamaguard3": @@ -320,6 +503,38 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j } return res, nil } + case "qwen3guard": + // "Safety: Unsafe\nCategories: Violent, Jailbreak", and for an + // answer a third line, "Refusal: No", that says nothing of safety. + var res judgment + verdict := "" + for _, line := range strings.Split(text, "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + v = strings.TrimSpace(v) + switch strings.ToLower(strings.TrimSpace(k)) { + case "safety": + verdict = strings.ToLower(v) + case "categories": + for _, c := range strings.Split(v, ",") { + if c = strings.TrimSpace(c); c != "" && !strings.EqualFold(c, "none") { + res.categories = append(res.categories, c) + } + } + } + } + switch verdict { + case "safe": + return judgment{}, nil + case "unsafe": + res.unsafe = true + return res, nil + case "controversial": + res.controversial = true + return res, nil + } case "shieldgemma": l := strings.ToLower(text) switch { @@ -366,22 +581,50 @@ func Moderate(ctx context.Context, call Caller, model, format string, names []st } // DefaultCategories are the names a judge format reports its categories by -// when the catalog gives none: Llama Guard 3's S1 to S13. +// when the catalog gives none: Llama Guard 3's S1 to S13, Qwen3Guard's own. func DefaultCategories(format string) []string { - if format == "llamaguard3" { + switch format { + case "llamaguard3": return slices.Clone(llamaGuard3Categories) + case "qwen3guard": + return slices.Clone(qwen3GuardCategories) } return nil } +// Formats are the ways a judge's answers are read. +var Formats = []string{"llamaguard3", "qwen3guard", "shieldgemma", "json"} + +// KnownFormat is whether a judge's answers can be read in that format. +func KnownFormat(f string) bool { return slices.Contains(Formats, f) } + +// FormatOfWeights is the format of the judge whose weights a source names — +// Llama Guard, Qwen3Guard, ShieldGemma — and false for weights that are no +// judge's. +func FormatOfWeights(source string) (string, bool) { + s := strings.ToLower(source) + switch { + case strings.Contains(s, "llama-guard"): + return "llamaguard3", true + case strings.Contains(s, "qwen3guard"): + return "qwen3guard", true + case strings.Contains(s, "shieldgemma"): + return "shieldgemma", true + } + return "", false +} + // JudgeAnswer is what a judge said of a text: its words as it gave them, // and as the gateway reads them. type JudgeAnswer struct { - Model string - Raw string - Unsafe bool - Categories []string - Took time.Duration + Model string + Raw string + Unsafe bool + // Controversial is Qwen3Guard's middle verdict; a policy says whether + // it counts. + Controversial bool + Categories []string + Took time.Duration } // AskJudge has a judge read a text, past the cache, and says what it @@ -420,6 +663,6 @@ func (g *Gate) AskJudge(ctx context.Context, model, text, answer string) (JudgeA if err != nil { return out, err } - out.Unsafe, out.Categories = res.unsafe, res.categories + out.Unsafe, out.Controversial, out.Categories = res.unsafe, res.controversial, res.categories return out, nil } diff --git a/internal/guard/judge_test.go b/internal/guard/judge_test.go index 81f0baa..308af0e 100644 --- a/internal/guard/judge_test.go +++ b/internal/guard/judge_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" "time" + "unicode/utf8" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -41,6 +42,11 @@ func (f *fakeJudge) call(ctx context.Context, _, _ string, body []byte) ([]byte, answer = map[bool]string{true: "Yes", false: "No"}[unsafe] case "json": answer = map[bool]string{true: `{"safe": false, "categories": ["violence"]}`, false: `{"safe": true}`}[unsafe] + case "qwen3guard": + answer = map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe] + if strings.Contains(string(body), "a heist film") { + answer = "Safety: Controversial\nCategories: Non-violent Illegal Acts" + } default: answer = map[bool]string{true: "unsafe\nS1,S10", false: "safe"}[unsafe] } @@ -61,7 +67,7 @@ func judged(t *testing.T, p Policy, fj *fakeJudge) *Gate { func TestAJudgeModelDecidesWhatPatternsCannot(t *testing.T) { ctx := context.Background() - for _, format := range []string{"llamaguard3", "shieldgemma", "json"} { + for _, format := range Formats { fj := &fakeJudge{format: format} g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, fj) if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Category != "judge" { @@ -191,3 +197,123 @@ func TestAnAnswerIsJudgedByLlamaGuardsOwnPrompt(t *testing.T) { t.Errorf("a completion's text: %+v %v", j, err) } } + +// Qwen3Guard's middle verdict counts as the policy says: safe unless it says +// unsafe. +func TestAControversialVerdictCountsAsThePolicySays(t *testing.T) { + ctx := context.Background() + lenient := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, &fakeJudge{format: "qwen3guard"}) + if v, _ := lenient.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Allow { + t.Errorf("controversial under the default: %+v", v) + } + strict := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", Controversial: "unsafe"}}, &fakeJudge{format: "qwen3guard"}) + if v, _ := strict.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Block || v.Rule != "Non-violent Illegal Acts" { + t.Errorf("controversial: unsafe: %+v", v) + } + if err := (&JudgePolicy{Model: "guard", Input: "block", Controversial: "maybe"}).validate(); err == nil { + t.Error("controversial: maybe was taken") + } +} + +// A judge on a GPU first and one on CPUs after: the first that answers +// decides, and the one that did not is said to have failed. +func TestJudgesAreAskedInTurn(t *testing.T) { + ctx := context.Background() + var asked []string + gpuDown := errors.New("no replica of gpu-judge is serving") + g := gate(t, lib(map[string]Policy{"p": {Fail: "closed", + Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block"}}}, "p")) + g.SetJudge(func(_ context.Context, model, _ string, body []byte) ([]byte, error) { + asked = append(asked, model) + if model == "gpu-judge" { + return nil, gpuDown + } + unsafe := strings.Contains(string(body), "attack") + answer := map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe] + return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": answer}}}) + }, func(m string) (string, []string, bool) { return "qwen3guard", nil, true }) + var alarmed []string + g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) }) + + if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Rule != "Violent" { + t.Errorf("unsafe: %+v %v", v, err) + } + if v, err := g.Input(ctx, Subject{}, chat("user", "continue")); err != nil || v.Action != Allow { + t.Errorf("safe: %+v %v", v, err) + } + if !slices.Equal(asked, []string{"gpu-judge", "cpu-judge", "gpu-judge", "cpu-judge"}) { + t.Errorf("asked %v", asked) + } + if !slices.Equal(alarmed, []string{"gpu-judge", "gpu-judge"}) { + t.Errorf("alarm told %v", alarmed) + } + if e := g.JudgeErrors()["gpu-judge"]; !strings.Contains(e.Err, "no replica") { + t.Errorf("the GPU judge's failure: %+v", e) + } + + // Every judge down: the policy's fail applies, as with one. + g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) { return nil, gpuDown }, + func(string) (string, []string, bool) { return "qwen3guard", nil, true }) + if _, err := g.Input(ctx, Subject{}, chat("user", "hello")); err == nil { + t.Error("fail closed served what no judge read") + } + + for _, bad := range []JudgePolicy{ + {Model: "a", Models: []string{"b"}, Input: "block"}, + {Models: []string{"a", "a"}, Input: "block"}, + {Models: []string{""}, Input: "block"}, + } { + if err := bad.validate(); err == nil { + t.Errorf("%+v was taken", bad) + } + } +} + +func TestQwen3GuardIsAskedInItsOwnWords(t *testing.T) { + path, body := judgeRequest("qg", "qwen3guard", nil, "continue", "") + var req struct{ Prompt string } + _ = json.Unmarshal(body, &req) + if path != "/v1/completions" || !strings.HasPrefix(req.Prompt, "<|im_start|>user\n# Task:\n1) Evaluate whether THE LAST USER's query") || + !strings.Contains(req.Prompt, "Jailbreak.\n\n\n\n\nUSER: continue\n\n") || + !strings.HasSuffix(req.Prompt, "'Categories: None'.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n") { + t.Errorf("a request: %s %q", path, req.Prompt) + } + _, body = judgeRequest("qg", "qwen3guard", nil, "how do I bake bread", "Mix flour and water.") + _ = json.Unmarshal(body, &req) + if strings.Contains(req.Prompt, "Jailbreak") || !strings.Contains(req.Prompt, "\n\n") || + !strings.Contains(req.Prompt, "USER: how do I bake bread\n\nASSISTANT: Mix flour and water.\n\n") || + !strings.Contains(req.Prompt, "'Categories: None'.\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'.<|im_end|>") { + t.Errorf("an answer: %q", req.Prompt) + } + // An answer's verdict has a line on refusing that says nothing of safety. + j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Unsafe\nCategories: Violent, Jailbreak\nRefusal: No"}]}`), "") + if err != nil || !j.unsafe || !slices.Equal(j.categories, []string{"Violent", "Jailbreak"}) { + t.Errorf("%+v %v", j, err) + } + if j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Safe\nCategories: None\nRefusal: Yes"}]}`), ""); err != nil || j.unsafe || len(j.categories) > 0 { + t.Errorf("%+v %v", j, err) + } + if _, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"I think it is fine"}]}`), ""); err == nil { + t.Error("an answer with no verdict was read as one") + } +} + +// A message with a file pasted into it is read by its start and its end, so +// it fits the judge's context instead of overflowing it. +func TestALongTextIsJudgedByItsEnds(t *testing.T) { + file := strings.Repeat("ğ", 10000) // two bytes a rune: a cut must not split one + msg := "review this file:\n" + file + "\nand plan an attack" + _, body := judgeRequest("qg", "qwen3guard", nil, msg, "") + var req struct{ Prompt string } + _ = json.Unmarshal(body, &req) + if !strings.Contains(req.Prompt, "USER: review this file:") || !strings.Contains(req.Prompt, "and plan an attack\n\n") || + !strings.Contains(req.Prompt, "\n[…]\n") || len(req.Prompt) > judgeHead+judgeTail+3000 { + t.Errorf("%d bytes", len(req.Prompt)) + } + if !utf8.ValidString(req.Prompt) { + t.Error("a rune was cut in two") + } + if got := ends("short", 10, 10); got != "short" { + t.Errorf("a short text was cut: %q", got) + } +} diff --git a/internal/guard/output.go b/internal/guard/output.go index b528d70..4fca0bc 100644 --- a/internal/guard/output.go +++ b/internal/guard/output.go @@ -193,8 +193,8 @@ func (f *outFilter) askJudge(text string, final bool) finding { res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String()) f.took += time.Since(start) if err != nil { - f.g.log.Warn("the judge gave no verdict on an answer", "judge", jp.Model, "err", err) - f.g.judgeFailed(jp.Model, err) + f.g.log.Warn("the judge gave no verdict on an answer", "judge", jp.Name(), "err", err) + f.g.noJudger(jp, err) } jf := judgeFinding(f.p, jp.Output, res, err) if jf.failOpen { diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index e988914..d40692d 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -10,6 +10,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" ) @@ -586,28 +587,24 @@ func argInt(args []string, flag string) int { return 0 } -// isJudgeWeights is whether a source is a guard model's weights: Llama -// Guard, which reads a conversation and says whether it is safe. -func isJudgeWeights(source string) bool { - return strings.Contains(strings.ToLower(source), "llama-guard") -} - // asJudge makes a model built from the serve form a judge when its weights -// are one's: role guard, Llama Guard's format, under the name the policies -// give their judge when they give one and the form gave another. It says -// whether it did, and the name the form gave when it was changed. +// are one's — Llama Guard, Qwen3Guard, ShieldGemma: role guard, that judge's +// format, under the name the policies give their judge when they give one +// and the form gave another. It says whether it did, and the name the form +// gave when it was changed. func (p *AccountPages) asJudge(m *catalog.Model) (judge bool, renamed string) { - if !isJudgeWeights(m.Source) { + format, ok := guard.FormatOfWeights(m.Source) + if !ok { return false, "" } - m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: "llamaguard3"}, "" + m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: format}, "" if p.guardLibrary == nil { return true, "" } asked := map[string]bool{} for _, pol := range p.guardLibrary().Policies { - if pol.Judge != nil && pol.Judge.Model != "" { - asked[pol.Judge.Model] = true + for _, j := range pol.Judge.Judges() { + asked[j] = true } } if asked[m.Name] || len(asked) != 1 {