From 83e1444981f0216862d1d147b0b1bc6ab73d6bf3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 15:18:51 +0200 Subject: [PATCH] Regenerate the configuration reference with the judge and the guard role Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/reference.md | 83 ++++++++++++++++++++++++++++++++ internal/docgen/reference_gen.go | 14 ++++++ internal/docgen/render.go | 1 + 3 files changed, 98 insertions(+) diff --git a/docs/reference.md b/docs/reference.md index bfeba26..337a519 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -67,6 +67,13 @@ rather than repeated. | `url` | `string` | [External](#external) | | `model` | `string` | [External](#external) | | `api_key_file` | `string` | [External](#external) | +| `format` | `string` | [GuardModel](#guardmodel) | +| `categories` | `[]string` | [GuardModel](#guardmodel) | +| `model` | `string` | [JudgePolicy](#judgepolicy) | +| `input` | `string` | [JudgePolicy](#judgepolicy) | +| `output` | `string` | [JudgePolicy](#judgepolicy) | +| `categories` | `[]string` | [JudgePolicy](#judgepolicy) | +| `latency_budget` | `time.Duration` | [JudgePolicy](#judgepolicy) | | `contact` | `string` | [Landing](#landing) | | `notice` | `string` | [Landing](#landing) | | `bin` | `string` | [LlamaCppConfig](#llamacppconfig) | @@ -93,6 +100,8 @@ rather than repeated. | `context_length` | `int` | [Model](#model) | | `max_output` | `int` | [Model](#model) | | `capabilities` | `Capabilities` | [Model](#model) | +| `role` | `string` | [Model](#model) | +| `guard` | `*GuardModel` | [Model](#model) | | `origin` | `*Origin` | [Model](#model) | | `prometheus` | `string` | [Monitoring](#monitoring) | | `window` | `time.Duration` | [Monitoring](#monitoring) | @@ -127,6 +136,7 @@ rather than repeated. | `drop_client_system` | `bool` | [Policy](#policy) | | `require_capabilities` | `bool` | [Policy](#policy) | | `validate_structured` | `bool` | [Policy](#policy) | +| `judge` | `*JudgePolicy` | [Policy](#policy) | | `stream` | `string` | [Policy](#policy) | | `enabled` | `bool` | [Prefix](#prefix) | | `max_nodes` | `int` | [Prefix](#prefix) | @@ -625,6 +635,56 @@ APIKeyFile holds the provider's key, on the control plane. A file rather than the key itself, so the key is in neither the catalog, the store nor the audit log; empty for a provider that asks for none. +## GuardModel + +### `format` + +`string` + +Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and +the categories), shieldgemma (Google's ShieldGemma: Yes or No), or +json ({"safe": false, "categories": ["..."]}). + +### `categories` + +`[]string` + +Categories the model is asked about, for shieldgemma — each a policy +sentence — and the names reported for llamaguard3's S1, S2, …. + +## JudgePolicy + +### `model` + +`string` + +Model is a catalog model with role: guard. + +### `input` + +`string` + +Input and Output are what an unsafe verdict does: flag or block. +Empty does not ask the judge about that direction. + +### `output` + +`string` + +### `categories` + +`[]string` + +Categories, when set, are the only ones that count: Llama Guard's +codes (S1, S10) or the names the model reports. + +### `latency_budget` + +`time.Duration` + +LatencyBudget is how long the judge is waited for before the policy's +fail applies. Zero is 250ms. + ## Landing ### `contact` @@ -848,6 +908,21 @@ Capabilities is what this model can do beyond plain completion. Takes a [Capabilities](#capabilities) block. +### `role` + +`string` + +Role "guard" is a model that judges the gateway's traffic for the +guardrails rather than serving clients: hidden from /v1/models and the +chat page, refused to any key that names it, critical by default, and +always running. Guard says how its answers are read. + +### `guard` + +`*GuardModel` + +Takes a [GuardModel](#guardmodel) block. + ### `origin` `*Origin` @@ -1078,6 +1153,14 @@ request asked for in response_format: one that does not fit is stopped — or, streamed a sentence at a time, flagged, having been sent already. +### `judge` + +`*JudgePolicy` + +Judge has a judge model on the cluster read the text too. + +Takes a [JudgePolicy](#judgepolicy) block. + ### `stream` `string` diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 0ddf61f..71d1a4e 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -72,6 +72,17 @@ var Reference = []Block{ {Key: "model", Type: "string", Doc: "Model is what the provider calls it; requests are rewritten to it.", Nested: ""}, {Key: "api_key_file", Type: "string", Doc: "APIKeyFile holds the provider's key, on the control plane. A file\nrather than the key itself, so the key is in neither the catalog, the\nstore nor the audit log; empty for a provider that asks for none.", Nested: ""}, }}, + {Package: "catalog", Name: "GuardModel", Title: "GuardModel", Doc: "", Fields: []Field{ + {Key: "format", Type: "string", Doc: "Format is llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and\nthe categories), shieldgemma (Google's ShieldGemma: Yes or No), or\njson ({\"safe\": false, \"categories\": [\"...\"]}).", Nested: ""}, + {Key: "categories", Type: "[]string", Doc: "Categories the model is asked about, for shieldgemma — each a policy\nsentence — and the names reported for llamaguard3's S1, S2, ….", Nested: ""}, + }}, + {Package: "guard", Name: "JudgePolicy", Title: "JudgePolicy", Doc: "", Fields: []Field{ + {Key: "model", Type: "string", Doc: "Model is a catalog model with role: guard.", Nested: ""}, + {Key: "input", Type: "string", Doc: "Input and Output are what an unsafe verdict does: flag or block.\nEmpty does not ask the judge about that direction.", Nested: ""}, + {Key: "output", Type: "string", Doc: "", Nested: ""}, + {Key: "categories", Type: "[]string", Doc: "Categories, when set, are the only ones that count: Llama Guard's\ncodes (S1, S10) or the names the model reports.", Nested: ""}, + {Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long the judge is waited for before the policy's\nfail applies. Zero is 250ms.", Nested: ""}, + }}, {Package: "config", Name: "Landing", Title: "Landing", Doc: "", Fields: []Field{ {Key: "contact", Type: "string", Doc: "Contact is where someone without a key should go — an address, a URL, a\nchannel name. Rendered as a link when it is one, as text otherwise.", Nested: ""}, {Key: "notice", Type: "string", Doc: "Notice is an optional line of your own, for whatever the reader needs to\nknow that this page cannot guess.", Nested: ""}, @@ -105,6 +116,8 @@ var Reference = []Block{ {Key: "context_length", Type: "int", Doc: "ContextLength is how many tokens one request may use, prompt and\ncompletion together, and MaxOutput bounds the completion alone. Zero\nmeans unknown, which is what every model was until now.\n\nThe gateway knew neither, so an over-long prompt was not refused: it was\nadmitted, given a replica, and failed at the engine, and a client sizing\nits prompts from /v1/models had nothing to read. Worse, the number is not\nthe one an operator writes in the engine's arguments — llama.cpp divides\n--ctx-size across its parallel slots, so a pool of 8192 across four slots\nis 2048 a request, and nothing said so.", Nested: ""}, {Key: "max_output", Type: "int", Doc: "", Nested: ""}, {Key: "capabilities", Type: "Capabilities", Doc: "Capabilities is what this model can do beyond plain completion.", Nested: "catalog.Capabilities"}, + {Key: "role", Type: "string", Doc: "Role \"guard\" is a model that judges the gateway's traffic for the\nguardrails rather than serving clients: hidden from /v1/models and the\nchat page, refused to any key that names it, critical by default, and\nalways running. Guard says how its answers are read.", Nested: ""}, + {Key: "guard", Type: "*GuardModel", Doc: "", Nested: "catalog.GuardModel"}, {Key: "origin", Type: "*Origin", Doc: "Origin is the registry repository the weights came from, at which\ncommit, and each file's checksum there. Written when a model is served\nfrom the registry; for one that was not, worked out from the mirror's\nrecord of where it fetched the file. It is what lets the control plane\nsay that the repository has published different weights since, and\nmove the model to them.", Nested: "catalog.Origin"}, }}, {Package: "config", Name: "Monitoring", Title: "Monitoring", Doc: "", Fields: []Field{ @@ -153,6 +166,7 @@ var Reference = []Block{ {Key: "drop_client_system", Type: "bool", Doc: "DropClientSystem removes the caller's own system and developer\nmessages, so only the pinned prompt instructs the model.", Nested: ""}, {Key: "require_capabilities", Type: "bool", Doc: "RequireCapabilities refuses a request offering tools, or sending\nimages, to a model whose catalog entry does not say it takes them.", Nested: ""}, {Key: "validate_structured", Type: "bool", Doc: "ValidateStructured checks an answer against the JSON schema the\nrequest asked for in response_format: one that does not fit is\nstopped — or, streamed a sentence at a time, flagged, having been\nsent already.", Nested: ""}, + {Key: "judge", Type: "*JudgePolicy", Doc: "Judge has a judge model on the cluster read the text too.", Nested: "guard.JudgePolicy"}, {Key: "stream", Type: "string", Doc: "Stream: sentence scans a streamed answer a sentence at a time,\nreleasing each once it passes; buffer_all holds the whole answer\nuntil it is judged. Empty is sentence.", Nested: ""}, }}, {Package: "config", Name: "Prefix", Title: "Prefix", Doc: "", Fields: []Field{ diff --git a/internal/docgen/render.go b/internal/docgen/render.go index baf891a..e529608 100644 --- a/internal/docgen/render.go +++ b/internal/docgen/render.go @@ -27,6 +27,7 @@ var Sources = []string{ "internal/router/admission/controller.go", "internal/accounting/cost.go", "internal/guard/policy.go", + "internal/guard/judge.go", } // notReference are files with koanf tags that an operator never writes.