package guard import ( "context" "crypto/sha256" "encoding/json" "errors" "fmt" "slices" "strings" "sync" "time" "unicode/utf8" "gitea.mynodes.xyz/ilker/llmgw/internal/guard/detect" ) // JudgePolicy has a judge model on the cluster read text for what patterns // cannot: violence, self-harm, abuse — whatever the model was trained on. type JudgePolicy struct { // Model is a catalog model with role: guard. Model string `koanf:"model"` // Models are judges asked in turn, instead of Model: the first that // answers gives the verdict. A judge on a GPU first and one on CPUs // after it is the GPU's speed while it is there, and the CPUs' when it // is not — busy, failed, or the node down. Models []string `koanf:"models"` // Input and Output are what an unsafe verdict does: flag or block. // Empty does not ask the judge about that direction. Input string `koanf:"input"` Output string `koanf:"output"` // Categories, when set, are the only ones that count: Llama Guard's // codes (S1, S10) or the names the model reports. Categories []string `koanf:"categories"` // Controversial is what Qwen3Guard's third verdict counts as: safe // (the default) or unsafe. Controversial string `koanf:"controversial"` // LatencyBudget is how long each judge is waited for before the next is // asked, or, after the last, the policy's fail applies. Zero is 250ms. LatencyBudget time.Duration `koanf:"latency_budget"` } // Judges are the judge models asked, in the order they are asked. func (j *JudgePolicy) Judges() []string { if j == nil { return nil } if len(j.Models) > 0 { return j.Models } if j.Model != "" { return []string{j.Model} } return nil } // Name is the judges as one name, for logs and errors: "a, else b". func (j *JudgePolicy) Name() string { return strings.Join(j.Judges(), ", else ") } // DefaultJudgeBudget is how long a judge is waited for when a policy does // not say. const DefaultJudgeBudget = 250 * time.Millisecond func (j *JudgePolicy) budget() time.Duration { if j.LatencyBudget > 0 { return j.LatencyBudget } return DefaultJudgeBudget } func (j *JudgePolicy) validate() error { switch { case j.Model == "" && len(j.Models) == 0: return errors.New("judge.model (or judge.models, judges asked in turn) is required") case j.Model != "" && len(j.Models) > 0: return errors.New("judge: give model or models, not both") } seen := map[string]bool{} for _, m := range j.Models { if strings.TrimSpace(m) == "" { return errors.New("judge.models has an empty name") } if seen[m] { return fmt.Errorf("judge.models names %s twice", m) } seen[m] = true } switch j.Controversial { case "", "safe", "unsafe": default: return fmt.Errorf("judge.controversial is %q: safe or unsafe", j.Controversial) } for k, a := range map[string]string{"input": j.Input, "output": j.Output} { switch a { case "", "flag", "block": default: return fmt.Errorf("judge.%s is %q: flag or block", k, a) } } if j.Input == "" && j.Output == "" { return errors.New("judge: set input or output (flag or block), or it judges nothing") } return nil } // mergeJudge keeps the first judges asked for, the strictest actions, every // category and the shortest wait. func mergeJudge(a, b *JudgePolicy) *JudgePolicy { if b == nil { return a } if a == nil { c := *b c.Categories = slices.Clone(b.Categories) c.Models = slices.Clone(b.Models) return &c } out := *a out.Models = slices.Clone(a.Models) out.Input = stricterOf(a.Input, b.Input, []string{"", "flag", "block"}) out.Output = stricterOf(a.Output, b.Output, []string{"", "flag", "block"}) out.Controversial = stricterOf(a.Controversial, b.Controversial, []string{"", "safe", "unsafe"}) out.Categories = union(slices.Clone(a.Categories), b.Categories) if b.LatencyBudget > 0 && (a.LatencyBudget == 0 || b.LatencyBudget < a.LatencyBudget) { out.LatencyBudget = b.LatencyBudget } return &out } // Caller sends a body to a model on the cluster at a path — // /v1/chat/completions or /v1/completions — and returns the answer's body: // the router, past admission, accounting and guardrails. type Caller func(ctx context.Context, model, path string, body []byte) ([]byte, error) // ModelFormat says how a judge model's answers are read, and the category // names it reports; ok is false for a model that is not a judge. type ModelFormat func(model string) (format string, categories []string, ok bool) // judgment is what a judge said. type judgment struct { unsafe bool // controversial is Qwen3Guard's middle verdict, which the policy says // how to count. controversial bool categories []string // by is the judge that said it. by string } // judge asks a judge model about text, with a cache of recent answers. type judge struct { call Caller format ModelFormat m *gateMetrics // failed is told of each judge that gave no verdict, whether or not // another after it did. failed func(model string, err error) mu sync.Mutex cache map[[32]byte]cached } type cached struct { j judgment at time.Time } const ( judgeCacheTTL = 5 * time.Minute judgeCacheSize = 10_000 ) // SetJudge lets policies with a judge: section ask a model on the cluster. func (g *Gate) SetJudge(call Caller, format ModelFormat) { g.judger = &judge{call: call, format: format, m: g.m, failed: g.judgeFailed, cache: map[[32]byte]cached{}} } // errNoJudge is a policy naming a judge the gateway cannot ask, and // errJudgeFormat a judge whose answer could not be read. var ( errNoJudge = errors.New("no judge model to ask") errJudgeFormat = errors.New("the judge's answer could not be read") ) // ask has the judges read conversation turns — the user's, and the // assistant's answer when judging output — each in turn until one answers. // The error is the last judge's, when none did. func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) { if j == nil { return judgment{}, errNoJudge } models := jp.Judges() key := sha256.Sum256([]byte(strings.Join(models, "\x01") + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer))) j.mu.Lock() if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL { j.mu.Unlock() return c.j, nil } j.mu.Unlock() err := errNoJudge for _, model := range models { var res judgment res, err = j.askOne(ctx, jp, model, user, answer) if err == nil { j.remember(key, res) return res, nil } if j.failed != nil { j.failed(model, err) } if ctx.Err() != nil { // The request itself is gone: no judge after this one is wanted. break } } return judgment{}, err } // askOne asks one judge, waiting the policy's budget for it. func (j *judge) askOne(ctx context.Context, jp *JudgePolicy, model, user, answer string) (res judgment, err error) { start := time.Now() defer func() { if j.m != nil { j.m.judged(model, time.Since(start), err) } }() format, names, ok := j.format(model) if !ok { return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, model) } ctx, cancel := context.WithTimeout(ctx, jp.budget()) defer cancel() path, body := judgeRequest(model, format, names, user, answer) out, err := j.call(ctx, model, path, body) if err != nil { return judgment{}, err } res, err = parseJudgment(format, names, out, answer) if err != nil { return judgment{}, err } res.by = model return res, nil } func (j *judge) remember(key [32]byte, res judgment) { j.mu.Lock() if len(j.cache) >= judgeCacheSize { for k, c := range j.cache { if time.Since(c.at) >= judgeCacheTTL || len(j.cache) >= judgeCacheSize { delete(j.cache, k) } } } j.cache[key] = cached{j: res, at: time.Now()} j.mu.Unlock() } // counts reports whether a judgment is one the policy acts on. func (jp *JudgePolicy) counts(res judgment) (bool, string) { if !res.unsafe && !(res.controversial && jp.Controversial == "unsafe") { return false, "" } if len(jp.Categories) == 0 { return true, strings.Join(res.categories, ",") } for _, c := range res.categories { if slices.ContainsFunc(jp.Categories, func(w string) bool { return strings.EqualFold(w, c) }) { return true, c } } return false, "" } // llamaGuard3Categories are Llama Guard 3's hazards, in the order of their // codes. var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex Crimes", "Child Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property", "Indiscriminate Weapons", "Hate", "Self-Harm", "Sexual Content", "Elections"} // llamaGuard3Answer is Llama Guard 3's prompt for judging an answer, written // out rather than left to the model's chat template: an engine given a // conversation that ends on the assistant's turn — llama.cpp — takes it as a // reply to go on with, and the verdict comes out as the reply's next words, or // not at all. func llamaGuard3Answer(categories []string, user, answer string) string { if len(categories) == 0 { categories = llamaGuard3Categories } var b strings.Builder // No <|begin_of_text|>: the engine puts it first itself. b.WriteString("<|start_header_id|>user<|end_header_id|>\n\nTask: Check if there is unsafe content in " + "'Agent' messages in conversations according our safety policy with the below categories.\n\n" + "\n") for i, c := range categories { fmt.Fprintf(&b, "S%d: %s.\n", i+1, c) } b.WriteString("\n\n\n\n") if user != "" { b.WriteString("User: " + user + "\n\n") } b.WriteString("Agent: " + answer + "\n\n\n\n" + "Provide your safety assessment for ONLY THE LAST Agent message in the above conversation:\n" + " - First line must read 'safe' or 'unsafe'.\n" + " - If unsafe, a second line must include a comma-separated list of violated categories." + "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n") return b.String() } // qwen3GuardCategories are Qwen3Guard's own, as it names them in a verdict. // Jailbreak is judged in a user's message only. var qwen3GuardCategories = []string{"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "PII", "Suicide & Self-Harm", "Unethical Acts", "Politically Sensitive Topics", "Copyright Violation", "Jailbreak"} // qwen3Guard is Qwen3Guard-Gen's prompt, as its chat template writes it, // for a user's message or, with answer, the assistant's reply to it. Written // out for the reason llamaGuard3Answer is, and for both directions so that a // judge served without --jinja reads the same words. func qwen3Guard(user, answer string) string { who, task, policy, cats, refusal, last := "USER's query", "", "", qwen3GuardCategories, "", "" if answer == "" { task = "1) Evaluate whether THE LAST USER's query belongs to safe or unsafe content based on our safety policy.\n" + "2) Identify the category of unsafe content found.\n" policy = "**Safety: Safe**: The USER's query does not contain any prohibited content or content that could lead to prohibited content.\n" + "**Safety: Unsafe**: The USER's query contains prohibited content.\n" + "**Safety: Controversial**: The USER's query does not explicitly contain prohibited content, but its intent, context, " + "or potential responses could be misinterpreted, misused, or exploited to generate unsafe content under certain conditions.\n" } else { who = "ASSISTANT's response" cats = cats[:len(cats)-1] task = "1) Evaluate whether THE LAST ASSISTANT's response belongs to safe or unsafe content based on our safety policy.\n" + "2) Identify the category of unsafe content found.\n" + "3) Determine if THE LAST ASSISTANT's response is a refusal to the USER's query.\n" policy = "**Safety: Safe**: The ASSISTANT's response does not contain any prohibited content or content that could lead to prohibited content.\n" + "**Safety: Unsafe**: The ASSISTANT's response contains prohibited content.\n" + "**Safety: Controversial**: The ASSISTANT's response is not explicitly harmful, but could become risky or offensive depending on context or audience.\n" refusal = "\n" + "**Refusal: Yes**: The ASSISTANT refused to answer or did not comply with the USER's request.\n" + "**Refusal: No**: The ASSISTANT directly answered or complied with the USER's request.\n" + "\n\n" last = "\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'." } var b strings.Builder b.WriteString("<|im_start|>user\n# Task:\n") b.WriteString(task) b.WriteString("\n\n") b.WriteString(policy) b.WriteString("\n\n\n") for _, c := range cats { b.WriteString(c) b.WriteString(".\n") } b.WriteString("\n\n") b.WriteString(refusal) b.WriteString("\n\nUSER: ") b.WriteString(user) if answer != "" { b.WriteString("\n\nASSISTANT: ") b.WriteString(answer) } b.WriteString("\n\n\n\nProvide your safety assessment for ONLY THE LAST **") b.WriteString(who) b.WriteString("** in the above conversation:\n" + " - The first line must be one of: 'Safety: Safe', 'Safety: Unsafe', 'Safety: Controversial'.\n" + " - The second line should start with 'Categories:' followed by a list of any unsafe content categories, " + "separated by commas. If the content is safe, use 'Categories: None'.") b.WriteString(last) b.WriteString("<|im_end|>\n<|im_start|>assistant\n\n\n\n\n") return b.String() } // What of a text a judge reads: its start and its end, so a long one still // fits the judge's context. A slot of 4096 tokens, the judges' own, holds // the prompt and about 6000 characters of code or prose; a request an editor // sent with a whole file pasted into it overflowed it, the judge answered // nothing, and under fail: closed the request was refused. const ( judgeHead = 1500 judgeTail = 4500 judgeAnswerHead = 1500 judgeAnswerTail = 3000 judgeAskedHead = 500 judgeAskedTail = 1000 ) // ends is s, or its first head and last tail bytes when it is longer, cut // on rune boundaries. func ends(s string, head, tail int) string { if len(s) <= head+tail { return s } h := head for h > 0 && !utf8.RuneStart(s[h]) { h-- } t := len(s) - tail for t < len(s) && !utf8.RuneStart(s[t]) { t++ } return s[:h] + "\n[…]\n" + s[t:] } // judgeRequest is the request a judge model is sent, and where. func judgeRequest(model, format string, categories []string, user, answer string) (string, []byte) { if answer == "" { user = ends(user, judgeHead, judgeTail) } else { user, answer = ends(user, judgeAskedHead, judgeAskedTail), ends(answer, judgeAnswerHead, judgeAnswerTail) } if format == "llamaguard3" && answer != "" { b, _ := json.Marshal(map[string]any{"model": model, "prompt": llamaGuard3Answer(categories, user, answer), "max_tokens": 16, "temperature": 0, "stream": false}) return "/v1/completions", b } if format == "qwen3guard" { b, _ := json.Marshal(map[string]any{"model": model, "prompt": qwen3Guard(user, answer), "max_tokens": 32, "temperature": 0, "stream": false}) return "/v1/completions", b } var msgs []map[string]string switch format { case "llamaguard3": // The model's own chat template turns the turns into its prompt. msgs = append(msgs, map[string]string{"role": "user", "content": user}) if answer != "" { msgs = append(msgs, map[string]string{"role": "assistant", "content": answer}) } case "shieldgemma": who, text := "Human Question", user if answer != "" { who, text = "Chatbot Response", answer } principles := "* " + strings.Join(categories, "\n* ") if len(categories) == 0 { principles = "* No dangerous, hateful, harassing or sexually explicit content." } msgs = append(msgs, map[string]string{"role": "user", "content": "You are a policy expert trying to help determine " + "whether a message is in violation of the defined safety policies.\n\n\n" + who + ": " + text + "\n\n\nOur safety principle is defined in the below:\n\n" + principles + "\n\nDoes the " + strings.ToLower(who) + " violate the above principle? Your answer must start with 'Yes' or 'No'."}) default: // json subject := "user message" text := user if answer != "" { subject, text = "assistant answer", answer } msgs = append(msgs, map[string]string{"role": "system", "content": "Classify whether the " + subject + " is unsafe. " + `Answer with JSON only: {"safe": true|false, "categories": ["..."]}.`}, map[string]string{"role": "user", "content": text}) } b, _ := json.Marshal(map[string]any{"model": model, "messages": msgs, "max_tokens": 32, "temperature": 0, "stream": false}) return "/v1/chat/completions", b } // parseJudgment reads a judge model's answer. // // answer is the text that was judged, when it was an answer: llama.cpp // takes a conversation ending on the assistant's turn as a reply to go on // with, and its answer then begins with that turn — "Rome.safe". func parseJudgment(format string, names []string, body []byte, answer string) (judgment, error) { var resp struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` Text string `json:"text"` } `json:"choices"` } if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { return judgment{}, fmt.Errorf("%w: not a completion", errJudgeFormat) } text := strings.TrimSpace(resp.Choices[0].Message.Content + resp.Choices[0].Text) if a := strings.TrimSpace(answer); a != "" { text = strings.TrimSpace(strings.TrimPrefix(text, a)) } switch format { case "llamaguard3": if len(names) == 0 { // Its own: S1 is Violent Crimes, S9 Indiscriminate Weapons. names = llamaGuard3Categories } lines := strings.Split(text, "\n") switch strings.ToLower(strings.TrimSpace(lines[0])) { case "safe": return judgment{}, nil case "unsafe": res := judgment{unsafe: true} if len(lines) > 1 { for _, c := range strings.Split(lines[1], ",") { c = strings.TrimSpace(c) if c == "" { continue } res.categories = append(res.categories, c) // And the name it stands for, when the model has them. var n int if _, err := fmt.Sscanf(c, "S%d", &n); err == nil && n >= 1 && n <= len(names) { res.categories = append(res.categories, names[n-1]) } } } return res, nil } case "qwen3guard": // "Safety: Unsafe\nCategories: Violent, Jailbreak", and for an // answer a third line, "Refusal: No", that says nothing of safety. var res judgment verdict := "" for _, line := range strings.Split(text, "\n") { k, v, ok := strings.Cut(line, ":") if !ok { continue } v = strings.TrimSpace(v) switch strings.ToLower(strings.TrimSpace(k)) { case "safety": verdict = strings.ToLower(v) case "categories": for _, c := range strings.Split(v, ",") { if c = strings.TrimSpace(c); c != "" && !strings.EqualFold(c, "none") { res.categories = append(res.categories, c) } } } } switch verdict { case "safe": return judgment{}, nil case "unsafe": res.unsafe = true return res, nil case "controversial": res.controversial = true return res, nil } case "shieldgemma": l := strings.ToLower(text) switch { case strings.HasPrefix(l, "yes"): return judgment{unsafe: true, categories: []string{"policy"}}, nil case strings.HasPrefix(l, "no"): return judgment{}, nil } default: start, end := strings.Index(text, "{"), strings.LastIndex(text, "}") if start >= 0 && end > start { var v struct { Safe *bool `json:"safe"` Categories []string `json:"categories"` } if json.Unmarshal([]byte(text[start:end+1]), &v) == nil && v.Safe != nil { return judgment{unsafe: !*v.Safe, categories: v.Categories}, nil } } } // The judge's own answer, cut short, so an operator can see what form it // came in: a verdict, not the text judged. seen := []rune(text) if len(seen) > 40 { seen = seen[:40] } return judgment{}, fmt.Errorf("%w: not in the form %s has: %q", errJudgeFormat, format, string(seen)) } // Moderate asks a judge model about one text, as a user's message, outside // any policy: for /v1/moderations. It answers whether the model found it // unsafe, and under which of its categories. func Moderate(ctx context.Context, call Caller, model, format string, names []string, text string) (bool, []string, error) { path, body := judgeRequest(model, format, names, text, "") out, err := call(ctx, model, path, body) if err != nil { return false, nil, err } res, err := parseJudgment(format, names, out, "") if err != nil { return false, nil, err } return res.unsafe, res.categories, nil } // DefaultCategories are the names a judge format reports its categories by // when the catalog gives none: Llama Guard 3's S1 to S13, Qwen3Guard's own. func DefaultCategories(format string) []string { switch format { case "llamaguard3": return slices.Clone(llamaGuard3Categories) case "qwen3guard": return slices.Clone(qwen3GuardCategories) } return nil } // Formats are the ways a judge's answers are read. var Formats = []string{"llamaguard3", "qwen3guard", "shieldgemma", "json"} // KnownFormat is whether a judge's answers can be read in that format. func KnownFormat(f string) bool { return slices.Contains(Formats, f) } // FormatOfWeights is the format of the judge whose weights a source names — // Llama Guard, Qwen3Guard, ShieldGemma — and false for weights that are no // judge's. func FormatOfWeights(source string) (string, bool) { s := strings.ToLower(source) switch { case strings.Contains(s, "llama-guard"): return "llamaguard3", true case strings.Contains(s, "qwen3guard"): return "qwen3guard", true case strings.Contains(s, "shieldgemma"): return "shieldgemma", true } return "", false } // JudgeAnswer is what a judge said of a text: its words as it gave them, // and as the gateway reads them. type JudgeAnswer struct { Model string Raw string Unsafe bool // Controversial is Qwen3Guard's middle verdict; a policy says whether // it counts. Controversial bool Categories []string Took time.Duration } // AskJudge has a judge read a text, past the cache, and says what it // answered: for an operator finding out why ordinary requests are refused. // With answer, the judge reads it as the assistant's reply to text. func (g *Gate) AskJudge(ctx context.Context, model, text, answer string) (JudgeAnswer, error) { out := JudgeAnswer{Model: model} if g.judger == nil { return out, errNoJudge } format, names, ok := g.judger.format(model) if !ok { return out, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, model) } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() start := time.Now() path, body := judgeRequest(model, format, names, text, answer) reply, err := g.judger.call(ctx, model, path, body) out.Took = time.Since(start) if err != nil { return out, err } var resp struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` Text string `json:"text"` } `json:"choices"` } if json.Unmarshal(reply, &resp) == nil && len(resp.Choices) > 0 { out.Raw = strings.TrimSpace(resp.Choices[0].Message.Content + resp.Choices[0].Text) } res, err := parseJudgment(format, names, reply, answer) if err != nil { return out, err } out.Unsafe, out.Controversial, out.Categories = res.unsafe, res.controversial, res.categories return out, nil }