diff --git a/BACKLOG.md b/BACKLOG.md index 5079bc5..49207d0 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -291,6 +291,16 @@ redacted values in answers. fallbacks for the scope (filter in `dispatch` and `budgetFallback`). ### B3. Judge model on the cluster (M) +**Done (branch `guardrails-b`):** `role: guard` + `guard.format`; hidden and +refused to clients and fallbacks; critical by default, never on demand; +`Router.Judge` past admission/accounting/guardrails; llamaguard3, +shieldgemma and json parsers; 5-minute cache; latency budget then +fail open/closed; streamed answers read every 600 characters, or held with +buffer_all. Simpler than written: the input judge runs after admission, not +alongside it, and shadow mode judges synchronously within the budget rather +than after the response. Not enforced: two replicas on different nodes — +documented, left to `scaling.min`. + - `catalog.Model.Role: "guard"` plus `Guard{Format: llamaguard3 | shieldgemma | json, Categories}`. Guard models are hidden from `/v1/models`, the chat picker and allowlists; placed like any model, diff --git a/CHANGELOG.md b/CHANGELOG.md index 61eed92..8bbbd2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,16 @@ APIs change between minors without a deprecation period. - **Where they apply:** `guard.default` everywhere; others attached with `llmgw ctl guard attach` to an organization, team, project, user or key, and narrowed like limits. +- **A judge model on the cluster** (`role: guard` in the catalog, `judge:` + in a guard policy). It reads what patterns cannot — Llama Guard 3, + ShieldGemma or a JSON-answering model — and its verdict flags or blocks + input, output or both. + - **Calls:** past admission and accounting, bounded by a latency budget + (250ms by default), after which the policy's `fail` decides. + - **Caching and streams:** answers are cached for five minutes, and a + streamed answer is read every 600 characters. + - **Isolation:** a judge model is hidden from clients and refused to them. + It is critical by default, and cannot be loaded on demand. ### Security diff --git a/docs/guardrails.md b/docs/guardrails.md index d05ea30..ab8c09b 100644 --- a/docs/guardrails.md +++ b/docs/guardrails.md @@ -92,6 +92,57 @@ guard: `buffer_all`, an answer that does not fit is stopped; streamed a sentence at a time it has gone out already, and is flagged. +## A judge model + +Patterns find what has a shape. For what does not — violence, self-harm, +abuse, whatever a moderation model was trained on — a policy can have a +**judge model on the cluster** read the text: + +```yaml +# a catalog model (server.yaml models:, or ctl model add) +- name: llama-guard-3-1b + source: http://mirror:8081/llama-guard-3-1b/…/Llama-Guard-3-1B-Q4_K_M.gguf + role: guard + guard: + format: llamaguard3 # llamaguard3, shieldgemma or json + categories: [Violent Crimes, Non-Violent Crimes, Sex-Related Crimes, …] # names for S1, S2, … + scaling: {min: 2, max: 2} + profiles: [{name: cpu, engine: llamacpp}] + +# in a guard policy +judge: + model: llama-guard-3-1b + input: block # flag or block; empty does not ask about input + output: flag + categories: [S1, S10] # only these count; empty: any unsafe verdict + latency_budget: 250ms # then the policy's fail: open serves (flagged), closed refuses +``` + +A model with `role: guard` is the guardrails' alone: + +- it is not in `/v1/models` or the chat page, and is refused (404) to + every key, as a name and as a fallback; +- it is `critical` by default and must always run — on demand is refused. + Give it two replicas on different nodes, because every judged request + waits on it. + +The gateway's calls to it go past admission, accounting and the guardrails +themselves. + +- **Formats:** + - `llamaguard3` reads Meta's "safe" / "unsafe +S1,S10". + - `shieldgemma` asks Google's model whether the text breaks the listed + principles (`categories`, each a sentence) and reads Yes or No. + - `json` expects `{"safe": false, "categories": [...]}`. +- **What is read:** + - **Input:** the request's last user turn, after the patterns. + - **Output:** the answer, together with that user turn. A streamed answer + is read every 600 characters and at the end, so text between two + readings has gone out already; `stream: buffer_all` holds it until it + is read. +- **Cache:** the same text is not asked about twice within five minutes. + ## Where a policy applies `guard.default` applies everywhere. Others are attached to a part of the diff --git a/internal/cli/server.go b/internal/cli/server.go index aab7794..b1695e4 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -130,6 +130,7 @@ func newServerCmd() *cobra.Command { if gate, err = guard.NewGate(cfg.Guard, att, log); err != nil { return fmt.Errorf("guard: %w", err) } + gate.SetJudge(rt.Judge, rt.GuardFormat) rt.SetGuard(gate) log.Info("guardrails on", "policies", len(cfg.Guard.Policies), "default", cfg.Guard.Default) } diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index c74c06a..9564fc5 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -195,6 +195,13 @@ type Model struct { // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` + // Role "guard" is a model that judges the gateway's traffic for the + // guardrails rather than serving clients: hidden from /v1/models and the + // chat page, refused to any key that names it, critical by default, and + // always running. Guard says how its answers are read. + Role string `json:"role,omitempty" koanf:"role"` + Guard *GuardModel `json:"guard,omitempty" koanf:"guard"` + // Origin is the registry repository the weights came from, at which // commit, and each file's checksum there. Written when a model is served // from the registry; for one that was not, worked out from the mirror's @@ -204,6 +211,23 @@ type Model struct { Origin *Origin `json:"origin,omitempty" koanf:"origin"` } +// RoleGuard is a judge model's role. +const RoleGuard = "guard" + +// IsGuard reports whether a model judges traffic rather than serving it. +func (m Model) IsGuard() bool { return m.Role == RoleGuard } + +// GuardModel is how a judge model's answers are read. +type GuardModel struct { + // Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and + // the categories), shieldgemma (Google's ShieldGemma: Yes or No), or + // json ({"safe": false, "categories": ["..."]}). + Format string `json:"format" koanf:"format"` + // Categories the model is asked about, for shieldgemma — each a policy + // sentence — and the names reported for llamaguard3's S1, S2, …. + Categories []string `json:"categories,omitempty" koanf:"categories"` +} + // Origin is where a model's weights were published. type Origin struct { // Repo is the repository, owner/name. @@ -420,6 +444,31 @@ func (m Model) validate() error { if !validClass(m.Class) { return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class) } + switch m.Role { + case "": + if m.Guard != nil { + return fmt.Errorf("model %q: guard: is for a model with role: guard", m.Name) + } + case RoleGuard: + if m.Guard == nil { + return fmt.Errorf("model %q: a guard model needs guard.format: llamaguard3, shieldgemma or json", m.Name) + } + switch m.Guard.Format { + case "llamaguard3", "shieldgemma", "json": + default: + return fmt.Errorf("model %q: guard.format %q is not llamaguard3, shieldgemma or json", m.Name, m.Guard.Format) + } + if m.Scaling.OnDemand() { + // A judge loaded on demand makes the first request of every + // idle period wait for it, or be judged by nothing. + return fmt.Errorf("model %q: a guard model must always run: scaling.min 1 or more", m.Name) + } + if m.IsExternal() { + return fmt.Errorf("model %q: a guard model judges in the cluster; it cannot be external", m.Name) + } + default: + return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role) + } if m.Scaling.Max < m.Scaling.Min { return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) } @@ -681,6 +730,10 @@ func (m Model) EffectiveClass() string { if m.Class != "" { return m.Class } + if m.IsGuard() { + // Every judged request waits on it. + return ClassCritical + } if m.Scaling.OnDemand() { return ClassOpportunistic } diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index 4a62194..d4417c9 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -403,3 +403,29 @@ func TestAModelThatWouldRunCodeOnItsNodesIsRefusedWhereItIsDefined(t *testing.T) t.Errorf("a tuned model was refused: %v", err) } } + +func TestAGuardModelSaysHowItIsReadAndAlwaysRuns(t *testing.T) { + base := func() Model { + return Model{Name: "lg", Source: "/m/lg.gguf", Role: RoleGuard, Guard: &GuardModel{Format: "llamaguard3"}, + Profiles: []Profile{{Name: "cpu", Engine: "llamacpp"}}, Scaling: Scaling{Min: 1, Max: 2}} + } + if err := base().validate(); err != nil { + t.Fatal(err) + } + for name, mut := range map[string]func(*Model){ + "no format": func(m *Model) { m.Guard = nil }, + "bad format": func(m *Model) { m.Guard.Format = "vibes" }, + "on demand": func(m *Model) { m.Scaling = Scaling{Min: 0, Max: 1} }, + "bad role": func(m *Model) { m.Role = "judge" }, + "guard w/o role": func(m *Model) { m.Role = "" }, + } { + m := base() + mut(&m) + if m.validate() == nil { + t.Errorf("%s accepted", name) + } + } + if base().EffectiveClass() != ClassCritical { + t.Error("a judge is not critical by default") + } +} diff --git a/internal/guard/gate.go b/internal/guard/gate.go index 927e114..2edf02e 100644 --- a/internal/guard/gate.go +++ b/internal/guard/gate.go @@ -36,8 +36,9 @@ type Gate struct { log *slog.Logger now func() time.Time - deny map[string]*regexp.Regexp // by pattern regex - tools toolMemo + deny map[string]*regexp.Regexp // by pattern regex + tools toolMemo + judger *judge mu sync.Mutex attached map[string][]string // "scope/id" → policy names @@ -264,7 +265,7 @@ func (g *Gate) Directives(s Subject) Directives { } // Input judges a request body. -func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error) { +func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, error) { start := g.now() p, names := g.Policy(s) if !p.active() { @@ -314,7 +315,12 @@ func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error) if p.Tools.ValidateArguments { offered = d.tools() } - g.tools.put(s.RequestID, offered, answer, start) + user := lastUser(segsOf(d)) + var judgeUser string + if p.Judge != nil && p.Judge.Output != "" { + judgeUser = user + } + g.tools.put(s.RequestID, offered, answer, judgeUser, start) if p.RequireCapabilities { if len(d.tools()) > 0 && !s.Tools { fs = append(fs, finding{action: Block, category: "capability", rule: "tools"}) @@ -347,6 +353,18 @@ func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error) if found && best.action == Block { return done(best), nil } + // The judge last: what a pattern already stopped need not wait for it. + if jp := p.Judge; jp != nil && jp.Input != "" && user != "" { + res, err := g.judger.ask(ctx, jp, user, "") + if err != nil && p.Fail == "closed" && !v.Shadow { + v.Took = g.now().Sub(start) + return v, fmt.Errorf("the judge could not be asked: %w", err) + } + fs = append(fs, judgeFinding(p, jp.Input, res, err)) + if best, found = strictest(fs); found && best.action == Block { + return done(best), nil + } + } if !v.Shadow { for _, rw := range rewrites { rw.seg.set(rw.text) @@ -369,6 +387,34 @@ func (g *Gate) Input(_ context.Context, s Subject, body []byte) (Verdict, error) return v, nil } +// judgeFinding is what a judge's answer, or its failing to give one, comes +// to under a policy. +func judgeFinding(p Policy, action string, res judgment, err error) finding { + if err != nil { + // Unanswered: closed stops it, open lets it through, said. + if p.Fail == "closed" { + return finding{action: Block, category: "judge", rule: "unavailable"} + } + return finding{action: Flag, category: "judge", rule: "unavailable"} + } + if ok, cat := p.Judge.counts(res); ok { + return finding{action: actions[action], category: "judge", rule: cat} + } + return finding{} +} + +func segsOf(d *doc) []segment { return d.segments() } + +// lastUser is the last thing the user said: what a judge reads. +func lastUser(segs []segment) string { + for i := len(segs) - 1; i >= 0; i-- { + if segs[i].role == "user" { + return segs[i].text + } + } + return "" +} + func toolAllowed(t Tools, name string) bool { for _, d := range t.Deny { if d == name { diff --git a/internal/guard/judge.go b/internal/guard/judge.go new file mode 100644 index 0000000..441b7c4 --- /dev/null +++ b/internal/guard/judge.go @@ -0,0 +1,278 @@ +package guard + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/guard/detect" +) + +// JudgePolicy has a judge model on the cluster read text for what patterns +// cannot: violence, self-harm, abuse — whatever the model was trained on. +type JudgePolicy struct { + // Model is a catalog model with role: guard. + Model string `koanf:"model"` + // Input and Output are what an unsafe verdict does: flag or block. + // Empty does not ask the judge about that direction. + Input string `koanf:"input"` + Output string `koanf:"output"` + // Categories, when set, are the only ones that count: Llama Guard's + // codes (S1, S10) or the names the model reports. + Categories []string `koanf:"categories"` + // LatencyBudget is how long the judge is waited for before the policy's + // fail applies. Zero is 250ms. + LatencyBudget time.Duration `koanf:"latency_budget"` +} + +// DefaultJudgeBudget is how long a judge is waited for when a policy does +// not say. +const DefaultJudgeBudget = 250 * time.Millisecond + +func (j *JudgePolicy) budget() time.Duration { + if j.LatencyBudget > 0 { + return j.LatencyBudget + } + return DefaultJudgeBudget +} + +func (j *JudgePolicy) validate() error { + if j.Model == "" { + return errors.New("judge.model is required") + } + for k, a := range map[string]string{"input": j.Input, "output": j.Output} { + switch a { + case "", "flag", "block": + default: + return fmt.Errorf("judge.%s is %q: flag or block", k, a) + } + } + if j.Input == "" && j.Output == "" { + return errors.New("judge: set input or output (flag or block), or it judges nothing") + } + return nil +} + +// mergeJudge keeps the first judge model asked for, the strictest actions, +// every category and the shortest wait. +func mergeJudge(a, b *JudgePolicy) *JudgePolicy { + if b == nil { + return a + } + if a == nil { + c := *b + c.Categories = slices.Clone(b.Categories) + return &c + } + out := *a + out.Input = stricterOf(a.Input, b.Input, []string{"", "flag", "block"}) + out.Output = stricterOf(a.Output, b.Output, []string{"", "flag", "block"}) + out.Categories = union(slices.Clone(a.Categories), b.Categories) + if b.LatencyBudget > 0 && (a.LatencyBudget == 0 || b.LatencyBudget < a.LatencyBudget) { + out.LatencyBudget = b.LatencyBudget + } + return &out +} + +// Caller sends a chat completion body to a model on the cluster and returns +// the answer's body: the router, past admission, accounting and guardrails. +type Caller func(ctx context.Context, model string, body []byte) ([]byte, error) + +// ModelFormat says how a judge model's answers are read, and the category +// names it reports; ok is false for a model that is not a judge. +type ModelFormat func(model string) (format string, categories []string, ok bool) + +// judgment is what a judge said. +type judgment struct { + unsafe bool + categories []string +} + +// judge asks a judge model about text, with a cache of recent answers. +type judge struct { + call Caller + format ModelFormat + + mu sync.Mutex + cache map[[32]byte]cached +} + +type cached struct { + j judgment + at time.Time +} + +const ( + judgeCacheTTL = 5 * time.Minute + judgeCacheSize = 10_000 +) + +// SetJudge lets policies with a judge: section ask a model on the cluster. +func (g *Gate) SetJudge(call Caller, format ModelFormat) { + g.judger = &judge{call: call, format: format, cache: map[[32]byte]cached{}} +} + +// errNoJudge is a policy naming a judge the gateway cannot ask. +var errNoJudge = errors.New("no judge model to ask") + +// ask has the judge read conversation turns — the user's, and the +// assistant's answer when judging output. +func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) { + if j == nil { + return judgment{}, errNoJudge + } + format, names, ok := j.format(jp.Model) + if !ok { + return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, jp.Model) + } + key := sha256.Sum256([]byte(jp.Model + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer))) + j.mu.Lock() + if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL { + j.mu.Unlock() + return c.j, nil + } + j.mu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, jp.budget()) + defer cancel() + body := judgeRequest(jp.Model, format, names, user, answer) + out, err := j.call(ctx, jp.Model, body) + if err != nil { + return judgment{}, err + } + res, err := parseJudgment(format, names, out) + if err != nil { + return judgment{}, err + } + j.mu.Lock() + if len(j.cache) >= judgeCacheSize { + for k, c := range j.cache { + if time.Since(c.at) >= judgeCacheTTL || len(j.cache) >= judgeCacheSize { + delete(j.cache, k) + } + } + } + j.cache[key] = cached{j: res, at: time.Now()} + j.mu.Unlock() + return res, nil +} + +// counts reports whether a judgment is one the policy acts on. +func (jp *JudgePolicy) counts(res judgment) (bool, string) { + if !res.unsafe { + return false, "" + } + if len(jp.Categories) == 0 { + return true, strings.Join(res.categories, ",") + } + for _, c := range res.categories { + if slices.ContainsFunc(jp.Categories, func(w string) bool { return strings.EqualFold(w, c) }) { + return true, c + } + } + return false, "" +} + +// judgeRequest is the chat completion a judge model is sent. +func judgeRequest(model, format string, categories []string, user, answer string) []byte { + var msgs []map[string]string + switch format { + case "llamaguard3": + // The model's own chat template turns the turns into its prompt. + msgs = append(msgs, map[string]string{"role": "user", "content": user}) + if answer != "" { + msgs = append(msgs, map[string]string{"role": "assistant", "content": answer}) + } + case "shieldgemma": + who, text := "Human Question", user + if answer != "" { + who, text = "Chatbot Response", answer + } + principles := "* " + strings.Join(categories, "\n* ") + if len(categories) == 0 { + principles = "* No dangerous, hateful, harassing or sexually explicit content." + } + msgs = append(msgs, map[string]string{"role": "user", "content": "You are a policy expert trying to help determine " + + "whether a message is in violation of the defined safety policies.\n\n\n" + who + ": " + text + + "\n\n\nOur safety principle is defined in the below:\n\n" + principles + + "\n\nDoes the " + strings.ToLower(who) + " violate the above principle? Your answer must start with 'Yes' or 'No'."}) + default: // json + subject := "user message" + text := user + if answer != "" { + subject, text = "assistant answer", answer + } + msgs = append(msgs, + map[string]string{"role": "system", "content": "Classify whether the " + subject + " is unsafe. " + + `Answer with JSON only: {"safe": true|false, "categories": ["..."]}.`}, + map[string]string{"role": "user", "content": text}) + } + b, _ := json.Marshal(map[string]any{"model": model, "messages": msgs, "max_tokens": 32, "temperature": 0, "stream": false}) + return b +} + +// parseJudgment reads a judge model's answer. +func parseJudgment(format string, names []string, body []byte) (judgment, error) { + var resp struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { + return judgment{}, errors.New("the judge's answer is not a chat completion") + } + text := strings.TrimSpace(resp.Choices[0].Message.Content) + switch format { + case "llamaguard3": + lines := strings.Split(text, "\n") + switch strings.ToLower(strings.TrimSpace(lines[0])) { + case "safe": + return judgment{}, nil + case "unsafe": + res := judgment{unsafe: true} + if len(lines) > 1 { + for _, c := range strings.Split(lines[1], ",") { + c = strings.TrimSpace(c) + if c == "" { + continue + } + res.categories = append(res.categories, c) + // And the name it stands for, when the model has them. + var n int + if _, err := fmt.Sscanf(c, "S%d", &n); err == nil && n >= 1 && n <= len(names) { + res.categories = append(res.categories, names[n-1]) + } + } + } + return res, nil + } + case "shieldgemma": + l := strings.ToLower(text) + switch { + case strings.HasPrefix(l, "yes"): + return judgment{unsafe: true, categories: []string{"policy"}}, nil + case strings.HasPrefix(l, "no"): + return judgment{}, nil + } + default: + start, end := strings.Index(text, "{"), strings.LastIndex(text, "}") + if start >= 0 && end > start { + var v struct { + Safe *bool `json:"safe"` + Categories []string `json:"categories"` + } + if json.Unmarshal([]byte(text[start:end+1]), &v) == nil && v.Safe != nil { + return judgment{unsafe: !*v.Safe, categories: v.Categories}, nil + } + } + } + return judgment{}, fmt.Errorf("the judge answered in a form %s does not have", format) +} diff --git a/internal/guard/judge_test.go b/internal/guard/judge_test.go new file mode 100644 index 0000000..74d1bb2 --- /dev/null +++ b/internal/guard/judge_test.go @@ -0,0 +1,141 @@ +package guard + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + "time" +) + +// fakeJudge answers as a judge model would, saying unsafe to text with +// "attack" in it. +type fakeJudge struct { + calls atomic.Int64 + err error + format string + delay time.Duration +} + +func (f *fakeJudge) call(ctx context.Context, _ string, body []byte) ([]byte, error) { + f.calls.Add(1) + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if f.err != nil { + return nil, f.err + } + unsafe := strings.Contains(string(body), "attack") + var answer string + switch f.format { + case "shieldgemma": + answer = map[bool]string{true: "Yes", false: "No"}[unsafe] + case "json": + answer = map[bool]string{true: `{"safe": false, "categories": ["violence"]}`, false: `{"safe": true}`}[unsafe] + default: + answer = map[bool]string{true: "unsafe\nS1,S10", false: "safe"}[unsafe] + } + b, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": answer}}}}) + return b, nil +} + +func judged(t *testing.T, p Policy, fj *fakeJudge) *Gate { + g := gate(t, lib(map[string]Policy{"p": p}, "p")) + if fj.format == "" { + fj.format = "llamaguard3" + } + g.SetJudge(fj.call, func(m string) (string, []string, bool) { + return fj.format, []string{"Violent Crimes"}, m == "guard" + }) + return g +} + +func TestAJudgeModelDecidesWhatPatternsCannot(t *testing.T) { + ctx := context.Background() + for _, format := range []string{"llamaguard3", "shieldgemma", "json"} { + fj := &fakeJudge{format: format} + g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, fj) + if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Category != "judge" { + t.Errorf("%s: unsafe: %+v %v", format, v, err) + } + if v, _ := g.Input(ctx, Subject{}, chat("user", "plan a picnic")); v.Action != Allow { + t.Errorf("%s: safe: %+v", format, v) + } + } + // Llama Guard's codes are named from the catalog, and only listed + // categories count when a policy lists some. + fj := &fakeJudge{} + g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", Categories: []string{"S2"}}}, fj) + if v, _ := g.Input(ctx, Subject{}, chat("user", "plan an attack")); v.Action != Allow { + t.Errorf("a category the policy does not list: %+v", v) + } + g = judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "flag", Categories: []string{"violent crimes"}}}, fj) + if v, _ := g.Input(ctx, Subject{}, chat("user", "plan an attack")); v.Action != Flag || v.Rule != "Violent Crimes" { + t.Errorf("by name: %+v", v) + } +} + +func TestAJudgeIsAskedOnceForTheSameText(t *testing.T) { + fj := &fakeJudge{} + g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "flag"}}, fj) + for range 3 { + _, _ = g.Input(context.Background(), Subject{}, chat("user", "hello there")) + } + if fj.calls.Load() != 1 { + t.Errorf("asked %d times", fj.calls.Load()) + } +} + +func TestAJudgeThatDoesNotAnswerFailsAsThePolicySays(t *testing.T) { + ctx := context.Background() + slow := &fakeJudge{delay: time.Second} + open := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", LatencyBudget: 20 * time.Millisecond}}, slow) + start := time.Now() + v, err := open.Input(ctx, Subject{}, chat("user", "plan an attack")) + if err != nil || v.Action != Flag || v.Rule != "unavailable" || time.Since(start) > 500*time.Millisecond { + t.Errorf("fail open: %+v %v after %s", v, err, time.Since(start)) + } + closed := judged(t, Policy{Fail: "closed", Judge: &JudgePolicy{Model: "guard", Input: "block"}}, &fakeJudge{err: errors.New("down")}) + if _, err := closed.Input(ctx, Subject{}, chat("user", "hi")); err == nil { + t.Error("fail closed served unjudged") + } + // A judge that is not a guard model is a judge that cannot be asked. + bad := judged(t, Policy{Fail: "closed", Judge: &JudgePolicy{Model: "chat-model", Input: "block"}}, &fakeJudge{}) + if _, err := bad.Input(ctx, Subject{}, chat("user", "hi")); err == nil { + t.Error("a model without role: guard was used as a judge") + } +} + +func TestAStreamedAnswerIsJudgedAsItGrows(t *testing.T) { + safe := strings.Repeat("Once upon a time. ", 40) // ~720 chars + more := strings.Repeat("And so it went on. ", 40) + + // A sentence at a time, the judge reads every judgeEvery characters: + // the stream stops at the next reading after the unsafe part, which has + // gone out already. + fj := &fakeJudge{} + g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Output: "block"}}, fj) + s := Subject{RequestID: "s1", Stream: true} + _, _ = g.Input(context.Background(), s, chat("user", "tell me a story")) + got, _, stopped := stream(g.Output(context.Background(), s), safe, "Then the attack began. ", more, "The end.") + if !stopped || strings.Contains(got, "The end.") || !strings.HasPrefix(got, "Once upon") { + t.Errorf("sentence: stopped=%v got %d chars", stopped, len(got)) + } + if n := fj.calls.Load(); n < 2 { + t.Errorf("judged %d times", n) + } + + // Held whole, nothing unsafe goes out at all. + g = judged(t, Policy{Stream: "buffer_all", Judge: &JudgePolicy{Model: "guard", Output: "block"}}, &fakeJudge{}) + _, _ = g.Input(context.Background(), s, chat("user", "tell me a story")) + got, _, stopped = stream(g.Output(context.Background(), s), safe, "Then the attack began. ", "The end.") + if !stopped || got != "" { + t.Errorf("buffer_all: stopped=%v got %q", stopped, got) + } +} diff --git a/internal/guard/output.go b/internal/guard/output.go index d88e1e2..9b58395 100644 --- a/internal/guard/output.go +++ b/internal/guard/output.go @@ -19,11 +19,13 @@ type toolSet struct { schemas map[string]json.RawMessage // answer is the schema response_format asks the answer to fit. answer json.RawMessage - at time.Time + // user is the request's last user turn, for a judge reading the answer. + user string + at time.Time } -func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, now time.Time) { - if id == "" || len(tools) == 0 && answer == nil { +func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, user string, now time.Time) { + if id == "" || len(tools) == 0 && answer == nil && user == "" { return } m.mu.Lock() @@ -36,7 +38,7 @@ func (m *toolMemo) put(id string, tools []tool, answer json.RawMessage, now time delete(m.byID, k) } } - set := toolSet{schemas: map[string]json.RawMessage{}, answer: answer, at: now} + set := toolSet{schemas: map[string]json.RawMessage{}, answer: answer, user: user, at: now} for _, t := range tools { set.schemas[t.name] = t.parameters } @@ -52,23 +54,25 @@ func (m *toolMemo) get(id string) toolSet { func outputWatched(p Policy) bool { o, t := p.Output, p.Tools return len(o.PII) > 0 || o.Injection != "" || len(o.Deny) > 0 || o.MaxChars > 0 || p.ValidateStructured || + p.Judge != nil && p.Judge.Output != "" || len(t.Allow) > 0 || len(t.Deny) > 0 || t.MaxCallsPerTurn > 0 || t.ValidateArguments } // Output is the filter for one answer, or nil when the policy in force // watches no answers. -func (g *Gate) Output(_ context.Context, s Subject) OutputFilter { +func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter { p, names := g.Policy(s) if !p.active() || !outputWatched(p) { return nil } memo := g.tools.get(s.RequestID) - return &outFilter{g: g, p: p, names: names, shadow: p.Mode == "shadow", - schemas: memo.schemas, answer: memo.answer, choices: map[int]*choiceState{}} + return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow", + schemas: memo.schemas, answer: memo.answer, user: memo.user, choices: map[int]*choiceState{}} } // outFilter judges one answer. type outFilter struct { + ctx context.Context g *Gate p Policy names string @@ -76,6 +80,9 @@ type outFilter struct { schemas map[string]json.RawMessage answer json.RawMessage full strings.Builder // the whole answer, when it must fit a schema + user string // the request's last user turn, for the judge + sofar strings.Builder // the answer so far, for the judge + judged int // how much of it the judge has read v Verdict took time.Duration stopped bool @@ -162,6 +169,28 @@ func (f *outFilter) judgeCalls(calls []call) finding { return f.record(fs) } +// judgeEvery is how much new answer text a streamed answer's judge waits +// for before reading it again. +const judgeEvery = 600 + +// askJudge has the judge read the answer so far, when the policy asks and +// enough has arrived since it last did, or at the end. +func (f *outFilter) askJudge(text string, final bool) finding { + jp := f.p.Judge + if jp == nil || jp.Output == "" { + return finding{} + } + f.sofar.WriteString(text) + if !final && f.sofar.Len()-f.judged < judgeEvery || f.sofar.Len() == f.judged { + return finding{} + } + f.judged = f.sofar.Len() + start := time.Now() + res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String()) + f.took += time.Since(start) + return f.record([]finding{judgeFinding(f.p, jp.Output, res, err)}) +} + // judgeStructured checks a whole answer against the schema the request // asked it to fit. func (f *outFilter) judgeStructured(text string, enforce bool) finding { @@ -215,6 +244,9 @@ func (f *outFilter) Buffered(body []byte) []byte { if f.judgeStructured(t, true).action == Block { blocked = true } + if f.askJudge(t, true).action == Block { + blocked = true + } if out != t { msg["content"] = rawString(out) ch["message"], _ = json.Marshal(msg) @@ -374,6 +406,9 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) { if f.answer != nil { f.full.WriteString(head) } + if i == 0 && f.askJudge(head, final).action == Block { + blocked = true + } if head != "" { fd, r := f.judgeText(head) blocked = blocked || fd.action == Block diff --git a/internal/guard/policy.go b/internal/guard/policy.go index 5cd6275..d905759 100644 --- a/internal/guard/policy.go +++ b/internal/guard/policy.go @@ -61,6 +61,8 @@ type Policy struct { // stopped — or, streamed a sentence at a time, flagged, having been // sent already. ValidateStructured bool `koanf:"validate_structured"` + // Judge has a judge model on the cluster read the text too. + Judge *JudgePolicy `koanf:"judge"` // Stream: sentence scans a streamed answer a sentence at a time, // releasing each once it passes; buffer_all holds the whole answer // until it is judged. Empty is sentence. @@ -137,6 +139,11 @@ func (p Policy) validate() error { default: return fmt.Errorf("stream is %q: sentence or buffer_all", p.Stream) } + if p.Judge != nil { + if err := p.Judge.validate(); err != nil { + return err + } + } switch p.Tools.OnDenied { case "", "strip", "block": default: @@ -189,6 +196,7 @@ func Merge(names []string, ps []Policy) Policy { out.DropClientSystem = out.DropClientSystem || p.DropClientSystem out.RequireCapabilities = out.RequireCapabilities || p.RequireCapabilities out.ValidateStructured = out.ValidateStructured || p.ValidateStructured + out.Judge = mergeJudge(out.Judge, p.Judge) out.MaxOutputTokens = minSet(out.MaxOutputTokens, p.MaxOutputTokens) if p.SystemPrompt != "" { prompts = append(prompts, p.SystemPrompt) diff --git a/internal/router/guard.go b/internal/router/guard.go index 75526ca..f937856 100644 --- a/internal/router/guard.go +++ b/internal/router/guard.go @@ -146,7 +146,7 @@ func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guar // outputFilter attaches the guard's filter for one attempt to the proxy // request, and returns it so its verdict can be read afterwards. func (r *Router) outputFilter(ctx context.Context, req *parsedRequest, model string, preq *proxy.Request) guard.OutputFilter { - if r.guard == nil { + if r.guard == nil || req.internal { return nil } f := r.guard.Output(ctx, r.subject(req, model)) diff --git a/internal/router/guard_test.go b/internal/router/guard_test.go index 12f5a6c..a19630f 100644 --- a/internal/router/guard_test.go +++ b/internal/router/guard_test.go @@ -6,11 +6,13 @@ import ( "encoding/json" "errors" "net/http" + "net/http/httptest" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) @@ -195,3 +197,33 @@ func TestTheGateJudgesRealTraffic(t *testing.T) { t.Errorf("%v %s", rec.Header(), rec.Body.String()) } } + +func TestAJudgeModelIsNobodysToAskButTheGuardrails(t *testing.T) { + e := newFakeEngine(t, "llama-guard") + cfg := baseConfig(config.Endpoint{ID: "j", URL: e.srv.URL, Models: []string{"llama-guard"}}) + judge := catModel("llama-guard", []string{"lg"}) + judge.Role, judge.Guard = catalog.RoleGuard, &catalog.GuardModel{Format: "llamaguard3"} + cfg.Models = []catalog.Model{judge} + rt := newTestRouter(t, cfg) + waitForEndpoint(t, rt) + + for _, name := range []string{"llama-guard", "lg"} { + if rec := post(t, rt.Handler(), map[string]any{"model": name, "messages": []any{}}, nil); rec.Code != http.StatusNotFound { + t.Errorf("a client asked %s: %d %s", name, rec.Code, rec.Body.String()) + } + } + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, req) + if strings.Contains(rec.Body.String(), "llama-guard") { + t.Errorf("listed: %s", rec.Body.String()) + } + calls := e.calls.Load() + out, err := rt.Judge(t.Context(), "llama-guard", []byte(`{"model":"llama-guard","messages":[{"role":"user","content":"x"}]}`)) + if err != nil || !strings.Contains(string(out), `"hi"`) || e.calls.Load() != calls+1 { + t.Errorf("the guardrails asking: %s %v", out, err) + } + if f, _, ok := rt.GuardFormat("lg"); !ok || f != "llamaguard3" { + t.Errorf("format %q %v", f, ok) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index d3239c9..c8f5b95 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -301,6 +301,9 @@ type parsedRequest struct { guardIn guard.Verdict // noStore is a policy that keeps nothing of this request. noStore bool + // internal is a request the gateway makes of itself — a judge's — which + // no guard filters. + internal bool } // inferenceBody covers the fields the router reads across all inference @@ -359,6 +362,13 @@ func (r *Router) handleInference(path string) http.HandlerFunc { } } + if r.isGuardModel(parsed.Model) { + // Said as a model that is not there: it is not, for clients. + writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found", + fmt.Sprintf("the model %q does not exist", parsed.Model)) + r.m.observeRequest("", "", http.StatusNotFound, time.Since(start), nil, 0) + return + } if p, ok := auth.FromContext(req.Context()); ok { if !r.permits(p, parsed.Model) { auth.WriteForbiddenModel(w, parsed.Model) @@ -471,6 +481,10 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // around the allowlist; matching only the canonical name would silently ignore // an allowlist written in the names clients actually use. func (r *Router) permits(p auth.Principal, requested string) bool { + // A judge model is the guardrails', not anybody's to ask. + if r.isGuardModel(requested) { + return false + } // An adapter named in the allowlist allows that adapter: not its model, // and not the model's other adapters, which the checks below would let // through if the adapter's own name were not asked about first. @@ -788,7 +802,7 @@ func noEndpointFor(model string, served []string) string { // handing it a simulated model would send someone's first request to an engine // that replies in Greek letters. func (r *Router) exampleModels() []string { - names := r.servedModels() + names := slices.DeleteFunc(r.servedModels(), r.isGuardModel) slices.SortStableFunc(names, func(a, b string) int { as, bs := r.modelMap()[a].Simulated(), r.modelMap()[b].Simulated() switch { @@ -803,6 +817,12 @@ func (r *Router) exampleModels() []string { return names } +// isGuardModel reports whether a name is a judge model, by any of its names. +func (r *Router) isGuardModel(name string) bool { + _, entry := r.resolve(name) + return entry.IsGuard() +} + // servedModels lists the canonical models at least one endpoint serves. func (r *Router) servedModels() []string { seen := map[string]struct{}{} @@ -840,7 +860,7 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { principal, authed := auth.FromContext(req.Context()) names := make([]string, 0, len(seen)) for n := range seen { - if authed && !r.permits(principal, n) { + if authed && !r.permits(principal, n) || r.isGuardModel(n) { continue } names = append(names, n) diff --git a/internal/router/judge.go b/internal/router/judge.go new file mode 100644 index 0000000..39e709b --- /dev/null +++ b/internal/router/judge.go @@ -0,0 +1,77 @@ +package router + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + + "gitea.mynodes.xyz/ilker/llmgw/internal/guard" +) + +// Judge sends a chat completion to a judge model and returns its answer: for +// the guardrails, which ask a model on the cluster about the traffic they +// judge. It goes past admission — a judge call queued behind the tenants it +// is judging would wait on them — and past accounting and the guardrails +// themselves, but through the scheduler and the replicas like any request. +func (r *Router) Judge(ctx context.Context, model string, body []byte) ([]byte, error) { + hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req, err := r.parse(hreq, "/v1/chat/completions") + if err != nil { + return nil, err + } + req.internal = true + req.Stream = false + req.RequestID = requestID("") + req.Header = http.Header{"X-Request-Id": []string{req.RequestID}} + w := &bufferWriter{header: http.Header{}} + res := r.dispatch(ctx, w, req) + if res.err != nil { + return nil, res.err + } + if w.status != 0 && w.status != http.StatusOK { + return nil, fmt.Errorf("the judge %s answered %d", model, w.status) + } + return w.buf.Bytes(), nil +} + +// GuardFormat is how a judge model's answers are read, from its catalog +// entry: for guard.Gate.SetJudge. +func (r *Router) GuardFormat(model string) (string, []string, bool) { + _, entry := r.resolve(model) + if !entry.IsGuard() || entry.Guard == nil { + return "", nil, false + } + return entry.Guard.Format, entry.Guard.Categories, true +} + +var _ guard.Caller = (*Router)(nil).Judge + +// bufferWriter is a ResponseWriter that keeps what is written, for a request +// the gateway makes of itself. +type bufferWriter struct { + header http.Header + status int + buf bytes.Buffer +} + +func (b *bufferWriter) Header() http.Header { return b.header } +func (b *bufferWriter) WriteHeader(code int) { + if b.status == 0 { + b.status = code + } +} +func (b *bufferWriter) Write(p []byte) (int, error) { + if b.status == 0 { + b.status = http.StatusOK + } + if b.buf.Len()+len(p) > 1<<20 { + return 0, errors.New("a judge's answer over 1 MiB") + } + return b.buf.Write(p) +} +func (b *bufferWriter) Flush() {} diff --git a/internal/router/router.go b/internal/router/router.go index 5870b80..2139992 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -742,7 +742,7 @@ func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parse for _, fb := range entry.Fallbacks { // A fallback is another model, and a key allowed only the one it // asked for is not served another because that one was full. - if authed && !r.permits(p, fb) { + if authed && !r.permits(p, fb) || r.isGuardModel(fb) { continue } // Nor sent outside the cluster under a policy that keeps nothing.