With restore: true on a guardrail policy's input, what redact rewrote as placeholders comes back in the answer: the model reads "[EMAIL_1]", the client reads the address it sent. A streamed placeholder split across events is held until it is whole, and what is held is sent at the end. The swap runs before the output filter, so the output rules judge the answer as the client reads it and a placeholder they make of a value the model invented is never taken for the request's own. The map lives in the Verdict for the request only; nothing stores, logs or traces it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
193 lines
5.1 KiB
Go
193 lines
5.1 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// Putting a request's own redacted values back into its answer: the model
|
|
// read "[EMAIL_1]", the client reads the address it sent. A guardrail policy
|
|
// asks for it with restore: true on its input rules.
|
|
//
|
|
// Before the output guardrails, not after: they then judge the answer as the
|
|
// client will read it, so a policy that redacts personal data in answers
|
|
// still does — and a placeholder the output guardrails make of a value the
|
|
// model invented is never mistaken for one of the request's own.
|
|
|
|
// restorer swaps placeholders back, in a whole answer or a streamed one.
|
|
type restorer struct {
|
|
values map[string]string
|
|
longest int
|
|
// held is, per choice, the tail of the text so far that may be the start
|
|
// of a placeholder the next event completes: "[EMA" before "IL_1]".
|
|
held map[int]string
|
|
// last is the most recent event, the shape a final release is sent in.
|
|
last map[string]any
|
|
}
|
|
|
|
func newRestorer(values map[string]string) *restorer {
|
|
r := &restorer{values: values, held: map[int]string{}}
|
|
for ph := range values {
|
|
r.longest = max(r.longest, len(ph))
|
|
}
|
|
return r
|
|
}
|
|
|
|
// text swaps every whole placeholder in s.
|
|
func (r *restorer) text(s string) string {
|
|
if !strings.Contains(s, "[") {
|
|
return s
|
|
}
|
|
for ph, v := range r.values {
|
|
s = strings.ReplaceAll(s, ph, v)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// split is s with its placeholders swapped, less a tail that could still
|
|
// become one: what to send now, and what to hold for the next event.
|
|
func (r *restorer) split(s string) (send, hold string) {
|
|
s = r.text(s)
|
|
i := strings.LastIndexByte(s, '[')
|
|
if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest {
|
|
return s, ""
|
|
}
|
|
tail := s[i:]
|
|
for ph := range r.values {
|
|
if strings.HasPrefix(ph, tail) {
|
|
return s[:i], tail
|
|
}
|
|
}
|
|
return s, ""
|
|
}
|
|
|
|
// body swaps the placeholders in a whole JSON answer. A placeholder is only
|
|
// ever inside a string there, so each value goes in escaped for one.
|
|
func (r *restorer) body(b []byte) []byte {
|
|
if !bytes.Contains(b, []byte("[")) {
|
|
return b
|
|
}
|
|
for ph, v := range r.values {
|
|
q, _ := json.Marshal(v)
|
|
b = bytes.ReplaceAll(b, []byte(ph), q[1:len(q)-1])
|
|
}
|
|
return b
|
|
}
|
|
|
|
// OnEvent swaps the placeholders in one streamed chat-completion event,
|
|
// holding back the start of one an event has split.
|
|
func (r *restorer) OnEvent(data string) ([]string, bool) {
|
|
if !strings.Contains(data, "[") && len(r.held) == 0 {
|
|
return []string{data}, false
|
|
}
|
|
var ev map[string]any
|
|
dec := json.NewDecoder(strings.NewReader(data))
|
|
dec.UseNumber()
|
|
if err := dec.Decode(&ev); err != nil {
|
|
return []string{data}, false
|
|
}
|
|
choices, _ := ev["choices"].([]any)
|
|
for i, c := range choices {
|
|
ch, _ := c.(map[string]any)
|
|
delta, _ := ch["delta"].(map[string]any)
|
|
content, ok := delta["content"].(string)
|
|
if !ok {
|
|
continue
|
|
}
|
|
idx := i
|
|
if n, ok := ch["index"].(json.Number); ok {
|
|
if v, err := n.Int64(); err == nil {
|
|
idx = int(v)
|
|
}
|
|
}
|
|
send, hold := r.split(r.held[idx] + content)
|
|
// The last event of a choice releases everything.
|
|
if ch["finish_reason"] != nil {
|
|
send, hold = send+hold, ""
|
|
}
|
|
delta["content"] = send
|
|
if hold == "" {
|
|
delete(r.held, idx)
|
|
} else {
|
|
r.held[idx] = hold
|
|
}
|
|
}
|
|
r.last = ev
|
|
out, err := json.Marshal(ev)
|
|
if err != nil {
|
|
return []string{data}, false
|
|
}
|
|
return []string{string(out)}, false
|
|
}
|
|
|
|
// OnEnd releases what is still held, as one more event per choice.
|
|
func (r *restorer) OnEnd() []string {
|
|
if len(r.held) == 0 || r.last == nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for idx, text := range r.held {
|
|
ev := map[string]any{}
|
|
for k, v := range r.last {
|
|
if k != "choices" && k != "usage" {
|
|
ev[k] = v
|
|
}
|
|
}
|
|
ev["choices"] = []any{map[string]any{"index": idx, "delta": map[string]any{"content": text}, "finish_reason": nil}}
|
|
if b, err := json.Marshal(ev); err == nil {
|
|
out = append(out, string(b))
|
|
}
|
|
}
|
|
r.held = map[int]string{}
|
|
return out
|
|
}
|
|
|
|
// streamChain runs one stream filter's output through another.
|
|
type streamChain struct{ first, then proxy.StreamFilter }
|
|
|
|
func (c streamChain) OnEvent(data string) ([]string, bool) {
|
|
first, stop := c.first.OnEvent(data)
|
|
var out []string
|
|
for _, d := range first {
|
|
e, s := c.then.OnEvent(d)
|
|
out = append(out, e...)
|
|
stop = stop || s
|
|
}
|
|
return out, stop
|
|
}
|
|
|
|
func (c streamChain) OnEnd() []string {
|
|
var out []string
|
|
for _, d := range c.first.OnEnd() {
|
|
e, _ := c.then.OnEvent(d)
|
|
out = append(out, e...)
|
|
}
|
|
return append(out, c.then.OnEnd()...)
|
|
}
|
|
|
|
// restoreInto puts the input's placeholders back in this attempt's answer,
|
|
// ahead of whatever output filter the attempt already has.
|
|
func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
|
|
if len(values) == 0 {
|
|
return
|
|
}
|
|
rs := newRestorer(values)
|
|
if f := preq.Filter; f != nil {
|
|
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
|
|
} else {
|
|
preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) }
|
|
}
|
|
if !stream {
|
|
return
|
|
}
|
|
if sf := preq.StreamFilter; sf != nil {
|
|
preq.StreamFilter = streamChain{first: rs, then: sf}
|
|
} else {
|
|
preq.StreamFilter = rs
|
|
}
|
|
}
|