Files
llmgw/internal/router/restore.go
T
Ilker ManapandClaude Opus 5.5 81685ecd75 A request's redacted values are put back in its answer
With restore: true on a guardrail policy's input, what redact rewrote as
placeholders comes back in the answer: the model reads "[EMAIL_1]", the
client reads the address it sent. A streamed placeholder split across
events is held until it is whole, and what is held is sent at the end.
The swap runs before the output filter, so the output rules judge the
answer as the client reads it and a placeholder they make of a value the
model invented is never taken for the request's own. The map lives in the
Verdict for the request only; nothing stores, logs or traces it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 16:41:53 +01:00

193 lines
5.1 KiB
Go

package router
import (
"bytes"
"encoding/json"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// Putting a request's own redacted values back into its answer: the model
// read "[EMAIL_1]", the client reads the address it sent. A guardrail policy
// asks for it with restore: true on its input rules.
//
// Before the output guardrails, not after: they then judge the answer as the
// client will read it, so a policy that redacts personal data in answers
// still does — and a placeholder the output guardrails make of a value the
// model invented is never mistaken for one of the request's own.
// restorer swaps placeholders back, in a whole answer or a streamed one.
type restorer struct {
values map[string]string
longest int
// held is, per choice, the tail of the text so far that may be the start
// of a placeholder the next event completes: "[EMA" before "IL_1]".
held map[int]string
// last is the most recent event, the shape a final release is sent in.
last map[string]any
}
func newRestorer(values map[string]string) *restorer {
r := &restorer{values: values, held: map[int]string{}}
for ph := range values {
r.longest = max(r.longest, len(ph))
}
return r
}
// text swaps every whole placeholder in s.
func (r *restorer) text(s string) string {
if !strings.Contains(s, "[") {
return s
}
for ph, v := range r.values {
s = strings.ReplaceAll(s, ph, v)
}
return s
}
// split is s with its placeholders swapped, less a tail that could still
// become one: what to send now, and what to hold for the next event.
func (r *restorer) split(s string) (send, hold string) {
s = r.text(s)
i := strings.LastIndexByte(s, '[')
if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest {
return s, ""
}
tail := s[i:]
for ph := range r.values {
if strings.HasPrefix(ph, tail) {
return s[:i], tail
}
}
return s, ""
}
// body swaps the placeholders in a whole JSON answer. A placeholder is only
// ever inside a string there, so each value goes in escaped for one.
func (r *restorer) body(b []byte) []byte {
if !bytes.Contains(b, []byte("[")) {
return b
}
for ph, v := range r.values {
q, _ := json.Marshal(v)
b = bytes.ReplaceAll(b, []byte(ph), q[1:len(q)-1])
}
return b
}
// OnEvent swaps the placeholders in one streamed chat-completion event,
// holding back the start of one an event has split.
func (r *restorer) OnEvent(data string) ([]string, bool) {
if !strings.Contains(data, "[") && len(r.held) == 0 {
return []string{data}, false
}
var ev map[string]any
dec := json.NewDecoder(strings.NewReader(data))
dec.UseNumber()
if err := dec.Decode(&ev); err != nil {
return []string{data}, false
}
choices, _ := ev["choices"].([]any)
for i, c := range choices {
ch, _ := c.(map[string]any)
delta, _ := ch["delta"].(map[string]any)
content, ok := delta["content"].(string)
if !ok {
continue
}
idx := i
if n, ok := ch["index"].(json.Number); ok {
if v, err := n.Int64(); err == nil {
idx = int(v)
}
}
send, hold := r.split(r.held[idx] + content)
// The last event of a choice releases everything.
if ch["finish_reason"] != nil {
send, hold = send+hold, ""
}
delta["content"] = send
if hold == "" {
delete(r.held, idx)
} else {
r.held[idx] = hold
}
}
r.last = ev
out, err := json.Marshal(ev)
if err != nil {
return []string{data}, false
}
return []string{string(out)}, false
}
// OnEnd releases what is still held, as one more event per choice.
func (r *restorer) OnEnd() []string {
if len(r.held) == 0 || r.last == nil {
return nil
}
var out []string
for idx, text := range r.held {
ev := map[string]any{}
for k, v := range r.last {
if k != "choices" && k != "usage" {
ev[k] = v
}
}
ev["choices"] = []any{map[string]any{"index": idx, "delta": map[string]any{"content": text}, "finish_reason": nil}}
if b, err := json.Marshal(ev); err == nil {
out = append(out, string(b))
}
}
r.held = map[int]string{}
return out
}
// streamChain runs one stream filter's output through another.
type streamChain struct{ first, then proxy.StreamFilter }
func (c streamChain) OnEvent(data string) ([]string, bool) {
first, stop := c.first.OnEvent(data)
var out []string
for _, d := range first {
e, s := c.then.OnEvent(d)
out = append(out, e...)
stop = stop || s
}
return out, stop
}
func (c streamChain) OnEnd() []string {
var out []string
for _, d := range c.first.OnEnd() {
e, _ := c.then.OnEvent(d)
out = append(out, e...)
}
return append(out, c.then.OnEnd()...)
}
// restoreInto puts the input's placeholders back in this attempt's answer,
// ahead of whatever output filter the attempt already has.
func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
if len(values) == 0 {
return
}
rs := newRestorer(values)
if f := preq.Filter; f != nil {
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
} else {
preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) }
}
if !stream {
return
}
if sf := preq.StreamFilter; sf != nil {
preq.StreamFilter = streamChain{first: rs, then: sf}
} else {
preq.StreamFilter = rs
}
}