package router import ( "bytes" "encoding/json" "net/http" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" ) // Putting a request's own redacted values back into its answer: the model // read "[EMAIL_1]", the client reads the address it sent. A guardrail policy // asks for it with restore: true on its input rules. // // Before the output guardrails, not after: they then judge the answer as the // client will read it, so a policy that redacts personal data in answers // still does — and a placeholder the output guardrails make of a value the // model invented is never mistaken for one of the request's own. // restorer swaps placeholders back, in a whole answer or a streamed one. type restorer struct { values map[string]string longest int // held is, per choice, the tail of the text so far that may be the start // of a placeholder the next event completes: "[EMA" before "IL_1]". held map[int]string // last is the most recent event, the shape a final release is sent in. last map[string]any } func newRestorer(values map[string]string) *restorer { r := &restorer{values: values, held: map[int]string{}} for ph := range values { r.longest = max(r.longest, len(ph)) } return r } // text swaps every whole placeholder in s. func (r *restorer) text(s string) string { if !strings.Contains(s, "[") { return s } for ph, v := range r.values { s = strings.ReplaceAll(s, ph, v) } return s } // split is s with its placeholders swapped, less a tail that could still // become one: what to send now, and what to hold for the next event. func (r *restorer) split(s string) (send, hold string) { s = r.text(s) i := strings.LastIndexByte(s, '[') if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest { return s, "" } tail := s[i:] for ph := range r.values { if strings.HasPrefix(ph, tail) { return s[:i], tail } } return s, "" } // body swaps the placeholders in a whole JSON answer. A placeholder is only // ever inside a string there, so each value goes in escaped for one. func (r *restorer) body(b []byte) []byte { if !bytes.Contains(b, []byte("[")) { return b } for ph, v := range r.values { q, _ := json.Marshal(v) b = bytes.ReplaceAll(b, []byte(ph), q[1:len(q)-1]) } return b } // OnEvent swaps the placeholders in one streamed chat-completion event, // holding back the start of one an event has split. func (r *restorer) OnEvent(data string) ([]string, bool) { if !strings.Contains(data, "[") && len(r.held) == 0 { return []string{data}, false } var ev map[string]any dec := json.NewDecoder(strings.NewReader(data)) dec.UseNumber() if err := dec.Decode(&ev); err != nil { return []string{data}, false } choices, _ := ev["choices"].([]any) for i, c := range choices { ch, _ := c.(map[string]any) delta, _ := ch["delta"].(map[string]any) content, ok := delta["content"].(string) if !ok { continue } idx := i if n, ok := ch["index"].(json.Number); ok { if v, err := n.Int64(); err == nil { idx = int(v) } } send, hold := r.split(r.held[idx] + content) // The last event of a choice releases everything. if ch["finish_reason"] != nil { send, hold = send+hold, "" } delta["content"] = send if hold == "" { delete(r.held, idx) } else { r.held[idx] = hold } } r.last = ev out, err := json.Marshal(ev) if err != nil { return []string{data}, false } return []string{string(out)}, false } // OnEnd releases what is still held, as one more event per choice. func (r *restorer) OnEnd() []string { if len(r.held) == 0 || r.last == nil { return nil } var out []string for idx, text := range r.held { ev := map[string]any{} for k, v := range r.last { if k != "choices" && k != "usage" { ev[k] = v } } ev["choices"] = []any{map[string]any{"index": idx, "delta": map[string]any{"content": text}, "finish_reason": nil}} if b, err := json.Marshal(ev); err == nil { out = append(out, string(b)) } } r.held = map[int]string{} return out } // streamChain runs one stream filter's output through another. type streamChain struct{ first, then proxy.StreamFilter } func (c streamChain) OnEvent(data string) ([]string, bool) { first, stop := c.first.OnEvent(data) var out []string for _, d := range first { e, s := c.then.OnEvent(d) out = append(out, e...) stop = stop || s } return out, stop } func (c streamChain) OnEnd() []string { var out []string for _, d := range c.first.OnEnd() { e, _ := c.then.OnEvent(d) out = append(out, e...) } return append(out, c.then.OnEnd()...) } // restoreInto puts the input's placeholders back in this attempt's answer, // ahead of whatever output filter the attempt already has. func restoreInto(values map[string]string, stream bool, preq *proxy.Request) { if len(values) == 0 { return } rs := newRestorer(values) if f := preq.Filter; f != nil { preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) } } else { preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) } } if !stream { return } if sf := preq.StreamFilter; sf != nil { preq.StreamFilter = streamChain{first: rs, then: sf} } else { preq.StreamFilter = rs } }