package router import ( "bytes" "encoding/json" "net/http" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" ) // Putting a request's own redacted values back into its answer: the model // read "[EMAIL_1]", the client reads the address it sent. A guardrail policy // asks for it with restore: true on its input rules. // // Before the output guardrails, not after: they then judge the answer as the // client will read it, so a policy that redacts personal data in answers // still does — and a placeholder the output guardrails make of a value the // model invented is never mistaken for one of the request's own. // restorer swaps placeholders back, in a whole answer or a streamed one. type restorer struct { values map[string]string longest int // jsonContent is an answer asked for as JSON (response_format): its // content is JSON, and a value goes into it escaped, as into a tool // call's arguments. jsonContent bool // held is, per choice and field, the tail of the text so far that may // be the start of a placeholder the next event completes: "[EMA" before // "IL_1]". Keyed by heldKey. held map[heldKey]string // last is the most recent event, the shape a final release is sent in. last map[string]any } // heldKey is where held text belongs: a choice, and in it the content, the // reasoning, a completion's text, or a tool call's arguments (tool >= 0). type heldKey struct { choice int field string tool int } func newRestorer(values map[string]string) *restorer { r := &restorer{values: values, held: map[heldKey]string{}} for ph := range values { r.longest = max(r.longest, len(ph)) } return r } // text swaps every whole placeholder in s. In a tool call's arguments, // which are JSON written into a string, a value goes in escaped for JSON: // O"Brien would otherwise end the argument early. func (r *restorer) text(s string, inJSON bool) string { if !strings.Contains(s, "[") { return s } for ph, v := range r.values { if inJSON { q, _ := json.Marshal(v) v = string(q[1 : len(q)-1]) } s = strings.ReplaceAll(s, ph, v) } return s } // split is s with its placeholders swapped, less a tail that could still // become one: what to send now, and what to hold for the next event. func (r *restorer) split(s string, inJSON bool) (send, hold string) { s = r.text(s, inJSON) i := strings.LastIndexByte(s, '[') if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest { return s, "" } tail := s[i:] for ph := range r.values { if strings.HasPrefix(ph, tail) { return s[:i], tail } } return s, "" } // body swaps the placeholders in a whole JSON answer: in every string, and // in a tool call's arguments escaped for the JSON they are. func (r *restorer) body(b []byte) []byte { if !bytes.Contains(b, []byte("[")) { return b } d := json.NewDecoder(bytes.NewReader(b)) d.UseNumber() var v any if err := d.Decode(&v); err != nil { return b } out, err := json.Marshal(r.walk(v, "")) if err != nil { return b } return out } func (r *restorer) walk(v any, key string) any { switch x := v.(type) { case map[string]any: for k, e := range x { x[k] = r.walk(e, k) } case []any: for i, e := range x { x[i] = r.walk(e, key) } case string: return r.text(x, key == "arguments" || (r.jsonContent && key == "content")) } return v } // OnEvent swaps the placeholders in one streamed event — a chat delta's // content, reasoning and tool calls' arguments, a completion's text — // holding back the start of one an event has split. func (r *restorer) OnEvent(data string) ([]string, bool) { if !strings.Contains(data, "[") && len(r.held) == 0 { return []string{data}, false } var ev map[string]any dec := json.NewDecoder(strings.NewReader(data)) dec.UseNumber() if err := dec.Decode(&ev); err != nil { return []string{data}, false } choices, _ := ev["choices"].([]any) for i, c := range choices { ch, _ := c.(map[string]any) if ch == nil { continue } idx := i if n, ok := ch["index"].(json.Number); ok { if v, err := n.Int64(); err == nil { idx = int(v) } } finished := ch["finish_reason"] != nil delta, _ := ch["delta"].(map[string]any) if delta == nil && finished && r.heldFor(idx) { delta = map[string]any{} ch["delta"] = delta } swap := func(container map[string]any, field string, k heldKey, inJSON bool) { s, ok := container[field].(string) if !ok && !(finished && r.held[k] != "") { return } send, hold := r.split(r.held[k]+s, inJSON) // The last event of a choice releases everything. if finished { send, hold = send+hold, "" } container[field] = send if hold == "" { delete(r.held, k) } else { r.held[k] = hold } } if delta != nil { swap(delta, "content", heldKey{idx, "content", -1}, r.jsonContent) swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false) calls, _ := delta["tool_calls"].([]any) for j, tc := range calls { call, _ := tc.(map[string]any) fn, _ := call["function"].(map[string]any) if fn == nil { continue } t := j if n, ok := call["index"].(json.Number); ok { if v, err := n.Int64(); err == nil { t = int(v) } } swap(fn, "arguments", heldKey{idx, "arguments", t}, true) } } if _, ok := ch["text"]; ok { swap(ch, "text", heldKey{idx, "text", -1}, false) } } r.last = ev out, err := json.Marshal(ev) if err != nil { return []string{data}, false } return []string{string(out)}, false } func (r *restorer) heldFor(choice int) bool { for k := range r.held { if k.choice == choice { return true } } return false } // OnEnd releases what is still held, as one more event for each. func (r *restorer) OnEnd() []string { if len(r.held) == 0 || r.last == nil { return nil } var out []string for k, text := range r.held { ev := map[string]any{} for key, v := range r.last { if key != "choices" && key != "usage" { ev[key] = v } } ch := map[string]any{"index": k.choice, "finish_reason": nil} switch k.field { case "text": ch["text"] = text case "arguments": ch["delta"] = map[string]any{"tool_calls": []any{map[string]any{"index": k.tool, "function": map[string]any{"arguments": text}}}} default: ch["delta"] = map[string]any{k.field: text} } ev["choices"] = []any{ch} if b, err := json.Marshal(ev); err == nil { out = append(out, string(b)) } } r.held = map[heldKey]string{} return out } // streamChain runs one stream filter's output through another. type streamChain struct{ first, then proxy.StreamFilter } func (c streamChain) OnEvent(data string) ([]string, bool) { first, stop := c.first.OnEvent(data) var out []string for _, d := range first { e, s := c.then.OnEvent(d) out = append(out, e...) stop = stop || s } return out, stop } func (c streamChain) OnEnd() []string { var out []string for _, d := range c.first.OnEnd() { e, _ := c.then.OnEvent(d) out = append(out, e...) } return append(out, c.then.OnEnd()...) } // restoreInto puts the input's placeholders back in this attempt's answer, // ahead of whatever output filter the attempt already has. func restoreInto(values map[string]string, stream, jsonContent bool, preq *proxy.Request) { if len(values) == 0 { return } rs := newRestorer(values) rs.jsonContent = jsonContent if f := preq.Filter; f != nil { preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) } } else { preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) } } if !stream { return } if sf := preq.StreamFilter; sf != nil { preq.StreamFilter = streamChain{first: rs, then: sf} } else { preq.StreamFilter = rs } } // jsonAnswer reports whether a request asks for its answer as JSON. func jsonAnswer(body []byte) bool { var b struct { ResponseFormat *struct { Type string `json:"type"` } `json:"response_format"` } return json.Unmarshal(body, &b) == nil && b.ResponseFormat != nil && (b.ResponseFormat.Type == "json_object" || b.ResponseFormat.Type == "json_schema") }