Files
llmgw/internal/router/restore.go
T
Ilker ManapandClaude Opus 5.5 c09d5ad4ee Outside services read answers redacted and whole; a card's own lanes
From the second review.
- An output service reads the answer with the values restore put back
  taken out again (guard.Subject.Restored), as an input service reads the
  request redacted.
- With n above 1, the service reads the first choice once it is whole,
  not at whichever choice finished first.
- Restored values go into an answer asked for as JSON (response_format)
  escaped for it, as into tool-call arguments.
- The embedding failure is logged once a minute through an outage, not
  once.
- A card's own lane count and fastest generation come from the kernel
  (max_link_width, max_link_speed): the driver's width.max is what card
  and slot allow together, so an x16 card in an x8 slot read x8 of x8 and
  was never said.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 13:22:39 +01:00

298 lines
8.0 KiB
Go

package router
import (
"bytes"
"encoding/json"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// Putting a request's own redacted values back into its answer: the model
// read "[EMAIL_1]", the client reads the address it sent. A guardrail policy
// asks for it with restore: true on its input rules.
//
// Before the output guardrails, not after: they then judge the answer as the
// client will read it, so a policy that redacts personal data in answers
// still does — and a placeholder the output guardrails make of a value the
// model invented is never mistaken for one of the request's own.
// restorer swaps placeholders back, in a whole answer or a streamed one.
type restorer struct {
values map[string]string
longest int
// jsonContent is an answer asked for as JSON (response_format): its
// content is JSON, and a value goes into it escaped, as into a tool
// call's arguments.
jsonContent bool
// held is, per choice and field, the tail of the text so far that may
// be the start of a placeholder the next event completes: "[EMA" before
// "IL_1]". Keyed by heldKey.
held map[heldKey]string
// last is the most recent event, the shape a final release is sent in.
last map[string]any
}
// heldKey is where held text belongs: a choice, and in it the content, the
// reasoning, a completion's text, or a tool call's arguments (tool >= 0).
type heldKey struct {
choice int
field string
tool int
}
func newRestorer(values map[string]string) *restorer {
r := &restorer{values: values, held: map[heldKey]string{}}
for ph := range values {
r.longest = max(r.longest, len(ph))
}
return r
}
// text swaps every whole placeholder in s. In a tool call's arguments,
// which are JSON written into a string, a value goes in escaped for JSON:
// O"Brien would otherwise end the argument early.
func (r *restorer) text(s string, inJSON bool) string {
if !strings.Contains(s, "[") {
return s
}
for ph, v := range r.values {
if inJSON {
q, _ := json.Marshal(v)
v = string(q[1 : len(q)-1])
}
s = strings.ReplaceAll(s, ph, v)
}
return s
}
// split is s with its placeholders swapped, less a tail that could still
// become one: what to send now, and what to hold for the next event.
func (r *restorer) split(s string, inJSON bool) (send, hold string) {
s = r.text(s, inJSON)
i := strings.LastIndexByte(s, '[')
if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest {
return s, ""
}
tail := s[i:]
for ph := range r.values {
if strings.HasPrefix(ph, tail) {
return s[:i], tail
}
}
return s, ""
}
// body swaps the placeholders in a whole JSON answer: in every string, and
// in a tool call's arguments escaped for the JSON they are.
func (r *restorer) body(b []byte) []byte {
if !bytes.Contains(b, []byte("[")) {
return b
}
d := json.NewDecoder(bytes.NewReader(b))
d.UseNumber()
var v any
if err := d.Decode(&v); err != nil {
return b
}
out, err := json.Marshal(r.walk(v, ""))
if err != nil {
return b
}
return out
}
func (r *restorer) walk(v any, key string) any {
switch x := v.(type) {
case map[string]any:
for k, e := range x {
x[k] = r.walk(e, k)
}
case []any:
for i, e := range x {
x[i] = r.walk(e, key)
}
case string:
return r.text(x, key == "arguments" || (r.jsonContent && key == "content"))
}
return v
}
// OnEvent swaps the placeholders in one streamed event — a chat delta's
// content, reasoning and tool calls' arguments, a completion's text —
// holding back the start of one an event has split.
func (r *restorer) OnEvent(data string) ([]string, bool) {
if !strings.Contains(data, "[") && len(r.held) == 0 {
return []string{data}, false
}
var ev map[string]any
dec := json.NewDecoder(strings.NewReader(data))
dec.UseNumber()
if err := dec.Decode(&ev); err != nil {
return []string{data}, false
}
choices, _ := ev["choices"].([]any)
for i, c := range choices {
ch, _ := c.(map[string]any)
if ch == nil {
continue
}
idx := i
if n, ok := ch["index"].(json.Number); ok {
if v, err := n.Int64(); err == nil {
idx = int(v)
}
}
finished := ch["finish_reason"] != nil
delta, _ := ch["delta"].(map[string]any)
if delta == nil && finished && r.heldFor(idx) {
delta = map[string]any{}
ch["delta"] = delta
}
swap := func(container map[string]any, field string, k heldKey, inJSON bool) {
s, ok := container[field].(string)
if !ok && !(finished && r.held[k] != "") {
return
}
send, hold := r.split(r.held[k]+s, inJSON)
// The last event of a choice releases everything.
if finished {
send, hold = send+hold, ""
}
container[field] = send
if hold == "" {
delete(r.held, k)
} else {
r.held[k] = hold
}
}
if delta != nil {
swap(delta, "content", heldKey{idx, "content", -1}, r.jsonContent)
swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false)
calls, _ := delta["tool_calls"].([]any)
for j, tc := range calls {
call, _ := tc.(map[string]any)
fn, _ := call["function"].(map[string]any)
if fn == nil {
continue
}
t := j
if n, ok := call["index"].(json.Number); ok {
if v, err := n.Int64(); err == nil {
t = int(v)
}
}
swap(fn, "arguments", heldKey{idx, "arguments", t}, true)
}
}
if _, ok := ch["text"]; ok {
swap(ch, "text", heldKey{idx, "text", -1}, false)
}
}
r.last = ev
out, err := json.Marshal(ev)
if err != nil {
return []string{data}, false
}
return []string{string(out)}, false
}
func (r *restorer) heldFor(choice int) bool {
for k := range r.held {
if k.choice == choice {
return true
}
}
return false
}
// OnEnd releases what is still held, as one more event for each.
func (r *restorer) OnEnd() []string {
if len(r.held) == 0 || r.last == nil {
return nil
}
var out []string
for k, text := range r.held {
ev := map[string]any{}
for key, v := range r.last {
if key != "choices" && key != "usage" {
ev[key] = v
}
}
ch := map[string]any{"index": k.choice, "finish_reason": nil}
switch k.field {
case "text":
ch["text"] = text
case "arguments":
ch["delta"] = map[string]any{"tool_calls": []any{map[string]any{"index": k.tool,
"function": map[string]any{"arguments": text}}}}
default:
ch["delta"] = map[string]any{k.field: text}
}
ev["choices"] = []any{ch}
if b, err := json.Marshal(ev); err == nil {
out = append(out, string(b))
}
}
r.held = map[heldKey]string{}
return out
}
// streamChain runs one stream filter's output through another.
type streamChain struct{ first, then proxy.StreamFilter }
func (c streamChain) OnEvent(data string) ([]string, bool) {
first, stop := c.first.OnEvent(data)
var out []string
for _, d := range first {
e, s := c.then.OnEvent(d)
out = append(out, e...)
stop = stop || s
}
return out, stop
}
func (c streamChain) OnEnd() []string {
var out []string
for _, d := range c.first.OnEnd() {
e, _ := c.then.OnEvent(d)
out = append(out, e...)
}
return append(out, c.then.OnEnd()...)
}
// restoreInto puts the input's placeholders back in this attempt's answer,
// ahead of whatever output filter the attempt already has.
func restoreInto(values map[string]string, stream, jsonContent bool, preq *proxy.Request) {
if len(values) == 0 {
return
}
rs := newRestorer(values)
rs.jsonContent = jsonContent
if f := preq.Filter; f != nil {
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
} else {
preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) }
}
if !stream {
return
}
if sf := preq.StreamFilter; sf != nil {
preq.StreamFilter = streamChain{first: rs, then: sf}
} else {
preq.StreamFilter = rs
}
}
// jsonAnswer reports whether a request asks for its answer as JSON.
func jsonAnswer(body []byte) bool {
var b struct {
ResponseFormat *struct {
Type string `json:"type"`
} `json:"response_format"`
}
return json.Unmarshal(body, &b) == nil && b.ResponseFormat != nil &&
(b.ResponseFormat.Type == "json_object" || b.ResponseFormat.Type == "json_schema")
}