Files
llmgw/internal/router/restore.go
T
Ilker ManapandClaude Opus 5.5 54c6b66cb9 Fixes from review: the similar cache, restore, outside services, latency
- The similar-question cache embeds a question as its project would: a
  project kept in the cluster has external fallbacks refused, and an
  embedding model outside the cluster is not asked at all. The vectors are
  compared outside the cache's lock. The question's bound is in characters.
- The restorer puts values back in streamed tool calls' arguments (escaped
  for the JSON they are), completions' text and reasoning, each held across
  events on its own, and releases a held tail with the finish event. A
  whole answer is walked as JSON, so a value with a quote no longer breaks
  tool-call arguments.
- An outside guardrail service reads the user's turn as redacted, and a
  streamed answer once rather than at the finish chunk and again at the
  end. guardrails.md says it can stop a stream only under buffer_all.
- A replica's smoothed time to first token is forgotten after five minutes
  without an answer: one found slow and then given nothing was never tried
  again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:15:43 +01:00

282 lines
7.3 KiB
Go

package router
import (
"bytes"
"encoding/json"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// Putting a request's own redacted values back into its answer: the model
// read "[EMAIL_1]", the client reads the address it sent. A guardrail policy
// asks for it with restore: true on its input rules.
//
// Before the output guardrails, not after: they then judge the answer as the
// client will read it, so a policy that redacts personal data in answers
// still does — and a placeholder the output guardrails make of a value the
// model invented is never mistaken for one of the request's own.
// restorer swaps placeholders back, in a whole answer or a streamed one.
type restorer struct {
values map[string]string
longest int
// held is, per choice and field, the tail of the text so far that may
// be the start of a placeholder the next event completes: "[EMA" before
// "IL_1]". Keyed by heldKey.
held map[heldKey]string
// last is the most recent event, the shape a final release is sent in.
last map[string]any
}
// heldKey is where held text belongs: a choice, and in it the content, the
// reasoning, a completion's text, or a tool call's arguments (tool >= 0).
type heldKey struct {
choice int
field string
tool int
}
func newRestorer(values map[string]string) *restorer {
r := &restorer{values: values, held: map[heldKey]string{}}
for ph := range values {
r.longest = max(r.longest, len(ph))
}
return r
}
// text swaps every whole placeholder in s. In a tool call's arguments,
// which are JSON written into a string, a value goes in escaped for JSON:
// O"Brien would otherwise end the argument early.
func (r *restorer) text(s string, inJSON bool) string {
if !strings.Contains(s, "[") {
return s
}
for ph, v := range r.values {
if inJSON {
q, _ := json.Marshal(v)
v = string(q[1 : len(q)-1])
}
s = strings.ReplaceAll(s, ph, v)
}
return s
}
// split is s with its placeholders swapped, less a tail that could still
// become one: what to send now, and what to hold for the next event.
func (r *restorer) split(s string, inJSON bool) (send, hold string) {
s = r.text(s, inJSON)
i := strings.LastIndexByte(s, '[')
if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest {
return s, ""
}
tail := s[i:]
for ph := range r.values {
if strings.HasPrefix(ph, tail) {
return s[:i], tail
}
}
return s, ""
}
// body swaps the placeholders in a whole JSON answer: in every string, and
// in a tool call's arguments escaped for the JSON they are.
func (r *restorer) body(b []byte) []byte {
if !bytes.Contains(b, []byte("[")) {
return b
}
d := json.NewDecoder(bytes.NewReader(b))
d.UseNumber()
var v any
if err := d.Decode(&v); err != nil {
return b
}
out, err := json.Marshal(r.walk(v, ""))
if err != nil {
return b
}
return out
}
func (r *restorer) walk(v any, key string) any {
switch x := v.(type) {
case map[string]any:
for k, e := range x {
x[k] = r.walk(e, k)
}
case []any:
for i, e := range x {
x[i] = r.walk(e, key)
}
case string:
return r.text(x, key == "arguments")
}
return v
}
// OnEvent swaps the placeholders in one streamed event — a chat delta's
// content, reasoning and tool calls' arguments, a completion's text —
// holding back the start of one an event has split.
func (r *restorer) OnEvent(data string) ([]string, bool) {
if !strings.Contains(data, "[") && len(r.held) == 0 {
return []string{data}, false
}
var ev map[string]any
dec := json.NewDecoder(strings.NewReader(data))
dec.UseNumber()
if err := dec.Decode(&ev); err != nil {
return []string{data}, false
}
choices, _ := ev["choices"].([]any)
for i, c := range choices {
ch, _ := c.(map[string]any)
if ch == nil {
continue
}
idx := i
if n, ok := ch["index"].(json.Number); ok {
if v, err := n.Int64(); err == nil {
idx = int(v)
}
}
finished := ch["finish_reason"] != nil
delta, _ := ch["delta"].(map[string]any)
if delta == nil && finished && r.heldFor(idx) {
delta = map[string]any{}
ch["delta"] = delta
}
swap := func(container map[string]any, field string, k heldKey, inJSON bool) {
s, ok := container[field].(string)
if !ok && !(finished && r.held[k] != "") {
return
}
send, hold := r.split(r.held[k]+s, inJSON)
// The last event of a choice releases everything.
if finished {
send, hold = send+hold, ""
}
container[field] = send
if hold == "" {
delete(r.held, k)
} else {
r.held[k] = hold
}
}
if delta != nil {
swap(delta, "content", heldKey{idx, "content", -1}, false)
swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false)
calls, _ := delta["tool_calls"].([]any)
for j, tc := range calls {
call, _ := tc.(map[string]any)
fn, _ := call["function"].(map[string]any)
if fn == nil {
continue
}
t := j
if n, ok := call["index"].(json.Number); ok {
if v, err := n.Int64(); err == nil {
t = int(v)
}
}
swap(fn, "arguments", heldKey{idx, "arguments", t}, true)
}
}
if _, ok := ch["text"]; ok {
swap(ch, "text", heldKey{idx, "text", -1}, false)
}
}
r.last = ev
out, err := json.Marshal(ev)
if err != nil {
return []string{data}, false
}
return []string{string(out)}, false
}
func (r *restorer) heldFor(choice int) bool {
for k := range r.held {
if k.choice == choice {
return true
}
}
return false
}
// OnEnd releases what is still held, as one more event for each.
func (r *restorer) OnEnd() []string {
if len(r.held) == 0 || r.last == nil {
return nil
}
var out []string
for k, text := range r.held {
ev := map[string]any{}
for key, v := range r.last {
if key != "choices" && key != "usage" {
ev[key] = v
}
}
ch := map[string]any{"index": k.choice, "finish_reason": nil}
switch k.field {
case "text":
ch["text"] = text
case "arguments":
ch["delta"] = map[string]any{"tool_calls": []any{map[string]any{"index": k.tool,
"function": map[string]any{"arguments": text}}}}
default:
ch["delta"] = map[string]any{k.field: text}
}
ev["choices"] = []any{ch}
if b, err := json.Marshal(ev); err == nil {
out = append(out, string(b))
}
}
r.held = map[heldKey]string{}
return out
}
// streamChain runs one stream filter's output through another.
type streamChain struct{ first, then proxy.StreamFilter }
func (c streamChain) OnEvent(data string) ([]string, bool) {
first, stop := c.first.OnEvent(data)
var out []string
for _, d := range first {
e, s := c.then.OnEvent(d)
out = append(out, e...)
stop = stop || s
}
return out, stop
}
func (c streamChain) OnEnd() []string {
var out []string
for _, d := range c.first.OnEnd() {
e, _ := c.then.OnEvent(d)
out = append(out, e...)
}
return append(out, c.then.OnEnd()...)
}
// restoreInto puts the input's placeholders back in this attempt's answer,
// ahead of whatever output filter the attempt already has.
func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
if len(values) == 0 {
return
}
rs := newRestorer(values)
if f := preq.Filter; f != nil {
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
} else {
preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) }
}
if !stream {
return
}
if sf := preq.StreamFilter; sf != nil {
preq.StreamFilter = streamChain{first: rs, then: sf}
} else {
preq.StreamFilter = rs
}
}