CI / test (push) Successful in 9m16s
From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
298 lines
8.0 KiB
Go
298 lines
8.0 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// Putting a request's own redacted values back into its answer: the model
|
|
// read "[EMAIL_1]", the client reads the address it sent. A guardrail policy
|
|
// asks for it with restore: true on its input rules.
|
|
//
|
|
// Before the output guardrails, not after: they then judge the answer as the
|
|
// client will read it, so a policy that redacts personal data in answers
|
|
// still does — and a placeholder the output guardrails make of a value the
|
|
// model invented is never mistaken for one of the request's own.
|
|
|
|
// restorer swaps placeholders back, in a whole answer or a streamed one.
|
|
type restorer struct {
|
|
values map[string]string
|
|
longest int
|
|
// jsonContent is an answer asked for as JSON (response_format): its
|
|
// content is JSON, and a value goes into it escaped, as into a tool
|
|
// call's arguments.
|
|
jsonContent bool
|
|
// held is, per choice and field, the tail of the text so far that may
|
|
// be the start of a placeholder the next event completes: "[EMA" before
|
|
// "IL_1]". Keyed by heldKey.
|
|
held map[heldKey]string
|
|
// last is the most recent event, the shape a final release is sent in.
|
|
last map[string]any
|
|
}
|
|
|
|
// heldKey is where held text belongs: a choice, and in it the content, the
|
|
// reasoning, a completion's text, or a tool call's arguments (tool >= 0).
|
|
type heldKey struct {
|
|
choice int
|
|
field string
|
|
tool int
|
|
}
|
|
|
|
func newRestorer(values map[string]string) *restorer {
|
|
r := &restorer{values: values, held: map[heldKey]string{}}
|
|
for ph := range values {
|
|
r.longest = max(r.longest, len(ph))
|
|
}
|
|
return r
|
|
}
|
|
|
|
// text swaps every whole placeholder in s. In a tool call's arguments,
|
|
// which are JSON written into a string, a value goes in escaped for JSON:
|
|
// O"Brien would otherwise end the argument early.
|
|
func (r *restorer) text(s string, inJSON bool) string {
|
|
if !strings.Contains(s, "[") {
|
|
return s
|
|
}
|
|
for ph, v := range r.values {
|
|
if inJSON {
|
|
q, _ := json.Marshal(v)
|
|
v = string(q[1 : len(q)-1])
|
|
}
|
|
s = strings.ReplaceAll(s, ph, v)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// split is s with its placeholders swapped, less a tail that could still
|
|
// become one: what to send now, and what to hold for the next event.
|
|
func (r *restorer) split(s string, inJSON bool) (send, hold string) {
|
|
s = r.text(s, inJSON)
|
|
i := strings.LastIndexByte(s, '[')
|
|
if i < 0 || strings.IndexByte(s[i:], ']') >= 0 || len(s)-i >= r.longest {
|
|
return s, ""
|
|
}
|
|
tail := s[i:]
|
|
for ph := range r.values {
|
|
if strings.HasPrefix(ph, tail) {
|
|
return s[:i], tail
|
|
}
|
|
}
|
|
return s, ""
|
|
}
|
|
|
|
// body swaps the placeholders in a whole JSON answer: in every string, and
|
|
// in a tool call's arguments escaped for the JSON they are.
|
|
func (r *restorer) body(b []byte) []byte {
|
|
if !bytes.Contains(b, []byte("[")) {
|
|
return b
|
|
}
|
|
d := json.NewDecoder(bytes.NewReader(b))
|
|
d.UseNumber()
|
|
var v any
|
|
if err := d.Decode(&v); err != nil {
|
|
return b
|
|
}
|
|
out, err := json.Marshal(r.walk(v, ""))
|
|
if err != nil {
|
|
return b
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (r *restorer) walk(v any, key string) any {
|
|
switch x := v.(type) {
|
|
case map[string]any:
|
|
for k, e := range x {
|
|
x[k] = r.walk(e, k)
|
|
}
|
|
case []any:
|
|
for i, e := range x {
|
|
x[i] = r.walk(e, key)
|
|
}
|
|
case string:
|
|
return r.text(x, key == "arguments" || (r.jsonContent && key == "content"))
|
|
}
|
|
return v
|
|
}
|
|
|
|
// OnEvent swaps the placeholders in one streamed event — a chat delta's
|
|
// content, reasoning and tool calls' arguments, a completion's text —
|
|
// holding back the start of one an event has split.
|
|
func (r *restorer) OnEvent(data string) ([]string, bool) {
|
|
if !strings.Contains(data, "[") && len(r.held) == 0 {
|
|
return []string{data}, false
|
|
}
|
|
var ev map[string]any
|
|
dec := json.NewDecoder(strings.NewReader(data))
|
|
dec.UseNumber()
|
|
if err := dec.Decode(&ev); err != nil {
|
|
return []string{data}, false
|
|
}
|
|
choices, _ := ev["choices"].([]any)
|
|
for i, c := range choices {
|
|
ch, _ := c.(map[string]any)
|
|
if ch == nil {
|
|
continue
|
|
}
|
|
idx := i
|
|
if n, ok := ch["index"].(json.Number); ok {
|
|
if v, err := n.Int64(); err == nil {
|
|
idx = int(v)
|
|
}
|
|
}
|
|
finished := ch["finish_reason"] != nil
|
|
delta, _ := ch["delta"].(map[string]any)
|
|
if delta == nil && finished && r.heldFor(idx) {
|
|
delta = map[string]any{}
|
|
ch["delta"] = delta
|
|
}
|
|
swap := func(container map[string]any, field string, k heldKey, inJSON bool) {
|
|
s, ok := container[field].(string)
|
|
if !ok && !(finished && r.held[k] != "") {
|
|
return
|
|
}
|
|
send, hold := r.split(r.held[k]+s, inJSON)
|
|
// The last event of a choice releases everything.
|
|
if finished {
|
|
send, hold = send+hold, ""
|
|
}
|
|
container[field] = send
|
|
if hold == "" {
|
|
delete(r.held, k)
|
|
} else {
|
|
r.held[k] = hold
|
|
}
|
|
}
|
|
if delta != nil {
|
|
swap(delta, "content", heldKey{idx, "content", -1}, r.jsonContent)
|
|
swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false)
|
|
calls, _ := delta["tool_calls"].([]any)
|
|
for j, tc := range calls {
|
|
call, _ := tc.(map[string]any)
|
|
fn, _ := call["function"].(map[string]any)
|
|
if fn == nil {
|
|
continue
|
|
}
|
|
t := j
|
|
if n, ok := call["index"].(json.Number); ok {
|
|
if v, err := n.Int64(); err == nil {
|
|
t = int(v)
|
|
}
|
|
}
|
|
swap(fn, "arguments", heldKey{idx, "arguments", t}, true)
|
|
}
|
|
}
|
|
if _, ok := ch["text"]; ok {
|
|
swap(ch, "text", heldKey{idx, "text", -1}, false)
|
|
}
|
|
}
|
|
r.last = ev
|
|
out, err := json.Marshal(ev)
|
|
if err != nil {
|
|
return []string{data}, false
|
|
}
|
|
return []string{string(out)}, false
|
|
}
|
|
|
|
func (r *restorer) heldFor(choice int) bool {
|
|
for k := range r.held {
|
|
if k.choice == choice {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// OnEnd releases what is still held, as one more event for each.
|
|
func (r *restorer) OnEnd() []string {
|
|
if len(r.held) == 0 || r.last == nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for k, text := range r.held {
|
|
ev := map[string]any{}
|
|
for key, v := range r.last {
|
|
if key != "choices" && key != "usage" {
|
|
ev[key] = v
|
|
}
|
|
}
|
|
ch := map[string]any{"index": k.choice, "finish_reason": nil}
|
|
switch k.field {
|
|
case "text":
|
|
ch["text"] = text
|
|
case "arguments":
|
|
ch["delta"] = map[string]any{"tool_calls": []any{map[string]any{"index": k.tool,
|
|
"function": map[string]any{"arguments": text}}}}
|
|
default:
|
|
ch["delta"] = map[string]any{k.field: text}
|
|
}
|
|
ev["choices"] = []any{ch}
|
|
if b, err := json.Marshal(ev); err == nil {
|
|
out = append(out, string(b))
|
|
}
|
|
}
|
|
r.held = map[heldKey]string{}
|
|
return out
|
|
}
|
|
|
|
// streamChain runs one stream filter's output through another.
|
|
type streamChain struct{ first, then proxy.StreamFilter }
|
|
|
|
func (c streamChain) OnEvent(data string) ([]string, bool) {
|
|
first, stop := c.first.OnEvent(data)
|
|
var out []string
|
|
for _, d := range first {
|
|
e, s := c.then.OnEvent(d)
|
|
out = append(out, e...)
|
|
stop = stop || s
|
|
}
|
|
return out, stop
|
|
}
|
|
|
|
func (c streamChain) OnEnd() []string {
|
|
var out []string
|
|
for _, d := range c.first.OnEnd() {
|
|
e, _ := c.then.OnEvent(d)
|
|
out = append(out, e...)
|
|
}
|
|
return append(out, c.then.OnEnd()...)
|
|
}
|
|
|
|
// restoreInto puts the input's placeholders back in this attempt's answer,
|
|
// ahead of whatever output filter the attempt already has.
|
|
func restoreInto(values map[string]string, stream, jsonContent bool, preq *proxy.Request) {
|
|
if len(values) == 0 {
|
|
return
|
|
}
|
|
rs := newRestorer(values)
|
|
rs.jsonContent = jsonContent
|
|
if f := preq.Filter; f != nil {
|
|
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
|
|
} else {
|
|
preq.Filter = func(b []byte, _ http.Header) []byte { return rs.body(b) }
|
|
}
|
|
if !stream {
|
|
return
|
|
}
|
|
if sf := preq.StreamFilter; sf != nil {
|
|
preq.StreamFilter = streamChain{first: rs, then: sf}
|
|
} else {
|
|
preq.StreamFilter = rs
|
|
}
|
|
}
|
|
|
|
// jsonAnswer reports whether a request asks for its answer as JSON.
|
|
func jsonAnswer(body []byte) bool {
|
|
var b struct {
|
|
ResponseFormat *struct {
|
|
Type string `json:"type"`
|
|
} `json:"response_format"`
|
|
}
|
|
return json.Unmarshal(body, &b) == nil && b.ResponseFormat != nil &&
|
|
(b.ResponseFormat.Type == "json_object" || b.ResponseFormat.Type == "json_schema")
|
|
}
|