- The similar-question cache embeds a question as its project would: a project kept in the cluster has external fallbacks refused, and an embedding model outside the cluster is not asked at all. The vectors are compared outside the cache's lock. The question's bound is in characters. - The restorer puts values back in streamed tool calls' arguments (escaped for the JSON they are), completions' text and reasoning, each held across events on its own, and releases a held tail with the finish event. A whole answer is walked as JSON, so a value with a quote no longer breaks tool-call arguments. - An outside guardrail service reads the user's turn as redacted, and a streamed answer once rather than at the finish chunk and again at the end. guardrails.md says it can stop a stream only under buffer_all. - A replica's smoothed time to first token is forgotten after five minutes without an answer: one found slow and then given nothing was never tried again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
8.7 KiB
Go
215 lines
8.7 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// restoreGuard redacts alice's address as the gate does, and asks for it
|
|
// back in the answer.
|
|
type restoreGuard struct{}
|
|
|
|
func (restoreGuard) Input(_ context.Context, _ guard.Subject, body []byte) (guard.Verdict, error) {
|
|
if !bytes.Contains(body, []byte("alice@example.com")) {
|
|
return guard.Verdict{}, nil
|
|
}
|
|
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
|
|
Body: bytes.ReplaceAll(body, []byte("alice@example.com"), []byte("[EMAIL_1]")),
|
|
Restore: map[string]string{"[EMAIL_1]": `alice@example.com`}}, nil
|
|
}
|
|
func (restoreGuard) Output(context.Context, guard.Subject) guard.OutputFilter { return nil }
|
|
func (restoreGuard) Directives(guard.Subject) guard.Directives { return guard.Directives{} }
|
|
|
|
// echoEngine answers with the placeholder, in one piece or, streamed, split
|
|
// across two events.
|
|
func echoEngine(t *testing.T) *httptest.Server {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
|
|
fmt.Fprint(w, "vllm:num_requests_running 0\nvllm:num_requests_waiting 0\nvllm:kv_cache_usage_perc 0.1\n")
|
|
})
|
|
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
|
b := new(bytes.Buffer)
|
|
_, _ = b.ReadFrom(r.Body)
|
|
if bytes.Contains(b.Bytes(), []byte("alice@example.com")) {
|
|
t.Errorf("the engine saw the address: %s", b)
|
|
}
|
|
if !bytes.Contains(b.Bytes(), []byte(`"stream":true`)) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, `{"model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"Sent to [EMAIL_1]."},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":4}}`)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
for _, piece := range []string{"Sent to [EMA", "IL_1]", "."} {
|
|
fmt.Fprintf(w, "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":%q},\"finish_reason\":null}]}\n\n", piece)
|
|
}
|
|
fmt.Fprint(w, "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n")
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestTheAnswerGetsBackTheValuesItsRequestHadRedacted(t *testing.T) {
|
|
srv := echoEngine(t)
|
|
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}))
|
|
rt.SetGuard(restoreGuard{})
|
|
|
|
rec := post(t, rt.Handler(), say("write to alice@example.com"), nil)
|
|
if !strings.Contains(rec.Body.String(), "Sent to alice@example.com.") || strings.Contains(rec.Body.String(), "[EMAIL_1]") {
|
|
t.Errorf("one-shot answer: %s", rec.Body.String())
|
|
}
|
|
|
|
streamed := say("write to alice@example.com")
|
|
streamed["stream"] = true
|
|
rec = post(t, rt.Handler(), streamed, nil)
|
|
var text strings.Builder
|
|
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
|
if i := strings.Index(line, `"content":"`); i >= 0 {
|
|
rest := line[i+len(`"content":"`):]
|
|
text.WriteString(rest[:strings.Index(rest, `"`)])
|
|
}
|
|
}
|
|
if text.String() != "Sent to alice@example.com." {
|
|
t.Errorf("streamed answer reads %q\n%s", text.String(), rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestARestorerHoldsOnlyWhatCouldBeAPlaceholder(t *testing.T) {
|
|
r := newRestorer(map[string]string{"[EMAIL_1]": "a@b.c", "[PHONE_1]": "+46 70"})
|
|
for _, c := range []struct{ in, send, hold string }{
|
|
{"Dear [EMA", "Dear ", "[EMA"},
|
|
{"a [1] b", "a [1] b", ""},
|
|
{"x [EMAIL_1] y", "x a@b.c y", ""},
|
|
{"end [", "end ", "["},
|
|
{"[X", "[X", ""},
|
|
} {
|
|
send, hold := r.split(c.in, false)
|
|
if send != c.send || hold != c.hold {
|
|
t.Errorf("split(%q) = %q, %q; want %q, %q", c.in, send, hold, c.send, c.hold)
|
|
}
|
|
}
|
|
// A value with a quote stays valid JSON in a whole answer.
|
|
q := newRestorer(map[string]string{"[PERSON_1]": `O"Brien`})
|
|
if got := string(q.body([]byte(`{"c":"hi [PERSON_1]"}`))); got != `{"c":"hi O\"Brien"}` {
|
|
t.Errorf("body: %s", got)
|
|
}
|
|
// What is held at the end is still sent.
|
|
e := newRestorer(map[string]string{"[EMAIL_1]": "a@b.c"})
|
|
e.OnEvent(`{"choices":[{"index":0,"delta":{"content":"to [EM"}}]}`)
|
|
if out := e.OnEnd(); len(out) != 1 || !strings.Contains(out[0], `"content":"[EM"`) {
|
|
t.Errorf("OnEnd: %v", out)
|
|
}
|
|
}
|
|
|
|
// anyEmailGuard redacts whichever address a request has to [EMAIL_1], and
|
|
// asks for it back: two callers' requests then read the same.
|
|
type anyEmailGuard struct{}
|
|
|
|
var anEmail = regexp.MustCompile(`[a-z]+@example\.com`)
|
|
|
|
func (anyEmailGuard) Input(_ context.Context, _ guard.Subject, body []byte) (guard.Verdict, error) {
|
|
addr := anEmail.Find(body)
|
|
if addr == nil {
|
|
return guard.Verdict{}, nil
|
|
}
|
|
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
|
|
Body: anEmail.ReplaceAll(body, []byte("[EMAIL_1]")),
|
|
Restore: map[string]string{"[EMAIL_1]": string(addr)}}, nil
|
|
}
|
|
func (anyEmailGuard) Output(context.Context, guard.Subject) guard.OutputFilter { return nil }
|
|
func (anyEmailGuard) Directives(guard.Subject) guard.Directives { return guard.Directives{} }
|
|
|
|
func TestAnAnswerWithACallersValuesPutBackIsNeverReusedForAnother(t *testing.T) {
|
|
// Redacted, alice's request and bob's read the same; the answer alice
|
|
// got has her address in it again, and must not be bob's.
|
|
srv := echoEngine(t)
|
|
rt, secret := cachingRouter(t, srv.URL, false)
|
|
rt.SetGuard(anyEmailGuard{})
|
|
hdr := map[string]string{"Authorization": "Bearer " + secret}
|
|
|
|
first := post(t, rt.Handler(), ask(0, "write to alice@example.com"), hdr)
|
|
second := post(t, rt.Handler(), ask(0, "write to bob@example.com"), hdr)
|
|
if strings.Contains(second.Body.String(), "alice") {
|
|
t.Fatalf("bob was given alice's answer: %s", second.Body.String())
|
|
}
|
|
if !strings.Contains(second.Body.String(), "bob@example.com") {
|
|
t.Errorf("bob's answer lacks his own address: %s", second.Body.String())
|
|
}
|
|
if got := first.Header().Get(cacheHeader) + "/" + second.Header().Get(cacheHeader); got != "bypass/bypass" {
|
|
t.Errorf("x-llmgw-cache %s, want bypass for both", got)
|
|
}
|
|
}
|
|
|
|
func TestAToolCallsArgumentsGetTheirValuesBackStreamedOrNot(t *testing.T) {
|
|
r := newRestorer(map[string]string{"[NAME_1]": `O"Brien`, "[EMAIL_1]": "a@example.com"})
|
|
// Whole: the value goes into the arguments escaped for the JSON they are.
|
|
body := r.body([]byte(`{"choices":[{"message":{"content":"to [NAME_1]","tool_calls":[{"function":{"name":"send",` +
|
|
`"arguments":"{\"to\":\"[EMAIL_1]\",\"name\":\"[NAME_1]\"}"}}]}}]}`))
|
|
var whole struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string
|
|
ToolCalls []struct{ Function struct{ Arguments string } } `json:"tool_calls"`
|
|
}
|
|
}
|
|
}
|
|
if err := json.Unmarshal(body, &whole); err != nil {
|
|
t.Fatalf("%v: %s", err, body)
|
|
}
|
|
var args map[string]string
|
|
if err := json.Unmarshal([]byte(whole.Choices[0].Message.ToolCalls[0].Function.Arguments), &args); err != nil ||
|
|
args["to"] != "a@example.com" || args["name"] != `O"Brien` || whole.Choices[0].Message.Content != `to O"Brien` {
|
|
t.Fatalf("arguments %q (%v), content %q", whole.Choices[0].Message.ToolCalls[0].Function.Arguments, err, whole.Choices[0].Message.Content)
|
|
}
|
|
|
|
// Streamed, the placeholder split across two events.
|
|
var argsText strings.Builder
|
|
for _, piece := range []string{`{"to":"[EMA`, `IL_1]"}`} {
|
|
ev, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"index": 0, "finish_reason": nil,
|
|
"delta": map[string]any{"tool_calls": []any{map[string]any{"index": 0, "function": map[string]any{"arguments": piece}}}}}}})
|
|
out, _ := r.OnEvent(string(ev))
|
|
for _, o := range out {
|
|
var e struct {
|
|
Choices []struct {
|
|
Delta struct {
|
|
ToolCalls []struct{ Function struct{ Arguments string } } `json:"tool_calls"`
|
|
}
|
|
}
|
|
}
|
|
_ = json.Unmarshal([]byte(o), &e)
|
|
for _, c := range e.Choices {
|
|
for _, tc := range c.Delta.ToolCalls {
|
|
argsText.WriteString(tc.Function.Arguments)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if argsText.String() != `{"to":"a@example.com"}` {
|
|
t.Errorf("streamed arguments read %q", argsText.String())
|
|
}
|
|
}
|
|
|
|
func TestHeldTextGoesOutWithTheFinishEvent(t *testing.T) {
|
|
// The answer ends on what could have begun a placeholder; the finish
|
|
// event, whose delta is empty, carries it rather than an event after.
|
|
r := newRestorer(map[string]string{"[EMAIL_1]": "a@example.com"})
|
|
out1, _ := r.OnEvent(`{"choices":[{"index":0,"delta":{"content":"see [EMA"},"finish_reason":null}]}`)
|
|
out2, _ := r.OnEvent(`{"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`)
|
|
if len(r.OnEnd()) != 0 {
|
|
t.Error("held text was released after the finish event")
|
|
}
|
|
if !strings.Contains(out1[0], `"see "`) || !strings.Contains(out2[0], `"content":"[EMA"`) {
|
|
t.Errorf("%s / %s", out1, out2)
|
|
}
|
|
}
|