Files
llmgw/internal/router/similar_test.go
T
Ilker ManapandClaude Opus 5.5 7a04248148
CI / test (push) Successful in 9m50s
A reused answer never carries another caller's values
With restore: true and the answer cache on, alice's "write to
alice@example.com" and bob's "write to bob@example.com" redact to the
same request, and bob was given the answer kept for alice, her address put
back in it. A request with restore values is now never looked up in the
cache nor kept in it.

The similar-question cache compared the last message's text and blanked
its parts: two pictures asked about in the same words were one question.
A last message with any part that is not text is now reused only when the
request is the same.

Found in review; both tests fail without the fix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 22:03:32 +01:00

165 lines
6.5 KiB
Go

package router
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// similarRouter is cachingRouter whose project also reuses answers to
// questions 0.95 alike, embedded by "emb": an engine that places each
// question where the test says, beside the chat engine.
func similarRouter(t *testing.T, places map[string][]float32) (*Router, string, *fakeEngine, *atomic.Int64) {
t.Helper()
e := newFakeEngine(t, "qwen2.5-14b-instruct")
target, _ := url.Parse(e.srv.URL)
chat := httputil.NewSingleHostReverseProxy(target)
var embeds atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("POST /v1/embeddings", func(w http.ResponseWriter, r *http.Request) {
embeds.Add(1)
var in struct {
Input []string `json:"input"`
}
b, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(b, &in)
v, ok := places[in.Input[0]]
if !ok {
v = []float32{0, 0, 1}
}
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"index": 0, "embedding": v}}})
})
mux.Handle("/", chat)
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
rt, secret := pricedRouter(t, srv.URL, "emb")
rt.cacheEmbedModel = "emb"
if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"),
store.AnswerCache{TTL: time.Hour, Similar: 0.95}); err != nil {
t.Fatal(err)
}
return rt, secret, e, &embeds
}
func TestASimilarQuestionGetsTheAnswerAlreadyGiven(t *testing.T) {
rt, secret, e, embeds := similarRouter(t, map[string][]float32{
"how do I reset my password": {1, 0.1, 0},
"how can I reset my password?": {1, 0.12, 0},
"what is the capital of Sweden": {0, 1, 0},
"how do I reset my password, then": {1, 0.11, 0},
})
h := rt.Handler()
hdr := func(id string) map[string]string {
return map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id}
}
first := post(t, h, ask(0, "how do I reset my password"), hdr("first"))
if first.Code != 200 || first.Header().Get(cacheHeader) != "miss" {
t.Fatalf("first: %d %q %s", first.Code, first.Header().Get(cacheHeader), first.Body.String())
}
second := post(t, h, ask(0, "how can I reset my password?"), hdr("second"))
if second.Code != 200 || second.Header().Get(cacheHeader) != "similar" {
t.Fatalf("second: %d %q %s", second.Code, second.Header().Get(cacheHeader), second.Body.String())
}
if s := second.Header().Get(similarityHeader); !strings.HasPrefix(s, "0.99") && s != "1.000" {
t.Errorf("similarity %q", s)
}
if second.Body.String() != first.Body.String() {
t.Errorf("the similar question got another answer:\n%s\n%s", first.Body, second.Body)
}
if u := usageOf(t, rt, "second"); !u.CacheHit || u.CostMicros != 0 {
t.Errorf("a similar hit is recorded as %+v", u)
}
other := post(t, h, ask(0, "what is the capital of Sweden"), hdr("other"))
if other.Header().Get(cacheHeader) != "miss" {
t.Errorf("an unlike question: %q", other.Header().Get(cacheHeader))
}
// The same question after another system prompt is another conversation.
sys := ask(0, "how do I reset my password, then")
sys["messages"] = append([]any{map[string]any{"role": "system", "content": "You are a pirate."}}, sys["messages"].([]any)...)
if rec := post(t, h, sys, hdr("system")); rec.Header().Get(cacheHeader) != "miss" {
t.Errorf("a like question under another system prompt: %q", rec.Header().Get(cacheHeader))
}
// A request answered exactly is not embedded at all.
before := embeds.Load()
if rec := post(t, h, ask(0, "how do I reset my password"), hdr("exact")); rec.Header().Get(cacheHeader) != "hit" {
t.Errorf("the same question: %q", rec.Header().Get(cacheHeader))
}
if embeds.Load() != before {
t.Error("an exact hit waited on the embedding model")
}
if got := e.calls.Load(); got != 3 {
t.Errorf("the chat model was asked %d times, want 3 (first, other, system)", got)
}
}
func TestWithoutAnEmbeddingModelOnlyTheSameRequestIsReused(t *testing.T) {
rt, secret, e, embeds := similarRouter(t, map[string][]float32{
"how do I reset my password": {1, 0.1, 0},
"how can I reset my password?": {1, 0.12, 0},
})
rt.cacheEmbedModel = ""
h := rt.Handler()
hdr := map[string]string{"Authorization": "Bearer " + secret}
post(t, h, ask(0, "how do I reset my password"), hdr)
if rec := post(t, h, ask(0, "how can I reset my password?"), hdr); rec.Header().Get(cacheHeader) != "miss" {
t.Errorf("without an embedding model: %q", rec.Header().Get(cacheHeader))
}
if e.calls.Load() != 2 || embeds.Load() != 0 {
t.Errorf("chat %d, embeddings %d", e.calls.Load(), embeds.Load())
}
}
func TestTheConversationAQuestionIsAskedInIsAllButItsLastMessage(t *testing.T) {
body := func(last string, earlier ...string) []byte {
var msgs []any
for _, s := range earlier {
msgs = append(msgs, map[string]any{"role": "user", "content": s}, map[string]any{"role": "assistant", "content": "ok"})
}
msgs = append(msgs, map[string]any{"role": "user", "content": last})
b, _ := json.Marshal(map[string]any{"model": "m", "messages": msgs, "temperature": 0})
return b
}
g1, q1, ok1 := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hi"))
g2, _, ok2 := similarGroup("p", "/v1/chat/completions", "m", body("b?", "hi"))
g3, _, _ := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hello"))
g4, _, _ := similarGroup("q", "/v1/chat/completions", "m", body("a?", "hi"))
if !ok1 || !ok2 || q1 != "a?" || g1 != g2 {
t.Errorf("two questions after the same turns are in different conversations")
}
if g1 == g3 || g1 == g4 {
t.Errorf("other turns or another project share a conversation")
}
tool, _ := json.Marshal(map[string]any{"model": "m", "messages": []any{
map[string]any{"role": "user", "content": "x"}, map[string]any{"role": "tool", "content": "42"}}})
if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", tool); ok {
t.Error("a tool's result is taken for a question")
}
}
func TestAQuestionAboutAPictureIsNotTakenForAnotherPicture(t *testing.T) {
pic := func(url string) []byte {
b, _ := json.Marshal(map[string]any{"model": "m", "temperature": 0, "messages": []any{map[string]any{
"role": "user", "content": []any{
map[string]any{"type": "text", "text": "what is in this picture?"},
map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}},
}}}})
return b
}
if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", pic("data:image/png;base64,AAAA")); ok {
t.Error("a question with a picture in it is compared by its text alone")
}
}