CI / test (push) Successful in 9m50s
With restore: true and the answer cache on, alice's "write to alice@example.com" and bob's "write to bob@example.com" redact to the same request, and bob was given the answer kept for alice, her address put back in it. A request with restore values is now never looked up in the cache nor kept in it. The similar-question cache compared the last message's text and blanked its parts: two pictures asked about in the same words were one question. A last message with any part that is not text is now reused only when the request is the same. Found in review; both tests fail without the fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
165 lines
6.5 KiB
Go
165 lines
6.5 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// similarRouter is cachingRouter whose project also reuses answers to
|
|
// questions 0.95 alike, embedded by "emb": an engine that places each
|
|
// question where the test says, beside the chat engine.
|
|
func similarRouter(t *testing.T, places map[string][]float32) (*Router, string, *fakeEngine, *atomic.Int64) {
|
|
t.Helper()
|
|
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
|
target, _ := url.Parse(e.srv.URL)
|
|
chat := httputil.NewSingleHostReverseProxy(target)
|
|
var embeds atomic.Int64
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("POST /v1/embeddings", func(w http.ResponseWriter, r *http.Request) {
|
|
embeds.Add(1)
|
|
var in struct {
|
|
Input []string `json:"input"`
|
|
}
|
|
b, _ := io.ReadAll(r.Body)
|
|
_ = json.Unmarshal(b, &in)
|
|
v, ok := places[in.Input[0]]
|
|
if !ok {
|
|
v = []float32{0, 0, 1}
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"index": 0, "embedding": v}}})
|
|
})
|
|
mux.Handle("/", chat)
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
|
|
rt, secret := pricedRouter(t, srv.URL, "emb")
|
|
rt.cacheEmbedModel = "emb"
|
|
if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"),
|
|
store.AnswerCache{TTL: time.Hour, Similar: 0.95}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rt, secret, e, &embeds
|
|
}
|
|
|
|
func TestASimilarQuestionGetsTheAnswerAlreadyGiven(t *testing.T) {
|
|
rt, secret, e, embeds := similarRouter(t, map[string][]float32{
|
|
"how do I reset my password": {1, 0.1, 0},
|
|
"how can I reset my password?": {1, 0.12, 0},
|
|
"what is the capital of Sweden": {0, 1, 0},
|
|
"how do I reset my password, then": {1, 0.11, 0},
|
|
})
|
|
h := rt.Handler()
|
|
hdr := func(id string) map[string]string {
|
|
return map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id}
|
|
}
|
|
|
|
first := post(t, h, ask(0, "how do I reset my password"), hdr("first"))
|
|
if first.Code != 200 || first.Header().Get(cacheHeader) != "miss" {
|
|
t.Fatalf("first: %d %q %s", first.Code, first.Header().Get(cacheHeader), first.Body.String())
|
|
}
|
|
second := post(t, h, ask(0, "how can I reset my password?"), hdr("second"))
|
|
if second.Code != 200 || second.Header().Get(cacheHeader) != "similar" {
|
|
t.Fatalf("second: %d %q %s", second.Code, second.Header().Get(cacheHeader), second.Body.String())
|
|
}
|
|
if s := second.Header().Get(similarityHeader); !strings.HasPrefix(s, "0.99") && s != "1.000" {
|
|
t.Errorf("similarity %q", s)
|
|
}
|
|
if second.Body.String() != first.Body.String() {
|
|
t.Errorf("the similar question got another answer:\n%s\n%s", first.Body, second.Body)
|
|
}
|
|
if u := usageOf(t, rt, "second"); !u.CacheHit || u.CostMicros != 0 {
|
|
t.Errorf("a similar hit is recorded as %+v", u)
|
|
}
|
|
|
|
other := post(t, h, ask(0, "what is the capital of Sweden"), hdr("other"))
|
|
if other.Header().Get(cacheHeader) != "miss" {
|
|
t.Errorf("an unlike question: %q", other.Header().Get(cacheHeader))
|
|
}
|
|
|
|
// The same question after another system prompt is another conversation.
|
|
sys := ask(0, "how do I reset my password, then")
|
|
sys["messages"] = append([]any{map[string]any{"role": "system", "content": "You are a pirate."}}, sys["messages"].([]any)...)
|
|
if rec := post(t, h, sys, hdr("system")); rec.Header().Get(cacheHeader) != "miss" {
|
|
t.Errorf("a like question under another system prompt: %q", rec.Header().Get(cacheHeader))
|
|
}
|
|
// A request answered exactly is not embedded at all.
|
|
before := embeds.Load()
|
|
if rec := post(t, h, ask(0, "how do I reset my password"), hdr("exact")); rec.Header().Get(cacheHeader) != "hit" {
|
|
t.Errorf("the same question: %q", rec.Header().Get(cacheHeader))
|
|
}
|
|
if embeds.Load() != before {
|
|
t.Error("an exact hit waited on the embedding model")
|
|
}
|
|
if got := e.calls.Load(); got != 3 {
|
|
t.Errorf("the chat model was asked %d times, want 3 (first, other, system)", got)
|
|
}
|
|
}
|
|
|
|
func TestWithoutAnEmbeddingModelOnlyTheSameRequestIsReused(t *testing.T) {
|
|
rt, secret, e, embeds := similarRouter(t, map[string][]float32{
|
|
"how do I reset my password": {1, 0.1, 0},
|
|
"how can I reset my password?": {1, 0.12, 0},
|
|
})
|
|
rt.cacheEmbedModel = ""
|
|
h := rt.Handler()
|
|
hdr := map[string]string{"Authorization": "Bearer " + secret}
|
|
post(t, h, ask(0, "how do I reset my password"), hdr)
|
|
if rec := post(t, h, ask(0, "how can I reset my password?"), hdr); rec.Header().Get(cacheHeader) != "miss" {
|
|
t.Errorf("without an embedding model: %q", rec.Header().Get(cacheHeader))
|
|
}
|
|
if e.calls.Load() != 2 || embeds.Load() != 0 {
|
|
t.Errorf("chat %d, embeddings %d", e.calls.Load(), embeds.Load())
|
|
}
|
|
}
|
|
|
|
func TestTheConversationAQuestionIsAskedInIsAllButItsLastMessage(t *testing.T) {
|
|
body := func(last string, earlier ...string) []byte {
|
|
var msgs []any
|
|
for _, s := range earlier {
|
|
msgs = append(msgs, map[string]any{"role": "user", "content": s}, map[string]any{"role": "assistant", "content": "ok"})
|
|
}
|
|
msgs = append(msgs, map[string]any{"role": "user", "content": last})
|
|
b, _ := json.Marshal(map[string]any{"model": "m", "messages": msgs, "temperature": 0})
|
|
return b
|
|
}
|
|
g1, q1, ok1 := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hi"))
|
|
g2, _, ok2 := similarGroup("p", "/v1/chat/completions", "m", body("b?", "hi"))
|
|
g3, _, _ := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hello"))
|
|
g4, _, _ := similarGroup("q", "/v1/chat/completions", "m", body("a?", "hi"))
|
|
if !ok1 || !ok2 || q1 != "a?" || g1 != g2 {
|
|
t.Errorf("two questions after the same turns are in different conversations")
|
|
}
|
|
if g1 == g3 || g1 == g4 {
|
|
t.Errorf("other turns or another project share a conversation")
|
|
}
|
|
tool, _ := json.Marshal(map[string]any{"model": "m", "messages": []any{
|
|
map[string]any{"role": "user", "content": "x"}, map[string]any{"role": "tool", "content": "42"}}})
|
|
if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", tool); ok {
|
|
t.Error("a tool's result is taken for a question")
|
|
}
|
|
}
|
|
|
|
func TestAQuestionAboutAPictureIsNotTakenForAnotherPicture(t *testing.T) {
|
|
pic := func(url string) []byte {
|
|
b, _ := json.Marshal(map[string]any{"model": "m", "temperature": 0, "messages": []any{map[string]any{
|
|
"role": "user", "content": []any{
|
|
map[string]any{"type": "text", "text": "what is in this picture?"},
|
|
map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}},
|
|
}}}})
|
|
return b
|
|
}
|
|
if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", pic("data:image/png;base64,AAAA")); ok {
|
|
t.Error("a question with a picture in it is compared by its text alone")
|
|
}
|
|
}
|