package router import ( "encoding/json" "io" "net/http" "net/http/httptest" "net/http/httputil" "net/url" "strings" "sync/atomic" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // similarRouter is cachingRouter whose project also reuses answers to // questions 0.95 alike, embedded by "emb": an engine that places each // question where the test says, beside the chat engine. func similarRouter(t *testing.T, places map[string][]float32) (*Router, string, *fakeEngine, *atomic.Int64) { t.Helper() e := newFakeEngine(t, "qwen2.5-14b-instruct") target, _ := url.Parse(e.srv.URL) chat := httputil.NewSingleHostReverseProxy(target) var embeds atomic.Int64 mux := http.NewServeMux() mux.HandleFunc("POST /v1/embeddings", func(w http.ResponseWriter, r *http.Request) { embeds.Add(1) var in struct { Input []string `json:"input"` } b, _ := io.ReadAll(r.Body) _ = json.Unmarshal(b, &in) v, ok := places[in.Input[0]] if !ok { v = []float32{0, 0, 1} } _ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"index": 0, "embedding": v}}}) }) mux.Handle("/", chat) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) rt, secret := pricedRouter(t, srv.URL, "emb") rt.cacheEmbedModel = "emb" if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), store.AnswerCache{TTL: time.Hour, Similar: 0.95}); err != nil { t.Fatal(err) } return rt, secret, e, &embeds } func TestASimilarQuestionGetsTheAnswerAlreadyGiven(t *testing.T) { rt, secret, e, embeds := similarRouter(t, map[string][]float32{ "how do I reset my password": {1, 0.1, 0}, "how can I reset my password?": {1, 0.12, 0}, "what is the capital of Sweden": {0, 1, 0}, "how do I reset my password, then": {1, 0.11, 0}, }) h := rt.Handler() hdr := func(id string) map[string]string { return map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id} } first := post(t, h, ask(0, "how do I reset my password"), hdr("first")) if first.Code != 200 || first.Header().Get(cacheHeader) != "miss" { t.Fatalf("first: %d %q %s", first.Code, first.Header().Get(cacheHeader), first.Body.String()) } second := post(t, h, ask(0, "how can I reset my password?"), hdr("second")) if second.Code != 200 || second.Header().Get(cacheHeader) != "similar" { t.Fatalf("second: %d %q %s", second.Code, second.Header().Get(cacheHeader), second.Body.String()) } if s := second.Header().Get(similarityHeader); !strings.HasPrefix(s, "0.99") && s != "1.000" { t.Errorf("similarity %q", s) } if second.Body.String() != first.Body.String() { t.Errorf("the similar question got another answer:\n%s\n%s", first.Body, second.Body) } if u := usageOf(t, rt, "second"); !u.CacheHit || u.CostMicros != 0 { t.Errorf("a similar hit is recorded as %+v", u) } other := post(t, h, ask(0, "what is the capital of Sweden"), hdr("other")) if other.Header().Get(cacheHeader) != "miss" { t.Errorf("an unlike question: %q", other.Header().Get(cacheHeader)) } // The same question after another system prompt is another conversation. sys := ask(0, "how do I reset my password, then") sys["messages"] = append([]any{map[string]any{"role": "system", "content": "You are a pirate."}}, sys["messages"].([]any)...) if rec := post(t, h, sys, hdr("system")); rec.Header().Get(cacheHeader) != "miss" { t.Errorf("a like question under another system prompt: %q", rec.Header().Get(cacheHeader)) } // A request answered exactly is not embedded at all. before := embeds.Load() if rec := post(t, h, ask(0, "how do I reset my password"), hdr("exact")); rec.Header().Get(cacheHeader) != "hit" { t.Errorf("the same question: %q", rec.Header().Get(cacheHeader)) } if embeds.Load() != before { t.Error("an exact hit waited on the embedding model") } if got := e.calls.Load(); got != 3 { t.Errorf("the chat model was asked %d times, want 3 (first, other, system)", got) } } func TestWithoutAnEmbeddingModelOnlyTheSameRequestIsReused(t *testing.T) { rt, secret, e, embeds := similarRouter(t, map[string][]float32{ "how do I reset my password": {1, 0.1, 0}, "how can I reset my password?": {1, 0.12, 0}, }) rt.cacheEmbedModel = "" h := rt.Handler() hdr := map[string]string{"Authorization": "Bearer " + secret} post(t, h, ask(0, "how do I reset my password"), hdr) if rec := post(t, h, ask(0, "how can I reset my password?"), hdr); rec.Header().Get(cacheHeader) != "miss" { t.Errorf("without an embedding model: %q", rec.Header().Get(cacheHeader)) } if e.calls.Load() != 2 || embeds.Load() != 0 { t.Errorf("chat %d, embeddings %d", e.calls.Load(), embeds.Load()) } } func TestTheConversationAQuestionIsAskedInIsAllButItsLastMessage(t *testing.T) { body := func(last string, earlier ...string) []byte { var msgs []any for _, s := range earlier { msgs = append(msgs, map[string]any{"role": "user", "content": s}, map[string]any{"role": "assistant", "content": "ok"}) } msgs = append(msgs, map[string]any{"role": "user", "content": last}) b, _ := json.Marshal(map[string]any{"model": "m", "messages": msgs, "temperature": 0}) return b } g1, q1, ok1 := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hi")) g2, _, ok2 := similarGroup("p", "/v1/chat/completions", "m", body("b?", "hi")) g3, _, _ := similarGroup("p", "/v1/chat/completions", "m", body("a?", "hello")) g4, _, _ := similarGroup("q", "/v1/chat/completions", "m", body("a?", "hi")) if !ok1 || !ok2 || q1 != "a?" || g1 != g2 { t.Errorf("two questions after the same turns are in different conversations") } if g1 == g3 || g1 == g4 { t.Errorf("other turns or another project share a conversation") } tool, _ := json.Marshal(map[string]any{"model": "m", "messages": []any{ map[string]any{"role": "user", "content": "x"}, map[string]any{"role": "tool", "content": "42"}}}) if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", tool); ok { t.Error("a tool's result is taken for a question") } } func TestAQuestionAboutAPictureIsNotTakenForAnotherPicture(t *testing.T) { pic := func(url string) []byte { b, _ := json.Marshal(map[string]any{"model": "m", "temperature": 0, "messages": []any{map[string]any{ "role": "user", "content": []any{ map[string]any{"type": "text", "text": "what is in this picture?"}, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}}, }}}}) return b } if _, _, ok := similarGroup("p", "/v1/chat/completions", "m", pic("data:image/png;base64,AAAA")); ok { t.Error("a question with a picture in it is compared by its text alone") } }