Answers to similar questions, not only the same one

llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-25 17:47:11 +01:00
1 parent ed0d1ec0ad
commit 3a3db8cd20
19 files changed
+514 -33

No files matched your search

+3 -3
View File
@@ -27,7 +27,7 @@ import (
func cachingRouter(t *testing.T, engineURL string, anyAnswer bool) (*Router, string) {
t.Helper()
rt, secret := pricedRouter(t, engineURL)
if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), time.Hour, anyAnswer); err != nil {
if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), store.AnswerCache{TTL: time.Hour, Any: anyAnswer}); err != nil {
t.Fatal(err)
}
return rt, secret
@@ -135,7 +135,7 @@ func TestAProjectThatReusesAnswersAnswersTheSameRequestWithoutTheModel(t *testin
}
// Turned off, nothing is reused, and nothing is said about it.
_ = rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), 0, false)
_ = rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), store.AnswerCache{})
rt.reqLogs.by = nil
calls := e.calls.Load()
if rec := post(t, h, ask(0, "what is 2+2"), auth("off")); rec.Header().Get(cacheHeader) != "" || e.calls.Load() != calls+1 {
@@ -164,7 +164,7 @@ func TestKeysOfOneProjectNeverSeeAnotherProjectsAnswers(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_ = rt.store.SetAnswerCache(t.Context(), other, time.Hour, true)
_ = rt.store.SetAnswerCache(t.Context(), other, store.AnswerCache{TTL: time.Hour, Any: true})
secret2 := store.KeyPrefix + strings.Repeat("b", 43)
sum := sha256.Sum256([]byte(secret2))
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",