Files
llmgw/internal/store/answercache_test.go
T
Ilker ManapandClaude Opus 5.5 3a3db8cd20
CI / test (push) Successful in 8m52s
Answers to similar questions, not only the same one
llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 17:47:11 +01:00

67 lines
2.2 KiB
Go

package store
import (
"testing"
"time"
)
// answerCacheRoundTrip is the same on both backends: a project's setting
// and a hit's mark on its usage record.
func answerCacheRoundTrip(t *testing.T, s *DB) {
t.Helper()
ctx := t.Context()
proj := tree(t, s)
_, k := mustKey(t, s, "k1", proj)
if c, err := s.AnswerCacheOf(ctx, proj); err != nil || c.TTL != 0 || c.Any {
t.Fatalf("a new project reuses answers: %+v %v", c, err)
}
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: 90 * time.Minute, Any: true}); err != nil {
t.Fatal(err)
}
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 90*time.Minute || !c.Any {
t.Errorf("set: %+v", c)
}
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: 0, Any: true}); err != nil {
t.Fatal(err)
}
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 0 || c.Any {
t.Errorf("off: %+v", c)
}
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: time.Hour, Similar: 0.95}); err != nil {
t.Fatal(err)
}
if c, _ := s.AnswerCacheOf(ctx, proj); c.Similar != 0.95 {
t.Errorf("similar: %+v", c)
}
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: time.Hour, Similar: 1.5}); err == nil {
t.Error("a similarity above 1 was set")
}
if err := s.SetAnswerCache(ctx, "no-such-project", AnswerCache{TTL: time.Hour, Any: false}); err == nil {
t.Error("a project that does not exist was set")
}
now := time.Now()
if err := s.WriteUsage(ctx, []Usage{
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", Status: 200, RequestID: "hit", CacheHit: true},
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", Status: 200, RequestID: "miss",
PromptTokens: 3, CompletionTokens: 4, CostMicros: 10},
}); err != nil {
t.Fatal(err)
}
if u, ok, err := s.UsageByRequest(ctx, "hit"); err != nil || !ok || !u.CacheHit {
t.Errorf("a hit: %+v %v", u, err)
}
if u, ok, err := s.UsageByRequest(ctx, "miss"); err != nil || !ok || u.CacheHit {
t.Errorf("a miss: %+v %v", u, err)
}
}
func TestAProjectsReuseOfAnswersIsKeptAndAHitIsMarked(t *testing.T) {
answerCacheRoundTrip(t, capacityStore(t))
}
func TestPostgresKeepsAProjectsReuseOfAnswers(t *testing.T) {
answerCacheRoundTrip(t, openPG(t))
}