Ilker ManapandClaude Opus 5.5
44f3a9e31d
llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on the console's project menu. The same request — project, endpoint, canonical model and the body without stream, stream_options, user, metadata, store and service_tier, keys sorted — is answered from the gateway's memory for the TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank, score) unless --any; only a whole 200 from the model asked for, not a fallback's, a canary's, or one a guardrail changed. Looked up after the limits and the input guardrails; the output policy judges a reused answer; no_store and stored conversations bypass it. Streamed answers are kept put back together and streamed back as chunks. x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost, marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total, llmgw_cache_bytes, llmgw_cache_entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
58 lines
1.7 KiB
Go
58 lines
1.7 KiB
Go
package store
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// answerCacheRoundTrip is the same on both backends: a project's setting
|
|
// and a hit's mark on its usage record.
|
|
func answerCacheRoundTrip(t *testing.T, s *DB) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
proj := tree(t, s)
|
|
_, k := mustKey(t, s, "k1", proj)
|
|
|
|
if c, err := s.AnswerCacheOf(ctx, proj); err != nil || c.TTL != 0 || c.Any {
|
|
t.Fatalf("a new project reuses answers: %+v %v", c, err)
|
|
}
|
|
if err := s.SetAnswerCache(ctx, proj, 90*time.Minute, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 90*time.Minute || !c.Any {
|
|
t.Errorf("set: %+v", c)
|
|
}
|
|
if err := s.SetAnswerCache(ctx, proj, 0, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 0 || c.Any {
|
|
t.Errorf("off: %+v", c)
|
|
}
|
|
if err := s.SetAnswerCache(ctx, "no-such-project", time.Hour, false); err == nil {
|
|
t.Error("a project that does not exist was set")
|
|
}
|
|
|
|
now := time.Now()
|
|
if err := s.WriteUsage(ctx, []Usage{
|
|
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", Status: 200, RequestID: "hit", CacheHit: true},
|
|
{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", Status: 200, RequestID: "miss",
|
|
PromptTokens: 3, CompletionTokens: 4, CostMicros: 10},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if u, ok, err := s.UsageByRequest(ctx, "hit"); err != nil || !ok || !u.CacheHit {
|
|
t.Errorf("a hit: %+v %v", u, err)
|
|
}
|
|
if u, ok, err := s.UsageByRequest(ctx, "miss"); err != nil || !ok || u.CacheHit {
|
|
t.Errorf("a miss: %+v %v", u, err)
|
|
}
|
|
}
|
|
|
|
func TestAProjectsReuseOfAnswersIsKeptAndAHitIsMarked(t *testing.T) {
|
|
answerCacheRoundTrip(t, capacityStore(t))
|
|
}
|
|
|
|
func TestPostgresKeepsAProjectsReuseOfAnswers(t *testing.T) {
|
|
answerCacheRoundTrip(t, openPG(t))
|
|
}
|