llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ed0d1ec0ad
commit
3a3db8cd20
19 files changed
+514
-33
No files matched your search
@@ -19,21 +19,29 @@ type AnswerCache struct {
|
||||
// Any reuses answers to requests sampled at a temperature above 0
|
||||
// with no seed, which would otherwise each be asked again.
|
||||
Any bool
|
||||
// Similar is the cosine similarity a chat question must reach to one
|
||||
// already answered, in the same conversation so far, to be given that
|
||||
// answer; 0 answers only the same request again.
|
||||
Similar float64
|
||||
}
|
||||
|
||||
// SetAnswerCache turns reusing answers on for a project, or off with ttl 0.
|
||||
// The TTL is kept in whole seconds, at least one.
|
||||
func (s *DB) SetAnswerCache(ctx context.Context, projectID string, ttl time.Duration, anyAnswer bool) error {
|
||||
func (s *DB) SetAnswerCache(ctx context.Context, projectID string, c AnswerCache) error {
|
||||
var secs any
|
||||
a := 0
|
||||
if ttl > 0 {
|
||||
secs = max(int64(ttl/time.Second), 1)
|
||||
if anyAnswer {
|
||||
a, similar := 0, 0.0
|
||||
if c.TTL > 0 {
|
||||
secs = max(int64(c.TTL/time.Second), 1)
|
||||
if c.Any {
|
||||
a = 1
|
||||
}
|
||||
similar = c.Similar
|
||||
}
|
||||
if similar < 0 || similar > 1 {
|
||||
return fmt.Errorf("store: similarity %v is not between 0 and 1", similar)
|
||||
}
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE projects SET cache_ttl_seconds = ?, cache_any = ? WHERE id = ?`, secs, a, projectID)
|
||||
`UPDATE projects SET cache_ttl_seconds = ?, cache_any = ?, cache_similar = ? WHERE id = ?`, secs, a, similar, projectID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -47,13 +55,14 @@ func (s *DB) SetAnswerCache(ctx context.Context, projectID string, ttl time.Dura
|
||||
func (s *DB) AnswerCacheOf(ctx context.Context, projectID string) (AnswerCache, error) {
|
||||
var secs sql.NullInt64
|
||||
var a int
|
||||
var similar float64
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT cache_ttl_seconds, cache_any FROM projects WHERE id = ?`, projectID).Scan(&secs, &a)
|
||||
`SELECT cache_ttl_seconds, cache_any, cache_similar FROM projects WHERE id = ?`, projectID).Scan(&secs, &a, &similar)
|
||||
if errors.Is(err, sql.ErrNoRows) || err == nil && (!secs.Valid || secs.Int64 <= 0) {
|
||||
return AnswerCache{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return AnswerCache{}, err
|
||||
}
|
||||
return AnswerCache{TTL: time.Duration(secs.Int64) * time.Second, Any: a == 1}, nil
|
||||
return AnswerCache{TTL: time.Duration(secs.Int64) * time.Second, Any: a == 1, Similar: similar}, nil
|
||||
}
|
||||
@@ -16,19 +16,28 @@ func answerCacheRoundTrip(t *testing.T, s *DB) {
|
||||
if c, err := s.AnswerCacheOf(ctx, proj); err != nil || c.TTL != 0 || c.Any {
|
||||
t.Fatalf("a new project reuses answers: %+v %v", c, err)
|
||||
}
|
||||
if err := s.SetAnswerCache(ctx, proj, 90*time.Minute, true); err != nil {
|
||||
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: 90 * time.Minute, Any: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 90*time.Minute || !c.Any {
|
||||
t.Errorf("set: %+v", c)
|
||||
}
|
||||
if err := s.SetAnswerCache(ctx, proj, 0, true); err != nil {
|
||||
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: 0, Any: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _ := s.AnswerCacheOf(ctx, proj); c.TTL != 0 || c.Any {
|
||||
t.Errorf("off: %+v", c)
|
||||
}
|
||||
if err := s.SetAnswerCache(ctx, "no-such-project", time.Hour, false); err == nil {
|
||||
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: time.Hour, Similar: 0.95}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _ := s.AnswerCacheOf(ctx, proj); c.Similar != 0.95 {
|
||||
t.Errorf("similar: %+v", c)
|
||||
}
|
||||
if err := s.SetAnswerCache(ctx, proj, AnswerCache{TTL: time.Hour, Similar: 1.5}); err == nil {
|
||||
t.Error("a similarity above 1 was set")
|
||||
}
|
||||
if err := s.SetAnswerCache(ctx, "no-such-project", AnswerCache{TTL: time.Hour, Any: false}); err == nil {
|
||||
t.Error("a project that does not exist was set")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- Answers to similar questions, for a project that asks: the cosine
|
||||
-- similarity a question must reach to another's to be given its answer.
|
||||
-- 0 is off, and only the same request is answered again. The questions and
|
||||
-- answers are held in each gateway's memory, never here.
|
||||
ALTER TABLE projects ADD COLUMN cache_similar DOUBLE PRECISION NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Answers to similar questions, for a project that asks: the cosine
|
||||
-- similarity a question must reach to another's to be given its answer.
|
||||
-- 0 is off, and only the same request is answered again. The questions and
|
||||
-- answers are held in each gateway's memory, never here.
|
||||
ALTER TABLE projects ADD COLUMN cache_similar REAL NOT NULL DEFAULT 0;
|
||||
Reference in new issue
Block a user