CI / test (push) Successful in 8m52s
llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
2.2 KiB
Go
69 lines
2.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// AnswerCache is how a project reuses answers: for how long one is served
|
|
// again to the same request, and whether that is any request's answer or
|
|
// only one that would come out the same every time. Zero TTL reuses none.
|
|
//
|
|
// Only the setting lives in the store. The answers are held in each
|
|
// gateway's memory, so turning this on writes no text anywhere.
|
|
type AnswerCache struct {
|
|
TTL time.Duration
|
|
// Any reuses answers to requests sampled at a temperature above 0
|
|
// with no seed, which would otherwise each be asked again.
|
|
Any bool
|
|
// Similar is the cosine similarity a chat question must reach to one
|
|
// already answered, in the same conversation so far, to be given that
|
|
// answer; 0 answers only the same request again.
|
|
Similar float64
|
|
}
|
|
|
|
// SetAnswerCache turns reusing answers on for a project, or off with ttl 0.
|
|
// The TTL is kept in whole seconds, at least one.
|
|
func (s *DB) SetAnswerCache(ctx context.Context, projectID string, c AnswerCache) error {
|
|
var secs any
|
|
a, similar := 0, 0.0
|
|
if c.TTL > 0 {
|
|
secs = max(int64(c.TTL/time.Second), 1)
|
|
if c.Any {
|
|
a = 1
|
|
}
|
|
similar = c.Similar
|
|
}
|
|
if similar < 0 || similar > 1 {
|
|
return fmt.Errorf("store: similarity %v is not between 0 and 1", similar)
|
|
}
|
|
res, err := s.db.ExecContext(ctx,
|
|
`UPDATE projects SET cache_ttl_seconds = ?, cache_any = ?, cache_similar = ? WHERE id = ?`, secs, a, similar, projectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return fmt.Errorf("store: no project %q", projectID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AnswerCacheOf is a project's setting; zero when it reuses none.
|
|
func (s *DB) AnswerCacheOf(ctx context.Context, projectID string) (AnswerCache, error) {
|
|
var secs sql.NullInt64
|
|
var a int
|
|
var similar float64
|
|
err := s.db.QueryRowContext(ctx,
|
|
`SELECT cache_ttl_seconds, cache_any, cache_similar FROM projects WHERE id = ?`, projectID).Scan(&secs, &a, &similar)
|
|
if errors.Is(err, sql.ErrNoRows) || err == nil && (!secs.Valid || secs.Int64 <= 0) {
|
|
return AnswerCache{}, nil
|
|
}
|
|
if err != nil {
|
|
return AnswerCache{}, err
|
|
}
|
|
return AnswerCache{TTL: time.Duration(secs.Int64) * time.Second, Any: a == 1, Similar: similar}, nil
|
|
}
|