Answers to similar questions, not only the same one
CI / test (push) Successful in 8m52s

llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-25 17:47:11 +01:00
1 parent ed0d1ec0ad
commit 3a3db8cd20
19 files changed
+514 -33

No files matched your search

+81 -9
View File
@@ -72,6 +72,10 @@ type cacheState struct {
// project is whose answer it is, and ttl how long it is reused.
project string
ttl time.Duration
// group and vec place it among answers to similar questions, for a
// project that reuses those; empty otherwise.
group string
vec []float32
}
// cachedAnswer is one reusable answer.
@@ -89,16 +93,23 @@ type cachedAnswer struct {
// project's TTL shortens the life of what it already has.
expires time.Time
size int64
// group is the conversation the question was asked in, everything of
// the request but its last user message; vec is that message embedded,
// of unit length. Empty for an answer reused only by the same request.
group string
vec []float32
}
// answerCache is a least-recently-used cache of answers, bounded in bytes.
type answerCache struct {
maxBytes, maxEntry int64
mu sync.Mutex
lru *list.List // of *cachedAnswer, most recently used at the front
by map[string]*list.Element
bytes int64
mu sync.Mutex
lru *list.List // of *cachedAnswer, most recently used at the front
by map[string]*list.Element
// groups are the answers to similar questions, by conversation.
groups map[string]map[*list.Element]struct{}
bytes int64
// swept is when expired answers were last cleared out.
swept time.Time
// size reports bytes and answers held, after each change.
@@ -117,7 +128,7 @@ func newAnswerCache(maxBytes, maxEntry int64, size func(int64, int)) *answerCach
size = func(int64, int) {}
}
return &answerCache{maxBytes: maxBytes, maxEntry: maxEntry, lru: list.New(),
by: map[string]*list.Element{}, size: size}
by: map[string]*list.Element{}, groups: map[string]map[*list.Element]struct{}{}, size: size}
}
// get is the answer kept under key, when it is younger than ttl and has
@@ -144,7 +155,8 @@ func (c *answerCache) get(key string, ttl time.Duration, now time.Time) *cachedA
func (c *answerCache) put(a *cachedAnswer, now time.Time) bool {
// What the entry costs besides its body: the key twice, the names, and
// the bookkeeping, roughly.
a.size = int64(len(a.body) + 2*len(a.key) + len(a.project) + len(a.model) + len(a.path) + 160)
a.size = int64(len(a.body) + 2*len(a.key) + len(a.project) + len(a.model) + len(a.path) + 160 +
len(a.group) + 4*len(a.vec))
if a.size > c.maxEntry {
return false
}
@@ -168,7 +180,14 @@ func (c *answerCache) put(a *cachedAnswer, now time.Time) bool {
for c.bytes+a.size > c.maxBytes && c.lru.Len() > 0 {
c.remove(c.lru.Back())
}
c.by[a.key] = c.lru.PushFront(a)
el := c.lru.PushFront(a)
c.by[a.key] = el
if a.group != "" {
if c.groups[a.group] == nil {
c.groups[a.group] = map[*list.Element]struct{}{}
}
c.groups[a.group][el] = struct{}{}
}
c.bytes += a.size
c.size(c.bytes, len(c.by))
return true
@@ -178,9 +197,38 @@ func (c *answerCache) remove(el *list.Element) {
a := el.Value.(*cachedAnswer)
c.lru.Remove(el)
delete(c.by, a.key)
if g := c.groups[a.group]; g != nil {
delete(g, el)
if len(g) == 0 {
delete(c.groups, a.group)
}
}
c.bytes -= a.size
}
// nearest is the answer in group whose question is most like vec, when
// that reaches least and the answer is younger than ttl, and how alike.
func (c *answerCache) nearest(group string, vec []float32, least float64, ttl time.Duration, now time.Time) (*cachedAnswer, float64) {
c.mu.Lock()
defer c.mu.Unlock()
var best *list.Element
bestSim := least
for el := range c.groups[group] {
a := el.Value.(*cachedAnswer)
if now.After(a.expires) || now.Sub(a.at) >= ttl || len(a.vec) != len(vec) {
continue
}
if s := dot(a.vec, vec); s >= bestSim {
best, bestSim = el, s
}
}
if best == nil {
return nil, 0
}
c.lru.MoveToFront(best)
return best.Value.(*cachedAnswer), bestSim
}
// answerCacheOf is a project's setting, read at most every requestLogTTL
// with its other settings. Unreadable, it reuses nothing: asking the model
// again is the safe way to be wrong.
@@ -238,6 +286,15 @@ func (r *Router) cacheLookup(ctx context.Context, w http.ResponseWriter, req *pa
}
result = "miss"
req.cache = cacheState{key: key, project: req.ProjectID, ttl: set.TTL}
if set.Similar > 0 {
if a, sim := r.similarLookup(ctx, req, model, set); a != nil {
result = "similar"
w.Header().Set("Age", strconv.Itoa(int(now.Sub(a.at).Seconds())))
w.Header().Set(similarityHeader, strconv.FormatFloat(sim, 'f', 3, 64))
req.cache = cacheState{}
return a
}
}
return nil
}
@@ -288,15 +345,29 @@ var answerFields = []string{"model", "stream", "stream_options", "user", "metada
// the answer taken out and its keys sorted. The project is spelled out
// rather than only hashed, so no answer is ever looked up across projects.
func answerKey(project, path, model string, body []byte) (string, bool) {
m, ok := decodeRequest(body)
if !ok {
return "", false
}
return keyOf(project, path, model, m)
}
// decodeRequest is a request as a map, its numbers kept as written and
// what does not change the answer taken out.
func decodeRequest(body []byte) (map[string]any, bool) {
d := json.NewDecoder(bytes.NewReader(body))
d.UseNumber()
var m map[string]any
if err := d.Decode(&m); err != nil || m == nil {
return "", false
return nil, false
}
for _, f := range answerFields {
delete(m, f)
}
return m, true
}
func keyOf(project, path, model string, m map[string]any) (string, bool) {
// encoding/json writes map keys sorted, at every depth.
norm, err := json.Marshal(canonicalNumbers(m))
if err != nil {
@@ -420,7 +491,8 @@ func (r *Router) cacheKeep(req *parsedRequest, res dispatchResult, status int, c
}
now := time.Now()
r.answers.put(&cachedAnswer{key: req.cache.key, project: req.cache.project, model: model,
path: req.Path, body: body, at: now, expires: now.Add(req.cache.ttl)}, now)
path: req.Path, body: body, at: now, expires: now.Add(req.cache.ttl),
group: req.cache.group, vec: req.cache.vec}, now)
}
// serveCached answers a request from the cache, as the model would have: