llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ed0d1ec0ad
commit
3a3db8cd20
19 files changed
+514
-33
No files matched your search
@@ -72,6 +72,10 @@ type cacheState struct {
|
||||
// project is whose answer it is, and ttl how long it is reused.
|
||||
project string
|
||||
ttl time.Duration
|
||||
// group and vec place it among answers to similar questions, for a
|
||||
// project that reuses those; empty otherwise.
|
||||
group string
|
||||
vec []float32
|
||||
}
|
||||
|
||||
// cachedAnswer is one reusable answer.
|
||||
@@ -89,16 +93,23 @@ type cachedAnswer struct {
|
||||
// project's TTL shortens the life of what it already has.
|
||||
expires time.Time
|
||||
size int64
|
||||
// group is the conversation the question was asked in, everything of
|
||||
// the request but its last user message; vec is that message embedded,
|
||||
// of unit length. Empty for an answer reused only by the same request.
|
||||
group string
|
||||
vec []float32
|
||||
}
|
||||
|
||||
// answerCache is a least-recently-used cache of answers, bounded in bytes.
|
||||
type answerCache struct {
|
||||
maxBytes, maxEntry int64
|
||||
|
||||
mu sync.Mutex
|
||||
lru *list.List // of *cachedAnswer, most recently used at the front
|
||||
by map[string]*list.Element
|
||||
bytes int64
|
||||
mu sync.Mutex
|
||||
lru *list.List // of *cachedAnswer, most recently used at the front
|
||||
by map[string]*list.Element
|
||||
// groups are the answers to similar questions, by conversation.
|
||||
groups map[string]map[*list.Element]struct{}
|
||||
bytes int64
|
||||
// swept is when expired answers were last cleared out.
|
||||
swept time.Time
|
||||
// size reports bytes and answers held, after each change.
|
||||
@@ -117,7 +128,7 @@ func newAnswerCache(maxBytes, maxEntry int64, size func(int64, int)) *answerCach
|
||||
size = func(int64, int) {}
|
||||
}
|
||||
return &answerCache{maxBytes: maxBytes, maxEntry: maxEntry, lru: list.New(),
|
||||
by: map[string]*list.Element{}, size: size}
|
||||
by: map[string]*list.Element{}, groups: map[string]map[*list.Element]struct{}{}, size: size}
|
||||
}
|
||||
|
||||
// get is the answer kept under key, when it is younger than ttl and has
|
||||
@@ -144,7 +155,8 @@ func (c *answerCache) get(key string, ttl time.Duration, now time.Time) *cachedA
|
||||
func (c *answerCache) put(a *cachedAnswer, now time.Time) bool {
|
||||
// What the entry costs besides its body: the key twice, the names, and
|
||||
// the bookkeeping, roughly.
|
||||
a.size = int64(len(a.body) + 2*len(a.key) + len(a.project) + len(a.model) + len(a.path) + 160)
|
||||
a.size = int64(len(a.body) + 2*len(a.key) + len(a.project) + len(a.model) + len(a.path) + 160 +
|
||||
len(a.group) + 4*len(a.vec))
|
||||
if a.size > c.maxEntry {
|
||||
return false
|
||||
}
|
||||
@@ -168,7 +180,14 @@ func (c *answerCache) put(a *cachedAnswer, now time.Time) bool {
|
||||
for c.bytes+a.size > c.maxBytes && c.lru.Len() > 0 {
|
||||
c.remove(c.lru.Back())
|
||||
}
|
||||
c.by[a.key] = c.lru.PushFront(a)
|
||||
el := c.lru.PushFront(a)
|
||||
c.by[a.key] = el
|
||||
if a.group != "" {
|
||||
if c.groups[a.group] == nil {
|
||||
c.groups[a.group] = map[*list.Element]struct{}{}
|
||||
}
|
||||
c.groups[a.group][el] = struct{}{}
|
||||
}
|
||||
c.bytes += a.size
|
||||
c.size(c.bytes, len(c.by))
|
||||
return true
|
||||
@@ -178,9 +197,38 @@ func (c *answerCache) remove(el *list.Element) {
|
||||
a := el.Value.(*cachedAnswer)
|
||||
c.lru.Remove(el)
|
||||
delete(c.by, a.key)
|
||||
if g := c.groups[a.group]; g != nil {
|
||||
delete(g, el)
|
||||
if len(g) == 0 {
|
||||
delete(c.groups, a.group)
|
||||
}
|
||||
}
|
||||
c.bytes -= a.size
|
||||
}
|
||||
|
||||
// nearest is the answer in group whose question is most like vec, when
|
||||
// that reaches least and the answer is younger than ttl, and how alike.
|
||||
func (c *answerCache) nearest(group string, vec []float32, least float64, ttl time.Duration, now time.Time) (*cachedAnswer, float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var best *list.Element
|
||||
bestSim := least
|
||||
for el := range c.groups[group] {
|
||||
a := el.Value.(*cachedAnswer)
|
||||
if now.After(a.expires) || now.Sub(a.at) >= ttl || len(a.vec) != len(vec) {
|
||||
continue
|
||||
}
|
||||
if s := dot(a.vec, vec); s >= bestSim {
|
||||
best, bestSim = el, s
|
||||
}
|
||||
}
|
||||
if best == nil {
|
||||
return nil, 0
|
||||
}
|
||||
c.lru.MoveToFront(best)
|
||||
return best.Value.(*cachedAnswer), bestSim
|
||||
}
|
||||
|
||||
// answerCacheOf is a project's setting, read at most every requestLogTTL
|
||||
// with its other settings. Unreadable, it reuses nothing: asking the model
|
||||
// again is the safe way to be wrong.
|
||||
@@ -238,6 +286,15 @@ func (r *Router) cacheLookup(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
}
|
||||
result = "miss"
|
||||
req.cache = cacheState{key: key, project: req.ProjectID, ttl: set.TTL}
|
||||
if set.Similar > 0 {
|
||||
if a, sim := r.similarLookup(ctx, req, model, set); a != nil {
|
||||
result = "similar"
|
||||
w.Header().Set("Age", strconv.Itoa(int(now.Sub(a.at).Seconds())))
|
||||
w.Header().Set(similarityHeader, strconv.FormatFloat(sim, 'f', 3, 64))
|
||||
req.cache = cacheState{}
|
||||
return a
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -288,15 +345,29 @@ var answerFields = []string{"model", "stream", "stream_options", "user", "metada
|
||||
// the answer taken out and its keys sorted. The project is spelled out
|
||||
// rather than only hashed, so no answer is ever looked up across projects.
|
||||
func answerKey(project, path, model string, body []byte) (string, bool) {
|
||||
m, ok := decodeRequest(body)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return keyOf(project, path, model, m)
|
||||
}
|
||||
|
||||
// decodeRequest is a request as a map, its numbers kept as written and
|
||||
// what does not change the answer taken out.
|
||||
func decodeRequest(body []byte) (map[string]any, bool) {
|
||||
d := json.NewDecoder(bytes.NewReader(body))
|
||||
d.UseNumber()
|
||||
var m map[string]any
|
||||
if err := d.Decode(&m); err != nil || m == nil {
|
||||
return "", false
|
||||
return nil, false
|
||||
}
|
||||
for _, f := range answerFields {
|
||||
delete(m, f)
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
func keyOf(project, path, model string, m map[string]any) (string, bool) {
|
||||
// encoding/json writes map keys sorted, at every depth.
|
||||
norm, err := json.Marshal(canonicalNumbers(m))
|
||||
if err != nil {
|
||||
@@ -420,7 +491,8 @@ func (r *Router) cacheKeep(req *parsedRequest, res dispatchResult, status int, c
|
||||
}
|
||||
now := time.Now()
|
||||
r.answers.put(&cachedAnswer{key: req.cache.key, project: req.cache.project, model: model,
|
||||
path: req.Path, body: body, at: now, expires: now.Add(req.cache.ttl)}, now)
|
||||
path: req.Path, body: body, at: now, expires: now.Add(req.cache.ttl),
|
||||
group: req.cache.group, vec: req.cache.vec}, now)
|
||||
}
|
||||
|
||||
// serveCached answers a request from the cache, as the model would have:
|
||||
|
||||
Reference in new issue
Block a user