llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
130 lines
4.4 KiB
Go
130 lines
4.4 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"time"
|
|
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Answers to similar questions, for a project that asks: a help desk asked
|
|
// "how do I reset my password" in forty wordings asks the model once. The
|
|
// last user message is embedded, and an answer kept for a question in the
|
|
// same conversation whose embedding reaches the project's similarity is
|
|
// given again. Everything else of the request — the system prompt, the
|
|
// earlier turns, tools, the sampling settings — must be the same, so a like
|
|
// question in another conversation is not taken for this one.
|
|
//
|
|
// Kept beside the exact answers, in this process's memory, for the reasons
|
|
// those are; an answer pushed out of the cache takes its question with it.
|
|
|
|
// similarityHeader says how alike a reused answer's question was.
|
|
const similarityHeader = "x-llmgw-cache-similarity"
|
|
|
|
// maxSimilarQuestion bounds the questions compared: a longer one is a
|
|
// document pasted in, where a changed figure or name leaves the embedding
|
|
// almost as it was. Those are reused only when they are the same.
|
|
const maxSimilarQuestion = 2000
|
|
|
|
// similarTimeout bounds asking for the embedding, which every miss waits on.
|
|
const similarTimeout = 2 * time.Second
|
|
|
|
// similarLookup is the answer to a question like this request's, and how
|
|
// alike. On a miss it leaves in req.cache where this request's answer is to
|
|
// be kept among the similar ones.
|
|
func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) {
|
|
if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" {
|
|
return nil, 0
|
|
}
|
|
group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody)
|
|
if !ok || question == "" || len(question) > maxSimilarQuestion {
|
|
return nil, 0
|
|
}
|
|
vec, err := r.embedQuestion(ctx, question)
|
|
if err != nil {
|
|
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
|
|
"model", r.cacheEmbedModel, "err", err)
|
|
return nil, 0
|
|
}
|
|
if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID {
|
|
return a, sim
|
|
}
|
|
req.cache.group, req.cache.vec = group, vec
|
|
return nil, 0
|
|
}
|
|
|
|
// similarGroup is the conversation a chat request asks its question in —
|
|
// the request with the last message's content left out, keyed like an
|
|
// answer — and the question. Not one when the last message is not the
|
|
// user's: a tool's result is not a question asked again in other words.
|
|
func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) {
|
|
m, ok := decodeRequest(body)
|
|
if !ok {
|
|
return "", "", false
|
|
}
|
|
msgs, _ := m["messages"].([]any)
|
|
if len(msgs) == 0 {
|
|
return "", "", false
|
|
}
|
|
last, _ := msgs[len(msgs)-1].(map[string]any)
|
|
if last == nil || last["role"] != "user" {
|
|
return "", "", false
|
|
}
|
|
question = lastUserText(body)
|
|
last["content"] = ""
|
|
group, ok = keyOf(project, path+"#similar", model, m)
|
|
return group, question, ok
|
|
}
|
|
|
|
// embedQuestion has router.cache.embedding_model embed a question, on the
|
|
// gateway's own account as a judge is asked, into a vector of unit length.
|
|
func (r *Router) embedQuestion(ctx context.Context, text string) ([]float32, error) {
|
|
if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" {
|
|
text = m.QueryPrefix + text
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, similarTimeout)
|
|
defer cancel()
|
|
body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}})
|
|
out, err := r.judgeOnce(ctx, "/v1/embeddings", body, nil, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var res struct {
|
|
Data []struct {
|
|
Embedding []float32 `json:"embedding"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
|
|
return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel)
|
|
}
|
|
return unit(res.Data[0].Embedding), nil
|
|
}
|
|
|
|
// unit is v scaled to length 1, so that cosine similarity is a dot product.
|
|
func unit(v []float32) []float32 {
|
|
var n float64
|
|
for _, x := range v {
|
|
n += float64(x) * float64(x)
|
|
}
|
|
if n == 0 {
|
|
return v
|
|
}
|
|
n = math.Sqrt(n)
|
|
out := make([]float32, len(v))
|
|
for i, x := range v {
|
|
out[i] = float32(float64(x) / n)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func dot(a, b []float32) float64 {
|
|
var s float64
|
|
for i := range a {
|
|
s += float64(a[i]) * float64(b[i])
|
|
}
|
|
return s
|
|
}
|