Files
llmgw/internal/router/similar.go
T
Ilker ManapandClaude Opus 5.5 3a3db8cd20
CI / test (push) Successful in 8m52s
Answers to similar questions, not only the same one
llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 17:47:11 +01:00

130 lines
4.4 KiB
Go

package router
import (
"context"
"encoding/json"
"fmt"
"math"
"time"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Answers to similar questions, for a project that asks: a help desk asked
// "how do I reset my password" in forty wordings asks the model once. The
// last user message is embedded, and an answer kept for a question in the
// same conversation whose embedding reaches the project's similarity is
// given again. Everything else of the request — the system prompt, the
// earlier turns, tools, the sampling settings — must be the same, so a like
// question in another conversation is not taken for this one.
//
// Kept beside the exact answers, in this process's memory, for the reasons
// those are; an answer pushed out of the cache takes its question with it.
// similarityHeader says how alike a reused answer's question was.
const similarityHeader = "x-llmgw-cache-similarity"
// maxSimilarQuestion bounds the questions compared: a longer one is a
// document pasted in, where a changed figure or name leaves the embedding
// almost as it was. Those are reused only when they are the same.
const maxSimilarQuestion = 2000
// similarTimeout bounds asking for the embedding, which every miss waits on.
const similarTimeout = 2 * time.Second
// similarLookup is the answer to a question like this request's, and how
// alike. On a miss it leaves in req.cache where this request's answer is to
// be kept among the similar ones.
func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) {
if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" {
return nil, 0
}
group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody)
if !ok || question == "" || len(question) > maxSimilarQuestion {
return nil, 0
}
vec, err := r.embedQuestion(ctx, question)
if err != nil {
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
"model", r.cacheEmbedModel, "err", err)
return nil, 0
}
if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID {
return a, sim
}
req.cache.group, req.cache.vec = group, vec
return nil, 0
}
// similarGroup is the conversation a chat request asks its question in —
// the request with the last message's content left out, keyed like an
// answer — and the question. Not one when the last message is not the
// user's: a tool's result is not a question asked again in other words.
func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) {
m, ok := decodeRequest(body)
if !ok {
return "", "", false
}
msgs, _ := m["messages"].([]any)
if len(msgs) == 0 {
return "", "", false
}
last, _ := msgs[len(msgs)-1].(map[string]any)
if last == nil || last["role"] != "user" {
return "", "", false
}
question = lastUserText(body)
last["content"] = ""
group, ok = keyOf(project, path+"#similar", model, m)
return group, question, ok
}
// embedQuestion has router.cache.embedding_model embed a question, on the
// gateway's own account as a judge is asked, into a vector of unit length.
func (r *Router) embedQuestion(ctx context.Context, text string) ([]float32, error) {
if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" {
text = m.QueryPrefix + text
}
ctx, cancel := context.WithTimeout(ctx, similarTimeout)
defer cancel()
body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}})
out, err := r.judgeOnce(ctx, "/v1/embeddings", body, nil, nil)
if err != nil {
return nil, err
}
var res struct {
Data []struct {
Embedding []float32 `json:"embedding"`
} `json:"data"`
}
if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel)
}
return unit(res.Data[0].Embedding), nil
}
// unit is v scaled to length 1, so that cosine similarity is a dot product.
func unit(v []float32) []float32 {
var n float64
for _, x := range v {
n += float64(x) * float64(x)
}
if n == 0 {
return v
}
n = math.Sqrt(n)
out := make([]float32, len(v))
for i, x := range v {
out[i] = float32(float64(x) / n)
}
return out
}
func dot(a, b []float32) float64 {
var s float64
for i := range a {
s += float64(a[i]) * float64(b[i])
}
return s
}