package router import ( "bytes" "context" "encoding/json" "fmt" "math" "net/http" "time" "unicode/utf8" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Answers to similar questions, for a project that asks: a help desk asked // "how do I reset my password" in forty wordings asks the model once. The // last user message is embedded, and an answer kept for a question in the // same conversation whose embedding reaches the project's similarity is // given again. Everything else of the request — the system prompt, the // earlier turns, tools, the sampling settings — must be the same, so a like // question in another conversation is not taken for this one. // // Kept beside the exact answers, in this process's memory, for the reasons // those are; an answer pushed out of the cache takes its question with it. // similarityHeader says how alike a reused answer's question was. const similarityHeader = "x-llmgw-cache-similarity" // maxSimilarQuestion bounds the questions compared, in characters: a longer one is a // document pasted in, where a changed figure or name leaves the embedding // almost as it was. Those are reused only when they are the same. const maxSimilarQuestion = 2000 // similarTimeout bounds asking for the embedding, which every miss waits on. const similarTimeout = 2 * time.Second // similarLookup is the answer to a question like this request's, and how // alike. On a miss it leaves in req.cache where this request's answer is to // be kept among the similar ones. func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) { if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" { return nil, 0 } group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody) if !ok || question == "" || utf8.RuneCountInString(question) > maxSimilarQuestion { return nil, 0 } // A project whose requests stay in the cluster keeps its questions in // it too: not embedded by a model outside it. if _, m := r.resolve(r.cacheEmbedModel); req.inCluster && m.IsExternal() { return nil, 0 } vec, err := r.embedQuestion(ctx, question, req.inCluster) if err != nil { // Once a minute at most: down, the model fails every miss. if r.embedWarned.Due(time.Now(), time.Minute) { r.log.Warn("a question could not be embedded for the cache; only the same request is reused", "model", r.cacheEmbedModel, "err", err) } return nil, 0 } if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID { return a, sim } req.cache.group, req.cache.vec = group, vec return nil, 0 } // similarGroup is the conversation a chat request asks its question in — // the request with the last message's content left out, keyed like an // answer — and the question. Not one when the last message is not the // user's: a tool's result is not a question asked again in other words. func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) { m, ok := decodeRequest(body) if !ok { return "", "", false } msgs, _ := m["messages"].([]any) if len(msgs) == 0 { return "", "", false } last, _ := msgs[len(msgs)-1].(map[string]any) if last == nil || last["role"] != "user" { return "", "", false } // A picture, a sound or a file is part of the question, and the text // beside it is not all of it: "what is in this picture?" asked of two // pictures is two questions. Reused only when the same, exactly. if parts, isList := last["content"].([]any); isList { for _, p := range parts { if part, _ := p.(map[string]any); part == nil || part["type"] != "text" { return "", "", false } } } question = lastUserText(body) last["content"] = "" group, ok = keyOf(project, path+"#similar", model, m) return group, question, ok } // embedQuestion has router.cache.embedding_model embed a question, on the // gateway's own account as a judge is asked, into a vector of unit length. func (r *Router) embedQuestion(ctx context.Context, text string, inCluster bool) ([]float32, error) { if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" { text = m.QueryPrefix + text } ctx, cancel := context.WithTimeout(ctx, similarTimeout) defer cancel() body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}}) hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/v1/embeddings", bytes.NewReader(body)) if err != nil { return nil, err } req, err := r.parse(hreq, "/v1/embeddings") if err != nil { return nil, err } // As the gateway's own, as a judge is asked; and kept in the cluster // when the question's project is, fallbacks included. req.internal, req.inCluster = true, inCluster req.RequestID = requestID("") req.Header = http.Header{"X-Request-Id": []string{req.RequestID}} w := &bufferWriter{header: http.Header{}} if res := r.dispatch(ctx, w, req); res.err != nil { return nil, res.err } if w.status != 0 && w.status != http.StatusOK { return nil, fmt.Errorf("%s answered %d", r.cacheEmbedModel, w.status) } out := w.buf.Bytes() var res struct { Data []struct { Embedding []float32 `json:"embedding"` } `json:"data"` } if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 { return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel) } return unit(res.Data[0].Embedding), nil } // unit is v scaled to length 1, so that cosine similarity is a dot product. func unit(v []float32) []float32 { var n float64 for _, x := range v { n += float64(x) * float64(x) } if n == 0 { return v } n = math.Sqrt(n) out := make([]float32, len(v)) for i, x := range v { out[i] = float32(float64(x) / n) } return out } func dot(a, b []float32) float64 { var s float64 for i := range a { s += float64(a[i]) * float64(b[i]) } return s }