Files
llmgw/internal/router/similar.go
T
Ilker ManapandClaude Opus 5.5 54c6b66cb9
CI / test (push) Successful in 9m48s
Fixes from review: the similar cache, restore, outside services, latency
- The similar-question cache embeds a question as its project would: a
  project kept in the cluster has external fallbacks refused, and an
  embedding model outside the cluster is not asked at all. The vectors are
  compared outside the cache's lock. The question's bound is in characters.
- The restorer puts values back in streamed tool calls' arguments (escaped
  for the JSON they are), completions' text and reasoning, each held across
  events on its own, and releases a held tail with the finish event. A
  whole answer is walked as JSON, so a value with a quote no longer breaks
  tool-call arguments.
- An outside guardrail service reads the user's turn as redacted, and a
  streamed answer once rather than at the finish chunk and again at the
  end. guardrails.md says it can stop a stream only under buffer_all.
- A replica's smoothed time to first token is forgotten after five minutes
  without an answer: one found slow and then given nothing was never tried
  again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:15:43 +01:00

165 lines
5.7 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"time"
"unicode/utf8"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Answers to similar questions, for a project that asks: a help desk asked
// "how do I reset my password" in forty wordings asks the model once. The
// last user message is embedded, and an answer kept for a question in the
// same conversation whose embedding reaches the project's similarity is
// given again. Everything else of the request — the system prompt, the
// earlier turns, tools, the sampling settings — must be the same, so a like
// question in another conversation is not taken for this one.
//
// Kept beside the exact answers, in this process's memory, for the reasons
// those are; an answer pushed out of the cache takes its question with it.
// similarityHeader says how alike a reused answer's question was.
const similarityHeader = "x-llmgw-cache-similarity"
// maxSimilarQuestion bounds the questions compared, in characters: a longer one is a
// document pasted in, where a changed figure or name leaves the embedding
// almost as it was. Those are reused only when they are the same.
const maxSimilarQuestion = 2000
// similarTimeout bounds asking for the embedding, which every miss waits on.
const similarTimeout = 2 * time.Second
// similarLookup is the answer to a question like this request's, and how
// alike. On a miss it leaves in req.cache where this request's answer is to
// be kept among the similar ones.
func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) {
if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" {
return nil, 0
}
group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody)
if !ok || question == "" || utf8.RuneCountInString(question) > maxSimilarQuestion {
return nil, 0
}
// A project whose requests stay in the cluster keeps its questions in
// it too: not embedded by a model outside it.
if _, m := r.resolve(r.cacheEmbedModel); req.inCluster && m.IsExternal() {
return nil, 0
}
vec, err := r.embedQuestion(ctx, question, req.inCluster)
if err != nil {
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
"model", r.cacheEmbedModel, "err", err)
return nil, 0
}
if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID {
return a, sim
}
req.cache.group, req.cache.vec = group, vec
return nil, 0
}
// similarGroup is the conversation a chat request asks its question in —
// the request with the last message's content left out, keyed like an
// answer — and the question. Not one when the last message is not the
// user's: a tool's result is not a question asked again in other words.
func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) {
m, ok := decodeRequest(body)
if !ok {
return "", "", false
}
msgs, _ := m["messages"].([]any)
if len(msgs) == 0 {
return "", "", false
}
last, _ := msgs[len(msgs)-1].(map[string]any)
if last == nil || last["role"] != "user" {
return "", "", false
}
// A picture, a sound or a file is part of the question, and the text
// beside it is not all of it: "what is in this picture?" asked of two
// pictures is two questions. Reused only when the same, exactly.
if parts, isList := last["content"].([]any); isList {
for _, p := range parts {
if part, _ := p.(map[string]any); part == nil || part["type"] != "text" {
return "", "", false
}
}
}
question = lastUserText(body)
last["content"] = ""
group, ok = keyOf(project, path+"#similar", model, m)
return group, question, ok
}
// embedQuestion has router.cache.embedding_model embed a question, on the
// gateway's own account as a judge is asked, into a vector of unit length.
func (r *Router) embedQuestion(ctx context.Context, text string, inCluster bool) ([]float32, error) {
if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" {
text = m.QueryPrefix + text
}
ctx, cancel := context.WithTimeout(ctx, similarTimeout)
defer cancel()
body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}})
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/v1/embeddings", bytes.NewReader(body))
if err != nil {
return nil, err
}
req, err := r.parse(hreq, "/v1/embeddings")
if err != nil {
return nil, err
}
// As the gateway's own, as a judge is asked; and kept in the cluster
// when the question's project is, fallbacks included.
req.internal, req.inCluster = true, inCluster
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
w := &bufferWriter{header: http.Header{}}
if res := r.dispatch(ctx, w, req); res.err != nil {
return nil, res.err
}
if w.status != 0 && w.status != http.StatusOK {
return nil, fmt.Errorf("%s answered %d", r.cacheEmbedModel, w.status)
}
out := w.buf.Bytes()
var res struct {
Data []struct {
Embedding []float32 `json:"embedding"`
} `json:"data"`
}
if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel)
}
return unit(res.Data[0].Embedding), nil
}
// unit is v scaled to length 1, so that cosine similarity is a dot product.
func unit(v []float32) []float32 {
var n float64
for _, x := range v {
n += float64(x) * float64(x)
}
if n == 0 {
return v
}
n = math.Sqrt(n)
out := make([]float32, len(v))
for i, x := range v {
out[i] = float32(float64(x) / n)
}
return out
}
func dot(a, b []float32) float64 {
var s float64
for i := range a {
s += float64(a[i]) * float64(b[i])
}
return s
}