Files
llmgw/internal/router/similar.go
T
Ilker ManapandClaude Opus 5.5 c09d5ad4ee
CI / test (push) Successful in 9m16s
Outside services read answers redacted and whole; a card's own lanes
From the second review.
- An output service reads the answer with the values restore put back
  taken out again (guard.Subject.Restored), as an input service reads the
  request redacted.
- With n above 1, the service reads the first choice once it is whole,
  not at whichever choice finished first.
- Restored values go into an answer asked for as JSON (response_format)
  escaped for it, as into tool-call arguments.
- The embedding failure is logged once a minute through an outage, not
  once.
- A card's own lane count and fastest generation come from the kernel
  (max_link_width, max_link_speed): the driver's width.max is what card
  and slot allow together, so an x16 card in an x8 slot read x8 of x8 and
  was never said.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 13:22:39 +01:00

168 lines
5.8 KiB
Go

package router
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"time"
"unicode/utf8"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Answers to similar questions, for a project that asks: a help desk asked
// "how do I reset my password" in forty wordings asks the model once. The
// last user message is embedded, and an answer kept for a question in the
// same conversation whose embedding reaches the project's similarity is
// given again. Everything else of the request — the system prompt, the
// earlier turns, tools, the sampling settings — must be the same, so a like
// question in another conversation is not taken for this one.
//
// Kept beside the exact answers, in this process's memory, for the reasons
// those are; an answer pushed out of the cache takes its question with it.
// similarityHeader says how alike a reused answer's question was.
const similarityHeader = "x-llmgw-cache-similarity"
// maxSimilarQuestion bounds the questions compared, in characters: a longer one is a
// document pasted in, where a changed figure or name leaves the embedding
// almost as it was. Those are reused only when they are the same.
const maxSimilarQuestion = 2000
// similarTimeout bounds asking for the embedding, which every miss waits on.
const similarTimeout = 2 * time.Second
// similarLookup is the answer to a question like this request's, and how
// alike. On a miss it leaves in req.cache where this request's answer is to
// be kept among the similar ones.
func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) {
if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" {
return nil, 0
}
group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody)
if !ok || question == "" || utf8.RuneCountInString(question) > maxSimilarQuestion {
return nil, 0
}
// A project whose requests stay in the cluster keeps its questions in
// it too: not embedded by a model outside it.
if _, m := r.resolve(r.cacheEmbedModel); req.inCluster && m.IsExternal() {
return nil, 0
}
vec, err := r.embedQuestion(ctx, question, req.inCluster)
if err != nil {
// Once a minute at most: down, the model fails every miss.
if r.embedWarned.Due(time.Now(), time.Minute) {
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
"model", r.cacheEmbedModel, "err", err)
}
return nil, 0
}
if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID {
return a, sim
}
req.cache.group, req.cache.vec = group, vec
return nil, 0
}
// similarGroup is the conversation a chat request asks its question in —
// the request with the last message's content left out, keyed like an
// answer — and the question. Not one when the last message is not the
// user's: a tool's result is not a question asked again in other words.
func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) {
m, ok := decodeRequest(body)
if !ok {
return "", "", false
}
msgs, _ := m["messages"].([]any)
if len(msgs) == 0 {
return "", "", false
}
last, _ := msgs[len(msgs)-1].(map[string]any)
if last == nil || last["role"] != "user" {
return "", "", false
}
// A picture, a sound or a file is part of the question, and the text
// beside it is not all of it: "what is in this picture?" asked of two
// pictures is two questions. Reused only when the same, exactly.
if parts, isList := last["content"].([]any); isList {
for _, p := range parts {
if part, _ := p.(map[string]any); part == nil || part["type"] != "text" {
return "", "", false
}
}
}
question = lastUserText(body)
last["content"] = ""
group, ok = keyOf(project, path+"#similar", model, m)
return group, question, ok
}
// embedQuestion has router.cache.embedding_model embed a question, on the
// gateway's own account as a judge is asked, into a vector of unit length.
func (r *Router) embedQuestion(ctx context.Context, text string, inCluster bool) ([]float32, error) {
if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" {
text = m.QueryPrefix + text
}
ctx, cancel := context.WithTimeout(ctx, similarTimeout)
defer cancel()
body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}})
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/v1/embeddings", bytes.NewReader(body))
if err != nil {
return nil, err
}
req, err := r.parse(hreq, "/v1/embeddings")
if err != nil {
return nil, err
}
// As the gateway's own, as a judge is asked; and kept in the cluster
// when the question's project is, fallbacks included.
req.internal, req.inCluster = true, inCluster
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
w := &bufferWriter{header: http.Header{}}
if res := r.dispatch(ctx, w, req); res.err != nil {
return nil, res.err
}
if w.status != 0 && w.status != http.StatusOK {
return nil, fmt.Errorf("%s answered %d", r.cacheEmbedModel, w.status)
}
out := w.buf.Bytes()
var res struct {
Data []struct {
Embedding []float32 `json:"embedding"`
} `json:"data"`
}
if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel)
}
return unit(res.Data[0].Embedding), nil
}
// unit is v scaled to length 1, so that cosine similarity is a dot product.
func unit(v []float32) []float32 {
var n float64
for _, x := range v {
n += float64(x) * float64(x)
}
if n == 0 {
return v
}
n = math.Sqrt(n)
out := make([]float32, len(v))
for i, x := range v {
out[i] = float32(float64(x) / n)
}
return out
}
func dot(a, b []float32) float64 {
var s float64
for i := range a {
s += float64(a[i]) * float64(b[i])
}
return s
}