From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
168 lines
5.8 KiB
Go
168 lines
5.8 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Answers to similar questions, for a project that asks: a help desk asked
|
|
// "how do I reset my password" in forty wordings asks the model once. The
|
|
// last user message is embedded, and an answer kept for a question in the
|
|
// same conversation whose embedding reaches the project's similarity is
|
|
// given again. Everything else of the request — the system prompt, the
|
|
// earlier turns, tools, the sampling settings — must be the same, so a like
|
|
// question in another conversation is not taken for this one.
|
|
//
|
|
// Kept beside the exact answers, in this process's memory, for the reasons
|
|
// those are; an answer pushed out of the cache takes its question with it.
|
|
|
|
// similarityHeader says how alike a reused answer's question was.
|
|
const similarityHeader = "x-llmgw-cache-similarity"
|
|
|
|
// maxSimilarQuestion bounds the questions compared, in characters: a longer one is a
|
|
// document pasted in, where a changed figure or name leaves the embedding
|
|
// almost as it was. Those are reused only when they are the same.
|
|
const maxSimilarQuestion = 2000
|
|
|
|
// similarTimeout bounds asking for the embedding, which every miss waits on.
|
|
const similarTimeout = 2 * time.Second
|
|
|
|
// similarLookup is the answer to a question like this request's, and how
|
|
// alike. On a miss it leaves in req.cache where this request's answer is to
|
|
// be kept among the similar ones.
|
|
func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model string, set storepkg.AnswerCache) (*cachedAnswer, float64) {
|
|
if r.cacheEmbedModel == "" || req.Path != "/v1/chat/completions" {
|
|
return nil, 0
|
|
}
|
|
group, question, ok := similarGroup(req.ProjectID, req.Path, model, req.UpstreamBody)
|
|
if !ok || question == "" || utf8.RuneCountInString(question) > maxSimilarQuestion {
|
|
return nil, 0
|
|
}
|
|
// A project whose requests stay in the cluster keeps its questions in
|
|
// it too: not embedded by a model outside it.
|
|
if _, m := r.resolve(r.cacheEmbedModel); req.inCluster && m.IsExternal() {
|
|
return nil, 0
|
|
}
|
|
vec, err := r.embedQuestion(ctx, question, req.inCluster)
|
|
if err != nil {
|
|
// Once a minute at most: down, the model fails every miss.
|
|
if r.embedWarned.Due(time.Now(), time.Minute) {
|
|
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
|
|
"model", r.cacheEmbedModel, "err", err)
|
|
}
|
|
return nil, 0
|
|
}
|
|
if a, sim := r.answers.nearest(group, vec, set.Similar, set.TTL, time.Now()); a != nil && a.project == req.ProjectID {
|
|
return a, sim
|
|
}
|
|
req.cache.group, req.cache.vec = group, vec
|
|
return nil, 0
|
|
}
|
|
|
|
// similarGroup is the conversation a chat request asks its question in —
|
|
// the request with the last message's content left out, keyed like an
|
|
// answer — and the question. Not one when the last message is not the
|
|
// user's: a tool's result is not a question asked again in other words.
|
|
func similarGroup(project, path, model string, body []byte) (group, question string, ok bool) {
|
|
m, ok := decodeRequest(body)
|
|
if !ok {
|
|
return "", "", false
|
|
}
|
|
msgs, _ := m["messages"].([]any)
|
|
if len(msgs) == 0 {
|
|
return "", "", false
|
|
}
|
|
last, _ := msgs[len(msgs)-1].(map[string]any)
|
|
if last == nil || last["role"] != "user" {
|
|
return "", "", false
|
|
}
|
|
// A picture, a sound or a file is part of the question, and the text
|
|
// beside it is not all of it: "what is in this picture?" asked of two
|
|
// pictures is two questions. Reused only when the same, exactly.
|
|
if parts, isList := last["content"].([]any); isList {
|
|
for _, p := range parts {
|
|
if part, _ := p.(map[string]any); part == nil || part["type"] != "text" {
|
|
return "", "", false
|
|
}
|
|
}
|
|
}
|
|
question = lastUserText(body)
|
|
last["content"] = ""
|
|
group, ok = keyOf(project, path+"#similar", model, m)
|
|
return group, question, ok
|
|
}
|
|
|
|
// embedQuestion has router.cache.embedding_model embed a question, on the
|
|
// gateway's own account as a judge is asked, into a vector of unit length.
|
|
func (r *Router) embedQuestion(ctx context.Context, text string, inCluster bool) ([]float32, error) {
|
|
if _, m := r.resolve(r.cacheEmbedModel); m.QueryPrefix != "" {
|
|
text = m.QueryPrefix + text
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, similarTimeout)
|
|
defer cancel()
|
|
body, _ := json.Marshal(map[string]any{"model": r.cacheEmbedModel, "input": []string{text}})
|
|
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/v1/embeddings", bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := r.parse(hreq, "/v1/embeddings")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// As the gateway's own, as a judge is asked; and kept in the cluster
|
|
// when the question's project is, fallbacks included.
|
|
req.internal, req.inCluster = true, inCluster
|
|
req.RequestID = requestID("")
|
|
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
|
|
w := &bufferWriter{header: http.Header{}}
|
|
if res := r.dispatch(ctx, w, req); res.err != nil {
|
|
return nil, res.err
|
|
}
|
|
if w.status != 0 && w.status != http.StatusOK {
|
|
return nil, fmt.Errorf("%s answered %d", r.cacheEmbedModel, w.status)
|
|
}
|
|
out := w.buf.Bytes()
|
|
var res struct {
|
|
Data []struct {
|
|
Embedding []float32 `json:"embedding"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(out, &res); err != nil || len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
|
|
return nil, fmt.Errorf("%s answered no embedding", r.cacheEmbedModel)
|
|
}
|
|
return unit(res.Data[0].Embedding), nil
|
|
}
|
|
|
|
// unit is v scaled to length 1, so that cosine similarity is a dot product.
|
|
func unit(v []float32) []float32 {
|
|
var n float64
|
|
for _, x := range v {
|
|
n += float64(x) * float64(x)
|
|
}
|
|
if n == 0 {
|
|
return v
|
|
}
|
|
n = math.Sqrt(n)
|
|
out := make([]float32, len(v))
|
|
for i, x := range v {
|
|
out[i] = float32(float64(x) / n)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func dot(a, b []float32) float64 {
|
|
var s float64
|
|
for i := range a {
|
|
s += float64(a[i]) * float64(b[i])
|
|
}
|
|
return s
|
|
}
|