Answers to similar questions, not only the same one

llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-25 17:47:11 +01:00
1 parent ed0d1ec0ad
commit 3a3db8cd20
19 files changed
+514 -33

No files matched your search

+21 -4
View File
@@ -5,6 +5,8 @@ import (
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func newAnswerCacheCmd(open opener) *cobra.Command {
@@ -12,6 +14,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
project string
ttl time.Duration
anyAnswer bool
similar float64
off bool
)
cmd := &cobra.Command{
@@ -28,15 +31,25 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
"a request continuing a stored conversation or under a guardrails no_store\n" +
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
"Off for every project until this is run. Without --ttl or --off, says how\n" +
"it stands. The answers are held in memory, router.cache bounds how much.",
"it stands. The answers are held in memory, router.cache bounds how much.\n\n" +
"--similar also answers a chat question that is like one already answered:\n" +
"its last user message embedded by router.cache.embedding_model, and given\n" +
"the answer whose question reaches that cosine similarity (0.95 is strict,\n" +
"0.9 looser), as x-llmgw-cache: similar. The system prompt, the earlier\n" +
"turns, tools and the sampling settings must still be the same.",
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
" llmgw keys cache --project Acme/support/desk --ttl 24h --similar 0.95\n" +
" llmgw keys cache --project Acme/ml/train --off",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
switch {
case off && (ttl != 0 || anyAnswer):
return fmt.Errorf("--off, or --ttl with --any: not both")
case off && (ttl != 0 || anyAnswer || similar != 0):
return fmt.Errorf("--off, or --ttl with --any or --similar: not both")
case similar < 0 || similar > 1:
return fmt.Errorf("--similar is a cosine similarity between 0 and 1, like 0.95")
case similar != 0 && ttl == 0:
return fmt.Errorf("--similar goes with --ttl: say how long answers are reused")
case ttl < 0:
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
case ttl > 0 && ttl < time.Second:
@@ -54,7 +67,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
return err
}
if off || ttl > 0 {
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
if err := s.SetAnswerCache(cmd.Context(), p.ID, store.AnswerCache{TTL: ttl, Any: anyAnswer, Similar: similar}); err != nil {
return err
}
}
@@ -71,6 +84,9 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
"and embeddings, rerank and score.\n", p.Path, c.TTL)
}
if c.TTL > 0 && c.Similar > 0 {
fmt.Fprintf(cmd.OutOrStdout(), "Chat questions reaching %.2f similarity to one answered get its answer too.\n", c.Similar)
}
if off || ttl > 0 {
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
}
@@ -80,6 +96,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
cmd.Flags().Float64Var(&similar, "similar", 0, "also answer chat questions this similar to one answered, like 0.95")
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
return cmd
}
+14 -1
View File
@@ -573,13 +573,20 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
if c.Any {
which = "any"
}
similar := "off"
if c.Similar > 0 {
similar = strconv.FormatFloat(c.Similar, 'f', -1, 64)
}
m.openForm(&formDialog{title: "Cache answers · " + r.path,
note: "Off by default. On, the same request to the same model within the time below is\n" +
"answered again from the gateway's memory, without the model, and costs nothing.\n" +
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.",
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.\n" +
"Similar: a chat question this alike to one answered gets its answer (needs\n" +
"router.cache.embedding_model).",
fields: []formField{
textF("reuse for", ttl, "like 1h; off reuses none"),
choiceF("answers", []string{"reproducible", "any"}, which),
textF("similar", similar, "like 0.95; off reuses only the same request"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
args := []string{"keys", "cache", "--project=" + r.id}
@@ -593,6 +600,12 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
if v[1] == "any" {
args = append(args, "--any")
}
if v[2] != "off" && v[2] != "" && v[2] != "0" {
if f, err := strconv.ParseFloat(v[2], 64); err != nil || f <= 0 || f > 1 {
return nil, fmt.Errorf("similar is a number like 0.95, or off")
}
args = append(args, "--similar="+v[2])
}
return m.runCLI("Cache answers", args...), nil
}})
return nil