llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ed0d1ec0ad
commit
3a3db8cd20
19 files changed
+514
-33
No files matched your search
@@ -5,6 +5,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
@@ -12,6 +14,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
project string
|
||||
ttl time.Duration
|
||||
anyAnswer bool
|
||||
similar float64
|
||||
off bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
@@ -28,15 +31,25 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
"a request continuing a stored conversation or under a guardrails no_store\n" +
|
||||
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
|
||||
"Off for every project until this is run. Without --ttl or --off, says how\n" +
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.",
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.\n\n" +
|
||||
"--similar also answers a chat question that is like one already answered:\n" +
|
||||
"its last user message embedded by router.cache.embedding_model, and given\n" +
|
||||
"the answer whose question reaches that cosine similarity (0.95 is strict,\n" +
|
||||
"0.9 looser), as x-llmgw-cache: similar. The system prompt, the earlier\n" +
|
||||
"turns, tools and the sampling settings must still be the same.",
|
||||
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
|
||||
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
|
||||
" llmgw keys cache --project Acme/support/desk --ttl 24h --similar 0.95\n" +
|
||||
" llmgw keys cache --project Acme/ml/train --off",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
switch {
|
||||
case off && (ttl != 0 || anyAnswer):
|
||||
return fmt.Errorf("--off, or --ttl with --any: not both")
|
||||
case off && (ttl != 0 || anyAnswer || similar != 0):
|
||||
return fmt.Errorf("--off, or --ttl with --any or --similar: not both")
|
||||
case similar < 0 || similar > 1:
|
||||
return fmt.Errorf("--similar is a cosine similarity between 0 and 1, like 0.95")
|
||||
case similar != 0 && ttl == 0:
|
||||
return fmt.Errorf("--similar goes with --ttl: say how long answers are reused")
|
||||
case ttl < 0:
|
||||
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
|
||||
case ttl > 0 && ttl < time.Second:
|
||||
@@ -54,7 +67,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, store.AnswerCache{TTL: ttl, Any: anyAnswer, Similar: similar}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -71,6 +84,9 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
|
||||
"and embeddings, rerank and score.\n", p.Path, c.TTL)
|
||||
}
|
||||
if c.TTL > 0 && c.Similar > 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Chat questions reaching %.2f similarity to one answered get its answer too.\n", c.Similar)
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
|
||||
}
|
||||
@@ -80,6 +96,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
||||
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
|
||||
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
|
||||
cmd.Flags().Float64Var(&similar, "similar", 0, "also answer chat questions this similar to one answered, like 0.95")
|
||||
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
|
||||
return cmd
|
||||
}
|
||||
@@ -573,13 +573,20 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
|
||||
if c.Any {
|
||||
which = "any"
|
||||
}
|
||||
similar := "off"
|
||||
if c.Similar > 0 {
|
||||
similar = strconv.FormatFloat(c.Similar, 'f', -1, 64)
|
||||
}
|
||||
m.openForm(&formDialog{title: "Cache answers · " + r.path,
|
||||
note: "Off by default. On, the same request to the same model within the time below is\n" +
|
||||
"answered again from the gateway's memory, without the model, and costs nothing.\n" +
|
||||
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.",
|
||||
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.\n" +
|
||||
"Similar: a chat question this alike to one answered gets its answer (needs\n" +
|
||||
"router.cache.embedding_model).",
|
||||
fields: []formField{
|
||||
textF("reuse for", ttl, "like 1h; off reuses none"),
|
||||
choiceF("answers", []string{"reproducible", "any"}, which),
|
||||
textF("similar", similar, "like 0.95; off reuses only the same request"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
args := []string{"keys", "cache", "--project=" + r.id}
|
||||
@@ -593,6 +600,12 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
|
||||
if v[1] == "any" {
|
||||
args = append(args, "--any")
|
||||
}
|
||||
if v[2] != "off" && v[2] != "" && v[2] != "0" {
|
||||
if f, err := strconv.ParseFloat(v[2], 64); err != nil || f <= 0 || f > 1 {
|
||||
return nil, fmt.Errorf("similar is a number like 0.95, or off")
|
||||
}
|
||||
args = append(args, "--similar="+v[2])
|
||||
}
|
||||
return m.runCLI("Cache answers", args...), nil
|
||||
}})
|
||||
return nil
|
||||
|
||||
Reference in new issue
Block a user