llmgw keys cache --similar 0.95 also gives a chat question the answer to one like it: the last user message embedded by router.cache.embedding_model (asked on the gateway's account, as a judge is, within 2 s), compared by cosine with the questions answered in the same conversation — system prompt, earlier turns, tools and sampling the same. The exact cache is asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar with x-llmgw-cache-similarity, charged as a hit. The vectors live with their answers in the gateway's memory, counted in max_bytes and evicted with them, not in a pgvector table as BACKLOG I1 planned: the cache keeps answer text out of the store on purpose. A question over 2,000 characters is reused only when the same. Migration 0051 holds the setting. BACKLOG I1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ed0d1ec0ad
commit
3a3db8cd20
19 files changed
+514
-33
No files matched your search
@@ -5,6 +5,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
@@ -12,6 +14,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
project string
|
||||
ttl time.Duration
|
||||
anyAnswer bool
|
||||
similar float64
|
||||
off bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
@@ -28,15 +31,25 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
"a request continuing a stored conversation or under a guardrails no_store\n" +
|
||||
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
|
||||
"Off for every project until this is run. Without --ttl or --off, says how\n" +
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.",
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.\n\n" +
|
||||
"--similar also answers a chat question that is like one already answered:\n" +
|
||||
"its last user message embedded by router.cache.embedding_model, and given\n" +
|
||||
"the answer whose question reaches that cosine similarity (0.95 is strict,\n" +
|
||||
"0.9 looser), as x-llmgw-cache: similar. The system prompt, the earlier\n" +
|
||||
"turns, tools and the sampling settings must still be the same.",
|
||||
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
|
||||
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
|
||||
" llmgw keys cache --project Acme/support/desk --ttl 24h --similar 0.95\n" +
|
||||
" llmgw keys cache --project Acme/ml/train --off",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
switch {
|
||||
case off && (ttl != 0 || anyAnswer):
|
||||
return fmt.Errorf("--off, or --ttl with --any: not both")
|
||||
case off && (ttl != 0 || anyAnswer || similar != 0):
|
||||
return fmt.Errorf("--off, or --ttl with --any or --similar: not both")
|
||||
case similar < 0 || similar > 1:
|
||||
return fmt.Errorf("--similar is a cosine similarity between 0 and 1, like 0.95")
|
||||
case similar != 0 && ttl == 0:
|
||||
return fmt.Errorf("--similar goes with --ttl: say how long answers are reused")
|
||||
case ttl < 0:
|
||||
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
|
||||
case ttl > 0 && ttl < time.Second:
|
||||
@@ -54,7 +67,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, store.AnswerCache{TTL: ttl, Any: anyAnswer, Similar: similar}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -71,6 +84,9 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
|
||||
"and embeddings, rerank and score.\n", p.Path, c.TTL)
|
||||
}
|
||||
if c.TTL > 0 && c.Similar > 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Chat questions reaching %.2f similarity to one answered get its answer too.\n", c.Similar)
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
|
||||
}
|
||||
@@ -80,6 +96,7 @@ func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
||||
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
|
||||
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
|
||||
cmd.Flags().Float64Var(&similar, "similar", 0, "also answer chat questions this similar to one answered, like 0.95")
|
||||
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
|
||||
return cmd
|
||||
}
|
||||
Reference in new issue
Block a user