Ilker ManapandClaude Opus 5.5
44f3a9e31d
llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on the console's project menu. The same request — project, endpoint, canonical model and the body without stream, stream_options, user, metadata, store and service_tier, keys sorted — is answered from the gateway's memory for the TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank, score) unless --any; only a whole 200 from the model asked for, not a fallback's, a canary's, or one a guardrail changed. Looked up after the limits and the input guardrails; the output policy judges a reused answer; no_store and stored conversations bypass it. Streamed answers are kept put back together and streamed back as chunks. x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost, marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total, llmgw_cache_bytes, llmgw_cache_entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
86 lines
3.2 KiB
Go
86 lines
3.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func newAnswerCacheCmd(open opener) *cobra.Command {
|
|
var (
|
|
project string
|
|
ttl time.Duration
|
|
anyAnswer bool
|
|
off bool
|
|
)
|
|
cmd := &cobra.Command{
|
|
Use: "cache",
|
|
Short: "Reuse a project's answers: the same request is answered again without the model",
|
|
Long: "With this on, a request of the project that is the same as one answered within\n" +
|
|
"--ttl, to the same model, gets that answer again, from the gateway's memory,\n" +
|
|
"rather than asking the model. It is answered with x-llmgw-cache: hit, recorded\n" +
|
|
"as a request with no tokens, and costs nothing against a budget.\n\n" +
|
|
"Only answers that would come out the same are reused: a chat or completion\n" +
|
|
"asked at temperature 0 or with a seed, and embeddings, rerank and score.\n" +
|
|
"--any reuses every answer, sampled or not. A client sending\n" +
|
|
"Cache-Control: no-cache or x-llmgw-cache: no is always asked afresh, and so is\n" +
|
|
"a request continuing a stored conversation or under a guardrails no_store\n" +
|
|
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
|
|
"Off for every project until this is run. Without --ttl or --off, says how\n" +
|
|
"it stands. The answers are held in memory, router.cache bounds how much.",
|
|
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
|
|
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
|
|
" llmgw keys cache --project Acme/ml/train --off",
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
switch {
|
|
case off && (ttl != 0 || anyAnswer):
|
|
return fmt.Errorf("--off, or --ttl with --any: not both")
|
|
case ttl < 0:
|
|
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
|
|
case ttl > 0 && ttl < time.Second:
|
|
return fmt.Errorf("--ttl is at least a second")
|
|
case anyAnswer && ttl == 0:
|
|
return fmt.Errorf("--any goes with --ttl: say how long answers are reused")
|
|
}
|
|
s, err := open(cmd.Context())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer s.Close()
|
|
p, err := resolveProject(cmd.Context(), s, project)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if off || ttl > 0 {
|
|
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
c, err := s.AnswerCacheOf(cmd.Context(), p.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch {
|
|
case c.TTL == 0:
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses no answers.\n", p.Path)
|
|
case c.Any:
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses every answer for %s, sampled or not.\n", p.Path, c.TTL)
|
|
default:
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
|
|
"and embeddings, rerank and score.\n", p.Path, c.TTL)
|
|
}
|
|
if off || ttl > 0 {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
|
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
|
|
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
|
|
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
|
|
return cmd
|
|
}
|