Reusing answers: a response cache per project, exact match, opt-in with a TTL
llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on the console's project menu. The same request — project, endpoint, canonical model and the body without stream, stream_options, user, metadata, store and service_tier, keys sorted — is answered from the gateway's memory for the TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank, score) unless --any; only a whole 200 from the model asked for, not a fallback's, a canary's, or one a guardrail changed. Looked up after the limits and the input guardrails; the output policy judges a reused answer; no_store and stored conversations bypass it. Streamed answers are kept put back together and streamed back as chunks. x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost, marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total, llmgw_cache_bytes, llmgw_cache_entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
44f3a9e31d
27 files changed
+2066
-11
No files matched your search
@@ -0,0 +1,85 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
var (
|
||||
project string
|
||||
ttl time.Duration
|
||||
anyAnswer bool
|
||||
off bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "cache",
|
||||
Short: "Reuse a project's answers: the same request is answered again without the model",
|
||||
Long: "With this on, a request of the project that is the same as one answered within\n" +
|
||||
"--ttl, to the same model, gets that answer again, from the gateway's memory,\n" +
|
||||
"rather than asking the model. It is answered with x-llmgw-cache: hit, recorded\n" +
|
||||
"as a request with no tokens, and costs nothing against a budget.\n\n" +
|
||||
"Only answers that would come out the same are reused: a chat or completion\n" +
|
||||
"asked at temperature 0 or with a seed, and embeddings, rerank and score.\n" +
|
||||
"--any reuses every answer, sampled or not. A client sending\n" +
|
||||
"Cache-Control: no-cache or x-llmgw-cache: no is always asked afresh, and so is\n" +
|
||||
"a request continuing a stored conversation or under a guardrails no_store\n" +
|
||||
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
|
||||
"Off for every project until this is run. Without --ttl or --off, says how\n" +
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.",
|
||||
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
|
||||
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
|
||||
" llmgw keys cache --project Acme/ml/train --off",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
switch {
|
||||
case off && (ttl != 0 || anyAnswer):
|
||||
return fmt.Errorf("--off, or --ttl with --any: not both")
|
||||
case ttl < 0:
|
||||
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
|
||||
case ttl > 0 && ttl < time.Second:
|
||||
return fmt.Errorf("--ttl is at least a second")
|
||||
case anyAnswer && ttl == 0:
|
||||
return fmt.Errorf("--any goes with --ttl: say how long answers are reused")
|
||||
}
|
||||
s, err := open(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
p, err := resolveProject(cmd.Context(), s, project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
c, err := s.AnswerCacheOf(cmd.Context(), p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case c.TTL == 0:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses no answers.\n", p.Path)
|
||||
case c.Any:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses every answer for %s, sampled or not.\n", p.Path, c.TTL)
|
||||
default:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
|
||||
"and embeddings, rerank and score.\n", p.Path, c.TTL)
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
||||
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
|
||||
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
|
||||
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAProjectIsToldToReuseItsAnswersAndToStop(t *testing.T) {
|
||||
db := filepath.Join(t.TempDir(), "s.db")
|
||||
mustRun(t, db, "org", "add", "org", "Acme")
|
||||
mustRun(t, db, "org", "add", "team", "ml", "--in", "Acme")
|
||||
mustRun(t, db, "org", "add", "project", "train", "--in", "Acme/ml")
|
||||
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train"); !strings.Contains(out, "reuses no answers") {
|
||||
t.Errorf("before: %s", out)
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train", "--ttl", "1h"); !strings.Contains(out, "for 1h0m0s: those asked at temperature 0") {
|
||||
t.Errorf("on: %s", out)
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--ttl", "24h", "--any"); !strings.Contains(out, "every answer for 24h0m0s") {
|
||||
t.Errorf("any: %s", out)
|
||||
}
|
||||
for _, bad := range [][]string{{"--off", "--ttl", "1h"}, {"--any"}, {"--ttl", "10ms"}} {
|
||||
if _, err := run(t, db, append([]string{"keys", "cache", "--project", "train"}, bad...)...); err == nil {
|
||||
t.Errorf("%v was accepted", bad)
|
||||
}
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--off"); !strings.Contains(out, "reuses no answers") {
|
||||
t.Errorf("off: %s", out)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,9 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
if v.CacheHit {
|
||||
fmt.Fprintf(out, "cache answered from the project's answer cache: nothing generated, nothing charged\n")
|
||||
}
|
||||
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||
if v.EndUser != "" {
|
||||
fmt.Fprintf(out, "end user %s (the client's own name for them)\n", v.EndUser)
|
||||
|
||||
@@ -44,6 +44,7 @@ func newTenancyCmd() *cobra.Command {
|
||||
newThreadsCmd(open),
|
||||
newRequestLogCmd(open),
|
||||
newResidencyCmd(open),
|
||||
newAnswerCacheCmd(open),
|
||||
newMemoryCmd(open),
|
||||
newMoveStoreCmd(),
|
||||
)
|
||||
|
||||
@@ -72,6 +72,8 @@ type peopleData struct {
|
||||
notes map[string]int // project id → hours kept, agent notes
|
||||
// requests is how each project keeps its requests and answers.
|
||||
requests map[string]store.RequestLog
|
||||
// cache is how each project reuses answers.
|
||||
cache map[string]store.AnswerCache
|
||||
// guardPolicies are server.yaml's guardrail policies, guardDefault those
|
||||
// applied everywhere, and guardAt which are attached where.
|
||||
guardPolicies []string
|
||||
@@ -125,6 +127,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
d.guardAt, _ = s.GuardAttachments(ctx)
|
||||
}
|
||||
d.threads, d.notes, d.requests = map[string]int{}, map[string]int{}, map[string]store.RequestLog{}
|
||||
d.cache = map[string]store.AnswerCache{}
|
||||
for _, r := range d.orgs {
|
||||
if r.level != "project" {
|
||||
continue
|
||||
@@ -138,6 +141,9 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
if l, err := s.RequestLogOf(ctx, r.id); err == nil {
|
||||
d.requests[r.id] = l
|
||||
}
|
||||
if c, err := s.AnswerCacheOf(ctx, r.id); err == nil {
|
||||
d.cache[r.id] = c
|
||||
}
|
||||
}
|
||||
return consolePeopleMsg(d)
|
||||
}
|
||||
@@ -552,6 +558,39 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
|
||||
return m.runCLI("Keep requests", append(args, "--retain-hours="+v[0], "--sample="+v[1])...), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
menuItem{"Cache answers…", func(m *consoleModel) tea.Cmd {
|
||||
c := m.people.cache[r.id]
|
||||
ttl, which := "off", "reproducible"
|
||||
if c.TTL > 0 {
|
||||
ttl = c.TTL.String()
|
||||
}
|
||||
if c.Any {
|
||||
which = "any"
|
||||
}
|
||||
m.openForm(&formDialog{title: "Cache answers · " + r.path,
|
||||
note: "Off by default. On, the same request to the same model within the time below is\n" +
|
||||
"answered again from the gateway's memory, without the model, and costs nothing.\n" +
|
||||
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.",
|
||||
fields: []formField{
|
||||
textF("reuse for", ttl, "like 1h; off reuses none"),
|
||||
choiceF("answers", []string{"reproducible", "any"}, which),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
args := []string{"keys", "cache", "--project=" + r.id}
|
||||
if v[0] == "off" || v[0] == "0" {
|
||||
return m.runCLI("Cache answers", append(args, "--off")...), nil
|
||||
}
|
||||
if d, err := time.ParseDuration(v[0]); err != nil || d < time.Second {
|
||||
return nil, fmt.Errorf("a duration like 1h, or off")
|
||||
}
|
||||
args = append(args, "--ttl="+v[0])
|
||||
if v[1] == "any" {
|
||||
args = append(args, "--any")
|
||||
}
|
||||
return m.runCLI("Cache answers", args...), nil
|
||||
}})
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
|
||||
|
||||
Reference in new issue
Block a user