Ilker ManapandClaude Opus 5.5
44f3a9e31d
llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on the console's project menu. The same request — project, endpoint, canonical model and the body without stream, stream_options, user, metadata, store and service_tier, keys sorted — is answered from the gateway's memory for the TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank, score) unless --any; only a whole 200 from the model asked for, not a fallback's, a canary's, or one a guardrail changed. Looked up after the limits and the input guardrails; the output policy judges a reused answer; no_store and stored conversations bypass it. Streamed answers are kept put back together and streamed back as chunks. x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost, marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total, llmgw_cache_bytes, llmgw_cache_entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
33 lines
1.2 KiB
Go
33 lines
1.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestAProjectIsToldToReuseItsAnswersAndToStop(t *testing.T) {
|
|
db := filepath.Join(t.TempDir(), "s.db")
|
|
mustRun(t, db, "org", "add", "org", "Acme")
|
|
mustRun(t, db, "org", "add", "team", "ml", "--in", "Acme")
|
|
mustRun(t, db, "org", "add", "project", "train", "--in", "Acme/ml")
|
|
|
|
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train"); !strings.Contains(out, "reuses no answers") {
|
|
t.Errorf("before: %s", out)
|
|
}
|
|
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train", "--ttl", "1h"); !strings.Contains(out, "for 1h0m0s: those asked at temperature 0") {
|
|
t.Errorf("on: %s", out)
|
|
}
|
|
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--ttl", "24h", "--any"); !strings.Contains(out, "every answer for 24h0m0s") {
|
|
t.Errorf("any: %s", out)
|
|
}
|
|
for _, bad := range [][]string{{"--off", "--ttl", "1h"}, {"--any"}, {"--ttl", "10ms"}} {
|
|
if _, err := run(t, db, append([]string{"keys", "cache", "--project", "train"}, bad...)...); err == nil {
|
|
t.Errorf("%v was accepted", bad)
|
|
}
|
|
}
|
|
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--off"); !strings.Contains(out, "reuses no answers") {
|
|
t.Errorf("off: %s", out)
|
|
}
|
|
}
|