Reusing answers: a response cache per project, exact match, opt-in with a TTL

llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on
the console's project menu. The same request — project, endpoint, canonical
model and the body without stream, stream_options, user, metadata, store and
service_tier, keys sorted — is answered from the gateway's memory for the
TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank,
score) unless --any; only a whole 200 from the model asked for, not a
fallback's, a canary's, or one a guardrail changed.

Looked up after the limits and the input guardrails; the output policy
judges a reused answer; no_store and stored conversations bypass it.
Streamed answers are kept put back together and streamed back as chunks.
x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or
x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost,
marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded
by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total,
llmgw_cache_bytes, llmgw_cache_entries.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:50:53 +02:00
1 parent 30f7f3474a
commit 44f3a9e31d
27 files changed
+2066 -11

No files matched your search

+11 -5
View File
@@ -435,18 +435,22 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_ms, end_user, tags)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
guard_action, guard_policy, guard_category, guard_ms, end_user, tags, cache_hit)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
for _, r := range records {
hit := 0
if r.CacheHit {
hit = 1
}
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardMs, r.EndUser, tagsJSON(r.Tags)); err != nil {
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit); err != nil {
return err
}
}
@@ -856,15 +860,16 @@ func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool,
var u Usage
var ts int64
var tags string
var hit int
err := s.db.QueryRowContext(ctx,
`SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens,
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_ms, end_user, tags
guard_action, guard_policy, guard_category, guard_ms, end_user, tags, cache_hit
FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID).
Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens,
&u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica,
&u.QueueMs, &u.TTFTMs, &u.GuardAction, &u.GuardPolicy, &u.GuardCategory, &u.GuardMs,
&u.EndUser, &tags)
&u.EndUser, &tags, &hit)
if errors.Is(err, sql.ErrNoRows) {
return Usage{}, false, nil
}
@@ -873,5 +878,6 @@ func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool,
}
u.At = time.UnixMilli(ts)
u.Tags = parseTags(tags)
u.CacheHit = hit == 1
return u, true, nil
}