Ilker ManapandClaude Opus 5.5
44f3a9e31d
llmgw keys cache --project P --ttl 1h [--any] | --off, and Cache answers… on the console's project menu. The same request — project, endpoint, canonical model and the body without stream, stream_options, user, metadata, store and service_tier, keys sorted — is answered from the gateway's memory for the TTL. Only reproducible answers (temperature 0 or a seed; embeddings, rerank, score) unless --any; only a whole 200 from the model asked for, not a fallback's, a canary's, or one a guardrail changed. Looked up after the limits and the input guardrails; the output policy judges a reused answer; no_store and stored conversations bypass it. Streamed answers are kept put back together and streamed back as chunks. x-llmgw-cache: hit|miss|bypass, Age on a hit, Cache-Control: no-cache or x-llmgw-cache: no to bypass. A hit is recorded with no tokens and no cost, marked cache_hit (migration 0027) and shown by ctl request. An LRU bounded by router.cache.max_bytes and max_entry_bytes; llmgw_cache_requests_total, llmgw_cache_bytes, llmgw_cache_entries. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
60 lines
1.8 KiB
Go
60 lines
1.8 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// AnswerCache is how a project reuses answers: for how long one is served
|
|
// again to the same request, and whether that is any request's answer or
|
|
// only one that would come out the same every time. Zero TTL reuses none.
|
|
//
|
|
// Only the setting lives in the store. The answers are held in each
|
|
// gateway's memory, so turning this on writes no text anywhere.
|
|
type AnswerCache struct {
|
|
TTL time.Duration
|
|
// Any reuses answers to requests sampled at a temperature above 0
|
|
// with no seed, which would otherwise each be asked again.
|
|
Any bool
|
|
}
|
|
|
|
// SetAnswerCache turns reusing answers on for a project, or off with ttl 0.
|
|
// The TTL is kept in whole seconds, at least one.
|
|
func (s *DB) SetAnswerCache(ctx context.Context, projectID string, ttl time.Duration, anyAnswer bool) error {
|
|
var secs any
|
|
a := 0
|
|
if ttl > 0 {
|
|
secs = max(int64(ttl/time.Second), 1)
|
|
if anyAnswer {
|
|
a = 1
|
|
}
|
|
}
|
|
res, err := s.db.ExecContext(ctx,
|
|
`UPDATE projects SET cache_ttl_seconds = ?, cache_any = ? WHERE id = ?`, secs, a, projectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return fmt.Errorf("store: no project %q", projectID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AnswerCacheOf is a project's setting; zero when it reuses none.
|
|
func (s *DB) AnswerCacheOf(ctx context.Context, projectID string) (AnswerCache, error) {
|
|
var secs sql.NullInt64
|
|
var a int
|
|
err := s.db.QueryRowContext(ctx,
|
|
`SELECT cache_ttl_seconds, cache_any FROM projects WHERE id = ?`, projectID).Scan(&secs, &a)
|
|
if errors.Is(err, sql.ErrNoRows) || err == nil && (!secs.Valid || secs.Int64 <= 0) {
|
|
return AnswerCache{}, nil
|
|
}
|
|
if err != nil {
|
|
return AnswerCache{}, err
|
|
}
|
|
return AnswerCache{TTL: time.Duration(secs.Int64) * time.Second, Any: a == 1}, nil
|
|
}
|