Requests kept and replayed: keys requests per project with retention and a share, ctl request --body and --replay

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 22:51:21 +02:00
1 parent 841ee21c12
commit 32ff1dbd6f
22 files changed
+757 -10

No files matched your search

+7
View File
@@ -294,6 +294,9 @@ type parsedRequest struct {
// user field. Both are recorded with the usage, never sent on.
Tags map[string]string
EndUser string
// logRetain is how long the request and its answer are kept, when its
// project keeps them and it fell in the share kept.
logRetain time.Duration
// Model is what will be dispatched, which is not always what was asked
// for: a caller out of budget may be moved to a model they can afford.
Model string
@@ -499,9 +502,13 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
return
}
r.decideRequestLog(req.Context(), parsed)
res := r.dispatch(req.Context(), w, parsed)
status := r.finish(w, parsed, res, start)
r.observeCanary(parsed, res, status, time.Since(start))
if parsed.logRetain > 0 {
go r.keepRequest(context.WithoutCancel(req.Context()), parsed, res, status)
}
r.observeOutput(parsed, res, res.judged)
span.SetAttributes(attribute.Int("http.response.status_code", status),
attribute.String("llmgw.served_model", res.model),
+152
View File
@@ -0,0 +1,152 @@
package router
import (
"bytes"
"context"
"fmt"
"hash/fnv"
"net/http"
"sync"
"time"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Keeping requests and answers, for a project that asks: what was sent to
// the model and what came back, for following one request up and for
// sending it again to another model.
// requestLogTTL is how long a project's setting is believed before it is
// read again.
const requestLogTTL = 30 * time.Second
type requestLogs struct {
mu sync.Mutex
by map[string]cachedLog
}
type cachedLog struct {
l storepkg.RequestLog
at time.Time
}
// requestLogOf is a project's setting, read at most every requestLogTTL.
func (r *Router) requestLogOf(ctx context.Context, project string) storepkg.RequestLog {
if r.store == nil || project == "" {
return storepkg.RequestLog{}
}
r.reqLogs.mu.Lock()
c, ok := r.reqLogs.by[project]
r.reqLogs.mu.Unlock()
if ok && time.Since(c.at) < requestLogTTL {
return c.l
}
l, err := r.store.RequestLogOf(ctx, project)
if err != nil {
r.log.Warn("could not read whether a project keeps its requests", "project", project, "err", err)
return c.l
}
r.reqLogs.mu.Lock()
if r.reqLogs.by == nil {
r.reqLogs.by = map[string]cachedLog{}
}
r.reqLogs.by[project] = cachedLog{l: l, at: time.Now()}
r.reqLogs.mu.Unlock()
return l
}
// sampled says whether a request falls in a share of them, by its id, so
// the same id always says the same.
func sampled(id string, share float64) bool {
if share >= 1 {
return true
}
h := fnv.New32a()
h.Write([]byte(id))
return float64(h.Sum32()%10000) < share*10000
}
// decideRequestLog marks a request to be kept, once it is known who sent
// it and that no policy forbids keeping it.
func (r *Router) decideRequestLog(ctx context.Context, req *parsedRequest) {
if req.noStore || req.internal {
return
}
l := r.requestLogOf(ctx, req.ProjectID)
if l.Retain > 0 && sampled(req.RequestID, l.Sample) {
req.logRetain = l.Retain
}
}
// keepRequest writes a finished request and its answer, for one marked.
func (r *Router) keepRequest(ctx context.Context, req *parsedRequest, res dispatchResult, status int) {
if req.logRetain <= 0 {
return
}
served := res.model
if served == "" {
served = req.Model
}
now := time.Now()
b := storepkg.RequestBody{RequestID: req.RequestID, ProjectID: req.ProjectID, At: now,
ExpiresAt: now.Add(req.logRetain), Path: req.Path, Model: req.Model, ServedModel: served,
Status: status, Request: string(req.UpstreamBody), Response: res.outcome.Reply,
Truncated: res.outcome.ReplyTruncated}
if err := r.store.PutRequestBody(ctx, b); err != nil {
r.log.Warn("could not keep a request", "request_id", req.RequestID, "err", err)
}
}
// pruneRequestBodies removes kept requests past their retention, hourly.
func (r *Router) pruneRequestBodies(ctx context.Context) {
t := time.NewTicker(time.Hour)
defer t.Stop()
for {
if n, err := r.store.PruneRequestBodies(ctx, time.Now()); err != nil {
if ctx.Err() == nil {
r.log.Warn("could not prune kept requests", "err", err)
}
} else if n > 0 {
r.log.Info("removed kept requests past their retention", "requests", n)
}
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
// Replay sends a kept request again to a model, as the gateway itself: past
// admission, accounting and the guardrails, which judged it the first time.
// For comparing models on real traffic. It answers the status, the body,
// and how long it took.
func (r *Router) Replay(ctx context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
if !batchEndpoints[path] {
return 0, nil, 0, fmt.Errorf("a request to %s cannot be replayed", path)
}
body = withoutField(withModel(body, model), "stream")
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
if err != nil {
return 0, nil, 0, err
}
req, err := r.parse(hreq, path)
if err != nil {
return 0, nil, 0, err
}
req.internal, req.Stream = true, false
req.RequestID = requestID("")
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
w := &bufferWriter{header: http.Header{}}
start := time.Now()
res := r.dispatch(ctx, w, req)
took := time.Since(start)
if res.err != nil {
return 0, nil, took, res.err
}
status := w.status
if status == 0 {
status = http.StatusOK
}
return status, w.buf.Bytes(), took, nil
}
+96
View File
@@ -0,0 +1,96 @@
package router
import (
"fmt"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func keptBody(t *testing.T, rt *Router, id string) (store.RequestBody, bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for {
b, err := rt.store.RequestBodyByID(t.Context(), id)
if err == nil {
return b, true
}
if time.Now().After(deadline) {
return store.RequestBody{}, false
}
time.Sleep(20 * time.Millisecond)
}
}
func TestAProjectThatAsksKeepsItsRequestsAndTheyReplay(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
h := rt.Handler()
project := mustProjectOf(t, rt, "key-1")
send := func(id string) {
rec := post(t, h, map[string]any{"model": "q14", "messages": []any{
map[string]any{"role": "user", "content": "what is 2+2"}}},
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
}
send("before")
if _, ok := keptBody(t, rt, "before"); ok {
t.Fatal("a request was kept by a project that keeps none")
}
if err := rt.store.SetRequestLog(t.Context(), project, 24, 1); err != nil {
t.Fatal(err)
}
rt.reqLogs.by = nil
send("kept-1")
b, ok := keptBody(t, rt, "kept-1")
if !ok {
t.Fatal("not kept")
}
if !strings.Contains(b.Request, "what is 2+2") || b.Response != "hi" || b.Status != 200 ||
b.ServedModel != "qwen2.5-14b-instruct" || b.Path != "/v1/chat/completions" {
t.Errorf("kept %+v", b)
}
if b.ExpiresAt.Sub(b.At) != 24*time.Hour {
t.Errorf("kept for %s", b.ExpiresAt.Sub(b.At))
}
status, body, _, err := rt.Replay(t.Context(), b.Path, "q14", []byte(b.Request))
if err != nil || status != 200 || !strings.Contains(string(body), "hi") {
t.Errorf("replay: %d %s %v", status, body, err)
}
if _, _, _, err := rt.Replay(t.Context(), "/v1/files", "q14", nil); err == nil {
t.Error("a non-inference path was replayed")
}
// A small share keeps few.
_ = rt.store.SetRequestLog(t.Context(), project, 24, 0.05)
rt.reqLogs.by = nil
for i := range 100 {
send(fmt.Sprintf("s-%d", i))
}
time.Sleep(200 * time.Millisecond)
n := 0
for i := range 100 {
if _, err := rt.store.RequestBodyByID(t.Context(), fmt.Sprintf("s-%d", i)); err == nil {
n++
}
}
if n == 0 || n > 20 {
t.Errorf("kept %d of 100 at 5%%", n)
}
// Off: what was kept goes at the next prune.
_ = rt.store.SetRequestLog(t.Context(), project, 0, 0)
if _, err := rt.store.PruneRequestBodies(t.Context(), time.Now()); err != nil {
t.Fatal(err)
}
if _, err := rt.store.RequestBodyByID(t.Context(), "kept-1"); err == nil {
t.Error("a project that stopped keeping requests still has them after a prune")
}
}
+3
View File
@@ -70,6 +70,8 @@ type Router struct {
canaries canaryStats
// batchWake has the batch worker look for work at once.
batchWake chan struct{}
// reqLogs is each project's keeping of requests, as last read.
reqLogs requestLogs
// extraMetrics are collectors other components publish on /metrics.
extraMetrics []prometheus.Collector
@@ -448,6 +450,7 @@ func (r *Router) Start(ctx context.Context) {
}
if r.store != nil {
go r.runBatches(ctx)
go r.pruneRequestBodies(ctx)
go r.refreshTenants(ctx)
go r.purgeSessions(ctx)
}
+1 -1
View File
@@ -221,7 +221,7 @@ const maxStoredReply = 128 << 10
// Zero for every request not continuing a conversation, which is what keeps
// the cost of this feature off the path of requests that do not use it.
func (p *parsedRequest) captureBytes() int {
if p.ThreadID == "" || p.noStore {
if p.ThreadID == "" && p.logRetain <= 0 || p.noStore {
return 0
}
return maxStoredReply