Requests kept and replayed: keys requests per project with retention and a share, ctl request --body and --replay
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
841ee21c12
commit
32ff1dbd6f
22 files changed
+757
-10
No files matched your search
@@ -294,6 +294,9 @@ type parsedRequest struct {
|
||||
// user field. Both are recorded with the usage, never sent on.
|
||||
Tags map[string]string
|
||||
EndUser string
|
||||
// logRetain is how long the request and its answer are kept, when its
|
||||
// project keeps them and it fell in the share kept.
|
||||
logRetain time.Duration
|
||||
// Model is what will be dispatched, which is not always what was asked
|
||||
// for: a caller out of budget may be moved to a model they can afford.
|
||||
Model string
|
||||
@@ -499,9 +502,13 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
r.decideRequestLog(req.Context(), parsed)
|
||||
res := r.dispatch(req.Context(), w, parsed)
|
||||
status := r.finish(w, parsed, res, start)
|
||||
r.observeCanary(parsed, res, status, time.Since(start))
|
||||
if parsed.logRetain > 0 {
|
||||
go r.keepRequest(context.WithoutCancel(req.Context()), parsed, res, status)
|
||||
}
|
||||
r.observeOutput(parsed, res, res.judged)
|
||||
span.SetAttributes(attribute.Int("http.response.status_code", status),
|
||||
attribute.String("llmgw.served_model", res.model),
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Keeping requests and answers, for a project that asks: what was sent to
|
||||
// the model and what came back, for following one request up and for
|
||||
// sending it again to another model.
|
||||
|
||||
// requestLogTTL is how long a project's setting is believed before it is
|
||||
// read again.
|
||||
const requestLogTTL = 30 * time.Second
|
||||
|
||||
type requestLogs struct {
|
||||
mu sync.Mutex
|
||||
by map[string]cachedLog
|
||||
}
|
||||
|
||||
type cachedLog struct {
|
||||
l storepkg.RequestLog
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// requestLogOf is a project's setting, read at most every requestLogTTL.
|
||||
func (r *Router) requestLogOf(ctx context.Context, project string) storepkg.RequestLog {
|
||||
if r.store == nil || project == "" {
|
||||
return storepkg.RequestLog{}
|
||||
}
|
||||
r.reqLogs.mu.Lock()
|
||||
c, ok := r.reqLogs.by[project]
|
||||
r.reqLogs.mu.Unlock()
|
||||
if ok && time.Since(c.at) < requestLogTTL {
|
||||
return c.l
|
||||
}
|
||||
l, err := r.store.RequestLogOf(ctx, project)
|
||||
if err != nil {
|
||||
r.log.Warn("could not read whether a project keeps its requests", "project", project, "err", err)
|
||||
return c.l
|
||||
}
|
||||
r.reqLogs.mu.Lock()
|
||||
if r.reqLogs.by == nil {
|
||||
r.reqLogs.by = map[string]cachedLog{}
|
||||
}
|
||||
r.reqLogs.by[project] = cachedLog{l: l, at: time.Now()}
|
||||
r.reqLogs.mu.Unlock()
|
||||
return l
|
||||
}
|
||||
|
||||
// sampled says whether a request falls in a share of them, by its id, so
|
||||
// the same id always says the same.
|
||||
func sampled(id string, share float64) bool {
|
||||
if share >= 1 {
|
||||
return true
|
||||
}
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(id))
|
||||
return float64(h.Sum32()%10000) < share*10000
|
||||
}
|
||||
|
||||
// decideRequestLog marks a request to be kept, once it is known who sent
|
||||
// it and that no policy forbids keeping it.
|
||||
func (r *Router) decideRequestLog(ctx context.Context, req *parsedRequest) {
|
||||
if req.noStore || req.internal {
|
||||
return
|
||||
}
|
||||
l := r.requestLogOf(ctx, req.ProjectID)
|
||||
if l.Retain > 0 && sampled(req.RequestID, l.Sample) {
|
||||
req.logRetain = l.Retain
|
||||
}
|
||||
}
|
||||
|
||||
// keepRequest writes a finished request and its answer, for one marked.
|
||||
func (r *Router) keepRequest(ctx context.Context, req *parsedRequest, res dispatchResult, status int) {
|
||||
if req.logRetain <= 0 {
|
||||
return
|
||||
}
|
||||
served := res.model
|
||||
if served == "" {
|
||||
served = req.Model
|
||||
}
|
||||
now := time.Now()
|
||||
b := storepkg.RequestBody{RequestID: req.RequestID, ProjectID: req.ProjectID, At: now,
|
||||
ExpiresAt: now.Add(req.logRetain), Path: req.Path, Model: req.Model, ServedModel: served,
|
||||
Status: status, Request: string(req.UpstreamBody), Response: res.outcome.Reply,
|
||||
Truncated: res.outcome.ReplyTruncated}
|
||||
if err := r.store.PutRequestBody(ctx, b); err != nil {
|
||||
r.log.Warn("could not keep a request", "request_id", req.RequestID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// pruneRequestBodies removes kept requests past their retention, hourly.
|
||||
func (r *Router) pruneRequestBodies(ctx context.Context) {
|
||||
t := time.NewTicker(time.Hour)
|
||||
defer t.Stop()
|
||||
for {
|
||||
if n, err := r.store.PruneRequestBodies(ctx, time.Now()); err != nil {
|
||||
if ctx.Err() == nil {
|
||||
r.log.Warn("could not prune kept requests", "err", err)
|
||||
}
|
||||
} else if n > 0 {
|
||||
r.log.Info("removed kept requests past their retention", "requests", n)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Replay sends a kept request again to a model, as the gateway itself: past
|
||||
// admission, accounting and the guardrails, which judged it the first time.
|
||||
// For comparing models on real traffic. It answers the status, the body,
|
||||
// and how long it took.
|
||||
func (r *Router) Replay(ctx context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
|
||||
if !batchEndpoints[path] {
|
||||
return 0, nil, 0, fmt.Errorf("a request to %s cannot be replayed", path)
|
||||
}
|
||||
body = withoutField(withModel(body, model), "stream")
|
||||
hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, nil, 0, err
|
||||
}
|
||||
req, err := r.parse(hreq, path)
|
||||
if err != nil {
|
||||
return 0, nil, 0, err
|
||||
}
|
||||
req.internal, req.Stream = true, false
|
||||
req.RequestID = requestID("")
|
||||
req.Header = http.Header{"X-Request-Id": []string{req.RequestID}}
|
||||
w := &bufferWriter{header: http.Header{}}
|
||||
start := time.Now()
|
||||
res := r.dispatch(ctx, w, req)
|
||||
took := time.Since(start)
|
||||
if res.err != nil {
|
||||
return 0, nil, took, res.err
|
||||
}
|
||||
status := w.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
return status, w.buf.Bytes(), took, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func keptBody(t *testing.T, rt *Router, id string) (store.RequestBody, bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
b, err := rt.store.RequestBodyByID(t.Context(), id)
|
||||
if err == nil {
|
||||
return b, true
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return store.RequestBody{}, false
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProjectThatAsksKeepsItsRequestsAndTheyReplay(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
h := rt.Handler()
|
||||
project := mustProjectOf(t, rt, "key-1")
|
||||
|
||||
send := func(id string) {
|
||||
rec := post(t, h, map[string]any{"model": "q14", "messages": []any{
|
||||
map[string]any{"role": "user", "content": "what is 2+2"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
send("before")
|
||||
if _, ok := keptBody(t, rt, "before"); ok {
|
||||
t.Fatal("a request was kept by a project that keeps none")
|
||||
}
|
||||
|
||||
if err := rt.store.SetRequestLog(t.Context(), project, 24, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rt.reqLogs.by = nil
|
||||
send("kept-1")
|
||||
b, ok := keptBody(t, rt, "kept-1")
|
||||
if !ok {
|
||||
t.Fatal("not kept")
|
||||
}
|
||||
if !strings.Contains(b.Request, "what is 2+2") || b.Response != "hi" || b.Status != 200 ||
|
||||
b.ServedModel != "qwen2.5-14b-instruct" || b.Path != "/v1/chat/completions" {
|
||||
t.Errorf("kept %+v", b)
|
||||
}
|
||||
if b.ExpiresAt.Sub(b.At) != 24*time.Hour {
|
||||
t.Errorf("kept for %s", b.ExpiresAt.Sub(b.At))
|
||||
}
|
||||
|
||||
status, body, _, err := rt.Replay(t.Context(), b.Path, "q14", []byte(b.Request))
|
||||
if err != nil || status != 200 || !strings.Contains(string(body), "hi") {
|
||||
t.Errorf("replay: %d %s %v", status, body, err)
|
||||
}
|
||||
if _, _, _, err := rt.Replay(t.Context(), "/v1/files", "q14", nil); err == nil {
|
||||
t.Error("a non-inference path was replayed")
|
||||
}
|
||||
|
||||
// A small share keeps few.
|
||||
_ = rt.store.SetRequestLog(t.Context(), project, 24, 0.05)
|
||||
rt.reqLogs.by = nil
|
||||
for i := range 100 {
|
||||
send(fmt.Sprintf("s-%d", i))
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
n := 0
|
||||
for i := range 100 {
|
||||
if _, err := rt.store.RequestBodyByID(t.Context(), fmt.Sprintf("s-%d", i)); err == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n == 0 || n > 20 {
|
||||
t.Errorf("kept %d of 100 at 5%%", n)
|
||||
}
|
||||
|
||||
// Off: what was kept goes at the next prune.
|
||||
_ = rt.store.SetRequestLog(t.Context(), project, 0, 0)
|
||||
if _, err := rt.store.PruneRequestBodies(t.Context(), time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rt.store.RequestBodyByID(t.Context(), "kept-1"); err == nil {
|
||||
t.Error("a project that stopped keeping requests still has them after a prune")
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,8 @@ type Router struct {
|
||||
canaries canaryStats
|
||||
// batchWake has the batch worker look for work at once.
|
||||
batchWake chan struct{}
|
||||
// reqLogs is each project's keeping of requests, as last read.
|
||||
reqLogs requestLogs
|
||||
|
||||
// extraMetrics are collectors other components publish on /metrics.
|
||||
extraMetrics []prometheus.Collector
|
||||
@@ -448,6 +450,7 @@ func (r *Router) Start(ctx context.Context) {
|
||||
}
|
||||
if r.store != nil {
|
||||
go r.runBatches(ctx)
|
||||
go r.pruneRequestBodies(ctx)
|
||||
go r.refreshTenants(ctx)
|
||||
go r.purgeSessions(ctx)
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ const maxStoredReply = 128 << 10
|
||||
// Zero for every request not continuing a conversation, which is what keeps
|
||||
// the cost of this feature off the path of requests that do not use it.
|
||||
func (p *parsedRequest) captureBytes() int {
|
||||
if p.ThreadID == "" || p.noStore {
|
||||
if p.ThreadID == "" && p.logRetain <= 0 || p.noStore {
|
||||
return 0
|
||||
}
|
||||
return maxStoredReply
|
||||
|
||||
Reference in new issue
Block a user