Requests kept and replayed: keys requests per project with retention and a share, ctl request --body and --replay
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
841ee21c12
commit
32ff1dbd6f
22 files changed
+757
-10
No files matched your search
@@ -0,0 +1,96 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func keptBody(t *testing.T, rt *Router, id string) (store.RequestBody, bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
b, err := rt.store.RequestBodyByID(t.Context(), id)
|
||||
if err == nil {
|
||||
return b, true
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return store.RequestBody{}, false
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProjectThatAsksKeepsItsRequestsAndTheyReplay(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
h := rt.Handler()
|
||||
project := mustProjectOf(t, rt, "key-1")
|
||||
|
||||
send := func(id string) {
|
||||
rec := post(t, h, map[string]any{"model": "q14", "messages": []any{
|
||||
map[string]any{"role": "user", "content": "what is 2+2"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
send("before")
|
||||
if _, ok := keptBody(t, rt, "before"); ok {
|
||||
t.Fatal("a request was kept by a project that keeps none")
|
||||
}
|
||||
|
||||
if err := rt.store.SetRequestLog(t.Context(), project, 24, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rt.reqLogs.by = nil
|
||||
send("kept-1")
|
||||
b, ok := keptBody(t, rt, "kept-1")
|
||||
if !ok {
|
||||
t.Fatal("not kept")
|
||||
}
|
||||
if !strings.Contains(b.Request, "what is 2+2") || b.Response != "hi" || b.Status != 200 ||
|
||||
b.ServedModel != "qwen2.5-14b-instruct" || b.Path != "/v1/chat/completions" {
|
||||
t.Errorf("kept %+v", b)
|
||||
}
|
||||
if b.ExpiresAt.Sub(b.At) != 24*time.Hour {
|
||||
t.Errorf("kept for %s", b.ExpiresAt.Sub(b.At))
|
||||
}
|
||||
|
||||
status, body, _, err := rt.Replay(t.Context(), b.Path, "q14", []byte(b.Request))
|
||||
if err != nil || status != 200 || !strings.Contains(string(body), "hi") {
|
||||
t.Errorf("replay: %d %s %v", status, body, err)
|
||||
}
|
||||
if _, _, _, err := rt.Replay(t.Context(), "/v1/files", "q14", nil); err == nil {
|
||||
t.Error("a non-inference path was replayed")
|
||||
}
|
||||
|
||||
// A small share keeps few.
|
||||
_ = rt.store.SetRequestLog(t.Context(), project, 24, 0.05)
|
||||
rt.reqLogs.by = nil
|
||||
for i := range 100 {
|
||||
send(fmt.Sprintf("s-%d", i))
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
n := 0
|
||||
for i := range 100 {
|
||||
if _, err := rt.store.RequestBodyByID(t.Context(), fmt.Sprintf("s-%d", i)); err == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n == 0 || n > 20 {
|
||||
t.Errorf("kept %d of 100 at 5%%", n)
|
||||
}
|
||||
|
||||
// Off: what was kept goes at the next prune.
|
||||
_ = rt.store.SetRequestLog(t.Context(), project, 0, 0)
|
||||
if _, err := rt.store.PruneRequestBodies(t.Context(), time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rt.store.RequestBodyByID(t.Context(), "kept-1"); err == nil {
|
||||
t.Error("a project that stopped keeping requests still has them after a prune")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user