Ilker ManapandClaude Opus 5.5
32ff1dbd6f
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
97 lines
2.7 KiB
Go
97 lines
2.7 KiB
Go
package router
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func keptBody(t *testing.T, rt *Router, id string) (store.RequestBody, bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
b, err := rt.store.RequestBodyByID(t.Context(), id)
|
|
if err == nil {
|
|
return b, true
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return store.RequestBody{}, false
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestAProjectThatAsksKeepsItsRequestsAndTheyReplay(t *testing.T) {
|
|
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
|
rt, secret := pricedRouter(t, e.srv.URL)
|
|
h := rt.Handler()
|
|
project := mustProjectOf(t, rt, "key-1")
|
|
|
|
send := func(id string) {
|
|
rec := post(t, h, map[string]any{"model": "q14", "messages": []any{
|
|
map[string]any{"role": "user", "content": "what is 2+2"}}},
|
|
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id})
|
|
if rec.Code != 200 {
|
|
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
send("before")
|
|
if _, ok := keptBody(t, rt, "before"); ok {
|
|
t.Fatal("a request was kept by a project that keeps none")
|
|
}
|
|
|
|
if err := rt.store.SetRequestLog(t.Context(), project, 24, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rt.reqLogs.by = nil
|
|
send("kept-1")
|
|
b, ok := keptBody(t, rt, "kept-1")
|
|
if !ok {
|
|
t.Fatal("not kept")
|
|
}
|
|
if !strings.Contains(b.Request, "what is 2+2") || b.Response != "hi" || b.Status != 200 ||
|
|
b.ServedModel != "qwen2.5-14b-instruct" || b.Path != "/v1/chat/completions" {
|
|
t.Errorf("kept %+v", b)
|
|
}
|
|
if b.ExpiresAt.Sub(b.At) != 24*time.Hour {
|
|
t.Errorf("kept for %s", b.ExpiresAt.Sub(b.At))
|
|
}
|
|
|
|
status, body, _, err := rt.Replay(t.Context(), b.Path, "q14", []byte(b.Request))
|
|
if err != nil || status != 200 || !strings.Contains(string(body), "hi") {
|
|
t.Errorf("replay: %d %s %v", status, body, err)
|
|
}
|
|
if _, _, _, err := rt.Replay(t.Context(), "/v1/files", "q14", nil); err == nil {
|
|
t.Error("a non-inference path was replayed")
|
|
}
|
|
|
|
// A small share keeps few.
|
|
_ = rt.store.SetRequestLog(t.Context(), project, 24, 0.05)
|
|
rt.reqLogs.by = nil
|
|
for i := range 100 {
|
|
send(fmt.Sprintf("s-%d", i))
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
n := 0
|
|
for i := range 100 {
|
|
if _, err := rt.store.RequestBodyByID(t.Context(), fmt.Sprintf("s-%d", i)); err == nil {
|
|
n++
|
|
}
|
|
}
|
|
if n == 0 || n > 20 {
|
|
t.Errorf("kept %d of 100 at 5%%", n)
|
|
}
|
|
|
|
// Off: what was kept goes at the next prune.
|
|
_ = rt.store.SetRequestLog(t.Context(), project, 0, 0)
|
|
if _, err := rt.store.PruneRequestBodies(t.Context(), time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := rt.store.RequestBodyByID(t.Context(), "kept-1"); err == nil {
|
|
t.Error("a project that stopped keeping requests still has them after a prune")
|
|
}
|
|
}
|