package router import ( "fmt" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) func keptBody(t *testing.T, rt *Router, id string) (store.RequestBody, bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) for { b, err := rt.store.RequestBodyByID(t.Context(), id) if err == nil { return b, true } if time.Now().After(deadline) { return store.RequestBody{}, false } time.Sleep(20 * time.Millisecond) } } func TestAProjectThatAsksKeepsItsRequestsAndTheyReplay(t *testing.T) { e := newFakeEngine(t, "qwen2.5-14b-instruct") rt, secret := pricedRouter(t, e.srv.URL) h := rt.Handler() project := mustProjectOf(t, rt, "key-1") send := func(id string) { rec := post(t, h, map[string]any{"model": "q14", "messages": []any{ map[string]any{"role": "user", "content": "what is 2+2"}}}, map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id}) if rec.Code != 200 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } } send("before") if _, ok := keptBody(t, rt, "before"); ok { t.Fatal("a request was kept by a project that keeps none") } if err := rt.store.SetRequestLog(t.Context(), project, 24, 1); err != nil { t.Fatal(err) } rt.reqLogs.by = nil send("kept-1") b, ok := keptBody(t, rt, "kept-1") if !ok { t.Fatal("not kept") } if !strings.Contains(b.Request, "what is 2+2") || b.Response != "hi" || b.Status != 200 || b.ServedModel != "qwen2.5-14b-instruct" || b.Path != "/v1/chat/completions" { t.Errorf("kept %+v", b) } if b.ExpiresAt.Sub(b.At) != 24*time.Hour { t.Errorf("kept for %s", b.ExpiresAt.Sub(b.At)) } status, body, _, err := rt.Replay(t.Context(), b.Path, "q14", []byte(b.Request)) if err != nil || status != 200 || !strings.Contains(string(body), "hi") { t.Errorf("replay: %d %s %v", status, body, err) } if _, _, _, err := rt.Replay(t.Context(), "/v1/files", "q14", nil); err == nil { t.Error("a non-inference path was replayed") } // A small share keeps few. _ = rt.store.SetRequestLog(t.Context(), project, 24, 0.05) rt.reqLogs.by = nil for i := range 100 { send(fmt.Sprintf("s-%d", i)) } time.Sleep(200 * time.Millisecond) n := 0 for i := range 100 { if _, err := rt.store.RequestBodyByID(t.Context(), fmt.Sprintf("s-%d", i)); err == nil { n++ } } if n == 0 || n > 20 { t.Errorf("kept %d of 100 at 5%%", n) } // Off: what was kept goes at the next prune. _ = rt.store.SetRequestLog(t.Context(), project, 0, 0) if _, err := rt.store.PruneRequestBodies(t.Context(), time.Now()); err != nil { t.Fatal(err) } if _, err := rt.store.RequestBodyByID(t.Context(), "kept-1"); err == nil { t.Error("a project that stopped keeping requests still has them after a prune") } }