package router import ( "bytes" "encoding/json" "fmt" "net/http" "net/http/httptest" "testing" "time" "github.com/prometheus/client_golang/prometheus/testutil" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" ) func TestTokenTiming(t *testing.T) { o := proxy.Outcome{Usage: &proxy.Usage{CompletionTokens: 11}, TTFT: 200 * time.Millisecond, Decode: time.Second} tpot, rate, ok := tokenTiming(true, o, 2*time.Second) if !ok || tpot != 100*time.Millisecond || rate != 10 { t.Errorf("streamed: %s %.1f %v", tpot, rate, ok) } if _, rate, ok := tokenTiming(false, o, 2*time.Second); !ok || rate != 5.5 { t.Errorf("whole: %.1f %v", rate, ok) } if _, _, ok := tokenTiming(true, proxy.Outcome{Usage: &proxy.Usage{CompletionTokens: 1}}, time.Second); ok { t.Error("one token has no time per token") } } func TestAModelPastItsSLOIsSaturatedForAdmission(t *testing.T) { e := newFakeEngine(t, "m") cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) m := catModel("m", nil) m.SLO = catalog.SLO{TTFTp99: 100 * time.Millisecond, TPOTp99: 50 * time.Millisecond} cfg.Models = []catalog.Model{m} rt := newTestRouter(t, cfg) waitForEndpoint(t, rt) sat, _ := rt.saturation("m", 0) if sat >= 1 { t.Fatalf("idle and unmeasured, saturation %.2f", sat) } for range sloMinSamples - 1 { rt.slo.add("m", 300*time.Millisecond, 20*time.Millisecond) } if p := rt.sloPressure("m"); p != 0 { t.Errorf("believed with too few: %.2f", p) } rt.slo.add("m", 300*time.Millisecond, 20*time.Millisecond) if sat, _ := rt.saturation("m", 0); sat < 2.9 { t.Errorf("first tokens at three times the SLO read as saturation %.2f", sat) } } func TestAStreamedAnswerIsTimedPerToken(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/chat/completions" { return } w.Header().Set("Content-Type", "text/event-stream") f := w.(http.Flusher) for i := range 5 { fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"t%d\"}}]}\n\n", i) f.Flush() time.Sleep(20 * time.Millisecond) } fmt.Fprint(w, "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":5}}\n\ndata: [DONE]\n\n") })) t.Cleanup(srv.Close) rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}})) b, _ := json.Marshal(map[string]any{"model": "m", "stream": true, "messages": []any{}}) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(b)) rec := httptest.NewRecorder() rt.Handler().ServeHTTP(rec, req) if rec.Code != 200 { t.Fatalf("%d", rec.Code) } if testutil.CollectAndCount(rt.m.token.tpot) != 1 || testutil.CollectAndCount(rt.m.token.gap) != 1 || testutil.CollectAndCount(rt.m.token.rate) != 1 { t.Error("not timed") } if _, tpot, _ := rt.slo.p99("m", time.Now()); tpot != 0 { t.Error("one request was believed as a p99") } }