Ilker ManapandClaude Opus 5.5
5b92ae0277
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
89 lines
3.0 KiB
Go
89 lines
3.0 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
func TestTokenTiming(t *testing.T) {
|
|
o := proxy.Outcome{Usage: &proxy.Usage{CompletionTokens: 11}, TTFT: 200 * time.Millisecond, Decode: time.Second}
|
|
tpot, rate, ok := tokenTiming(true, o, 2*time.Second)
|
|
if !ok || tpot != 100*time.Millisecond || rate != 10 {
|
|
t.Errorf("streamed: %s %.1f %v", tpot, rate, ok)
|
|
}
|
|
if _, rate, ok := tokenTiming(false, o, 2*time.Second); !ok || rate != 5.5 {
|
|
t.Errorf("whole: %.1f %v", rate, ok)
|
|
}
|
|
if _, _, ok := tokenTiming(true, proxy.Outcome{Usage: &proxy.Usage{CompletionTokens: 1}}, time.Second); ok {
|
|
t.Error("one token has no time per token")
|
|
}
|
|
}
|
|
|
|
func TestAModelPastItsSLOIsSaturatedForAdmission(t *testing.T) {
|
|
e := newFakeEngine(t, "m")
|
|
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})
|
|
m := catModel("m", nil)
|
|
m.SLO = catalog.SLO{TTFTp99: 100 * time.Millisecond, TPOTp99: 50 * time.Millisecond}
|
|
cfg.Models = []catalog.Model{m}
|
|
rt := newTestRouter(t, cfg)
|
|
waitForEndpoint(t, rt)
|
|
|
|
sat, _ := rt.saturation("m", 0)
|
|
if sat >= 1 {
|
|
t.Fatalf("idle and unmeasured, saturation %.2f", sat)
|
|
}
|
|
for range sloMinSamples - 1 {
|
|
rt.slo.add("m", 300*time.Millisecond, 20*time.Millisecond)
|
|
}
|
|
if p := rt.sloPressure("m"); p != 0 {
|
|
t.Errorf("believed with too few: %.2f", p)
|
|
}
|
|
rt.slo.add("m", 300*time.Millisecond, 20*time.Millisecond)
|
|
if sat, _ := rt.saturation("m", 0); sat < 2.9 {
|
|
t.Errorf("first tokens at three times the SLO read as saturation %.2f", sat)
|
|
}
|
|
}
|
|
|
|
func TestAStreamedAnswerIsTimedPerToken(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
f := w.(http.Flusher)
|
|
for i := range 5 {
|
|
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"content\":\"t%d\"}}]}\n\n", i)
|
|
f.Flush()
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
fmt.Fprint(w, "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":5}}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}))
|
|
b, _ := json.Marshal(map[string]any{"model": "m", "stream": true, "messages": []any{}})
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(b))
|
|
rec := httptest.NewRecorder()
|
|
rt.Handler().ServeHTTP(rec, req)
|
|
if rec.Code != 200 {
|
|
t.Fatalf("%d", rec.Code)
|
|
}
|
|
if testutil.CollectAndCount(rt.m.token.tpot) != 1 || testutil.CollectAndCount(rt.m.token.gap) != 1 ||
|
|
testutil.CollectAndCount(rt.m.token.rate) != 1 {
|
|
t.Error("not timed")
|
|
}
|
|
if _, tpot, _ := rt.slo.p99("m", time.Now()); tpot != 0 {
|
|
t.Error("one request was believed as a p99")
|
|
}
|
|
}
|