Files
llmgw/test/e2e/harness_test.go
T
ilkerandClaude Opus 5 a3f71aa5ee Phase 1: the router serves traffic
llmgw server is real now. It fronts a static set of engine replicas with an
OpenAI-compatible API, scoring endpoints through the pipeline added in the
previous commit.

- proxy: SSE relayed event by event with a flush after each, cancellation
  propagated upstream, and separate deadlines for dial, headers, first token,
  inter-token idle and total. The idle watchdog is what distinguishes a wedged
  engine from a slow one.
- router: dispatch with retry and model fallback. Retry only happens while
  nothing has reached the client; after that, splicing two responses together
  would be worse than the short stream the client already sees.
- config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides.
  Validation rejects duplicate endpoint ids and aliases that shadow a model,
  because both misroute silently at request time rather than failing loudly.
- metrics: requests, duration, TTFT, tokens, plus per-scorer values and
  per-filter drops on a separate admin listener with health and pprof.

Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across
3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV
blocks on every engine; router overhead p99 is about 0.9 ms against the plan's
2 ms budget.

The handlers live in internal/router rather than an internal/router/openai
package as the plan's tree sketched: they need the dispatch loop, and splitting
them apart would have meant an interface with exactly one implementation. Plan
§4 updated to match.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:41:13 +02:00

199 lines
4.5 KiB
Go

// Package e2e runs the router against real sim-engines over real HTTP.
//
// These tests exist to check the phase 1 exit criterion from the plan, so they
// deliberately use the whole stack: no fakes, no in-process shortcuts beyond
// httptest listeners.
package e2e
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
"gitea.mynodes.xyz/ilker/llmgw/internal/sim"
)
const testModel = "test-model"
type engine struct {
eng *sim.Engine
srv *httptest.Server
}
type harness struct {
engines []engine
router *httptest.Server
rt *router.Router
}
// simConfig is fast enough for tests while still exercising queueing.
func simConfig(name string, seed int64) sim.Config {
c := sim.Default()
c.Model = testModel
c.Seed = seed
c.Tick = time.Millisecond
c.MaxNumSeqs = 512
c.NumGPUBlocks = 200000
c.PrefillTokensPerSec = 5e6
c.DecodeTokensPerSec = 2000
c.BatchContention = 0
_ = name
return c
}
// start brings up n sim-engines behind a router.
func start(t *testing.T, n int, tune ...func(*config.Server)) *harness {
t.Helper()
h := &harness{}
cfg := config.Default()
cfg.Router.ScrapeInterval = 200 * time.Millisecond
cfg.Router.ScrapeTimeout = 150 * time.Millisecond
cfg.Router.StaleAfter = 2 * time.Second
for i := range n {
e := sim.New(simConfig(fmt.Sprintf("sim-%d", i), int64(i+1)))
srv := httptest.NewServer(sim.NewServer(e, slog.New(slog.DiscardHandler)))
h.engines = append(h.engines, engine{eng: e, srv: srv})
cfg.Endpoints = append(cfg.Endpoints, config.Endpoint{
ID: fmt.Sprintf("e%d", i),
URL: srv.URL,
Models: []string{testModel},
Pool: "test",
Weight: 1,
})
}
for _, f := range tune {
f(&cfg)
}
rt, err := router.New(cfg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
rt.Start(ctx)
h.rt = rt
h.router = httptest.NewServer(rt.Handler())
t.Cleanup(func() {
h.router.Close()
cancel()
rt.Stop()
for _, e := range h.engines {
e.srv.Close()
e.eng.Close()
}
})
// Let the first scrape land so scheduling starts from real data.
h.waitForFreshMetrics(t)
return h
}
func (h *harness) waitForFreshMetrics(t *testing.T) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
fresh := 0
for _, s := range h.rt.Registry().Snapshots(testModel) {
if !s.Stale {
fresh++
}
}
if fresh == len(h.engines) {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("endpoint metrics never became fresh")
}
func (h *harness) totalRunning() int {
n := 0
for _, e := range h.engines {
s := e.eng.Stats()
n += s.Running + s.Waiting
}
return n
}
// chatBody builds a streaming chat request.
func chatBody(prompt string, maxTokens int, stream bool) []byte {
b, _ := json.Marshal(map[string]any{
"model": testModel,
"messages": []map[string]string{{"role": "user", "content": prompt}},
"max_tokens": maxTokens,
"stream": stream,
})
return b
}
// streamRequest sends a streaming request and counts the content chunks.
func streamRequest(ctx context.Context, url string, body []byte, hdr map[string]string) (chunks int, ttfb time.Duration, err error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
return 0, 0, err
}
req.Header.Set("Content-Type", "application/json")
for k, v := range hdr {
req.Header.Set(k, v)
}
start := time.Now()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return 0, 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<12))
return 0, 0, fmt.Errorf("status %d: %s", resp.StatusCode, b)
}
br := bufio.NewReader(resp.Body)
for {
line, rerr := br.ReadString('\n')
if strings.HasPrefix(line, "data: ") {
if chunks == 0 {
ttfb = time.Since(start)
}
if !strings.Contains(line, "[DONE]") {
chunks++
}
}
if rerr != nil {
if rerr == io.EOF {
return chunks, ttfb, nil
}
return chunks, ttfb, rerr
}
}
}
// envInt reads an integer from the environment, falling back to def.
func envInt(key string, def int) int {
v := os.Getenv(key)
if v == "" {
return def
}
var n int
if _, err := fmt.Sscanf(v, "%d", &n); err != nil || n <= 0 {
return def
}
return n
}