Phase 1: the router serves traffic
CI / test (push) Successful in 1m1s

llmgw server is real now. It fronts a static set of engine replicas with an
OpenAI-compatible API, scoring endpoints through the pipeline added in the
previous commit.

- proxy: SSE relayed event by event with a flush after each, cancellation
  propagated upstream, and separate deadlines for dial, headers, first token,
  inter-token idle and total. The idle watchdog is what distinguishes a wedged
  engine from a slow one.
- router: dispatch with retry and model fallback. Retry only happens while
  nothing has reached the client; after that, splicing two responses together
  would be worse than the short stream the client already sees.
- config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides.
  Validation rejects duplicate endpoint ids and aliases that shadow a model,
  because both misroute silently at request time rather than failing loudly.
- metrics: requests, duration, TTFT, tokens, plus per-scorer values and
  per-filter drops on a separate admin listener with health and pprof.

Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across
3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV
blocks on every engine; router overhead p99 is about 0.9 ms against the plan's
2 ms budget.

The handlers live in internal/router rather than an internal/router/openai
package as the plan's tree sketched: they need the dispatch loop, and splitting
them apart would have meant an interface with exactly one implementation. Plan
§4 updated to match.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 01:41:13 +02:00
1 parent f0c1ca87f1
commit a3f71aa5ee
20 files changed
+3028 -26

No files matched your search

+198
View File
@@ -0,0 +1,198 @@
// Package e2e runs the router against real sim-engines over real HTTP.
//
// These tests exist to check the phase 1 exit criterion from the plan, so they
// deliberately use the whole stack: no fakes, no in-process shortcuts beyond
// httptest listeners.
package e2e
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
"gitea.mynodes.xyz/ilker/llmgw/internal/sim"
)
const testModel = "test-model"
type engine struct {
eng *sim.Engine
srv *httptest.Server
}
type harness struct {
engines []engine
router *httptest.Server
rt *router.Router
}
// simConfig is fast enough for tests while still exercising queueing.
func simConfig(name string, seed int64) sim.Config {
c := sim.Default()
c.Model = testModel
c.Seed = seed
c.Tick = time.Millisecond
c.MaxNumSeqs = 512
c.NumGPUBlocks = 200000
c.PrefillTokensPerSec = 5e6
c.DecodeTokensPerSec = 2000
c.BatchContention = 0
_ = name
return c
}
// start brings up n sim-engines behind a router.
func start(t *testing.T, n int, tune ...func(*config.Server)) *harness {
t.Helper()
h := &harness{}
cfg := config.Default()
cfg.Router.ScrapeInterval = 200 * time.Millisecond
cfg.Router.ScrapeTimeout = 150 * time.Millisecond
cfg.Router.StaleAfter = 2 * time.Second
for i := range n {
e := sim.New(simConfig(fmt.Sprintf("sim-%d", i), int64(i+1)))
srv := httptest.NewServer(sim.NewServer(e, slog.New(slog.DiscardHandler)))
h.engines = append(h.engines, engine{eng: e, srv: srv})
cfg.Endpoints = append(cfg.Endpoints, config.Endpoint{
ID: fmt.Sprintf("e%d", i),
URL: srv.URL,
Models: []string{testModel},
Pool: "test",
Weight: 1,
})
}
for _, f := range tune {
f(&cfg)
}
rt, err := router.New(cfg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
rt.Start(ctx)
h.rt = rt
h.router = httptest.NewServer(rt.Handler())
t.Cleanup(func() {
h.router.Close()
cancel()
rt.Stop()
for _, e := range h.engines {
e.srv.Close()
e.eng.Close()
}
})
// Let the first scrape land so scheduling starts from real data.
h.waitForFreshMetrics(t)
return h
}
func (h *harness) waitForFreshMetrics(t *testing.T) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
fresh := 0
for _, s := range h.rt.Registry().Snapshots(testModel) {
if !s.Stale {
fresh++
}
}
if fresh == len(h.engines) {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("endpoint metrics never became fresh")
}
func (h *harness) totalRunning() int {
n := 0
for _, e := range h.engines {
s := e.eng.Stats()
n += s.Running + s.Waiting
}
return n
}
// chatBody builds a streaming chat request.
func chatBody(prompt string, maxTokens int, stream bool) []byte {
b, _ := json.Marshal(map[string]any{
"model": testModel,
"messages": []map[string]string{{"role": "user", "content": prompt}},
"max_tokens": maxTokens,
"stream": stream,
})
return b
}
// streamRequest sends a streaming request and counts the content chunks.
func streamRequest(ctx context.Context, url string, body []byte, hdr map[string]string) (chunks int, ttfb time.Duration, err error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
return 0, 0, err
}
req.Header.Set("Content-Type", "application/json")
for k, v := range hdr {
req.Header.Set(k, v)
}
start := time.Now()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return 0, 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<12))
return 0, 0, fmt.Errorf("status %d: %s", resp.StatusCode, b)
}
br := bufio.NewReader(resp.Body)
for {
line, rerr := br.ReadString('\n')
if strings.HasPrefix(line, "data: ") {
if chunks == 0 {
ttfb = time.Since(start)
}
if !strings.Contains(line, "[DONE]") {
chunks++
}
}
if rerr != nil {
if rerr == io.EOF {
return chunks, ttfb, nil
}
return chunks, ttfb, rerr
}
}
}
// envInt reads an integer from the environment, falling back to def.
func envInt(key string, def int) int {
v := os.Getenv(key)
if v == "" {
return def
}
var n int
if _, err := fmt.Sscanf(v, "%d", &n); err != nil || n <= 0 {
return def
}
return n
}
+197
View File
@@ -0,0 +1,197 @@
package e2e
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"os"
"slices"
"sync"
"sync/atomic"
"testing"
"time"
)
func bytesReader(b []byte) *bytes.Reader { return bytes.NewReader(b) }
func drainBody(resp *http.Response) {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
// TestConcurrentStreams is the first half of the phase 1 exit criterion:
// 1000 concurrent streams behind 3 sim-engines, all of them completing.
func TestConcurrentStreams(t *testing.T) {
concurrency := envInt("LLMGW_E2E_CONCURRENCY", 1000)
if testing.Short() {
concurrency = 50
}
const tokens = 16
h := start(t, 3)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
var (
ok atomic.Int64
short atomic.Int64
failed atomic.Int64
firstErr atomic.Value
wg sync.WaitGroup
)
start := time.Now()
for i := range concurrency {
wg.Add(1)
go func() {
defer wg.Done()
n, _, err := streamRequest(ctx, h.router.URL,
chatBody(fmt.Sprintf("request %d", i), tokens, true),
map[string]string{"x-session-id": fmt.Sprintf("s%d", i%97)})
switch {
case err != nil:
failed.Add(1)
firstErr.CompareAndSwap(nil, err)
case n < tokens:
short.Add(1)
default:
ok.Add(1)
}
}()
}
wg.Wait()
elapsed := time.Since(start)
t.Logf("%d streams in %v (%.0f req/s): ok=%d short=%d failed=%d",
concurrency, elapsed.Round(time.Millisecond),
float64(concurrency)/elapsed.Seconds(), ok.Load(), short.Load(), failed.Load())
if f := failed.Load(); f > 0 {
t.Fatalf("%d of %d streams failed; first error: %v", f, concurrency, firstErr.Load())
}
if s := short.Load(); s > 0 {
t.Errorf("%d streams ended before %d tokens", s, tokens)
}
// Load must actually have been spread; a single engine taking everything
// would pass the checks above while defeating the point of the router.
used := 0
for _, e := range h.engines {
if st := e.eng.Stats(); st.PrefixQueries > 0 || st.CachedBlocks > 0 {
used++
}
}
if used < 2 {
t.Errorf("only %d of %d engines saw traffic", used, len(h.engines))
}
}
// TestClientDisconnectStopsUpstream is the second half of the exit criterion:
// when a client goes away, the GPU must stop working on its request.
func TestClientDisconnectStopsUpstream(t *testing.T) {
const streams = 40
h := start(t, 3)
ctx, cancel := context.WithCancel(context.Background())
var wg sync.WaitGroup
for i := range streams {
wg.Add(1)
go func() {
defer wg.Done()
// Long generations, so every request is still running when cancelled.
_, _, _ = streamRequest(ctx, h.router.URL,
chatBody(fmt.Sprintf("long request %d", i), 100000, true), nil)
}()
}
// Wait until the engines are actually busy before pulling the plug.
deadline := time.Now().Add(10 * time.Second)
for h.totalRunning() < streams && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if busy := h.totalRunning(); busy < streams {
t.Fatalf("only %d of %d requests reached the engines", busy, streams)
}
cancel()
wg.Wait()
// The engines must drain: cancellation has to propagate router -> engine.
deadline = time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if h.totalRunning() == 0 {
// And the KV blocks must come back, not just the request slots.
for _, e := range h.engines {
if u := e.eng.Stats().KVCacheUsage; u != 0 {
t.Fatalf("KV cache still held after cancellation: %v", u)
}
}
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("engines still working %d requests after every client disconnected", h.totalRunning())
}
// TestRouterOverhead is the third part of the exit criterion: the router should
// add little latency of its own.
//
// It measures the same request against an engine directly and through the
// router, and reports the p99 difference. The assertion is deliberately loose,
// because a shared CI machine cannot hold a 2 ms bound reliably; set
// LLMGW_E2E_STRICT=1 on quiet hardware to enforce the plan's actual target.
func TestRouterOverhead(t *testing.T) {
samples := envInt("LLMGW_E2E_SAMPLES", 300)
if testing.Short() {
samples = 50
}
h := start(t, 1)
direct := h.engines[0].srv.URL
body := chatBody("overhead probe", 1, false)
measure := func(url string) []time.Duration {
out := make([]time.Duration, 0, samples)
for range samples {
start := time.Now()
resp, err := http.Post(url+"/v1/chat/completions", "application/json", bytesReader(body)) //nolint:noctx // test
if err != nil {
t.Fatal(err)
}
drainBody(resp)
out = append(out, time.Since(start))
}
slices.Sort(out)
return out
}
// Warm both paths so connection setup does not land in the samples.
measure(direct)
measure(h.router.URL)
d := measure(direct)
r := measure(h.router.URL)
p := func(s []time.Duration, q float64) time.Duration {
return s[min(int(float64(len(s))*q), len(s)-1)]
}
overheadP50 := p(r, 0.50) - p(d, 0.50)
overheadP99 := p(r, 0.99) - p(d, 0.99)
t.Logf("direct p50=%v p99=%v | routed p50=%v p99=%v | overhead p50=%v p99=%v",
p(d, 0.50).Round(time.Microsecond), p(d, 0.99).Round(time.Microsecond),
p(r, 0.50).Round(time.Microsecond), p(r, 0.99).Round(time.Microsecond),
overheadP50.Round(time.Microsecond), overheadP99.Round(time.Microsecond))
limit := 25 * time.Millisecond
if os.Getenv("LLMGW_E2E_STRICT") == "1" {
limit = 2 * time.Millisecond
}
if overheadP99 > limit {
t.Errorf("router added %v at p99, over the %v budget", overheadP99, limit)
}
}