llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f0c1ca87f1
commit
a3f71aa5ee
20 files changed
+3028
-26
No files matched your search
@@ -0,0 +1,198 @@
|
||||
// Package e2e runs the router against real sim-engines over real HTTP.
|
||||
//
|
||||
// These tests exist to check the phase 1 exit criterion from the plan, so they
|
||||
// deliberately use the whole stack: no fakes, no in-process shortcuts beyond
|
||||
// httptest listeners.
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sim"
|
||||
)
|
||||
|
||||
const testModel = "test-model"
|
||||
|
||||
type engine struct {
|
||||
eng *sim.Engine
|
||||
srv *httptest.Server
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
engines []engine
|
||||
router *httptest.Server
|
||||
rt *router.Router
|
||||
}
|
||||
|
||||
// simConfig is fast enough for tests while still exercising queueing.
|
||||
func simConfig(name string, seed int64) sim.Config {
|
||||
c := sim.Default()
|
||||
c.Model = testModel
|
||||
c.Seed = seed
|
||||
c.Tick = time.Millisecond
|
||||
c.MaxNumSeqs = 512
|
||||
c.NumGPUBlocks = 200000
|
||||
c.PrefillTokensPerSec = 5e6
|
||||
c.DecodeTokensPerSec = 2000
|
||||
c.BatchContention = 0
|
||||
_ = name
|
||||
return c
|
||||
}
|
||||
|
||||
// start brings up n sim-engines behind a router.
|
||||
func start(t *testing.T, n int, tune ...func(*config.Server)) *harness {
|
||||
t.Helper()
|
||||
|
||||
h := &harness{}
|
||||
cfg := config.Default()
|
||||
cfg.Router.ScrapeInterval = 200 * time.Millisecond
|
||||
cfg.Router.ScrapeTimeout = 150 * time.Millisecond
|
||||
cfg.Router.StaleAfter = 2 * time.Second
|
||||
|
||||
for i := range n {
|
||||
e := sim.New(simConfig(fmt.Sprintf("sim-%d", i), int64(i+1)))
|
||||
srv := httptest.NewServer(sim.NewServer(e, slog.New(slog.DiscardHandler)))
|
||||
h.engines = append(h.engines, engine{eng: e, srv: srv})
|
||||
cfg.Endpoints = append(cfg.Endpoints, config.Endpoint{
|
||||
ID: fmt.Sprintf("e%d", i),
|
||||
URL: srv.URL,
|
||||
Models: []string{testModel},
|
||||
Pool: "test",
|
||||
Weight: 1,
|
||||
})
|
||||
}
|
||||
for _, f := range tune {
|
||||
f(&cfg)
|
||||
}
|
||||
|
||||
rt, err := router.New(cfg, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
rt.Start(ctx)
|
||||
|
||||
h.rt = rt
|
||||
h.router = httptest.NewServer(rt.Handler())
|
||||
|
||||
t.Cleanup(func() {
|
||||
h.router.Close()
|
||||
cancel()
|
||||
rt.Stop()
|
||||
for _, e := range h.engines {
|
||||
e.srv.Close()
|
||||
e.eng.Close()
|
||||
}
|
||||
})
|
||||
|
||||
// Let the first scrape land so scheduling starts from real data.
|
||||
h.waitForFreshMetrics(t)
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *harness) waitForFreshMetrics(t *testing.T) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
fresh := 0
|
||||
for _, s := range h.rt.Registry().Snapshots(testModel) {
|
||||
if !s.Stale {
|
||||
fresh++
|
||||
}
|
||||
}
|
||||
if fresh == len(h.engines) {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("endpoint metrics never became fresh")
|
||||
}
|
||||
|
||||
func (h *harness) totalRunning() int {
|
||||
n := 0
|
||||
for _, e := range h.engines {
|
||||
s := e.eng.Stats()
|
||||
n += s.Running + s.Waiting
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// chatBody builds a streaming chat request.
|
||||
func chatBody(prompt string, maxTokens int, stream bool) []byte {
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"model": testModel,
|
||||
"messages": []map[string]string{{"role": "user", "content": prompt}},
|
||||
"max_tokens": maxTokens,
|
||||
"stream": stream,
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
// streamRequest sends a streaming request and counts the content chunks.
|
||||
func streamRequest(ctx context.Context, url string, body []byte, hdr map[string]string) (chunks int, ttfb time.Duration, err error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<12))
|
||||
return 0, 0, fmt.Errorf("status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
br := bufio.NewReader(resp.Body)
|
||||
for {
|
||||
line, rerr := br.ReadString('\n')
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
if chunks == 0 {
|
||||
ttfb = time.Since(start)
|
||||
}
|
||||
if !strings.Contains(line, "[DONE]") {
|
||||
chunks++
|
||||
}
|
||||
}
|
||||
if rerr != nil {
|
||||
if rerr == io.EOF {
|
||||
return chunks, ttfb, nil
|
||||
}
|
||||
return chunks, ttfb, rerr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// envInt reads an integer from the environment, falling back to def.
|
||||
func envInt(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
var n int
|
||||
if _, err := fmt.Sscanf(v, "%d", &n); err != nil || n <= 0 {
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func bytesReader(b []byte) *bytes.Reader { return bytes.NewReader(b) }
|
||||
|
||||
func drainBody(resp *http.Response) {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
// TestConcurrentStreams is the first half of the phase 1 exit criterion:
|
||||
// 1000 concurrent streams behind 3 sim-engines, all of them completing.
|
||||
func TestConcurrentStreams(t *testing.T) {
|
||||
concurrency := envInt("LLMGW_E2E_CONCURRENCY", 1000)
|
||||
if testing.Short() {
|
||||
concurrency = 50
|
||||
}
|
||||
const tokens = 16
|
||||
|
||||
h := start(t, 3)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
ok atomic.Int64
|
||||
short atomic.Int64
|
||||
failed atomic.Int64
|
||||
firstErr atomic.Value
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
start := time.Now()
|
||||
for i := range concurrency {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
n, _, err := streamRequest(ctx, h.router.URL,
|
||||
chatBody(fmt.Sprintf("request %d", i), tokens, true),
|
||||
map[string]string{"x-session-id": fmt.Sprintf("s%d", i%97)})
|
||||
switch {
|
||||
case err != nil:
|
||||
failed.Add(1)
|
||||
firstErr.CompareAndSwap(nil, err)
|
||||
case n < tokens:
|
||||
short.Add(1)
|
||||
default:
|
||||
ok.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
t.Logf("%d streams in %v (%.0f req/s): ok=%d short=%d failed=%d",
|
||||
concurrency, elapsed.Round(time.Millisecond),
|
||||
float64(concurrency)/elapsed.Seconds(), ok.Load(), short.Load(), failed.Load())
|
||||
|
||||
if f := failed.Load(); f > 0 {
|
||||
t.Fatalf("%d of %d streams failed; first error: %v", f, concurrency, firstErr.Load())
|
||||
}
|
||||
if s := short.Load(); s > 0 {
|
||||
t.Errorf("%d streams ended before %d tokens", s, tokens)
|
||||
}
|
||||
|
||||
// Load must actually have been spread; a single engine taking everything
|
||||
// would pass the checks above while defeating the point of the router.
|
||||
used := 0
|
||||
for _, e := range h.engines {
|
||||
if st := e.eng.Stats(); st.PrefixQueries > 0 || st.CachedBlocks > 0 {
|
||||
used++
|
||||
}
|
||||
}
|
||||
if used < 2 {
|
||||
t.Errorf("only %d of %d engines saw traffic", used, len(h.engines))
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientDisconnectStopsUpstream is the second half of the exit criterion:
|
||||
// when a client goes away, the GPU must stop working on its request.
|
||||
func TestClientDisconnectStopsUpstream(t *testing.T) {
|
||||
const streams = 40
|
||||
|
||||
h := start(t, 3)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var wg sync.WaitGroup
|
||||
for i := range streams {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// Long generations, so every request is still running when cancelled.
|
||||
_, _, _ = streamRequest(ctx, h.router.URL,
|
||||
chatBody(fmt.Sprintf("long request %d", i), 100000, true), nil)
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait until the engines are actually busy before pulling the plug.
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for h.totalRunning() < streams && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if busy := h.totalRunning(); busy < streams {
|
||||
t.Fatalf("only %d of %d requests reached the engines", busy, streams)
|
||||
}
|
||||
|
||||
cancel()
|
||||
wg.Wait()
|
||||
|
||||
// The engines must drain: cancellation has to propagate router -> engine.
|
||||
deadline = time.Now().Add(15 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if h.totalRunning() == 0 {
|
||||
// And the KV blocks must come back, not just the request slots.
|
||||
for _, e := range h.engines {
|
||||
if u := e.eng.Stats().KVCacheUsage; u != 0 {
|
||||
t.Fatalf("KV cache still held after cancellation: %v", u)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("engines still working %d requests after every client disconnected", h.totalRunning())
|
||||
}
|
||||
|
||||
// TestRouterOverhead is the third part of the exit criterion: the router should
|
||||
// add little latency of its own.
|
||||
//
|
||||
// It measures the same request against an engine directly and through the
|
||||
// router, and reports the p99 difference. The assertion is deliberately loose,
|
||||
// because a shared CI machine cannot hold a 2 ms bound reliably; set
|
||||
// LLMGW_E2E_STRICT=1 on quiet hardware to enforce the plan's actual target.
|
||||
func TestRouterOverhead(t *testing.T) {
|
||||
samples := envInt("LLMGW_E2E_SAMPLES", 300)
|
||||
if testing.Short() {
|
||||
samples = 50
|
||||
}
|
||||
|
||||
h := start(t, 1)
|
||||
direct := h.engines[0].srv.URL
|
||||
body := chatBody("overhead probe", 1, false)
|
||||
|
||||
measure := func(url string) []time.Duration {
|
||||
out := make([]time.Duration, 0, samples)
|
||||
for range samples {
|
||||
start := time.Now()
|
||||
resp, err := http.Post(url+"/v1/chat/completions", "application/json", bytesReader(body)) //nolint:noctx // test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
drainBody(resp)
|
||||
out = append(out, time.Since(start))
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Warm both paths so connection setup does not land in the samples.
|
||||
measure(direct)
|
||||
measure(h.router.URL)
|
||||
|
||||
d := measure(direct)
|
||||
r := measure(h.router.URL)
|
||||
|
||||
p := func(s []time.Duration, q float64) time.Duration {
|
||||
return s[min(int(float64(len(s))*q), len(s)-1)]
|
||||
}
|
||||
overheadP50 := p(r, 0.50) - p(d, 0.50)
|
||||
overheadP99 := p(r, 0.99) - p(d, 0.99)
|
||||
|
||||
t.Logf("direct p50=%v p99=%v | routed p50=%v p99=%v | overhead p50=%v p99=%v",
|
||||
p(d, 0.50).Round(time.Microsecond), p(d, 0.99).Round(time.Microsecond),
|
||||
p(r, 0.50).Round(time.Microsecond), p(r, 0.99).Round(time.Microsecond),
|
||||
overheadP50.Round(time.Microsecond), overheadP99.Round(time.Microsecond))
|
||||
|
||||
limit := 25 * time.Millisecond
|
||||
if os.Getenv("LLMGW_E2E_STRICT") == "1" {
|
||||
limit = 2 * time.Millisecond
|
||||
}
|
||||
if overheadP99 > limit {
|
||||
t.Errorf("router added %v at p99, over the %v budget", overheadP99, limit)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user