Files
llmgw/internal/router/router_test.go
T
ilkerandClaude Opus 5 a3f71aa5ee
CI / test (push) Successful in 1m1s
Phase 1: the router serves traffic
llmgw server is real now. It fronts a static set of engine replicas with an
OpenAI-compatible API, scoring endpoints through the pipeline added in the
previous commit.

- proxy: SSE relayed event by event with a flush after each, cancellation
  propagated upstream, and separate deadlines for dial, headers, first token,
  inter-token idle and total. The idle watchdog is what distinguishes a wedged
  engine from a slow one.
- router: dispatch with retry and model fallback. Retry only happens while
  nothing has reached the client; after that, splicing two responses together
  would be worse than the short stream the client already sees.
- config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides.
  Validation rejects duplicate endpoint ids and aliases that shadow a model,
  because both misroute silently at request time rather than failing loudly.
- metrics: requests, duration, TTFT, tokens, plus per-scorer values and
  per-filter drops on a separate admin listener with health and pprof.

Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across
3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV
blocks on every engine; router overhead p99 is about 0.9 ms against the plan's
2 ms budget.

The handlers live in internal/router rather than an internal/router/openai
package as the plan's tree sketched: they need the dispatch loop, and splitting
them apart would have meant an interface with exactly one implementation. Plan
§4 updated to match.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:41:13 +02:00

316 lines
10 KiB
Go

package router
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
)
// fakeEngine serves the minimum an endpoint needs: the metric contract plus a
// scripted chat response.
type fakeEngine struct {
srv *httptest.Server
calls atomic.Int64
// status, when non-zero, is returned instead of a completion.
status atomic.Int64
}
func newFakeEngine(t *testing.T, model string) *fakeEngine {
t.Helper()
f := &fakeEngine{}
mux := http.NewServeMux()
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n")
fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n")
fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n")
})
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) {
f.calls.Add(1)
if code := int(f.status.Load()); code != 0 {
http.Error(w, "unavailable", code)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"model":%q,"choices":[{"message":{"content":"hi"}}],"usage":{"prompt_tokens":4,"completion_tokens":2}}`, model)
})
f.srv = httptest.NewServer(mux)
t.Cleanup(f.srv.Close)
return f
}
func newTestRouter(t *testing.T, cfg config.Server) *Router {
t.Helper()
rt, err := New(cfg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
rt.Start(t.Context())
t.Cleanup(rt.Stop)
return rt
}
func post(t *testing.T, h http.Handler, body any, hdr map[string]string) *httptest.ResponseRecorder {
t.Helper()
b, err := json.Marshal(body)
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(b))
req.Header.Set("Content-Type", "application/json")
for k, v := range hdr {
req.Header.Set(k, v)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
func baseConfig(eps ...config.Endpoint) config.Server {
c := config.Default()
c.Router.ScrapeInterval = 50 * time.Millisecond
c.Router.StaleAfter = 5 * time.Second
c.Endpoints = eps
return c
}
func TestRoutesToHealthyEndpoint(t *testing.T) {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{
ID: "a", URL: e.srv.URL, Models: []string{"m"},
}))
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if e.calls.Load() != 1 {
t.Errorf("engine received %d calls", e.calls.Load())
}
}
func TestRetriesOnAnotherEndpoint(t *testing.T) {
bad := newFakeEngine(t, "m")
bad.status.Store(http.StatusServiceUnavailable)
good := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(
config.Endpoint{ID: "bad", URL: bad.srv.URL, Models: []string{"m"}},
config.Endpoint{ID: "good", URL: good.srv.URL, Models: []string{"m"}},
))
// With two endpoints and two retries, the bad one must not sink the request.
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if good.calls.Load() == 0 {
t.Error("the healthy endpoint was never tried")
}
}
func TestNoRetryAfterBytesReachClient(t *testing.T) {
// An endpoint that starts streaming then dies must not be retried: splicing
// two responses together is worse than a short stream (plan §7.7).
var served atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
io.WriteString(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n"+
"# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n"+
"# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0\n")
})
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) {
served.Add(1)
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(http.StatusOK)
f := w.(http.Flusher)
io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n")
f.Flush()
// Then hang up mid-stream.
})
srv := httptest.NewServer(mux)
defer srv.Close()
rt := newTestRouter(t, baseConfig(
config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}},
config.Endpoint{ID: "b", URL: srv.URL, Models: []string{"m"}},
))
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}, "stream": true}, nil)
if !strings.Contains(rec.Body.String(), `"content":"a"`) {
t.Fatalf("partial stream not delivered: %q", rec.Body.String())
}
if n := served.Load(); n != 1 {
t.Errorf("request was sent %d times; it must not be retried once streaming began", n)
}
}
func TestFallbackModel(t *testing.T) {
primary := newFakeEngine(t, "primary")
primary.status.Store(http.StatusServiceUnavailable)
backup := newFakeEngine(t, "backup")
cfg := baseConfig(
config.Endpoint{ID: "p", URL: primary.srv.URL, Models: []string{"primary"}},
config.Endpoint{ID: "b", URL: backup.srv.URL, Models: []string{"backup"}},
)
cfg.Models = []config.Model{{Name: "primary", Fallbacks: []string{"backup"}}}
rt := newTestRouter(t, cfg)
rec := post(t, rt.Handler(), map[string]any{"model": "primary", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if backup.calls.Load() == 0 {
t.Fatal("the fallback model was never tried")
}
// The client asked for one model and got another; they must be told.
if got := rec.Header().Get("x-llmgw-served-model"); got != "backup" {
t.Errorf("x-llmgw-served-model=%q, expected \"backup\"", got)
}
}
func TestAliasResolution(t *testing.T) {
e := newFakeEngine(t, "llama-3.3-70b")
cfg := baseConfig(config.Endpoint{
ID: "a", URL: e.srv.URL, Models: []string{"llama-3.3-70b"},
})
cfg.Models = []config.Model{{Name: "llama-3.3-70b", Aliases: []string{"llama"}}}
rt := newTestRouter(t, cfg)
rec := post(t, rt.Handler(), map[string]any{"model": "llama", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("alias was not resolved: %d %s", rec.Code, rec.Body.String())
}
}
func TestUnknownModelIs503(t *testing.T) {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
rec := post(t, rt.Handler(), map[string]any{"model": "nope", "messages": []any{}}, nil)
if rec.Code != http.StatusServiceUnavailable {
t.Errorf("status %d, expected 503", rec.Code)
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("error body is not JSON: %q", rec.Body.String())
}
if _, ok := body["error"]; !ok {
t.Errorf("error body is not OpenAI-shaped: %v", body)
}
}
func TestMissingModelIs400(t *testing.T) {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
rec := post(t, rt.Handler(), map[string]any{"messages": []any{}}, nil)
if rec.Code != http.StatusBadRequest {
t.Errorf("status %d, expected 400", rec.Code)
}
}
func TestModelsEndpointListsAliases(t *testing.T) {
e := newFakeEngine(t, "llama-3.3-70b")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"llama-3.3-70b"}})
cfg.Models = []config.Model{{Name: "llama-3.3-70b", Aliases: []string{"llama"}}}
rt := newTestRouter(t, cfg)
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, req)
var out struct {
Data []struct {
ID string `json:"id"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
ids := map[string]bool{}
for _, d := range out.Data {
ids[d.ID] = true
}
if !ids["llama-3.3-70b"] || !ids["llama"] {
t.Errorf("both the model and its alias should be listed: %v", ids)
}
}
func TestInjectIncludeUsagePreservesFields(t *testing.T) {
// The one place the router rewrites a request body; nothing may be lost.
in := []byte(`{"model":"m","stream":true,"temperature":0.7,"tools":[{"x":1}]}`)
out, ok := injectIncludeUsage(in)
if !ok {
t.Fatal("injection declined a well-formed body")
}
var m map[string]any
if err := json.Unmarshal(out, &m); err != nil {
t.Fatal(err)
}
for _, k := range []string{"model", "stream", "temperature", "tools"} {
if _, present := m[k]; !present {
t.Errorf("field %q was dropped", k)
}
}
opts, _ := m["stream_options"].(map[string]any)
if opts["include_usage"] != true {
t.Errorf("include_usage not set: %v", m["stream_options"])
}
}
func TestInjectIncludeUsageKeepsExistingOptions(t *testing.T) {
in := []byte(`{"model":"m","stream":true,"stream_options":{"something":"else"}}`)
out, ok := injectIncludeUsage(in)
if !ok {
t.Fatal("injection declined")
}
var m map[string]any
if err := json.Unmarshal(out, &m); err != nil {
t.Fatal(err)
}
opts, ok := m["stream_options"].(map[string]any)
if !ok {
t.Fatalf("stream_options is not an object: %v", m["stream_options"])
}
if opts["something"] != "else" {
t.Error("an existing stream_options field was dropped")
}
if opts["include_usage"] != true {
t.Error("include_usage not set")
}
}
func TestPromptPrefixIsBounded(t *testing.T) {
long := strings.Repeat("x", prefixBytes*3)
b := inferenceBody{Messages: []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
}{{Role: "user", Content: json.RawMessage(`"` + long + `"`)}}}
if got := len(promptPrefix(&b)); got > prefixBytes {
t.Errorf("prefix is %d bytes, over the %d cap", got, prefixBytes)
}
}
func TestSessionHeaderFallsBackToUser(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/", nil)
if got := sessionID(req, "x-session-id", "user-42"); got != "user-42" {
t.Errorf("expected the user field to be used, got %q", got)
}
req.Header.Set("x-session-id", "sess-1")
if got := sessionID(req, "x-session-id", "user-42"); got != "sess-1" {
t.Errorf("the header should win, got %q", got)
}
}