CI / test (push) Successful in 1m1s
llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
151 lines
3.7 KiB
Go
151 lines
3.7 KiB
Go
package config
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func write(t *testing.T, content string) string {
|
|
t.Helper()
|
|
p := filepath.Join(t.TempDir(), "server.yaml")
|
|
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func TestLoadDefaultsWithoutFile(t *testing.T) {
|
|
cfg, err := Load("")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.Listen != ":8080" || cfg.Router.DefaultProfile != "chat" {
|
|
t.Errorf("defaults not applied: %+v", cfg)
|
|
}
|
|
}
|
|
|
|
func TestLoadOverlaysOnDefaults(t *testing.T) {
|
|
// Specifying one field must not blank out its neighbours.
|
|
p := write(t, `
|
|
listen: ":9999"
|
|
endpoints:
|
|
- id: a
|
|
url: http://127.0.0.1:8000
|
|
models: [m]
|
|
`)
|
|
cfg, err := Load(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.Listen != ":9999" {
|
|
t.Errorf("listen=%q", cfg.Listen)
|
|
}
|
|
if cfg.Router.ScrapeInterval != 2*time.Second {
|
|
t.Errorf("an unspecified default was lost: %v", cfg.Router.ScrapeInterval)
|
|
}
|
|
if len(cfg.Endpoints) != 1 || cfg.Endpoints[0].ID != "a" {
|
|
t.Errorf("endpoints=%+v", cfg.Endpoints)
|
|
}
|
|
}
|
|
|
|
func TestDurationsParse(t *testing.T) {
|
|
p := write(t, `
|
|
router:
|
|
scrape_interval: 500ms
|
|
timeouts:
|
|
idle: 45s
|
|
`)
|
|
cfg, err := Load(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.Router.ScrapeInterval != 500*time.Millisecond {
|
|
t.Errorf("scrape_interval=%v", cfg.Router.ScrapeInterval)
|
|
}
|
|
if cfg.Router.Timeouts.Idle != 45*time.Second {
|
|
t.Errorf("timeouts.idle=%v", cfg.Router.Timeouts.Idle)
|
|
}
|
|
}
|
|
|
|
func TestEnvOverride(t *testing.T) {
|
|
t.Setenv("LLMGW_ROUTER__RETRIES", "7")
|
|
cfg, err := Load("")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg.Router.Retries != 7 {
|
|
t.Errorf("retries=%d, expected the environment to win", cfg.Router.Retries)
|
|
}
|
|
}
|
|
|
|
func TestMissingFileIsAnError(t *testing.T) {
|
|
if _, err := Load("/nonexistent/server.yaml"); err == nil {
|
|
t.Error("expected an error for a missing config file")
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsBadEndpoints(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
ep Endpoint
|
|
want string
|
|
}{
|
|
{"no id", Endpoint{URL: "http://x", Models: []string{"m"}}, "id is required"},
|
|
{"no url", Endpoint{ID: "a", Models: []string{"m"}}, "url is required"},
|
|
{"bad scheme", Endpoint{ID: "a", URL: "127.0.0.1:8000", Models: []string{"m"}}, "http://"},
|
|
{"no models", Endpoint{ID: "a", URL: "http://x"}, "at least one model"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
s := Default()
|
|
s.Endpoints = []Endpoint{c.ep}
|
|
err := s.Validate()
|
|
if err == nil {
|
|
t.Fatal("expected an error")
|
|
}
|
|
if !strings.Contains(err.Error(), c.want) {
|
|
t.Errorf("error %q does not mention %q", err, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsDuplicateEndpointID(t *testing.T) {
|
|
s := Default()
|
|
s.Endpoints = []Endpoint{
|
|
{ID: "a", URL: "http://x", Models: []string{"m"}},
|
|
{ID: "a", URL: "http://y", Models: []string{"m"}},
|
|
}
|
|
if err := s.Validate(); err == nil || !strings.Contains(err.Error(), "twice") {
|
|
t.Errorf("duplicate id not rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsAliasCollision(t *testing.T) {
|
|
// An alias shadowing another model would silently misroute every request
|
|
// for that model, so it has to fail at load time.
|
|
s := Default()
|
|
s.Models = []Model{
|
|
{Name: "llama-3.3-70b", Aliases: []string{"qwen"}},
|
|
{Name: "qwen"},
|
|
}
|
|
err := s.Validate()
|
|
if err == nil || !strings.Contains(err.Error(), "collides") {
|
|
t.Errorf("alias collision not rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateAcceptsDistinctAliases(t *testing.T) {
|
|
s := Default()
|
|
s.Models = []Model{
|
|
{Name: "llama-3.3-70b", Aliases: []string{"llama", "llama3"}},
|
|
{Name: "qwen2.5-32b", Aliases: []string{"qwen"}},
|
|
}
|
|
if err := s.Validate(); err != nil {
|
|
t.Errorf("valid aliases rejected: %v", err)
|
|
}
|
|
}
|