CI / test (push) Successful in 1m1s
llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
219 lines
7.1 KiB
Go
219 lines
7.1 KiB
Go
// Package config loads llmgw's YAML configuration.
|
|
//
|
|
// Phase 1 only needs the router's half of it: the endpoint list is static here
|
|
// and will come from the control plane in phase 3 (plan §6.4). The shape is
|
|
// already the one the plan describes, so that later change stays additive.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/knadh/koanf/parsers/yaml"
|
|
env "github.com/knadh/koanf/providers/env/v2"
|
|
"github.com/knadh/koanf/providers/file"
|
|
"github.com/knadh/koanf/v2"
|
|
)
|
|
|
|
// Server is the whole server configuration.
|
|
type Server struct {
|
|
// Listen is the client-facing OpenAI API address.
|
|
Listen string `koanf:"listen"`
|
|
// AdminListen serves metrics, health and pprof. Kept separate so the
|
|
// client-facing port can be exposed without exposing diagnostics.
|
|
AdminListen string `koanf:"admin_listen"`
|
|
|
|
Router Router `koanf:"router"`
|
|
Models []Model `koanf:"models"`
|
|
Endpoints []Endpoint `koanf:"endpoints"`
|
|
}
|
|
|
|
// Router holds routing behaviour.
|
|
type Router struct {
|
|
ScrapeInterval time.Duration `koanf:"scrape_interval"`
|
|
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
|
|
// StaleAfter is how long an endpoint may go unscraped before being scored
|
|
// down (plan §7.6).
|
|
StaleAfter time.Duration `koanf:"stale_after"`
|
|
|
|
// Retries is how many *additional* endpoints may be tried after a failure,
|
|
// and only while nothing has been written to the client (plan §7.7).
|
|
Retries int `koanf:"retries"`
|
|
// Cooldown is how long a failing endpoint leaves the candidate pool.
|
|
Cooldown time.Duration `koanf:"cooldown"`
|
|
// FailureThreshold is the consecutive failure count that opens a breaker.
|
|
// 1 turns the breaker into a pure cooldown.
|
|
FailureThreshold int `koanf:"failure_threshold"`
|
|
|
|
// DefaultProfile names the routing profile used when a model picks none.
|
|
DefaultProfile string `koanf:"default_profile"`
|
|
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
|
|
SessionHeader string `koanf:"session_header"`
|
|
// InjectUsage asks engines for usage even when the client did not, so token
|
|
// accounting works regardless of client behaviour.
|
|
InjectUsage bool `koanf:"inject_usage"`
|
|
|
|
// MaxBodyBytes bounds the request body the router will read.
|
|
MaxBodyBytes int64 `koanf:"max_body_bytes"`
|
|
|
|
Timeouts Timeouts `koanf:"timeouts"`
|
|
Scoring Scoring `koanf:"scoring"`
|
|
}
|
|
|
|
// Timeouts bounds each phase of an upstream request.
|
|
type Timeouts struct {
|
|
Dial time.Duration `koanf:"dial"`
|
|
ResponseHeader time.Duration `koanf:"response_header"`
|
|
FirstToken time.Duration `koanf:"first_token"`
|
|
Idle time.Duration `koanf:"idle"`
|
|
Total time.Duration `koanf:"total"`
|
|
}
|
|
|
|
// Scoring holds the thresholds the scorers normalize against. These want to be
|
|
// measured with ctl bench (plan §10), not guessed.
|
|
type Scoring struct {
|
|
TargetWaiting float64 `koanf:"target_waiting"`
|
|
TargetInflight float64 `koanf:"target_inflight"`
|
|
KVCeiling float64 `koanf:"kv_ceiling"`
|
|
SessionLoadFactor float64 `koanf:"session_load_factor"`
|
|
}
|
|
|
|
// Model is a catalog entry as clients see it.
|
|
type Model struct {
|
|
Name string `koanf:"name"`
|
|
Aliases []string `koanf:"aliases"`
|
|
// Fallbacks are tried, in order, when this model has no usable endpoint.
|
|
Fallbacks []string `koanf:"fallbacks"`
|
|
// RoutingProfile overrides the router's default profile for this model.
|
|
RoutingProfile string `koanf:"routing_profile"`
|
|
}
|
|
|
|
// Endpoint is one engine replica. Static in phase 1.
|
|
type Endpoint struct {
|
|
ID string `koanf:"id"`
|
|
URL string `koanf:"url"`
|
|
Models []string `koanf:"models"`
|
|
Pool string `koanf:"pool"`
|
|
Weight float64 `koanf:"weight"`
|
|
Priority int `koanf:"priority"`
|
|
}
|
|
|
|
// Default returns the configuration used when nothing is specified.
|
|
func Default() Server {
|
|
return Server{
|
|
Listen: ":8080",
|
|
AdminListen: ":9090",
|
|
Router: Router{
|
|
ScrapeInterval: 2 * time.Second,
|
|
ScrapeTimeout: time.Second,
|
|
StaleAfter: 10 * time.Second,
|
|
Retries: 2,
|
|
Cooldown: 15 * time.Second,
|
|
FailureThreshold: 3,
|
|
DefaultProfile: "chat",
|
|
SessionHeader: "x-session-id",
|
|
InjectUsage: true,
|
|
MaxBodyBytes: 32 << 20,
|
|
Timeouts: Timeouts{
|
|
Dial: 5 * time.Second,
|
|
ResponseHeader: 60 * time.Second,
|
|
FirstToken: 120 * time.Second,
|
|
Idle: 60 * time.Second,
|
|
Total: 30 * time.Minute,
|
|
},
|
|
Scoring: Scoring{
|
|
TargetWaiting: 4,
|
|
TargetInflight: 8,
|
|
KVCeiling: 0.9,
|
|
SessionLoadFactor: 1.25,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load reads configuration from path, overlaying it on the defaults, then
|
|
// applies LLMGW_-prefixed environment overrides.
|
|
//
|
|
// An empty path loads the defaults, which is what makes `llmgw server` with a
|
|
// couple of --endpoint flags usable for a smoke test.
|
|
func Load(path string) (Server, error) {
|
|
cfg := Default()
|
|
k := koanf.New(".")
|
|
|
|
if path != "" {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return cfg, fmt.Errorf("config file: %w", err)
|
|
}
|
|
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
|
return cfg, fmt.Errorf("parsing %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
|
|
err := k.Load(env.Provider(".", env.Opt{
|
|
Prefix: "LLMGW_",
|
|
TransformFunc: func(key, value string) (string, any) {
|
|
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
|
|
return strings.ReplaceAll(key, "__", "."), value
|
|
},
|
|
}), nil)
|
|
if err != nil {
|
|
return cfg, fmt.Errorf("environment overrides: %w", err)
|
|
}
|
|
|
|
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
|
return cfg, fmt.Errorf("decoding configuration: %w", err)
|
|
}
|
|
return cfg, cfg.Validate()
|
|
}
|
|
|
|
// Validate rejects configurations that would otherwise fail confusingly at
|
|
// request time.
|
|
func (s Server) Validate() error {
|
|
if s.Listen == "" {
|
|
return fmt.Errorf("listen is empty")
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(s.Endpoints))
|
|
for i, e := range s.Endpoints {
|
|
switch {
|
|
case e.ID == "":
|
|
return fmt.Errorf("endpoints[%d]: id is required", i)
|
|
case e.URL == "":
|
|
return fmt.Errorf("endpoint %q: url is required", e.ID)
|
|
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
|
|
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
|
|
case len(e.Models) == 0:
|
|
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
|
|
}
|
|
if _, dup := seen[e.ID]; dup {
|
|
return fmt.Errorf("endpoint id %q is used twice", e.ID)
|
|
}
|
|
seen[e.ID] = struct{}{}
|
|
}
|
|
|
|
names := make(map[string]struct{}, len(s.Models))
|
|
for i, m := range s.Models {
|
|
if m.Name == "" {
|
|
return fmt.Errorf("models[%d]: name is required", i)
|
|
}
|
|
if _, dup := names[m.Name]; dup {
|
|
return fmt.Errorf("model %q is declared twice", m.Name)
|
|
}
|
|
names[m.Name] = struct{}{}
|
|
}
|
|
// Aliases share a namespace with names, so a collision must be caught here
|
|
// rather than silently shadowing a model at request time.
|
|
for _, m := range s.Models {
|
|
for _, a := range m.Aliases {
|
|
if _, clash := names[a]; clash {
|
|
return fmt.Errorf("model %q: alias %q collides with another model name", m.Name, a)
|
|
}
|
|
names[a] = struct{}{}
|
|
}
|
|
}
|
|
return nil
|
|
}
|