Files
llmgw/internal/config/config.go
T
ilkerandClaude Opus 5 dc38dd6096 Phase 1.5: admission control
Inserted between quota checks and scheduling: admission answers when a request
may proceed, the scheduler still answers where it goes. Keeping them apart is
the point — an engine queue is FIFO and knows nothing about tenants, so once a
batch job is in it, no endpoint selection rescues the request behind it.

- saturation detector: one number per pool, the worse of queue depth and KV
  occupancy, since either alone misleads.
- priority bands with per-band saturation ceilings, so the cluster degrades from
  the bottom up.
- tenant round-robin within a band: one caller sending a thousand requests takes
  one slot in the rotation, not a thousand.
- bounded queues with capacity (503) and TTL (504); cancelled requests leave the
  queue instead of holding capacity.
- metrics for queue depth, wait time, rejections and pool saturation.

Exit criterion, measured in test/e2e: with 500 concurrent bronze requests
running continuously, interactive p99 in the platinum band stays at 1.04-1.24x
idle against a 1.5x budget. With admission off the same load pushes it to
55-60x. Both directions are asserted, so the test fails if admission regresses
and also if the load generator stops being a real load.

Two lags had to be closed, and they needed different answers. Engine metrics are
scraped and lag an interval; router inflight does not, but it counts running
requests, and treating those as queue makes a healthy engine look overloaded
every interval. What is actually queued is inflight minus batch capacity, and
capacity is not in the metric contract, so it is learned as the high-water mark
of observed running. Separately, between admission granting a request and the
router dispatching it the request is invisible everywhere, so a simultaneous
burst passed the check en masse; the controller now counts its own outstanding
leases against dispatched inflight.

One plan correction: a saturation ceiling of 1.0 was documented as "never held
back". It is not — it means "flow up to the healthy limit". A band that must
never wait needs a ceiling above 1. Plan §7.2.2 updated.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:53:38 +02:00

259 lines
8.5 KiB
Go

// Package config loads llmgw's YAML configuration.
//
// Phase 1 only needs the router's half of it: the endpoint list is static here
// and will come from the control plane in phase 3 (plan §6.4). The shape is
// already the one the plan describes, so that later change stays additive.
package config
import (
"fmt"
"os"
"strings"
"time"
"github.com/knadh/koanf/parsers/yaml"
env "github.com/knadh/koanf/providers/env/v2"
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/v2"
)
// Server is the whole server configuration.
type Server struct {
// Listen is the client-facing OpenAI API address.
Listen string `koanf:"listen"`
// AdminListen serves metrics, health and pprof. Kept separate so the
// client-facing port can be exposed without exposing diagnostics.
AdminListen string `koanf:"admin_listen"`
Router Router `koanf:"router"`
Models []Model `koanf:"models"`
Endpoints []Endpoint `koanf:"endpoints"`
}
// Router holds routing behaviour.
type Router struct {
ScrapeInterval time.Duration `koanf:"scrape_interval"`
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
// StaleAfter is how long an endpoint may go unscraped before being scored
// down (plan §7.6).
StaleAfter time.Duration `koanf:"stale_after"`
// Retries is how many *additional* endpoints may be tried after a failure,
// and only while nothing has been written to the client (plan §7.7).
Retries int `koanf:"retries"`
// Cooldown is how long a failing endpoint leaves the candidate pool.
Cooldown time.Duration `koanf:"cooldown"`
// FailureThreshold is the consecutive failure count that opens a breaker.
// 1 turns the breaker into a pure cooldown.
FailureThreshold int `koanf:"failure_threshold"`
// DefaultProfile names the routing profile used when a model picks none.
DefaultProfile string `koanf:"default_profile"`
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
SessionHeader string `koanf:"session_header"`
// InjectUsage asks engines for usage even when the client did not, so token
// accounting works regardless of client behaviour.
InjectUsage bool `koanf:"inject_usage"`
// MaxBodyBytes bounds the request body the router will read.
MaxBodyBytes int64 `koanf:"max_body_bytes"`
Timeouts Timeouts `koanf:"timeouts"`
Scoring Scoring `koanf:"scoring"`
Admission Admission `koanf:"admission"`
}
// Admission configures the flow control layer (plan §7.2).
type Admission struct {
// Enabled turns the layer on. Off, every request goes straight through,
// which is the v0.2.0 behaviour.
Enabled bool `koanf:"enabled"`
Bands []Band `koanf:"bands"`
DefaultBand string `koanf:"default_band"`
// BandHeader lets a client declare its band. Until authentication lands in
// v1.0.0 this is trusted, so the port must not be exposed to callers who
// should not be able to promote themselves.
BandHeader string `koanf:"band_header"`
QueueTTL time.Duration `koanf:"queue_ttl"`
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
Interval time.Duration `koanf:"interval"`
}
// Band is one priority class.
type Band struct {
Name string `koanf:"name"`
Priority int `koanf:"priority"`
SaturationCeiling float64 `koanf:"saturation_ceiling"`
}
// Timeouts bounds each phase of an upstream request.
type Timeouts struct {
Dial time.Duration `koanf:"dial"`
ResponseHeader time.Duration `koanf:"response_header"`
FirstToken time.Duration `koanf:"first_token"`
Idle time.Duration `koanf:"idle"`
Total time.Duration `koanf:"total"`
}
// Scoring holds the thresholds the scorers normalize against. These want to be
// measured with ctl bench (plan §10), not guessed.
type Scoring struct {
TargetWaiting float64 `koanf:"target_waiting"`
TargetInflight float64 `koanf:"target_inflight"`
KVCeiling float64 `koanf:"kv_ceiling"`
SessionLoadFactor float64 `koanf:"session_load_factor"`
}
// Model is a catalog entry as clients see it.
type Model struct {
Name string `koanf:"name"`
Aliases []string `koanf:"aliases"`
// Fallbacks are tried, in order, when this model has no usable endpoint.
Fallbacks []string `koanf:"fallbacks"`
// RoutingProfile overrides the router's default profile for this model.
RoutingProfile string `koanf:"routing_profile"`
}
// Endpoint is one engine replica. Static in phase 1.
type Endpoint struct {
ID string `koanf:"id"`
URL string `koanf:"url"`
Models []string `koanf:"models"`
Pool string `koanf:"pool"`
Weight float64 `koanf:"weight"`
Priority int `koanf:"priority"`
}
// Default returns the configuration used when nothing is specified.
func Default() Server {
return Server{
Listen: ":8080",
AdminListen: ":9090",
Router: Router{
ScrapeInterval: 2 * time.Second,
ScrapeTimeout: time.Second,
StaleAfter: 10 * time.Second,
Retries: 2,
Cooldown: 15 * time.Second,
FailureThreshold: 3,
DefaultProfile: "chat",
SessionHeader: "x-session-id",
InjectUsage: true,
MaxBodyBytes: 32 << 20,
Timeouts: Timeouts{
Dial: 5 * time.Second,
ResponseHeader: 60 * time.Second,
FirstToken: 120 * time.Second,
Idle: 60 * time.Second,
Total: 30 * time.Minute,
},
Scoring: Scoring{
TargetWaiting: 4,
TargetInflight: 8,
KVCeiling: 0.9,
SessionLoadFactor: 1.25,
},
Admission: Admission{
Enabled: true,
BandHeader: "x-llmgw-band",
DefaultBand: "gold",
Bands: []Band{
{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
},
QueueTTL: 30 * time.Second,
QueueCapacityPerBand: 2000,
Interval: 10 * time.Millisecond,
},
},
}
}
// Load reads configuration from path, overlaying it on the defaults, then
// applies LLMGW_-prefixed environment overrides.
//
// An empty path loads the defaults, which is what makes `llmgw server` with a
// couple of --endpoint flags usable for a smoke test.
func Load(path string) (Server, error) {
cfg := Default()
k := koanf.New(".")
if path != "" {
if _, err := os.Stat(path); err != nil {
return cfg, fmt.Errorf("config file: %w", err)
}
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
return cfg, fmt.Errorf("parsing %s: %w", path, err)
}
}
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
err := k.Load(env.Provider(".", env.Opt{
Prefix: "LLMGW_",
TransformFunc: func(key, value string) (string, any) {
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
return strings.ReplaceAll(key, "__", "."), value
},
}), nil)
if err != nil {
return cfg, fmt.Errorf("environment overrides: %w", err)
}
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
return cfg, fmt.Errorf("decoding configuration: %w", err)
}
return cfg, cfg.Validate()
}
// Validate rejects configurations that would otherwise fail confusingly at
// request time.
func (s Server) Validate() error {
if s.Listen == "" {
return fmt.Errorf("listen is empty")
}
seen := make(map[string]struct{}, len(s.Endpoints))
for i, e := range s.Endpoints {
switch {
case e.ID == "":
return fmt.Errorf("endpoints[%d]: id is required", i)
case e.URL == "":
return fmt.Errorf("endpoint %q: url is required", e.ID)
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
case len(e.Models) == 0:
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
}
if _, dup := seen[e.ID]; dup {
return fmt.Errorf("endpoint id %q is used twice", e.ID)
}
seen[e.ID] = struct{}{}
}
names := make(map[string]struct{}, len(s.Models))
for i, m := range s.Models {
if m.Name == "" {
return fmt.Errorf("models[%d]: name is required", i)
}
if _, dup := names[m.Name]; dup {
return fmt.Errorf("model %q is declared twice", m.Name)
}
names[m.Name] = struct{}{}
}
// Aliases share a namespace with names, so a collision must be caught here
// rather than silently shadowing a model at request time.
for _, m := range s.Models {
for _, a := range m.Aliases {
if _, clash := names[a]; clash {
return fmt.Errorf("model %q: alias %q collides with another model name", m.Name, a)
}
names[a] = struct{}{}
}
}
return nil
}