# llmgw router configuration (phase 1). # # The endpoint list is static here. From phase 3 it comes from the control # plane instead and this section disappears; everything else stays. # # Any field can be overridden from the environment with an LLMGW_ prefix and # __ for nesting: LLMGW_ROUTER__RETRIES=3 listen: ":8080" # client-facing OpenAI API admin_listen: ":9090" # metrics, health, pprof — do not expose this publicly router: scrape_interval: 2s scrape_timeout: 1s # How long an endpoint may go unscraped before it is scored down. It is never # dropped for staleness alone: if the whole pool went stale there would be # nowhere left to send anything. stale_after: 10s # Additional endpoints to try after a failure, and only while nothing has # reached the client yet. retries: 2 # How long a failing endpoint leaves the candidate pool. A long fallback list # with no cooldown turns one endpoint's rate limit into a cluster-wide retry # storm. cooldown: 15s # Consecutive failures that open the breaker. Set to 1 for pure cooldown # behaviour. failure_threshold: 3 default_profile: chat # chat | batch | long-context | debug session_header: x-session-id # affinity key; falls back to the `user` field # Ask engines for usage even when the client did not, so token accounting # works regardless of client behaviour. The extra chunk is dropped on the way # back to a client that did not ask for it. inject_usage: true max_body_bytes: 33554432 timeouts: dial: 5s response_header: 60s first_token: 120s idle: 60s # between streamed tokens; catches a wedged engine total: 30m # What the scorers normalize against. Measure these with `ctl bench` rather # than guessing: target_waiting in particular governs how the whole system # behaves under load. Too low starves the GPU, too high brings back queueing # delay. scoring: target_waiting: 4 target_inflight: 8 kv_ceiling: 0.9 session_load_factor: 1.25 models: - name: llama-3.3-70b aliases: [llama] fallbacks: [qwen2.5-32b] routing_profile: chat - name: qwen2.5-32b aliases: [qwen] routing_profile: batch endpoints: - id: gpu-a-01 url: http://gpu-a-01:8000 models: [llama-3.3-70b] pool: ib-a/h100-80g weight: 100 # from ctl bench; do not estimate it by hand priority: 100 # higher pools are tried first - id: gpu-b-01 url: http://gpu-b-01:8000 models: [llama-3.3-70b, qwen2.5-32b] pool: ib-b/a100-80g weight: 45 priority: 50