# llmgw router configuration (phase 1).
#
# The endpoint list is static here. From phase 3 it comes from the control
# plane instead and this section disappears; everything else stays.
#
# Any field can be overridden from the environment with an LLMGW_ prefix and
# __ for nesting:  LLMGW_ROUTER__RETRIES=3

listen: ":8080"        # client-facing OpenAI API
admin_listen: ":9090"  # metrics, health, pprof — do not expose this publicly

router:
  scrape_interval: 2s
  scrape_timeout: 1s
  # How long an endpoint may go unscraped before it is scored down. It is never
  # dropped for staleness alone: if the whole pool went stale there would be
  # nowhere left to send anything.
  stale_after: 10s

  # Additional endpoints to try after a failure, and only while nothing has
  # reached the client yet.
  retries: 2
  # How long a failing endpoint leaves the candidate pool. A long fallback list
  # with no cooldown turns one endpoint's rate limit into a cluster-wide retry
  # storm.
  cooldown: 15s
  # Consecutive failures that open the breaker. Set to 1 for pure cooldown
  # behaviour.
  failure_threshold: 3

  default_profile: chat        # chat | batch | long-context | debug
  session_header: x-session-id # affinity key; falls back to the `user` field
  # Ask engines for usage even when the client did not, so token accounting
  # works regardless of client behaviour. The extra chunk is dropped on the way
  # back to a client that did not ask for it.
  inject_usage: true
  max_body_bytes: 33554432

  timeouts:
    dial: 5s
    response_header: 60s
    first_token: 120s
    idle: 60s      # between streamed tokens; catches a wedged engine
    total: 30m

  # Flow control: decides *when* a request may proceed, before scheduling
  # decides where it goes. Without it a batch workload starves interactive
  # traffic, because an engine queue is FIFO and knows nothing about tenants.
  admission:
    enabled: true
    # Until authentication lands, the band comes from a header and is trusted.
    # Do not expose this port to callers who should not be able to promote
    # themselves.
    band_header: x-llmgw-band
    default_band: gold
    # A band waits once pool saturation passes its ceiling, so the cluster
    # degrades from the bottom up. A ceiling of 1.0 means "flow up to the
    # healthy limit"; use a value above 1 for a band that must never wait.
    bands:
      - { name: platinum, priority: 400, saturation_ceiling: 1.00 }
      - { name: gold,     priority: 300, saturation_ceiling: 0.85 }
      - { name: silver,   priority: 200, saturation_ceiling: 0.60 }
      - { name: bronze,   priority: 100, saturation_ceiling: 0.30 }
    # Drop a request that has waited this long: the client has usually given up,
    # and sending the GPU work nobody is waiting for helps no one.
    queue_ttl: 30s
    # Reject immediately once a band's queue is this deep, rather than letting
    # it accumulate silently.
    queue_capacity_per_band: 2000
    interval: 10ms

  # What the scorers normalize against. Measure these with `ctl bench` rather
  # than guessing: target_waiting in particular governs how the whole system
  # behaves under load. Too low starves the GPU, too high brings back queueing
  # delay.
  scoring:
    target_waiting: 4
    target_inflight: 8
    kv_ceiling: 0.9
    session_load_factor: 1.25

models:
  - name: llama-3.3-70b
    aliases: [llama]
    fallbacks: [qwen2.5-32b]
    routing_profile: chat

  - name: qwen2.5-32b
    aliases: [qwen]
    routing_profile: batch

endpoints:
  - id: gpu-a-01
    url: http://gpu-a-01:8000
    models: [llama-3.3-70b]
    pool: ib-a/h100-80g
    weight: 100     # from ctl bench; do not estimate it by hand
    priority: 100   # higher pools are tried first

  - id: gpu-b-01
    url: http://gpu-b-01:8000
    models: [llama-3.3-70b, qwen2.5-32b]
    pool: ib-b/a100-80g
    weight: 45
    priority: 50