llmgw/deploy/examples/server.yaml

# llmgw router configuration (phase 1).
#
# The endpoint list is static here. From phase 3 it comes from the control
# plane instead and this section disappears; everything else stays.
#
# Any field can be overridden from the environment with an LLMGW_ prefix and
# __ for nesting:  LLMGW_ROUTER__RETRIES=3

listen: ":8080"        # client-facing OpenAI API
admin_listen: ":9090"  # metrics, health, pprof — do not expose this publicly
# Where node agents connect. Separate from the client port so the fleet-facing
# surface can be firewalled independently.
agent_listen: ":7443"

# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken
# from its certificate's common name and an agent claiming a different node id
# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach
# it can claim to be any node, so it must stay on a trusted network.
#
# These certificates come from llmgw's own authority, not a public one: public
# authorities stopped issuing the clientAuth usage, so no Let's Encrypt
# certificate can authenticate an agent.
#
#   llmgw ctl cert init-ca --dir /etc/llmgw/pki
#   llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki
#
# Every address an agent might dial has to appear in --host, bare IP addresses
# included: hostname verification reads subject alternative names and ignores
# the common name entirely.
#
# This covers the fleet-facing port only. The client-facing API above is public
# and belongs behind a publicly trusted certificate.
#
# agent_tls:
#   ca: /etc/llmgw/ca.pem
#   cert: /etc/llmgw/server.pem
#   key: /etc/llmgw/server-key.pem

router:
  scrape_interval: 2s
  scrape_timeout: 1s
  # How long an endpoint may go unscraped before it is scored down. It is never
  # dropped for staleness alone: if the whole pool went stale there would be
  # nowhere left to send anything.
  stale_after: 10s

  # Additional endpoints to try after a failure, and only while nothing has
  # reached the client yet.
  retries: 2
  # How long a failing endpoint leaves the candidate pool. A long fallback list
  # with no cooldown turns one endpoint's rate limit into a cluster-wide retry
  # storm.
  cooldown: 15s
  # Consecutive failures that open the breaker. Set to 1 for pure cooldown
  # behaviour.
  failure_threshold: 3

  default_profile: chat        # chat | batch | long-context | debug
  session_header: x-session-id # affinity key; falls back to the `user` field
  # Ask engines for usage even when the client did not, so token accounting
  # works regardless of client behaviour. The extra chunk is dropped on the way
  # back to a client that did not ask for it.
  inject_usage: true
  max_body_bytes: 33554432

  timeouts:
    dial: 5s
    response_header: 60s
    first_token: 120s
    idle: 60s      # between streamed tokens; catches a wedged engine
    total: 30m

  # Flow control: decides *when* a request may proceed, before scheduling
  # decides where it goes. Without it a batch workload starves interactive
  # traffic, because an engine queue is FIFO and knows nothing about tenants.
  admission:
    enabled: true
    # Until authentication lands, the band comes from a header and is trusted.
    # Do not expose this port to callers who should not be able to promote
    # themselves.
    band_header: x-llmgw-band
    default_band: gold
    # A band waits once pool saturation passes its ceiling, so the cluster
    # degrades from the bottom up. A ceiling of 1.0 means "flow up to the
    # healthy limit"; use a value above 1 for a band that must never wait.
    bands:
      - { name: platinum, priority: 400, saturation_ceiling: 1.00 }
      - { name: gold,     priority: 300, saturation_ceiling: 0.85 }
      - { name: silver,   priority: 200, saturation_ceiling: 0.60 }
      - { name: bronze,   priority: 100, saturation_ceiling: 0.30 }
    # Drop a request that has waited this long: the client has usually given up,
    # and sending the GPU work nobody is waiting for helps no one.
    queue_ttl: 30s
    # Reject immediately once a band's queue is this deep, rather than letting
    # it accumulate silently.
    queue_capacity_per_band: 2000
    interval: 10ms

  # What the scorers normalize against. Measure these with `ctl bench` rather
  # than guessing: target_waiting in particular governs how the whole system
  # behaves under load. Too low starves the GPU, too high brings back queueing
  # delay.
  scoring:
    target_waiting: 4
    target_inflight: 8
    kv_ceiling: 0.9
    session_load_factor: 1.25

# How aggressively the control plane reconciles.
control:
  interval: 5s
  base_backoff: 10s
  max_backoff: 10m
  # Node expiry is staged deliberately: a network blip must not tear down
  # replicas that are still serving, and rebuilding a multi-node replica costs
  # tens of minutes of model loading.
  unreachable_after: 15s
  lost_after: 60s
  port_range: [20000, 20999]
  mgmt_interface: eth0

models:
  - name: llama-3.3-70b
    aliases: [llama]
    fallbacks: [qwen2.5-32b]
    routing_profile: chat

  - name: qwen2.5-32b
    aliases: [qwen]
    routing_profile: batch

# A static replica list, used only when no model declares a profile. That is the
# router-only mode for testing against sim-engines; with the control plane
# running, agents provide the replicas instead.
#
# endpoints:
#   - id: sim-1
#     url: http://127.0.0.1:8001
#     models: [llama-3.3-70b]
#     pool: local
#     weight: 1