llmgw/deploy/examples/server.yaml

# llmgw router configuration (phase 1).
#
# The endpoint list is static here. From phase 3 it comes from the control
# plane instead and this section disappears; everything else stays.
#
# Any field can be overridden from the environment with an LLMGW_ prefix and
# __ for nesting:  LLMGW_ROUTER__RETRIES=3

listen: ":8080"        # client-facing OpenAI API
admin_listen: "127.0.0.1:9090"  # dashboard, management API, pprof — keep it local
# Metrics only, on their own port, so Prometheus can reach them without pprof
# being reachable too. Empty means off; metrics stay on admin_listen either way.
# metrics_listen: "0.0.0.0:9091"
# Where node agents connect. Separate from the client port so the fleet-facing
# surface can be firewalled independently.
agent_listen: ":7443"

# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken
# from its certificate's common name and an agent claiming a different node id
# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach
# it can claim to be any node, so it must stay on a trusted network.
#
# These certificates come from llmgw's own authority, not a public one: public
# authorities stopped issuing the clientAuth usage, so no Let's Encrypt
# certificate can authenticate an agent.
#
#   llmgw ctl cert init-ca --dir /etc/llmgw/pki
#   llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki
#
# Every address an agent might dial has to appear in --host, bare IP addresses
# included: hostname verification reads subject alternative names and ignores
# the common name entirely.
#
# This covers the fleet-facing port only. The client-facing API above is public
# and belongs behind a publicly trusted certificate.
#
# agent_tls:
#   ca: /etc/llmgw/ca.pem
#   cert: /etc/llmgw/server.pem
#   key: /etc/llmgw/server-key.pem

# The public page at "/". Someone who reaches this address without a key needs a
# route to getting one, and the gateway cannot guess who to send them to.
#
# landing:
#   contact: "platform@example.com"   # or an https:// URL, or "#platform on Slack"
#   notice: "Internal use only."

# API keys on the client-facing port (plan §7.9). Off by default, because a
# gateway on a trusted network with no keys issued still has to serve — but
# nothing else protects that port, so turning it on is what makes it safe to
# expose.
#
#   llmgw keys init --store /var/lib/llmgw/llmgw.db
#   llmgw keys create --store /var/lib/llmgw/llmgw.db --project default "laptop"
#
# The key is printed once. Only its hash is stored, so there is nothing to
# recover it from.
#
# auth:
#   enabled: true
#   store: /var/lib/llmgw/llmgw.db
#   # How long a resolved key is trusted before being read again. This is the
#   # price of keeping the database off the request path: it bounds how long a
#   # revoked key keeps working.
#   cache_ttl: 30s
#   # How long a rejected key is remembered as rejected, so a client looping
#   # with a bad key cannot become a database read per request.
#   negative_ttl: 5s
#
#   # Serve the signup form. What it creates is a pending account that can do
#   # nothing until `llmgw users approve` places it in a project, so this is not
#   # a way in — it only moves key creation from you to the person who needs one.
#   # Off, there is no form and you issue keys with `llmgw keys create --claim`.
#   signup_open: true
#
#   # What each model costs, in micro-units per thousand tokens. A unit is
#   # whatever you decide: self-hosted models have no per-token price, so it can
#   # be GPU-seconds, a notional rate, or 1000 everywhere — which turns budgets
#   # into plain token quotas.
#   #
#   # A model priced at zero is exempt from budget checks, which is what lets a
#   # key with nothing left still be served a cheap fallback.
#   default_price: { prompt_per_1k: 0, completion_per_1k: 0 }
#   pricing:
#     llama-3.3-70b: { prompt_per_1k: 1000, completion_per_1k: 3000 }
#     qwen2.5-32b:   { prompt_per_1k: 400,  completion_per_1k: 1200 }
#
#   # Usage rows are written in batches off the request path. Beyond buffer_size
#   # they are dropped rather than blocking a request: losing some accounting is
#   # bad, turning a stalled database into an outage is worse.
#   flush_interval: 200ms
#   batch_size: 500
#   buffer_size: 8192
#
#   # How long the record of each request is kept (what `ctl request` reads).
#   # Charts, totals and budgets come from five-minute sums that are kept, so
#   # this only bounds the table that grows by a row a request. 0 keeps all.
#   usage_retention: 2160h   # 90 days

router:
  scrape_interval: 2s
  scrape_timeout: 1s
  # How long an endpoint may go unscraped before it is scored down. It is never
  # dropped for staleness alone: if the whole pool went stale there would be
  # nowhere left to send anything.
  stale_after: 10s

  # Additional endpoints to try after a failure, and only while nothing has
  # reached the client yet.
  retries: 2
  # How long a failing endpoint leaves the candidate pool. A long fallback list
  # with no cooldown turns one endpoint's rate limit into a cluster-wide retry
  # storm.
  cooldown: 15s
  # Consecutive failures that open the breaker. Set to 1 for pure cooldown
  # behaviour.
  failure_threshold: 3

  default_profile: chat        # chat | batch | long-context | debug

  # Approximate prefix routing (plan §7.5 tier 1): send a request to the replica
  # that has probably already prefilled its prefix.
  #
  # It earns its keep when the working set does not fit in one replica's cache.
  # With a handful of prompts and a large cache, scattering is free — every
  # replica ends up holding everything — and this only adds constraint. Measured
  # with sim-engines whose caches were too small for the working set: cache hits
  # went from 22.5% to 81.7%.
  #
  # Watch llmgw_prefix_decisions_total to tune it. A high
  # load_balance_override count means the guard is doing its job and affinity is
  # not getting a chance; a high cache_miss_min_load means match_ratio or
  # min_match is too strict for your traffic.
  prefix:
    enabled: false
    # The shortest match worth acting on, in bytes. Below this the overlap is
    # coincidence — two English prompts share "The".
    min_match: 64
    # The share of the prompt that must match. A short shared header on a very
    # long prompt saves almost no prefill.
    match_ratio: 0.3
    # The imbalance guard, and both must trip before affinity is abandoned.
    # Absolute alone fires on an idle cluster where one replica has two requests
    # and another none; relative alone fires on a busy one where 200 against 180
    # is a rounding error. Without the guard the cache hit rate rises while p99
    # collapses.
    balance_abs: 8
    balance_rel: 1.5
    # An engine evicts on its own schedule, so an old record may describe a
    # cache that is gone.
    ttl: 10m
    max_nodes: 100000
  session_header: x-session-id # affinity key; falls back to the `user` field
  # Ask engines for usage even when the client did not, so token accounting
  # works regardless of client behaviour. The extra chunk is dropped on the way
  # back to a client that did not ask for it.
  inject_usage: true
  max_body_bytes: 33554432

  timeouts:
    dial: 5s
    response_header: 60s
    first_token: 120s
    idle: 60s      # between streamed tokens; catches a wedged engine
    total: 30m

  # Flow control: decides *when* a request may proceed, before scheduling
  # decides where it goes. Without it a batch workload starves interactive
  # traffic, because an engine queue is FIFO and knows nothing about tenants.
  admission:
    enabled: true
    # With auth on, the key decides the band and this header is ignored. With
    # auth off the header is trusted, so any client can promote itself into the
    # top band — do not expose the port in that configuration.
    band_header: x-llmgw-band
    default_band: gold
    # A band waits once pool saturation passes its ceiling, so the cluster
    # degrades from the bottom up. A ceiling of 1.0 means "flow up to the
    # healthy limit"; use a value above 1 for a band that must never wait.
    bands:
      - { name: platinum, priority: 400, saturation_ceiling: 1.00 }
      - { name: gold,     priority: 300, saturation_ceiling: 0.85 }
      - { name: silver,   priority: 200, saturation_ceiling: 0.60 }
      - { name: bronze,   priority: 100, saturation_ceiling: 0.30 }
    # Drop a request that has waited this long: the client has usually given up,
    # and sending the GPU work nobody is waiting for helps no one.
    queue_ttl: 30s
    # Reject immediately once a band's queue is this deep, rather than letting
    # it accumulate silently.
    queue_capacity_per_band: 2000
    interval: 10ms

  # What the scorers normalize against. Measure these with `ctl bench` rather
  # than guessing: target_waiting in particular governs how the whole system
  # behaves under load. Too low starves the GPU, too high brings back queueing
  # delay.
  scoring:
    target_waiting: 4
    target_inflight: 8
    kv_ceiling: 0.9
    session_load_factor: 1.25

# How aggressively the control plane reconciles.
# The engines every node installs, and at which version. Unset keeps the
# versions this release was checked against (llama.cpp b10991). Each agent
# picks the build for its own hardware and installs it itself; changing a
# version here reaches every node on its next connect.
# engines:
#   llamacpp: b10991

control:
  interval: 5s
  base_backoff: 10s
  max_backoff: 10m
  # Node expiry is staged deliberately: a network blip must not tear down
  # replicas that are still serving, and rebuilding a multi-node replica costs
  # tens of minutes of model loading.
  unreachable_after: 15s
  lost_after: 60s
  port_range: [20000, 20999]
  mgmt_interface: eth0

# Each model needs a source — a weights path or a HuggingFace id — and at least
# one profile if the control plane is to place it. Without profiles the model is
# catalogued but never started, which is the router-only mode below.
models:
  - name: llama-3.3-70b
    source: meta-llama/Llama-3.3-70B-Instruct
    aliases: [llama]
    fallbacks: [qwen2.5-32b]
    routing_profile: chat
    scaling: { min: 1, max: 4 }
    profiles:
      # Two nodes of four H100s each. A multi-node profile requires a fabric,
      # and a replica never spans two of them.
      - name: tp8
        engine: vllm
        gpu_class: h100-80g
        tp: 8
        nodes: 2
        gpus_per_node: 4
        priority: 100
        weight: 8
        startup_timeout: 30m
      # Where a whole node is free, one node is cheaper to schedule and faster
      # to start than two.
      - name: tp4-a100
        engine: vllm
        gpu_class: a100-40g
        tp: 4
        priority: 50
        weight: 3
        startup_timeout: 20m

  - name: qwen2.5-32b
    source: Qwen/Qwen2.5-32B-Instruct
    aliases: [qwen]
    routing_profile: batch
    scaling: { min: 1, max: 2 }
    profiles:
      - name: tp2
        engine: sglang
        gpu_class: h100-80g
        tp: 2
        priority: 100
        weight: 2
        startup_timeout: 15m

# A static replica list, used only when no model declares a profile. That is the
# router-only mode for testing against sim-engines; with the control plane
# running, agents provide the replicas instead.
#
# endpoints:
#   - id: sim-1
#     url: http://127.0.0.1:8001
#     models: [llama-3.3-70b]
#     pool: local
#     weight: 1