# llmgw agent configuration, one per GPU node.
#
# The agent dials out to the control plane and keeps one stream open. Engines
# keep serving if the agent restarts: the router talks to them directly, and
# orphan adoption picks them up again on the next start.

# Everything a node can find out about itself, it does: its id is its hostname,
# its address is the one on the route to the control plane, llama-server is
# looked for on PATH and in /opt/llama.cpp, GPUs are there when nvidia-smi is,
# weights are fetched into /var/lib/llmgw-weights and looked for there, and
# memory, disk and installed engines are re-read while the agent runs. So a
# plain CPU node needs only the two blocks below, and the same file on every
# node. What follows them are overrides, for when the defaults are wrong.

server: "http://llmgw-server.internal:7443"

# The agent's half of mutual TLS. The certificate's common name must be this
# node's id — its hostname, unless node_id says otherwise — because the control
# plane compares the two and refuses the stream when they disagree:
#
#   llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki
#
# With this set, `server` above must be an https:// address — an agent holding
# certificates and dialling http:// is refused at startup rather than connecting
# in the clear without noticing.
#
# tls:
#   ca: /etc/llmgw/ca.pem
#   cert: /etc/llmgw/agent.pem
#   key: /etc/llmgw/agent-key.pem

# ---- overrides ---------------------------------------------------------------

# node_id: gpu-a-01          # when it is not the hostname

# The address engines are reachable on, and the address NCCL bootstraps over.
# Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA.
# mgmt_addr: "10.0.0.11"
# mgmt_interface: eth0

# Free-form, for a profile's node_selector. Nothing needs them; do not restate
# what is measured (memory, GPUs, engines), because a label goes stale.
# labels:
#   rack: r12

heartbeat_interval: 5s
reconnect_base: 1s
reconnect_max: 30s

# Fabric labelling is explicit and there is no way around it. OpenSM's default
# subnet prefix is fe80:: on every fabric, so a matching prefix proves nothing
# and a wrong guess produces a replica that starts, passes health checks and
# then hangs on its first all-reduce (plan §5.2).
#
# A node with no fabrics is its own single-node island: it can host single-node
# replicas and will never be given a multi-node one.
fabrics:
  - name: ib-a
    hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3]
    rate_gbps: 200

discovery:
  # Worth setting where the driver lives outside the default paths, which
  # containerized driver installs often do.
  nvidia_smi: ""
  ib_root: /sys/class/infiniband
  # NVML is the only source of NVLink topology, which is what keeps a
  # tensor-parallel group off the PCIe path inside a node. It needs a binary
  # built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
  prefer_nvml: true

# Where this node keeps model weights besides /var/lib/llmgw-weights, which is
# where fetched weights land and is always looked in. Storage differs at
# every installation and llmgw deliberately does not own it: a Lustre mount, an
# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
# made by hand all look identical from here.
#
# What it buys is placement preferring a node that already holds the model over
# one that must first move tens of gigabytes. That matters most where fabrics
# are isolated from one another, because weights in one fabric are not weights
# in another and the only path between them is the management network.
#
# model_dirs:
#   - /var/lib/llmgw-models
#   - /mnt/lustre/models

# The agent installs the engines the control plane names (see `engines:` in
# server.yaml) under /opt/llmgw/engines: the build for this machine — CUDA when
# an NVIDIA driver is loaded — checked against its publisher's checksum, with
# the OS packages it needs, and run before the node takes any model. llama.cpp
# is installed this way today. Naming a binary below opts that engine out and
# uses yours instead; vLLM and SGLang still have to be named until the agent
# installs them too.
# engines_dir: /opt/llmgw/engines

engines:
  vllm:
    bin: /opt/venvs/vllm/bin/vllm
    # How a replica spanning nodes is launched. mp, the default, is what vLLM
    # documents today and needs nothing but vLLM; ray is for a vLLM too old
    # for mp's multi-node flags, and needs Ray on every node.
    launch_mode: mp       # "mp" or "ray"
    # ray_bin: /opt/venvs/vllm/bin/ray
  sglang:
    # SGLang is launched as a module, which is its documented entry point.
    python: /opt/venvs/sglang/bin/python

supervisor:
  work_dir: /var/lib/llmgw-agent
  # Engine output is the only diagnosis when a model fails to load, so keep a
  # useful tail of it.
  log_lines: 512
  # Engines need time to free GPU memory cleanly before being killed.
  stop_timeout: 60s
  health_interval: 2s