# llmgw agent configuration, one per GPU node.
#
# The agent dials out to the control plane and keeps one stream open. Engines
# keep serving if the agent restarts: the router talks to them directly, and
# orphan adoption picks them up again on the next start.

node_id: gpu-a-01          # defaults to the hostname
server: "http://llmgw-server.internal:7443"

# The agent's half of mutual TLS. The certificate's common name must be this
# node's id, because the control plane compares the two and refuses the stream
# when they disagree:
#
#   llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki
#
# With this set, `server` above must be an https:// address — an agent holding
# certificates and dialling http:// is refused at startup rather than connecting
# in the clear without noticing.
#
# tls:
#   ca: /etc/llmgw/ca.pem
#   cert: /etc/llmgw/agent.pem
#   key: /etc/llmgw/agent-key.pem

# The address engines are reachable on, and the address NCCL bootstraps over.
# Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA.
mgmt_addr: "10.0.0.11"
mgmt_interface: eth0

labels:
  rack: r12

heartbeat_interval: 5s
reconnect_base: 1s
reconnect_max: 30s

# Fabric labelling is explicit and there is no way around it. OpenSM's default
# subnet prefix is fe80:: on every fabric, so a matching prefix proves nothing
# and a wrong guess produces a replica that starts, passes health checks and
# then hangs on its first all-reduce (plan §5.2).
#
# A node with no fabrics is its own single-node island: it can host single-node
# replicas and will never be given a multi-node one.
fabrics:
  - name: ib-a
    hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3]
    rate_gbps: 200

discovery:
  # Worth setting where the driver lives outside the default paths, which
  # containerized driver installs often do.
  nvidia_smi: ""
  ib_root: /sys/class/infiniband
  # NVML is the only source of NVLink topology, which is what keeps a
  # tensor-parallel group off the PCIe path inside a node. It needs a binary
  # built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
  prefer_nvml: true
  skip_gpus: false

# Where this node keeps model weights.
#
# Unset by default, and unset means the node is not asked to look — which is
# not the same as having nothing, and constrains nothing. Storage differs at
# every installation and llmgw deliberately does not own it: a Lustre mount, an
# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
# made by hand all look identical from here.
#
# What it buys is placement preferring a node that already holds the model over
# one that must first move tens of gigabytes. That matters most where fabrics
# are isolated from one another, because weights in one fabric are not weights
# in another and the only path between them is the management network.
#
# model_dirs:
#   - /var/lib/llmgw-models
#   - /mnt/lustre/models

engines:
  vllm:
    bin: /opt/venvs/vllm/bin/vllm
    # Which multi-node mechanism works depends on the vLLM version, and the
    # flags have changed more than once. Check the version you deploy.
    launch_mode: ray      # "ray" or "mp"
    ray_bin: /opt/venvs/vllm/bin/ray
  sglang:
    # SGLang is launched as a module, which is its documented entry point.
    python: /opt/venvs/sglang/bin/python

supervisor:
  work_dir: /var/lib/llmgw-agent
  # Engine output is the only diagnosis when a model fails to load, so keep a
  # useful tail of it.
  log_lines: 512
  # Engines need time to free GPU memory cleanly before being killed.
  stop_timeout: 60s
  health_interval: 2s