Ilker ManapandClaude Opus 5.5
4c5a2fd6da
vLLM was installed by hand and named in each agent's configuration. The agent now installs it: a Python 3.12 environment built by uv, which is itself a GitHub release checked against its checksum, from PyPI resolved as it stood a day after that vLLM was published and installed with --require-hashes. The resolved list is kept in the environment. The environment is built relocatable under a scratch name and renamed into place once it runs, like an archive. The default is 0.30.0, whose flags were checked against the ones llmgw builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or newer, and 25 GB free while it installs; a node lacking one says which, and a node with no GPU skips it without being shown as a fault. gcc comes with it because Triton compiles its launcher at the first model load. SGLang installs the same way when the control plane names a version. llmgw doctor reports whether vLLM can be installed on the node. Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
295 lines
13 KiB
Bash
Executable File
295 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Give a machine an identity in an llmgw cluster.
|
|
#
|
|
# Run on the host that holds the certificate authority, because that is the one
|
|
# thing that cannot be delegated: this script issues the credential by which the
|
|
# control plane will believe the node is who it says it is.
|
|
#
|
|
# What it does, in the order it matters:
|
|
# 1. issue a client certificate whose common name is the node id
|
|
# 2. copy the CA root, the certificate and its key to the machine
|
|
# 3. trust the root system-wide, so other tooling there stops complaining
|
|
# 4. install the binary, the configuration and the unit
|
|
# 5. start the agent
|
|
#
|
|
# The node id, the VM name, the hostname and the certificate common name are all
|
|
# the same string. They do not have to be, and every time they are not, someone
|
|
# spends an afternoon working out which of the four a message is talking about.
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<USAGE
|
|
usage: $0 --node ID --addr HOST --server URL [options]
|
|
|
|
--node ID node id, hostname and certificate common name
|
|
--addr HOST how to reach the machine over ssh
|
|
--server URL the control plane's agent endpoint, e.g. https://10.0.0.140:7443;
|
|
several, comma separated, for a node that fails over
|
|
--fabric NAME InfiniBand fabric this node is attached to; repeatable as
|
|
--fabric NAME:hca1,hca2 . Omit for a node with no fabric,
|
|
which can host single-node replicas and will never be given
|
|
a multi-node one.
|
|
--mgmt-addr ADDR address engines are reachable on (default: the node's
|
|
address on the route to --server, found by the agent)
|
|
--mgmt-if NAME interface NCCL bootstraps over (default: the one holding
|
|
that address)
|
|
--llamacpp-bin P use this llama-server instead of the one the agent installs
|
|
--vllm-bin P use this vllm instead of the one the agent installs on a
|
|
GPU node (also set engines.vllm: "" in server.yaml)
|
|
--sglang-python P python interpreter of an SGLang virtualenv you installed
|
|
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
|
|
driver installs routinely put it somewhere else.
|
|
--ib-root DIR the sysfs InfiniBand directory (default /sys/class/infiniband)
|
|
--pki DIR where the authority lives (default /etc/llmgw/pki)
|
|
--binary PATH llmgw binary to install (default: the one running this).
|
|
Releases carry static linux/amd64 and linux/arm64 builds;
|
|
downloading one is better than copying whatever happens to
|
|
be on the machine you are running this from.
|
|
--releases URL where release binaries are downloaded from (default the
|
|
project's Gitea releases)
|
|
--ssh-user NAME (default ilker)
|
|
USAGE
|
|
exit 2
|
|
}
|
|
|
|
NODE= ADDR= SERVER= MGMT_OVERRIDE= MGMT_IF_OVERRIDE=
|
|
NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
|
|
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
|
|
SSH_WAIT=180
|
|
PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
|
|
RELEASES=https://gitea.mynodes.xyz/ilker/llmgw/releases/download
|
|
FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--node) NODE="$2"; shift 2;;
|
|
--addr) ADDR="$2"; shift 2;;
|
|
--server) SERVER="$2"; shift 2;;
|
|
--fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
|
|
--mgmt-addr) MGMT_OVERRIDE="$2"; shift 2;;
|
|
--mgmt-if) MGMT_IF_OVERRIDE="$2"; shift 2;;
|
|
--llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
|
|
--vllm-bin) VLLM_BIN="$2"; shift 2;;
|
|
--sglang-python) SGLANG_PYTHON="$2"; shift 2;;
|
|
--nvidia-smi) NVIDIA_SMI="$2"; shift 2;;
|
|
--ib-root) IB_ROOT="$2"; shift 2;;
|
|
--pki) PKI="$2"; shift 2;;
|
|
--binary) BINARY="$2"; shift 2;;
|
|
--releases) RELEASES="$2"; shift 2;;
|
|
--ssh-user) SSH_USER="$2"; shift 2;;
|
|
*) usage;;
|
|
esac
|
|
done
|
|
[ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage
|
|
BINARY="${BINARY:-$(command -v llmgw)}"
|
|
[ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; }
|
|
[ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; }
|
|
|
|
# accept-new, not "no": a machine provisioned a minute ago has a host key
|
|
# nobody has seen, and refusing it stopped onboarding dead with "Host key
|
|
# verification failed" — an error about ssh, in the middle of a script whose
|
|
# whole job is to talk to a new machine. accept-new learns an unknown key and
|
|
# still refuses a changed one, which is the distinction that matters: the risk
|
|
# being guarded against is a substituted machine, not a new one.
|
|
SSHOPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
|
|
SSH="ssh $SSHOPTS $SSH_USER@$ADDR"
|
|
say() { printf ' %s\n' "$*"; }
|
|
|
|
echo "onboarding $NODE at $ADDR"
|
|
|
|
# Waiting rather than failing: cloud-init restarts sshd after it regenerates
|
|
# the host keys, so a machine that answered a moment ago refuses connections
|
|
# for a few seconds. Observed while onboarding node05 — the agent was already
|
|
# registered and healthy while ssh was refusing.
|
|
wait_for_ssh() {
|
|
deadline=$(( $(date +%s) + SSH_WAIT ))
|
|
until $SSH true 2>/dev/null; do
|
|
[ "$(date +%s)" -lt "$deadline" ] || {
|
|
echo "cannot reach $SSH_USER@$ADDR over ssh after ${SSH_WAIT}s" >&2
|
|
exit 1
|
|
}
|
|
sleep 5
|
|
done
|
|
}
|
|
wait_for_ssh
|
|
|
|
# 1. The certificate. Issued fresh every time: re-onboarding a machine should
|
|
# not depend on finding an old key, and the old one keeps working until it
|
|
# is revoked, which is a separate decision.
|
|
say "issuing a certificate"
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
|
|
|
|
# 2 and 3. Identity, and trusting the root so future tooling on that machine
|
|
# does not treat our certificates as strangers.
|
|
say "installing the certificate and trusting the root"
|
|
scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
|
|
$SSH "set -e
|
|
sudo install -d -m 0755 /etc/llmgw
|
|
sudo install -m 0644 /tmp/ca.pem /etc/llmgw/ca.pem
|
|
sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
|
|
sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
|
|
sudo chown root:root /etc/llmgw/*
|
|
sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
|
|
sudo update-ca-certificates >/dev/null 2>&1 || true
|
|
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
|
|
|
|
# 4. The binary, the configuration and the unit.
|
|
# The node fetches the release CI built for this version, for its own
|
|
# architecture, and checks it against the release's checksums — rather than
|
|
# being handed whatever binary is on the machine running this script. A build
|
|
# that is not a release (a developer's, with commits past the tag) has no such
|
|
# file, and is copied as before.
|
|
TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
|
|
if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
|
|
say "installing llmgw $TAG from the release"
|
|
$SSH "set -e
|
|
arch=\$(dpkg --print-architecture)
|
|
name=llmgw-$TAG-linux-\$arch
|
|
cd /tmp && rm -f \$name checksums.txt
|
|
curl -fsSL -o \$name '$RELEASES/$TAG/'\$name
|
|
curl -fsSL -o checksums.txt '$RELEASES/$TAG/checksums.txt'
|
|
grep \" \$name\\\$\" checksums.txt | sha256sum -c - >/dev/null
|
|
sudo install -m 0755 \$name /usr/local/bin/llmgw
|
|
rm -f \$name checksums.txt"
|
|
else
|
|
say "installing the binary (${TAG:-unversioned}, not a release)"
|
|
scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
|
|
$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
|
|
fi
|
|
|
|
say "writing the configuration"
|
|
NODE_IS_HOSTNAME=no
|
|
[ "$($SSH hostname 2>/dev/null)" = "$NODE" ] && NODE_IS_HOSTNAME=yes
|
|
# Only what the node cannot find out for itself. Its name is its hostname, its
|
|
# address is the one on the route to the control plane, llama-server is looked
|
|
# for on PATH and where release archives are unpacked, GPUs are there when
|
|
# nvidia-smi is, and memory, disk and engines are re-read while it runs. Every
|
|
# setting written here is one that has to be kept true by hand — so an option
|
|
# below is written only when it was given, as an override.
|
|
{
|
|
echo "# $NODE, written by onboard-node.sh."
|
|
# Several addresses, comma separated, are every control plane the node may
|
|
# fail over to; a standby refuses it and it moves on to the next.
|
|
if [ "${SERVER#*,}" != "$SERVER" ]; then
|
|
echo "servers:"
|
|
printf '%s\n' "$SERVER" | tr ',' '\n' | sed 's/^ *//; s/^/ - "/; s/$/"/'
|
|
else
|
|
echo "server: \"$SERVER\""
|
|
fi
|
|
echo
|
|
echo "tls:"
|
|
echo " ca: /etc/llmgw/ca.pem"
|
|
echo " cert: /etc/llmgw/agent.pem"
|
|
echo " key: /etc/llmgw/agent-key.pem"
|
|
# The certificate's common name is --node, and the control plane refuses a
|
|
# stream whose node id differs from it. Said explicitly only when the
|
|
# hostname would say something else.
|
|
[ "$NODE_IS_HOSTNAME" = yes ] || echo "node_id: $NODE"
|
|
[ -n "$MGMT_OVERRIDE" ] && echo "mgmt_addr: \"$MGMT_OVERRIDE\""
|
|
[ -n "$MGMT_IF_OVERRIDE" ] && echo "mgmt_interface: $MGMT_IF_OVERRIDE"
|
|
if [ -n "$FABRIC_NAME" ]; then
|
|
# Fabric membership is the one thing that cannot be found: OpenSM's default
|
|
# subnet prefix is the same on every fabric, so a matching prefix proves
|
|
# nothing and a wrong guess hangs the first all-reduce.
|
|
echo "fabrics:"
|
|
echo " - name: $FABRIC_NAME"
|
|
echo " hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
|
|
fi
|
|
if [ -n "$NVIDIA_SMI" ] || [ "$IB_ROOT" != /sys/class/infiniband ]; then
|
|
echo "discovery:"
|
|
[ -n "$NVIDIA_SMI" ] && echo " nvidia_smi: \"$NVIDIA_SMI\""
|
|
[ "$IB_ROOT" != /sys/class/infiniband ] && echo " ib_root: $IB_ROOT"
|
|
fi
|
|
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
echo "engines:"
|
|
[ -n "$LLAMACPP_BIN" ] && printf ' llamacpp:\n bin: "%s"\n' "$LLAMACPP_BIN"
|
|
[ -n "$VLLM_BIN" ] && printf ' vllm:\n bin: "%s"\n' "$VLLM_BIN"
|
|
[ -n "$SGLANG_PYTHON" ] && printf ' sglang:\n python: "%s"\n' "$SGLANG_PYTHON"
|
|
fi
|
|
true
|
|
} > "$TMP/agent.yaml"
|
|
scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml"
|
|
|
|
cat > "$TMP/llmgw-agent.service" <<'UNIT'
|
|
[Unit]
|
|
Description=llmgw node agent
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
|
|
# Root, because the agent launches engines and needs device access. Engines
|
|
# outlive a restart of this unit and are adopted again on the next start, so
|
|
# KillMode must not take the whole process group with it.
|
|
KillMode=process
|
|
Restart=on-failure
|
|
RestartSec=2s
|
|
# RDMA pins the buffers it registers, and NCCL registers a great deal: with
|
|
# systemd's default 8 MB lock limit, an engine that drops the capability that
|
|
# lets root ignore it fails its first collective over InfiniBand.
|
|
LimitMEMLOCK=infinity
|
|
StateDirectory=llmgw-agent
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
scp -q "$TMP/llmgw-agent.service" "$SSH_USER@$ADDR:/tmp/"
|
|
|
|
# An engine path that is wrong, or right but unrunnable, is the failure this
|
|
# node type is most likely to have, and it is invisible: the agent registers
|
|
# healthy and every shard placed on it dies with "executable file not found"
|
|
# or a missing shared library. Checked here because it is cheap here and
|
|
# expensive later — llama-server on node05 was present, executable, and could
|
|
# not start for want of libgomp1, and the node reported ready throughout.
|
|
check_engine() {
|
|
[ -n "$2" ] || return 0
|
|
if $SSH "test -x '$2'" 2>/dev/null; then
|
|
if $SSH "'$2' --version >/dev/null 2>&1 || '$2' --help >/dev/null 2>&1"; then
|
|
say "$1: $2 runs"
|
|
return 0
|
|
fi
|
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
|
$1 at $2 exists but will not run. Usually a missing shared library:
|
|
ssh $SSH_USER@$ADDR ldd $2 | grep 'not found'"
|
|
return 0
|
|
fi
|
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
|
$1 is configured at $2, which is not an executable on $NODE."
|
|
}
|
|
ENGINE_PROBLEMS=
|
|
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
say "checking the engines"
|
|
check_engine llamacpp "$LLAMACPP_BIN"
|
|
check_engine vllm "$VLLM_BIN"
|
|
check_engine sglang "$SGLANG_PYTHON"
|
|
fi
|
|
|
|
say "starting the agent"
|
|
$SSH 'set -e
|
|
sudo install -m 0644 -o root -g root /tmp/agent.yaml /etc/llmgw/agent.yaml
|
|
sudo install -m 0644 /tmp/llmgw-agent.service /etc/systemd/system/llmgw-agent.service
|
|
rm -f /tmp/agent.yaml /tmp/llmgw-agent.service
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable --now llmgw-agent >/dev/null
|
|
sudo systemctl restart llmgw-agent'
|
|
|
|
echo
|
|
if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
say "the agent now installs the engines the control plane asks for: the"
|
|
say "build for this machine, checked against its publisher's checksum and run"
|
|
say "before the node is offered any model. 'llmgw ctl nodes' shows progress."
|
|
fi
|
|
if [ -n "$ENGINE_PROBLEMS" ]; then
|
|
echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS"
|
|
echo
|
|
echo "The node will register as ready regardless. Fix this before a replica"
|
|
echo "is placed on it, or the shard fails and the reason is three logs away."
|
|
echo
|
|
fi
|
|
echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds."
|
|
echo "If it does not, the control plane's log says why — a certificate whose"
|
|
echo "common name does not match the claimed node id is refused by design."
|