Ilker ManapandClaude Opus 5.5
5dac1854c2
Every fix so far became its own patch release, and the natural alternative — v0.10.3-a — sorts before v0.10.3 by SemVer's rules: a hyphen marks a pre-release, so every tool that orders versions would take the fix for the older one. A release candidate of the next version, v0.10.4-rc.1, sorts after v0.10.3 and before v0.10.4, which is what it is. CI already builds any v* tag. Its release notes now come from a "## Unreleased" section in CHANGELOG.md, where changes gather until the version they belong to is released, and the release is marked as a pre-release. Onboarding, in Go and in the shell script, counts a candidate as a release and downloads it rather than copying the operator's binary. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
295 lines
13 KiB
Bash
Executable File
295 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Give a machine an identity in an llmgw cluster.
|
|
#
|
|
# Run on the host that holds the certificate authority, because that is the one
|
|
# thing that cannot be delegated: this script issues the credential by which the
|
|
# control plane will believe the node is who it says it is.
|
|
#
|
|
# What it does, in the order it matters:
|
|
# 1. issue a client certificate whose common name is the node id
|
|
# 2. copy the CA root, the certificate and its key to the machine
|
|
# 3. trust the root system-wide, so other tooling there stops complaining
|
|
# 4. install the binary, the configuration and the unit
|
|
# 5. start the agent
|
|
#
|
|
# The node id, the VM name, the hostname and the certificate common name are all
|
|
# the same string. They do not have to be, and every time they are not, someone
|
|
# spends an afternoon working out which of the four a message is talking about.
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat >&2 <<USAGE
|
|
usage: $0 --node ID --addr HOST --server URL [options]
|
|
|
|
--node ID node id, hostname and certificate common name
|
|
--addr HOST how to reach the machine over ssh
|
|
--server URL the control plane's agent endpoint, e.g. https://10.0.0.140:7443;
|
|
several, comma separated, for a node that fails over
|
|
--fabric NAME InfiniBand fabric this node is attached to; repeatable as
|
|
--fabric NAME:hca1,hca2 . Omit for a node with no fabric,
|
|
which can host single-node replicas and will never be given
|
|
a multi-node one.
|
|
--mgmt-addr ADDR address engines are reachable on (default: the node's
|
|
address on the route to --server, found by the agent)
|
|
--mgmt-if NAME interface NCCL bootstraps over (default: the one holding
|
|
that address)
|
|
--llamacpp-bin P use this llama-server instead of the one the agent installs
|
|
--vllm-bin P use this vllm instead of the one the agent installs on a
|
|
GPU node (also set engines.vllm: "" in server.yaml)
|
|
--sglang-python P python interpreter of an SGLang virtualenv you installed
|
|
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
|
|
driver installs routinely put it somewhere else.
|
|
--ib-root DIR the sysfs InfiniBand directory (default /sys/class/infiniband)
|
|
--pki DIR where the authority lives (default /etc/llmgw/pki)
|
|
--binary PATH llmgw binary to install (default: the one running this).
|
|
Releases carry static linux/amd64 and linux/arm64 builds;
|
|
downloading one is better than copying whatever happens to
|
|
be on the machine you are running this from.
|
|
--releases URL where release binaries are downloaded from (default the
|
|
project's Gitea releases)
|
|
--ssh-user NAME (default ilker)
|
|
USAGE
|
|
exit 2
|
|
}
|
|
|
|
NODE= ADDR= SERVER= MGMT_OVERRIDE= MGMT_IF_OVERRIDE=
|
|
NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
|
|
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
|
|
SSH_WAIT=180
|
|
PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
|
|
RELEASES=https://gitea.mynodes.xyz/ilker/llmgw/releases/download
|
|
FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--node) NODE="$2"; shift 2;;
|
|
--addr) ADDR="$2"; shift 2;;
|
|
--server) SERVER="$2"; shift 2;;
|
|
--fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
|
|
--mgmt-addr) MGMT_OVERRIDE="$2"; shift 2;;
|
|
--mgmt-if) MGMT_IF_OVERRIDE="$2"; shift 2;;
|
|
--llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
|
|
--vllm-bin) VLLM_BIN="$2"; shift 2;;
|
|
--sglang-python) SGLANG_PYTHON="$2"; shift 2;;
|
|
--nvidia-smi) NVIDIA_SMI="$2"; shift 2;;
|
|
--ib-root) IB_ROOT="$2"; shift 2;;
|
|
--pki) PKI="$2"; shift 2;;
|
|
--binary) BINARY="$2"; shift 2;;
|
|
--releases) RELEASES="$2"; shift 2;;
|
|
--ssh-user) SSH_USER="$2"; shift 2;;
|
|
*) usage;;
|
|
esac
|
|
done
|
|
[ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage
|
|
BINARY="${BINARY:-$(command -v llmgw)}"
|
|
[ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; }
|
|
[ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; }
|
|
|
|
# accept-new, not "no": a machine provisioned a minute ago has a host key
|
|
# nobody has seen, and refusing it stopped onboarding dead with "Host key
|
|
# verification failed" — an error about ssh, in the middle of a script whose
|
|
# whole job is to talk to a new machine. accept-new learns an unknown key and
|
|
# still refuses a changed one, which is the distinction that matters: the risk
|
|
# being guarded against is a substituted machine, not a new one.
|
|
SSHOPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
|
|
SSH="ssh $SSHOPTS $SSH_USER@$ADDR"
|
|
say() { printf ' %s\n' "$*"; }
|
|
|
|
echo "onboarding $NODE at $ADDR"
|
|
|
|
# Waiting rather than failing: cloud-init restarts sshd after it regenerates
|
|
# the host keys, so a machine that answered a moment ago refuses connections
|
|
# for a few seconds. Observed while onboarding node05 — the agent was already
|
|
# registered and healthy while ssh was refusing.
|
|
wait_for_ssh() {
|
|
deadline=$(( $(date +%s) + SSH_WAIT ))
|
|
until $SSH true 2>/dev/null; do
|
|
[ "$(date +%s)" -lt "$deadline" ] || {
|
|
echo "cannot reach $SSH_USER@$ADDR over ssh after ${SSH_WAIT}s" >&2
|
|
exit 1
|
|
}
|
|
sleep 5
|
|
done
|
|
}
|
|
wait_for_ssh
|
|
|
|
# 1. The certificate. Issued fresh every time: re-onboarding a machine should
|
|
# not depend on finding an old key, and the old one keeps working until it
|
|
# is revoked, which is a separate decision.
|
|
say "issuing a certificate"
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
|
|
|
|
# 2 and 3. Identity, and trusting the root so future tooling on that machine
|
|
# does not treat our certificates as strangers.
|
|
say "installing the certificate and trusting the root"
|
|
scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
|
|
$SSH "set -e
|
|
sudo install -d -m 0755 /etc/llmgw
|
|
sudo install -m 0644 /tmp/ca.pem /etc/llmgw/ca.pem
|
|
sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
|
|
sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
|
|
sudo chown root:root /etc/llmgw/*
|
|
sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
|
|
sudo update-ca-certificates >/dev/null 2>&1 || true
|
|
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
|
|
|
|
# 4. The binary, the configuration and the unit.
|
|
# The node fetches the release CI built for this version, for its own
|
|
# architecture, and checks it against the release's checksums — rather than
|
|
# being handed whatever binary is on the machine running this script. A build
|
|
# that is not a release (a developer's, with commits past the tag) has no such
|
|
# file, and is copied as before.
|
|
TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
|
|
if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$'; then
|
|
say "installing llmgw $TAG from the release"
|
|
$SSH "set -e
|
|
arch=\$(dpkg --print-architecture)
|
|
name=llmgw-$TAG-linux-\$arch
|
|
cd /tmp && rm -f \$name checksums.txt
|
|
curl -fsSL -o \$name '$RELEASES/$TAG/'\$name
|
|
curl -fsSL -o checksums.txt '$RELEASES/$TAG/checksums.txt'
|
|
grep \" \$name\\\$\" checksums.txt | sha256sum -c - >/dev/null
|
|
sudo install -m 0755 \$name /usr/local/bin/llmgw
|
|
rm -f \$name checksums.txt"
|
|
else
|
|
say "installing the binary (${TAG:-unversioned}, not a release)"
|
|
scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
|
|
$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
|
|
fi
|
|
|
|
say "writing the configuration"
|
|
NODE_IS_HOSTNAME=no
|
|
[ "$($SSH hostname 2>/dev/null)" = "$NODE" ] && NODE_IS_HOSTNAME=yes
|
|
# Only what the node cannot find out for itself. Its name is its hostname, its
|
|
# address is the one on the route to the control plane, llama-server is looked
|
|
# for on PATH and where release archives are unpacked, GPUs are there when
|
|
# nvidia-smi is, and memory, disk and engines are re-read while it runs. Every
|
|
# setting written here is one that has to be kept true by hand — so an option
|
|
# below is written only when it was given, as an override.
|
|
{
|
|
echo "# $NODE, written by onboard-node.sh."
|
|
# Several addresses, comma separated, are every control plane the node may
|
|
# fail over to; a standby refuses it and it moves on to the next.
|
|
if [ "${SERVER#*,}" != "$SERVER" ]; then
|
|
echo "servers:"
|
|
printf '%s\n' "$SERVER" | tr ',' '\n' | sed 's/^ *//; s/^/ - "/; s/$/"/'
|
|
else
|
|
echo "server: \"$SERVER\""
|
|
fi
|
|
echo
|
|
echo "tls:"
|
|
echo " ca: /etc/llmgw/ca.pem"
|
|
echo " cert: /etc/llmgw/agent.pem"
|
|
echo " key: /etc/llmgw/agent-key.pem"
|
|
# The certificate's common name is --node, and the control plane refuses a
|
|
# stream whose node id differs from it. Said explicitly only when the
|
|
# hostname would say something else.
|
|
[ "$NODE_IS_HOSTNAME" = yes ] || echo "node_id: $NODE"
|
|
[ -n "$MGMT_OVERRIDE" ] && echo "mgmt_addr: \"$MGMT_OVERRIDE\""
|
|
[ -n "$MGMT_IF_OVERRIDE" ] && echo "mgmt_interface: $MGMT_IF_OVERRIDE"
|
|
if [ -n "$FABRIC_NAME" ]; then
|
|
# Fabric membership is the one thing that cannot be found: OpenSM's default
|
|
# subnet prefix is the same on every fabric, so a matching prefix proves
|
|
# nothing and a wrong guess hangs the first all-reduce.
|
|
echo "fabrics:"
|
|
echo " - name: $FABRIC_NAME"
|
|
echo " hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
|
|
fi
|
|
if [ -n "$NVIDIA_SMI" ] || [ "$IB_ROOT" != /sys/class/infiniband ]; then
|
|
echo "discovery:"
|
|
[ -n "$NVIDIA_SMI" ] && echo " nvidia_smi: \"$NVIDIA_SMI\""
|
|
[ "$IB_ROOT" != /sys/class/infiniband ] && echo " ib_root: $IB_ROOT"
|
|
fi
|
|
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
echo "engines:"
|
|
[ -n "$LLAMACPP_BIN" ] && printf ' llamacpp:\n bin: "%s"\n' "$LLAMACPP_BIN"
|
|
[ -n "$VLLM_BIN" ] && printf ' vllm:\n bin: "%s"\n' "$VLLM_BIN"
|
|
[ -n "$SGLANG_PYTHON" ] && printf ' sglang:\n python: "%s"\n' "$SGLANG_PYTHON"
|
|
fi
|
|
true
|
|
} > "$TMP/agent.yaml"
|
|
scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml"
|
|
|
|
cat > "$TMP/llmgw-agent.service" <<'UNIT'
|
|
[Unit]
|
|
Description=llmgw node agent
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
|
|
# Root, because the agent launches engines and needs device access. Engines
|
|
# outlive a restart of this unit and are adopted again on the next start, so
|
|
# KillMode must not take the whole process group with it.
|
|
KillMode=process
|
|
Restart=on-failure
|
|
RestartSec=2s
|
|
# RDMA pins the buffers it registers, and NCCL registers a great deal: with
|
|
# systemd's default 8 MB lock limit, an engine that drops the capability that
|
|
# lets root ignore it fails its first collective over InfiniBand.
|
|
LimitMEMLOCK=infinity
|
|
StateDirectory=llmgw-agent
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
scp -q "$TMP/llmgw-agent.service" "$SSH_USER@$ADDR:/tmp/"
|
|
|
|
# An engine path that is wrong, or right but unrunnable, is the failure this
|
|
# node type is most likely to have, and it is invisible: the agent registers
|
|
# healthy and every shard placed on it dies with "executable file not found"
|
|
# or a missing shared library. Checked here because it is cheap here and
|
|
# expensive later — llama-server on node05 was present, executable, and could
|
|
# not start for want of libgomp1, and the node reported ready throughout.
|
|
check_engine() {
|
|
[ -n "$2" ] || return 0
|
|
if $SSH "test -x '$2'" 2>/dev/null; then
|
|
if $SSH "'$2' --version >/dev/null 2>&1 || '$2' --help >/dev/null 2>&1"; then
|
|
say "$1: $2 runs"
|
|
return 0
|
|
fi
|
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
|
$1 at $2 exists but will not run. Usually a missing shared library:
|
|
ssh $SSH_USER@$ADDR ldd $2 | grep 'not found'"
|
|
return 0
|
|
fi
|
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
|
$1 is configured at $2, which is not an executable on $NODE."
|
|
}
|
|
ENGINE_PROBLEMS=
|
|
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
say "checking the engines"
|
|
check_engine llamacpp "$LLAMACPP_BIN"
|
|
check_engine vllm "$VLLM_BIN"
|
|
check_engine sglang "$SGLANG_PYTHON"
|
|
fi
|
|
|
|
say "starting the agent"
|
|
$SSH 'set -e
|
|
sudo install -m 0644 -o root -g root /tmp/agent.yaml /etc/llmgw/agent.yaml
|
|
sudo install -m 0644 /tmp/llmgw-agent.service /etc/systemd/system/llmgw-agent.service
|
|
rm -f /tmp/agent.yaml /tmp/llmgw-agent.service
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable --now llmgw-agent >/dev/null
|
|
sudo systemctl restart llmgw-agent'
|
|
|
|
echo
|
|
if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
|
say "the agent now installs the engines the control plane asks for: the"
|
|
say "build for this machine, checked against its publisher's checksum and run"
|
|
say "before the node is offered any model. 'llmgw ctl nodes' shows progress."
|
|
fi
|
|
if [ -n "$ENGINE_PROBLEMS" ]; then
|
|
echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS"
|
|
echo
|
|
echo "The node will register as ready regardless. Fix this before a replica"
|
|
echo "is placed on it, or the shard fails and the reason is three logs away."
|
|
echo
|
|
fi
|
|
echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds."
|
|
echo "If it does not, the control plane's log says why — a certificate whose"
|
|
echo "common name does not match the claimed node id is refused by design."
|