diff --git a/deploy/bootstrap/add-node.sh b/deploy/bootstrap/add-node.sh new file mode 100755 index 00000000..13d2eeb8 --- /dev/null +++ b/deploy/bootstrap/add-node.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# add-node.sh — build a node and put it in the fleet, in one command. +# +# provision-node.sh and onboard-node.sh stay separate because the two halves +# need different access: one the hypervisor, one the certificate authority. +# That is a real boundary and this script does not erase it — it drives both +# where one person holds both, which is the common case and was the case here. +# +# The reason to have it is not keystrokes. The VM name, the hostname, the node +# id and the certificate's common name must all be the same string, and the +# address appears in three places; split across two invocations run minutes +# apart, that agreement is maintained by whoever is typing. Here it is stated +# once. +set -eu + +usage() { + cat >&2 <<'USAGE' +usage: add-node.sh --name NODE --ip ADDR/CIDR --server URL [options] + + --name NODE node id, hostname and VM name; also the certificate's CN + --ip ADDR/CIDR e.g. 10.0.0.145/24 + --server URL control plane, e.g. https://10.0.0.140:7443 + + --vmid N Proxmox VMID (default: the last octet of --ip) + --cores N (default 4) + --memory MB (default 8192) + --disk GB (default 20) + --fabric NAME InfiniBand fabric; repeatable as NAME:hca1,hca2. + Omit for a node with no fabric. + --llamacpp-bin P path to llama-server on the node + --vllm-bin P path to vllm + --sglang-python P path to the python that has sglang + --binary PATH llmgw binary to install (default: the one on PATH) + --pki DIR certificate authority (default /etc/llmgw/pki) + --ssh-user U (default ilker) + --provision-only build the VM and stop + --onboard-only skip provisioning; the machine already exists +USAGE + exit 2 +} + +NAME= IP= SERVER= VMID= CORES=4 MEMORY=8192 DISK=20 +BINARY= PKI=/etc/llmgw/pki SSH_USER=ilker +PROVISION=1 ONBOARD=1 +set -- "$@" +PASS_THROUGH= +while [ $# -gt 0 ]; do + case "$1" in + --name) NAME="$2"; shift 2;; + --ip) IP="$2"; shift 2;; + --server) SERVER="$2"; shift 2;; + --vmid) VMID="$2"; shift 2;; + --cores) CORES="$2"; shift 2;; + --memory) MEMORY="$2"; shift 2;; + --disk) DISK="$2"; shift 2;; + --binary) BINARY="$2"; shift 2;; + --pki) PKI="$2"; shift 2;; + --ssh-user) SSH_USER="$2"; shift 2;; + --provision-only) ONBOARD=0; shift;; + --onboard-only) PROVISION=0; shift;; + # Everything the onboarding step understands and this one does not need to. + --fabric|--llamacpp-bin|--vllm-bin|--sglang-python|--ib-root|--nvidia-smi|--mgmt-addr|--mgmt-if) + PASS_THROUGH="$PASS_THROUGH $1 $2"; shift 2;; + -h|--help) usage;; + *) echo "unknown option: $1" >&2; usage;; + esac +done +[ -n "$NAME" ] && [ -n "$IP" ] && [ -n "$SERVER" ] || usage + +# The address without its prefix, which is what onboarding and ssh want. +ADDR="${IP%%/*}" +# The last octet as the VMID by default: it keeps `qm list` and `ctl nodes` in +# the same order, and one fewer number to choose is one fewer to get wrong. +[ -n "$VMID" ] || VMID="${ADDR##*.}" + +here() { dirname "$0"; } +PROVISION_SH="$(here)/provision-node.sh" +ONBOARD_SH="$(here)/onboard-node.sh" +[ -x "$PROVISION_SH" ] || PROVISION_SH="$(command -v llmgw-provision-node || true)" +[ -x "$ONBOARD_SH" ] || ONBOARD_SH="$(command -v llmgw-onboard-node || true)" + +if [ "$PROVISION" = 1 ]; then + [ -x "$PROVISION_SH" ] || { echo "cannot find provision-node.sh" >&2; exit 1; } + echo "== 1/2 building $NAME as VM $VMID" + "$PROVISION_SH" --vmid "$VMID" --name "$NAME" --ip "$IP" \ + --cores "$CORES" --memory "$MEMORY" --disk "$DISK" +fi + +if [ "$ONBOARD" = 0 ]; then + echo "$NAME built. Onboard it from the CA host when you are ready." + exit 0 +fi + +[ -x "$ONBOARD_SH" ] || { echo "cannot find onboard-node.sh" >&2; exit 1; } +echo +echo "== 2/2 onboarding $NAME" +# Onboarding waits for ssh itself, which is what makes running the two halves +# back to back safe: cloud-init is still regenerating host keys and restarting +# sshd for the first minute of a new machine's life. +# shellcheck disable=SC2086 +"$ONBOARD_SH" --node "$NAME" --addr "$ADDR" --server "$SERVER" \ + --pki "$PKI" --ssh-user "$SSH_USER" \ + ${BINARY:+--binary "$BINARY"} $PASS_THROUGH + +echo +echo "Check it with: llmgw ctl nodes" +echo +echo "A node is not finished until the engines its profiles name are installed" +echo "and the weights are on disk. Onboarding checks that a configured engine" +echo "runs, but it installs neither — see docs/node-onboarding.md." diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index 7e230ad2..3b79b009 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -50,6 +50,7 @@ USAGE NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0 NVIDIA_SMI= IB_ROOT=/sys/class/infiniband LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON= +SSH_WAIT=180 PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1" @@ -78,11 +79,33 @@ BINARY="${BINARY:-$(command -v llmgw)}" [ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; } [ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; } -SSH="ssh -o BatchMode=yes -o ConnectTimeout=10 $SSH_USER@$ADDR" +# accept-new, not "no": a machine provisioned a minute ago has a host key +# nobody has seen, and refusing it stopped onboarding dead with "Host key +# verification failed" — an error about ssh, in the middle of a script whose +# whole job is to talk to a new machine. accept-new learns an unknown key and +# still refuses a changed one, which is the distinction that matters: the risk +# being guarded against is a substituted machine, not a new one. +SSHOPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new" +SSH="ssh $SSHOPTS $SSH_USER@$ADDR" say() { printf ' %s\n' "$*"; } echo "onboarding $NODE at $ADDR" -$SSH true || { echo "cannot reach $SSH_USER@$ADDR over ssh" >&2; exit 1; } + +# Waiting rather than failing: cloud-init restarts sshd after it regenerates +# the host keys, so a machine that answered a moment ago refuses connections +# for a few seconds. Observed while onboarding node05 — the agent was already +# registered and healthy while ssh was refusing. +wait_for_ssh() { + deadline=$(( $(date +%s) + SSH_WAIT )) + until $SSH true 2>/dev/null; do + [ "$(date +%s)" -lt "$deadline" ] || { + echo "cannot reach $SSH_USER@$ADDR over ssh after ${SSH_WAIT}s" >&2 + exit 1 + } + sleep 5 + done +} +wait_for_ssh # 1. The certificate. Issued fresh every time: re-onboarding a machine should # not depend on finding an old key, and the old one keeps working until it @@ -95,7 +118,7 @@ trap 'rm -rf "$TMP"' EXIT # 2 and 3. Identity, and trusting the root so future tooling on that machine # does not treat our certificates as strangers. say "installing the certificate and trusting the root" -scp -q "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/" +scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/" $SSH "set -e sudo install -d -m 0755 /etc/llmgw sudo install -m 0644 /tmp/ca.pem /etc/llmgw/ca.pem @@ -108,7 +131,7 @@ $SSH "set -e # 4. The binary, the configuration and the unit. say "installing the binary" -scp -q "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw" +scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw" $SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw' say "writing the configuration" @@ -192,6 +215,35 @@ WantedBy=multi-user.target UNIT scp -q "$TMP/llmgw-agent.service" "$SSH_USER@$ADDR:/tmp/" +# An engine path that is wrong, or right but unrunnable, is the failure this +# node type is most likely to have, and it is invisible: the agent registers +# healthy and every shard placed on it dies with "executable file not found" +# or a missing shared library. Checked here because it is cheap here and +# expensive later — llama-server on node05 was present, executable, and could +# not start for want of libgomp1, and the node reported ready throughout. +check_engine() { + [ -n "$2" ] || return 0 + if $SSH "test -x '$2'" 2>/dev/null; then + if $SSH "'$2' --version >/dev/null 2>&1 || '$2' --help >/dev/null 2>&1"; then + say "$1: $2 runs" + return 0 + fi + ENGINE_PROBLEMS="$ENGINE_PROBLEMS + $1 at $2 exists but will not run. Usually a missing shared library: + ssh $SSH_USER@$ADDR ldd $2 | grep 'not found'" + return 0 + fi + ENGINE_PROBLEMS="$ENGINE_PROBLEMS + $1 is configured at $2, which is not an executable on $NODE." +} +ENGINE_PROBLEMS= +if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then + say "checking the engines" + check_engine llamacpp "$LLAMACPP_BIN" + check_engine vllm "$VLLM_BIN" + check_engine sglang "$SGLANG_PYTHON" +fi + say "starting the agent" $SSH 'set -e sudo install -m 0644 -o root -g root /tmp/agent.yaml /etc/llmgw/agent.yaml @@ -206,6 +258,13 @@ if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then say "no engine paths given; this node can only run the 'sim' engine until" say "one is installed and the configuration points at it" fi +if [ -n "$ENGINE_PROBLEMS" ]; then + echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS" + echo + echo "The node will register as ready regardless. Fix this before a replica" + echo "is placed on it, or the shard fails and the reason is three logs away." + echo +fi echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds." echo "If it does not, the control plane's log says why — a certificate whose" echo "common name does not match the claimed node id is refused by design." diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index 6748bd6d..b53caee3 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -1,8 +1,33 @@ # Adding a node -Written for whoever runs the cluster. Two scripts, because the two halves of -onboarding are done by different people with different access: one needs the -hypervisor, the other needs the certificate authority. +Written for whoever runs the cluster. + +## The short version + +Where one person holds both the hypervisor and the certificate authority — +which is the common case — one command does the whole thing in about +45 seconds: + +``` +llmgw-add-node \ + --name llmgw-node05 --ip 10.0.0.145/24 \ + --server https://10.0.0.140:7443 \ + --cores 8 --memory 32768 --disk 60 \ + --llamacpp-bin /opt/llama.cpp/llama-server +``` + +The VMID defaults to the last octet of the address, so `qm list` and +`llmgw ctl nodes` stay in the same order. `--provision-only` and +`--onboard-only` split it back into the two halves below when the two kinds +of access belong to different people. + +The rest of this page is what that command does, and what it still does not. + +## The two halves + +Two scripts, because the two halves of onboarding are done by different people +with different access: one needs the hypervisor, the other needs the +certificate authority. ## 1. The machine @@ -55,6 +80,16 @@ what makes a node's identity something it proves rather than something it picks. `authorized_keys` into the new machine, so a node built by the script above already satisfies it; an older machine may need the key added. +It waits for ssh rather than failing on it, and learns the host key of a machine +it has not seen (`accept-new`, which still refuses a *changed* key — the risk +being guarded against is a substituted machine, not a new one). Both are needed +to run the halves back to back: for the first minute of its life a new machine +is still regenerating host keys and restarting sshd, so it refuses connections +it accepted a moment earlier. + +It also checks that a configured engine actually runs, which is cheap here and +expensive later — see below. + ## What onboarding does not do It gives a machine an identity and an agent. It does **not** install inference @@ -62,6 +97,13 @@ engines or model weights, and a node missing either looks healthy in `llmgw ctl nodes` right up until a replica is placed on it and the shard dies with "executable file not found". +Onboarding does check that an engine you point it at can be executed, and says +so loudly when it cannot. That check exists because the failure is invisible +otherwise: `llama-server` on node05 was present, executable, and could not start +for want of `libgomp1`, and the node reported `ready` in `llmgw ctl nodes` the +whole time. Being told at onboarding beats finding out when a replica is placed +and the reason is three logs away. + So the node is not finished until: - the engines its catalog profiles name are installed (`vllm`, `sglang`,