diff --git a/deploy/bootstrap/add-node.sh b/deploy/bootstrap/add-node.sh index 13d2eeb..290dd89 100755 --- a/deploy/bootstrap/add-node.sh +++ b/deploy/bootstrap/add-node.sh @@ -23,6 +23,7 @@ usage: add-node.sh --name NODE --ip ADDR/CIDR --server URL [options] --vmid N Proxmox VMID (default: the last octet of --ip) --cores N (default 4) + --cpu TYPE QEMU CPU model (default host; see provision-node.sh) --memory MB (default 8192) --disk GB (default 20) --fabric NAME InfiniBand fabric; repeatable as NAME:hca1,hca2. @@ -39,7 +40,7 @@ USAGE exit 2 } -NAME= IP= SERVER= VMID= CORES=4 MEMORY=8192 DISK=20 +NAME= IP= SERVER= VMID= CORES=4 MEMORY=8192 DISK=20 CPU= BINARY= PKI=/etc/llmgw/pki SSH_USER=ilker PROVISION=1 ONBOARD=1 set -- "$@" @@ -51,6 +52,7 @@ while [ $# -gt 0 ]; do --server) SERVER="$2"; shift 2;; --vmid) VMID="$2"; shift 2;; --cores) CORES="$2"; shift 2;; + --cpu) CPU="$2"; shift 2;; --memory) MEMORY="$2"; shift 2;; --disk) DISK="$2"; shift 2;; --binary) BINARY="$2"; shift 2;; @@ -82,8 +84,9 @@ ONBOARD_SH="$(here)/onboard-node.sh" if [ "$PROVISION" = 1 ]; then [ -x "$PROVISION_SH" ] || { echo "cannot find provision-node.sh" >&2; exit 1; } echo "== 1/2 building $NAME as VM $VMID" + # shellcheck disable=SC2086 "$PROVISION_SH" --vmid "$VMID" --name "$NAME" --ip "$IP" \ - --cores "$CORES" --memory "$MEMORY" --disk "$DISK" + --cores "$CORES" --memory "$MEMORY" --disk "$DISK" ${CPU:+--cpu "$CPU"} fi if [ "$ONBOARD" = 0 ]; then diff --git a/deploy/bootstrap/provision-node.sh b/deploy/bootstrap/provision-node.sh index c980580..0136fd8 100755 --- a/deploy/bootstrap/provision-node.sh +++ b/deploy/bootstrap/provision-node.sh @@ -21,6 +21,7 @@ usage: $0 --vmid N --name NAME --ip ADDR [options] --ip ADDR address with prefix, e.g. 10.0.0.144/24 --gw ADDR gateway (default 10.0.0.1) --cores N (default 4) + --cpu TYPE QEMU CPU model (default host) --memory MB (default 8192) --disk GB (default 20) --bridge NAME (default vmbr0) @@ -33,6 +34,17 @@ USAGE } VMID= NAME= IP= GW=10.0.0.1 CORES=4 MEMORY=8192 DISK=20 +# host, not Proxmox's kvm64 default. kvm64 hides AVX, AVX2, FMA and even +# SSE4.2 from the guest, and an inference node without vector instructions is +# not slightly slower, it is unusable: llama.cpp falls back to its scalar +# backend and a 1.5B model dropped from 39 tokens a second to 5.8, with prompt +# processing 19x worse. It cost an afternoon to find because nothing is broken +# — every number is simply small. +# +# The trade is live migration: a VM pinned to the host CPU cannot move to a +# machine with a different one. For a node whose job is arithmetic, that is +# the right way round. +CPU=host BRIDGE=vmbr0 STORAGE=local-lvm USER=ilker IMAGE=/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img SSHKEYS=/root/.ssh/authorized_keys @@ -44,6 +56,7 @@ while [ $# -gt 0 ]; do --ip) IP="$2"; shift 2;; --gw) GW="$2"; shift 2;; --cores) CORES="$2"; shift 2;; + --cpu) CPU="$2"; shift 2;; --memory) MEMORY="$2"; shift 2;; --disk) DISK="$2"; shift 2;; --bridge) BRIDGE="$2"; shift 2;; @@ -64,7 +77,7 @@ fi echo "creating $NAME as VM $VMID at $IP" qm create "$VMID" \ - --name "$NAME" --cores "$CORES" --memory "$MEMORY" \ + --name "$NAME" --cores "$CORES" --memory "$MEMORY" --cpu "$CPU" \ --net0 "virtio,bridge=$BRIDGE" --ostype l26 --agent enabled=1 # Imported rather than installed: a cloud image plus cloud-init is the whole diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index b53caee..e865191 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -90,6 +90,32 @@ it accepted a moment earlier. It also checks that a configured engine actually runs, which is cheap here and expensive later — see below. +## The CPU model matters, and its default is wrong + +Nodes are provisioned with `--cpu host`. Proxmox's default, `kvm64`, hides +AVX, AVX2, FMA and even SSE4.2 from the guest, and llama.cpp then loads its +scalar backend instead of the one built for the host's instruction set. +Measured on node05, an 8-core VM on a Xeon E5-2680 v4, same weights, same +threads, the only change being the CPU model: + +| Qwen2.5-Coder | kvm64 tok/s | host tok/s | +| -------------- | ----------: | ---------: | +| 1.5B q4_K_M | 5.80 | 39.09 | +| 3B q4_K_M | 2.90 | 20.51 | +| 7B q4_K_M | 1.33 | 9.88 | + +Prompt processing is worse still: 7.6 tokens a second against 143.8, a factor +of nineteen. + +It is worth calling out because nothing looks broken. The node registers, the +engine starts, replies are correct — every number is simply small, and the +obvious conclusion is that CPU inference is hopeless rather than that the VM +cannot multiply efficiently. + +The trade is live migration: a VM pinned to the host CPU cannot move to a +machine with a different one. For a node whose job is arithmetic that is the +right way round, and `--cpu` overrides it where it is not. + ## What onboarding does not do It gives a machine an identity and an agent. It does **not** install inference