diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index c7ea358..8fbe542 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -24,8 +24,15 @@ jobs: for arch in amd64 arm64; do CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -trimpath \ -ldflags="-s -w -X main.version=${GITHUB_REF_NAME}" -o rollcall-linux-$arch . + # install bundle: the binary with deploy/ (bootstrap.sh, units) and agent/ (image build) + b=rollcall-${GITHUB_REF_NAME}-linux-$arch + mkdir -p "$b" + cp rollcall-linux-$arch "$b/rollcall" + cp -r ../../deploy ../../agent ../../README.md ../../INSTALL.md ../../CHANGELOG.md "$b/" + tar -czf "$b.tar.gz" "$b" + rm -rf "$b" done - sha256sum rollcall-linux-* > SHA256SUMS + sha256sum rollcall-linux-* rollcall-*.tar.gz > SHA256SUMS - name: Release notes from CHANGELOG.md env: @@ -57,7 +64,7 @@ jobs: body=$(python3 -c 'import json,sys; print(json.dumps({"tag_name": sys.argv[1], "name": sys.argv[1], "body": open("../../notes.md").read(), "draft": False, "prerelease": "-" in sys.argv[1]}))' "$TAG") id=$(curl -sfS -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \ -d "$body" "$API/releases" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') - for f in rollcall-linux-amd64 rollcall-linux-arm64 SHA256SUMS; do + for f in rollcall-linux-amd64 rollcall-linux-arm64 rollcall-*.tar.gz SHA256SUMS; do curl -sfS -X POST -H "Authorization: token $TOKEN" \ -F "attachment=@$f" "$API/releases/$id/assets?name=$f" >/dev/null echo "uploaded $f" diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..d22d969 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,158 @@ +# Installing Rollcall at a new site + +This follows a site from nothing to a deployed machine. It was written from a +clean run: a fresh Debian 13 VM became the region, discovered its interfaces, +was given a provisioning network, and an empty machine commissioned itself and +was deployed with Debian 13. + +The network does not have to be decided before you start. Day 0 installs the +region on its management address; the provisioning network is added when it +exists. + +## What you need + +| | | +|---|---| +| Region host | Debian 13, 2 CPUs, 4 GB RAM, 30 GB disk (plus ~1 GB per OS image), root | +| Network access | a default route; the internet or a mirror for packages and cloud images | +| Later: provisioning network | an interface of this host (or of a rack controller) on the VLAN where machines PXE-boot | +| Later: BMC network | the rack controller must reach the machines' BMCs (IPMI, Redfish) | +| Ports | DHCP 67/udp and TFTP 69/udp on the provisioning network; HTTP 80 for machines; HTTPS 443 for people and rack controllers | + +The release bundle `rollcall--linux-amd64.tar.gz` holds the binary, +`deploy/` and `agent/`. From a checkout, `deploy/bootstrap.sh` builds the binary +itself when Go is installed. + +## Day 0: install + +```sh +tar xzf rollcall-v0.2.0-linux-amd64.tar.gz && cd rollcall-v0.2.0-linux-amd64 +sudo ./deploy/bootstrap.sh --image-url https://gitea.example/OWNER/rollcall/releases/download/v0.2.0 +``` + +- **Packages, database, TFTP.** PostgreSQL with its own role and a random password, + dnsmasq, the BIOS and UEFI syslinux files, lldpd. +- **Configuration.** `/etc/rollcall/config.json` with generated tokens. The admin + API only answers over HTTPS from other hosts (`admin_http: loopback`). +- **Commissioning image.** Downloaded and checked against `SHA256SUMS`. For a + private repository set `IMAGE_TOKEN`. Without a release to download from, + `--image-url http://EXISTING-REGION/rollcall` copies it from another region, + and `--build-image` builds it here with live-build (about 20 minutes). +- **SSH keys.** The keys of the user running sudo go to root on deployed + machines; add more with `--ssh-key KEY_OR_FILE`. + +At the end the region answers on its management address: the web UI at +`https://ADDRESS`, no DHCP, no subnets. Check it: + +```sh +sudo rollcall doctor +``` + +On day 0 the only warning should be that the host serves no DHCP yet. + +## Find the provisioning network + +```sh +sudo rollcall net discover +``` + +``` +INTERFACE KIND LINK SPEED ADDRESSES SWITCH / PORT / VLAN DHCP SERVERS NEIGH +eth0* physical up - 10.0.0.116/24 - 10.0.0.1 4 +eth1 physical up - 10.214.0.1/24 - none 0 + +eth0 uplink: management network with its own DHCP (10.0.0.1); keep it for the region's access +eth1 candidate provisioning network 10.214.0.0/24: bootstrap.sh --iface eth1 +``` + +For each interface: link and speed, addresses and the subnet they imply, the +switch port and VLAN that LLDP reports, and DHCP servers that answer a probe. +The last column matters most: a DHCP server already on the provisioning VLAN +competes with Rollcall for the machines' PXE boot. + +| What discover shows | What to do | +|---|---| +| An interface with an address and no DHCP | the provisioning network: next step | +| Link up, no address, LLDP shows the VLAN | give the interface its address on that VLAN first | +| Another DHCP server on the provisioning VLAN | the network team turns it off for that VLAN, or keeps it and points it at this host (next-server, or a DHCP relay), then `--no-dhcp` | +| No link | cable it | + +## Add the provisioning network + +```sh +sudo ./deploy/bootstrap.sh --iface eth1 --os debian-13 +``` + +- **Server address.** The host's address on `eth1` becomes the server address + machines use. The management address stays in the certificate, so the web UI + keeps working there. +- **Subnet.** Modelled with ranges: reserved `.1-.20`, static `.50-.99` (where + deployed machines get their addresses), dynamic `.100-.200` (PXE and commissioning). +- **DHCP and PXE** on `eth1`, unless `--no-dhcp`. +- **Routing.** Without `--gateway`, this host routes for the machines: forwarding + plus masquerade out of the uplink. Give `--gateway IP` when the network has a router. +- **First OS image.** `--os debian-13` imports it now; a nightly timer keeps the + catalog current. + +The same can be done in the web UI (Network, New subnet) or the CLI +(`rollcall subnet add`, `range add`, `vlan add ... -dhcp`), or imported from +NetBox (`rollcall ipam import`). Several provisioning VLANs, bonds and VLAN +layouts per machine are described in the README, under Network. + +`rollcall doctor` should now report no failures. It also sends a DHCP +DISCOVER on the provisioning interface and fails if any other server answers. + +## First machines + +Set a machine to PXE-boot on the provisioning network and power it on. An +unknown machine commissions itself: it boots the live image, reports its +hardware, and appears as `ready`. + +```sh +sudo rollcall list +sudo rollcall deploy MACHINE debian-13 -hostname node01 -boot -watch +``` + +`-boot` power-cycles the machine through its BMC. Set the BMC first if +commissioning did not find it (`rollcall bmc MACHINE ...`, or the machine's page +in the web UI). Without a BMC, power the machine on by hand after `deploy`. + +A deployed machine gets the address from the static range, the domain, a route, +DNS through the rack controller, root SSH keys and trust in the Rollcall CA. + +## Admin access from your laptop + +```sh +# on the laptop, once, with the token (sudo jq -r .admin_token /etc/rollcall/config.json on the region) +ROLLCALL_URL=https://REGION ROLLCALL_TOKEN=... ROLLCALL_CA=ca.pem rollcall admin-cert request mylaptop +``` + +The key is made on the laptop; only a certificate request travels. The CLI then +works without the token. The `.p12` it writes goes into the browser's keychain, +and the web UI signs in by itself. Trust the CA (`/ca.pem`) in the browser once. + +## More segments: rack controllers + +A provisioning VLAN the region has no interface on gets its own rack controller: + +```sh +# on the region +sudo rollcall controller join-code -name hall-b -addr REGION:443 +# on the rack host (Debian 13, an interface on that VLAN), from the bundle +sudo ./rollcall rack join CODE -url http://RACK-ADDRESS-ON-THAT-VLAN +sudo sh deploy/install-rack.sh +``` + +Rack controllers talk to the region over mutual TLS and update themselves to the +region's version. + +## Integrations + +- **NetBox (or Infoblox) as the address authority.** `ipam` in `config.json`: see README, External IPAM. +- **Certificates for machines.** Delegate a name-constrained intermediate to an ACME + server: see README, Certificate authority. + +## Upgrades + +Replace the binary on the region (or let CD do it) and restart `rollcall`. Rack +controllers follow by themselves. `rollcall doctor` afterwards. diff --git a/README.md b/README.md index d936815..5f2d528 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ unknown server ──PXE──▶ dnsmasq (DHCP+TFTP) ──▶ pxelinux default ## Server +New site: **[INSTALL.md](INSTALL.md)** (`deploy/bootstrap.sh`, from day 0 without a network plan to the first deployed machine). The steps below are the manual equivalent. + ```sh # build cd cmd/rollcall && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o rollcall . diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 2e54596..218e1b6 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -17,19 +17,23 @@ # or import the prefixes from NetBox (rollcall ipam import). # # --name NAME rack controller name of this host (default: short hostname) -# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS +# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS - a +# region's http://HOST/rollcall, or a release's download URL, where the files are +# commissioning-* (deploy/publish-image.sh); IMAGE_TOKEN=... for a private repository # --build-image build the commissioning image here with live-build (~20 min) # --gateway IP default router for machines on the provisioning network # --nat UPLINK this host routes for them: forward + masquerade out of UPLINK # (default with --iface and no --gateway: the default route's interface) # --domain D DNS domain handed to machines (default rollcall.internal) # --no-dhcp model the network but leave DHCP to the existing server +# --ssh-key K SSH public key (or a file of them) for root on deployed machines; repeatable. +# Default: the keys of the user running sudo, then root's. # --os NAME import one OS image now (e.g. debian-13); a nightly timer syncs the catalog # --yes do not ask set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) ROOT=$(cd "$HERE/.." && pwd) -IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 +IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 SSHKEYS=() while [ $# -gt 0 ]; do case "$1" in --iface) IFACE=$2; shift ;; @@ -40,6 +44,7 @@ while [ $# -gt 0 ]; do --image-url) IMAGE_URL=${2%/}; shift ;; --build-image) BUILD=1 ;; --os) OS=$2; shift ;; + --ssh-key) SSHKEYS+=("$2"); shift ;; --no-dhcp) DHCP=0 ;; --yes|-y) YES=1 ;; -h|--help) sed -n '2,31p' "$0"; exit 0 ;; @@ -71,7 +76,11 @@ if [ -n "$IFACE" ]; then else [ -n "$UPLINK" ] || die "no default route: give this host network access first" ADDR=$(addr_of "$UPLINK" | cut -d/ -f1) - echo " no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets" + if [ -f "$CONF" ]; then + echo " existing installation: server address and network model are kept" + else + echo " no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets" + fi fi SERVER=http://$ADDR echo " region $SERVER, rack controller \"$RACK\"" @@ -160,7 +169,8 @@ P ok "$CONF written (tokens generated; the web UI is HTTPS-only: admin_http loopback)" else # an existing installation: the provisioning address may have arrived since - if python3 - "$CONF" "$SERVER" "$ADDR" "$FQDN" <<'P' + # (without --iface the server address stays what it is) + if [ -n "$IFACE" ] && python3 - "$CONF" "$SERVER" "$ADDR" "$FQDN" <<'P' import json, sys p, srv, addr, fqdn = sys.argv[1:] c = json.load(open(p)) @@ -177,10 +187,41 @@ if changed: sys.exit(0 if changed else 1) P then CHANGED=1; ok "server_url now $SERVER (was different), tls_names updated" - else ok "$CONF kept" + else SERVER=$(jq -r .server_url "$CONF"); ok "$CONF kept (server $SERVER)" fi fi +# SSH keys for deployed machines: given ones, else the operator's +keyfiles=() +if [ ${#SSHKEYS[@]} -eq 0 ] && grep -q '"ssh_keys": \[\]' "$CONF" 2>/dev/null; then + for f in "/home/${SUDO_USER:-none}/.ssh/authorized_keys" /root/.ssh/authorized_keys; do [ -r "$f" ] && keyfiles+=("$f"); done +fi +if [ ${#SSHKEYS[@]} -gt 0 ] || [ ${#keyfiles[@]} -gt 0 ]; then + n=$(python3 - "$CONF" "${SSHKEYS[@]}" -- "${keyfiles[@]}" <<'P2' +import json, os, re, sys +conf = sys.argv[1]; rest = sys.argv[2:]; sep = rest.index("--") +given, files = rest[:sep], rest[sep + 1:] +keyre = re.compile(r'((?:ssh-(?:ed25519|rsa|dss)|ecdsa-sha2-\S+|sk-\S+)\s+[A-Za-z0-9+/=]+(?:\s+\S+)?)') +def keys_of(text): + return [m.group(1).strip() for line in text.splitlines() for m in [keyre.search(line)] if m] +found = [] +for g in given: + found += keys_of(open(g).read()) if os.path.isfile(g) else keys_of(g) +for f in files: + found += keys_of(open(f).read()) +c = json.load(open(conf)); cv = c.setdefault("cluster_vars", {}) +have = cv.get("ssh_keys") or [] +new = [k for k in dict.fromkeys(found) if k not in have] +if new: + cv["ssh_keys"] = have + new + json.dump(c, open(conf, "w"), indent=2) +print(len(new)) +P2 +) + [ "$n" -gt 0 ] && { ok "$n SSH key(s) for root on deployed machines"; CHANGED=1; } || true +fi +[ "$(jq '.cluster_vars.ssh_keys | length' "$CONF")" -gt 0 ] || echo " note: no SSH keys for deployed machines yet (--ssh-key KEY)" >&2 + # ------------------------------------------------------------------ 6. services say "6/9 services" install -d /srv/rollcall/image @@ -196,8 +237,13 @@ if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashf ok "present, kept" elif [ -n "$IMAGE_URL" ]; then tmp=$(mktemp -d -p /srv/rollcall) - curl -fsSL -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "$IMAGE_URL/SHA256SUMS missing" - for f in vmlinuz initrd.img filesystem.squashfs; do curl -fsSL -o "$tmp/$f" "$IMAGE_URL/$f"; done + auth=(); [ -n "${IMAGE_TOKEN:-}" ] && auth=(-H "Authorization: token $IMAGE_TOKEN") + pre="" + if ! curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS"; then + pre=commissioning- # a release: the image files carry a prefix + curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/${pre}SHA256SUMS" || die "no SHA256SUMS or ${pre}SHA256SUMS at $IMAGE_URL" + fi + for f in vmlinuz initrd.img filesystem.squashfs; do curl -fSL --progress-bar "${auth[@]}" -o "$tmp/$f" "$IMAGE_URL/$pre$f"; done (cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch" mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp" chmod 0644 $IMG/* diff --git a/deploy/publish-image.sh b/deploy/publish-image.sh new file mode 100755 index 0000000..e07fec1 --- /dev/null +++ b/deploy/publish-image.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Publish this region's commissioning image as assets of a Gitea release, so new +# installations download it (bootstrap.sh --image-url .../releases/download/TAG). +# GITEA_TOKEN=... ./publish-image.sh v0.2.0 [https://gitea.example/api/v1/repos/OWNER/REPO] +# Files are uploaded as commissioning-{vmlinuz,initrd.img,filesystem.squashfs,SHA256SUMS}. +set -eu +TAG=${1:?usage: publish-image.sh TAG [API]} +API=${2:-https://gitea.mynodes.xyz/api/v1/repos/ilker/rollcall} +IMG=${IMG:-/srv/rollcall/image} +: "${GITEA_TOKEN:?set GITEA_TOKEN (write:repository)}" +cd "$IMG" +sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS +id=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/tags/$TAG" | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])') +existing=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets" | python3 -c 'import json,sys;[print(a["id"],a["name"]) for a in json.load(sys.stdin)]') +for f in vmlinuz initrd.img filesystem.squashfs SHA256SUMS; do + name=commissioning-$f + old=$(echo "$existing" | awk -v n="$name" '$2==n{print $1}') + [ -n "$old" ] && curl -fsS -X DELETE -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets/$old" >/dev/null + echo "uploading $name ($(du -h "$f" | cut -f1))" + curl -fsS -H "Authorization: token $GITEA_TOKEN" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" >/dev/null +done +echo "done: bootstrap.sh --image-url ${API%/api/v1/repos/*}/$(echo "$API" | sed 's#.*/repos/##')/releases/download/$TAG"