rollcall/deploy/bootstrap.sh

#!/bin/bash
# Rollcall region on a fresh Debian 13 host. Run as root from a checkout
# (deploy/) or an unpacked release (rollcall + deploy/ + agent/).
#
# Day 0 - nothing about the network is decided yet:
#   ./bootstrap.sh [--image-url URL | --build-image] [--name NAME]
#     installs everything and starts the region on this host's management
#     address: web UI and API work, no DHCP, no subnets. Then look around:
#       rollcall net discover     interfaces, switch ports (LLDP), DHCP servers, neighbours
#
# Later - once the provisioning network is known (run again, nothing is lost):
#   ./bootstrap.sh --iface IFACE [--gateway IP | --nat UPLINK] [--domain D] [--no-dhcp]
#     IFACE must have its address on that network; this host's address there
#     becomes the server URL machines use, the subnet is modelled with ranges
#     and (unless --no-dhcp) this host serves DHCP/PXE on it.
#   Or do the same from the web UI / CLI (rollcall subnet add, vlan add -dhcp),
#   or import the prefixes from NetBox (rollcall ipam import).
#
#   --name NAME   rack controller name of this host (default: short hostname)
#   --image-url   directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS - a
#                 region's http://HOST/rollcall, or a release's download URL, where the files are
#                 commissioning-* (deploy/publish-image.sh); IMAGE_TOKEN=... for a private repository
#   --build-image build the commissioning image here with live-build (~20 min)
#   --gateway IP  default router for machines on the provisioning network
#   --nat UPLINK  this host routes for them: forward + masquerade out of UPLINK
#                 (default with --iface and no --gateway: the default route's interface)
#   --domain D    DNS domain handed to machines (default rollcall.internal)
#   --no-dhcp     model the network but leave DHCP to the existing server (no PXE from here)
#   --proxy-dhcp  the site's DHCP server keeps handing out addresses; this host only answers
#                 PXE clients with where to boot from (no ranges, no NAT)
#   --ssh-key K   SSH public key (or a file of them) for root on deployed machines; repeatable.
#                 Default: the keys of the user running sudo, then root's.
#   --os NAME     import one OS image now (e.g. debian-13); a nightly timer syncs the catalog
#   --yes         do not ask
set -euo pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
ROOT=$(cd "$HERE/.." && pwd)
IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 SSHKEYS=()
while [ $# -gt 0 ]; do
  case "$1" in
    --iface) IFACE=$2; shift ;;
    --name) RACK=$2; shift ;;
    --domain) DOMAIN=$2; shift ;;
    --gateway) GATEWAY=$2; shift ;;
    --nat) NAT=$2; shift ;;
    --image-url) IMAGE_URL=${2%/}; shift ;;
    --build-image) BUILD=1 ;;
    --os) OS=$2; shift ;;
    --ssh-key) SSHKEYS+=("$2"); shift ;;
    --no-dhcp) DHCP=0 ;;
    --proxy-dhcp) DHCP=2 ;;
    --yes|-y) YES=1 ;;
    -h|--help) sed -n '2,31p' "$0"; exit 0 ;;
    *) echo "unknown option $1 (--help)" >&2; exit 2 ;;
  esac
  shift
done

say() { printf '\n\033[1m== %s\033[0m\n' "$*"; }
ok()  { printf '   \033[32mOK\033[0m %s\n' "$*"; }
die() { printf '   \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; }
[ "$(id -u)" = 0 ] || die "run as root"
OSNAME=$(. /etc/os-release; echo "$ID $VERSION_ID")
case "$OSNAME" in debian*) ;; *) echo "note: tested on Debian 13; this is $OSNAME" >&2 ;; esac
CONF=/etc/rollcall/config.json
UPLINK=$(ip -4 route show default | awk '{print $5; exit}')
addr_of() { ip -4 -o addr show dev "$1" scope global | awk '{print $4; exit}'; }

# ------------------------------------------------------------------ 0. where
say "0/9  this host"
if [ -n "$IFACE" ]; then
  CIDR_ADDR=$(addr_of "$IFACE")
  [ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; give it its address on the provisioning network first"
  read -r ADDR NET PREFIX FIRST <<<"$(python3 -c 'import ipaddress,sys;i=ipaddress.ip_interface(sys.argv[1]);print(i.ip,i.network,i.network.prefixlen,i.network.network_address)' "$CIDR_ADDR")"
  [ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; model it by hand (rollcall subnet add / range add)"
  case $DHCP in 1) DHCPTXT="served here" ;; 2) DHCPTXT="proxy (the site's server hands out addresses)" ;; *) DHCPTXT=off ;; esac
  if [ -z "$GATEWAY" ] && [ $DHCP != 2 ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi
  [ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not $IFACE"
  echo "   provisioning  $IFACE $CIDR_ADDR (subnet $NET), machines' gateway ${GATEWAY:-from the site DHCP server}${NAT:+ via NAT out of $NAT}, DHCP $DHCPTXT"
else
  [ -n "$UPLINK" ] || die "no default route: give this host network access first"
  ADDR=$(addr_of "$UPLINK" | cut -d/ -f1)
  if [ -f "$CONF" ]; then
    echo "   existing installation: server address and network model are kept"
  else
    echo "   no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets"
  fi
fi
SERVER=http://$ADDR
echo "   region        $SERVER, rack controller \"$RACK\""
if [ $YES = 0 ] && [ -t 0 ]; then
  read -r -p "   continue? [y/N] " a; case "$a" in y|Y|yes) ;; *) exit 1 ;; esac
fi

# ------------------------------------------------------------------ 1. packages
say "1/9  packages"
export DEBIAN_FRONTEND=noninteractive
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables lldpd"
[ $BUILD = 1 ] && PKGS="$PKGS live-build debootstrap squashfs-tools xorriso"
apt-get update -q >/dev/null
apt-get install -yq $PKGS >/dev/null
install -d /etc/dnsmasq.d
# until rollcall serves a VLAN, dnsmasq must not answer DNS/DHCP everywhere
# (rollcall's generated configuration adds the interfaces of the VLANs it serves)
printf '# rollcall bootstrap: answer only on loopback until rollcall serves a VLAN\n# (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\ninterface=lo\n' > /etc/dnsmasq.d/00-rollcall-base.conf
systemctl restart dnsmasq || true
ok "installed (lldpd shows which switch port and VLAN each interface is on)"

# ------------------------------------------------------------------ 2. database
say "2/9  PostgreSQL"
systemctl enable --now postgresql >/dev/null
PGPASS=""
if [ -f "$CONF" ]; then
  ok "configured before, kept"
else
  PGPASS=$(head -c 32 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
  if [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_roles where rolname='rollcall'")" = 1 ]; then
    runuser -u postgres -- psql -q -c "ALTER ROLE rollcall PASSWORD '$PGPASS'"
  else
    runuser -u postgres -- psql -q -c "CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'"
  fi
  [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_database where datname='rollcall'")" = 1 ] || runuser -u postgres -- psql -q -c "CREATE DATABASE rollcall OWNER rollcall"
  ok "role and database rollcall"
fi

# ------------------------------------------------------------------ 3. TFTP
say "3/9  TFTP boot files"
T=/srv/tftp
install -d $T $T/efi64 $T/pxelinux.cfg
cp -u /usr/lib/PXELINUX/pxelinux.0 $T/
for m in ldlinux.c32 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/bios/$m $T/; done
cp -u /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi $T/efi64/
for m in ldlinux.e64 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/efi64/$m $T/efi64/; done
ln -sfn ../pxelinux.cfg $T/efi64/pxelinux.cfg
ok "BIOS (pxelinux.0) and UEFI (efi64/syslinux.efi)"

# ------------------------------------------------------------------ 4. binary
say "4/9  rollcall binary"
BIN=""
for b in "$ROOT/rollcall" "$HERE/rollcall"; do [ -x "$b" ] && BIN=$b && break; done
if [ -z "$BIN" ] && [ -d "$ROOT/cmd/rollcall" ] && command -v go >/dev/null; then
  (cd "$ROOT/cmd/rollcall" && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo dev)" -o "$HERE/rollcall" .)
  BIN=$HERE/rollcall
fi
[ -n "$BIN" ] || die "no rollcall binary next to this script (unpack a release, or build: cd cmd/rollcall && go build)"
[ "$BIN" = "$HERE/rollcall" ] || cp "$BIN" "$HERE/rollcall"
ok "$("$HERE/rollcall" version)"

# ------------------------------------------------------------------ 5. config
say "5/9  configuration"
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
install -d -m 0750 -o root -g rollcall /etc/rollcall
FQDN=$(hostname -f 2>/dev/null || hostname)
CHANGED=0
if [ ! -f "$CONF" ]; then
  AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen)
  python3 - "$HERE/config.example.json" "$CONF" <<P
import json, sys
c = json.load(open(sys.argv[1]))
c.update({
  "listen": ":80", "server_url": "$SERVER",
  "database_url": "postgres://rollcall:$PGPASS@127.0.0.1:5432/rollcall?sslmode=disable",
  "admin_token": "$ADMIN", "agent_token": "$AGENT",
  "commission_append": "",   # derived from server_url, follows it when it changes
  "rack_name": "$RACK", "tls_listen": ":443", "tls_names": sorted({"$FQDN", "$ADDR"}), "admin_http": "loopback",
  "node_pools": {}, "bmc_hints": [],
})
c["cluster_vars"].update({"domain": "$DOMAIN", "cluster_name": "$RACK", "slurm_controller": "$RACK", "slurm_controller_ip": "$ADDR", "ssh_keys": []})
c["ipam"]["provider"] = ""
json.dump(c, open(sys.argv[2], "w"), indent=2)
P
  chown root:rollcall "$CONF"; chmod 0640 "$CONF"
  ok "$CONF written (tokens generated; the web UI is HTTPS-only: admin_http loopback)"
else
  # an existing installation: the provisioning address may have arrived since
  # (without --iface the server address stays what it is)
  if [ -n "$IFACE" ] && python3 - "$CONF" "$SERVER" "$ADDR" "$FQDN" <<'P'
import json, sys
p, srv, addr, fqdn = sys.argv[1:]
c = json.load(open(p))
old = c.get("server_url", "")
names = set(c.get("tls_names") or []) | {addr, fqdn}
changed = old != srv or set(c.get("tls_names") or []) != names
gen = c.get("commission_append", "")
if old and old in gen and "rollcall.token=" in gen:  # the old generated form: let it follow server_url
    c["commission_append"] = ""
    changed = True
c["server_url"], c["tls_names"] = srv, sorted(names)
if changed:
    json.dump(c, open(p, "w"), indent=2)
sys.exit(0 if changed else 1)
P
  then CHANGED=1; ok "server_url now $SERVER (was different), tls_names updated"
  else SERVER=$(jq -r .server_url "$CONF"); ok "$CONF kept (server $SERVER)"
  fi
fi

# SSH keys for deployed machines: given ones, else the operator's
keyfiles=()
if [ ${#SSHKEYS[@]} -eq 0 ] && grep -q '"ssh_keys": \[\]' "$CONF" 2>/dev/null; then
  for f in "/home/${SUDO_USER:-none}/.ssh/authorized_keys" /root/.ssh/authorized_keys; do [ -r "$f" ] && keyfiles+=("$f"); done
fi
if [ ${#SSHKEYS[@]} -gt 0 ] || [ ${#keyfiles[@]} -gt 0 ]; then
  n=$(python3 - "$CONF" "${SSHKEYS[@]}" -- "${keyfiles[@]}" <<'P2'
import json, os, re, sys
conf = sys.argv[1]; rest = sys.argv[2:]; sep = rest.index("--")
given, files = rest[:sep], rest[sep + 1:]
keyre = re.compile(r'((?:ssh-(?:ed25519|rsa|dss)|ecdsa-sha2-\S+|sk-\S+)\s+[A-Za-z0-9+/=]+(?:\s+\S+)?)')
def keys_of(text):
    return [m.group(1).strip() for line in text.splitlines() for m in [keyre.search(line)] if m]
found = []
for g in given:
    found += keys_of(open(g).read()) if os.path.isfile(g) else keys_of(g)
for f in files:
    found += keys_of(open(f).read())
c = json.load(open(conf)); cv = c.setdefault("cluster_vars", {})
have = cv.get("ssh_keys") or []
new = [k for k in dict.fromkeys(found) if k not in have]
if new:
    cv["ssh_keys"] = have + new
    json.dump(c, open(conf, "w"), indent=2)
print(len(new))
P2
)
  [ "$n" -gt 0 ] && { ok "$n SSH key(s) for root on deployed machines"; CHANGED=1; } || true
fi
[ "$(jq '.cluster_vars.ssh_keys | length' "$CONF")" -gt 0 ] || echo "   note: no SSH keys for deployed machines yet (--ssh-key KEY)" >&2

# ------------------------------------------------------------------ 6. services
say "6/9  services"
install -d /srv/rollcall/image
(cd "$HERE" && sh ./install.sh >/dev/null)
for _ in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done
curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall"
ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')"

# ------------------------------------------------------------------ 7. commissioning image
say "7/9  commissioning image"
IMG=/srv/rollcall/image
if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashfs ]; then
  ok "present, kept"
elif [ -n "$IMAGE_URL" ]; then
  tmp=$(mktemp -d -p /srv/rollcall)
  auth=(); [ -n "${IMAGE_TOKEN:-}" ] && auth=(-H "Authorization: token $IMAGE_TOKEN")
  # a release carries the image as commissioning-* (its plain SHA256SUMS is the binaries');
  # a region's /rollcall directory has the plain names
  pre=commissioning-
  if ! curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/${pre}SHA256SUMS"; then
    pre=""
    curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "no commissioning-SHA256SUMS or SHA256SUMS at $IMAGE_URL${IMAGE_TOKEN:+}$( [ -z "${IMAGE_TOKEN:-}" ] && echo ' (private repository? set IMAGE_TOKEN)')"
  fi
  for f in vmlinuz initrd.img filesystem.squashfs; do curl -fSL --progress-bar "${auth[@]}" -o "$tmp/$f" "$IMAGE_URL/$pre$f"; done
  (cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch"
  mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp"
  chmod 0644 $IMG/*
  ok "downloaded and verified from $IMAGE_URL"
elif [ $BUILD = 1 ]; then
  [ -x "$ROOT/agent/build-image.sh" ] || die "agent/build-image.sh not found next to deploy/"
  OUT=$IMG "$ROOT/agent/build-image.sh"
  (cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS)
  ok "built"
else
  echo "   not yet: rerun with --image-url URL or --build-image before machines commission" >&2
fi

# ------------------------------------------------------------------ 8. provisioning network
say "8/9  provisioning network"
if [ -z "$IFACE" ]; then
  echo "   not decided yet - nothing modelled, no DHCP served. When it is:"
  echo "     rollcall net discover                 what each interface sees (LLDP, DHCP servers, neighbours)"
  echo "     $0 --iface IFACE ...      or the web UI: Network -> New subnet"
else
  [ $CHANGED = 1 ] && systemctl restart rollcall && sleep 2
  off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; }
  FAB=provisioning
  rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
  if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
    if [ $DHCP = 2 ]; then
      # the site's DHCP server owns the addresses: no ranges (machines keep DHCP);
      # set a static range aside later (rollcall range add) to give them fixed ones
      rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning ${GATEWAY:+-gateway "$GATEWAY"} -domain "$DOMAIN" -name provisioning >/dev/null
      ok "subnet $NET (addresses from the site's DHCP server)"
    else
      rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
      rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
      rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
      rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
      ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)"
    fi
  else
    ok "subnet $NET exists, kept"
  fi
  if [ $DHCP = 1 ]; then
    rollcall vlan add "$FAB" 0 -dhcp -rack "$RACK" >/dev/null
    ok "DHCP/PXE on $IFACE"
  elif [ $DHCP = 2 ]; then
    rollcall vlan add "$FAB" 0 -proxy-dhcp -rack "$RACK" >/dev/null
    ok "proxy DHCP on $IFACE: PXE answers only; addresses from the site's DHCP server"
  fi
  if [ -n "$NAT" ]; then
    echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
    sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf
    install -d /etc/nftables.d
    printf 'table ip rollcall_nat {\n  chain postrouting {\n    type nat hook postrouting priority 100;\n    oifname "%s" ip saddr %s masquerade\n  }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft
    grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf
    nft -c -f /etc/nftables.conf && systemctl enable nftables >/dev/null 2>&1 && systemctl restart nftables
    ok "forwarding + masquerade $NET out of $NAT"
  fi
fi

# ------------------------------------------------------------------ 9. OS images
say "9/9  OS images"
if [ -n "$OS" ]; then
  runuser -u rollcall -- rollcall os sync "$OS" || echo "   os sync $OS failed; the nightly timer retries" >&2
else
  echo "   none imported now; rollcall-os-sync.timer syncs the catalog nightly (now: runuser -u rollcall -- rollcall os sync debian-13)"
fi

cat <<EOF

Done: rollcall region "$RACK" on $SERVER
  web UI   https://$ADDR   (trust the CA from /ca.pem: $(rollcall ca show | awk '/fingerprint/{print $2}'))
  admin    from your laptop: rollcall admin-cert request NAME   (token once: jq -r .admin_token $CONF)
  check    rollcall doctor
EOF
[ -z "$IFACE" ] && echo "  next     rollcall net discover   (then: $0 --iface IFACE)"
exit 0