Network model, IPAM, interface layouts and the region/rack split (MAAS-style)
- fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static), IPAM with adoption of addresses machines already have; rollcall fabric|vlan|subnet|range|ip - interface layouts (physical/bond/vlan, NICs by selector: boot, name, mac, fastest:N, ib:N) per machine or per tag; deployment allocates addresses and writes cloud-init network-config plus udev name rules (RHEL cloud images boot with net.ifnames=0); release frees them; rollcall layout, rollcall net show|set - region/rack: rollcall rack long-polls its desired state (per-MAC PXE files kept in the database, dnsmasq configuration and host files) and a BMC command queue, caches boot and OS images (kernels/initrds prefetched for TFTP), proxies the node API and tags requests with its name; the region runs an embedded rack controller; install-rack.sh, rollcall-rack.service - dnsmasq configuration generated per rack from the VLANs it serves (split pool for two racks); reload script restarts only on change and restores the last working config if dnsmasq does not come back (drilled in the lab) - web: Network page, per-machine network tab, layout action, tag layout; console: Network page, network section in the machine view, layout action - tests for candidates, plans (default, bond+VLAN, pinned, DHCP fallback), network-config, udev rules and dnsmasq rendering - lab: pve-vbmc address script finds the interface per BMC address Verified in the lab: embedded rack (10.213.77.0/24) and a remote rack controller on a second SDN segment (10.213.78.0/24): enlistment, commissioning, BMC through the rack's queue, image cache, deployment and phone-home through the rack; Debian 13, Ubuntu 24.04 and Rocky 9 with a static boot NIC and with an active-backup bond carrying VLAN 200. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e3d9373201
commit
736a7e38a3
31 files changed
+4357
-176
No files matched your search
@@ -12,6 +12,8 @@
|
||||
"images_dir": "/srv/rollcall/images",
|
||||
"overlays_dir": "/etc/rollcall/overlays",
|
||||
"dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
|
||||
"rack_name": "pxe-server",
|
||||
"rack_secret": "<RACK-SECRET: rollcall keygen; only needed for remote rack controllers>",
|
||||
"os_dir": "/srv/rollcall/os",
|
||||
"os_sources": "/etc/rollcall/os-sources.json",
|
||||
"node_pools": {"mgmt": "10.213.77.50-10.213.77.99"},
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
# Install a rollcall rack controller on this host (Debian/Ubuntu). Run as root
|
||||
# from this directory with the rollcall binary next to it and
|
||||
# /etc/rollcall/rack.json written (see rack.example.json). The region needs a
|
||||
# rack_secret in its config.json; the same value goes into rack.json.
|
||||
#
|
||||
# The rack controller serves DHCP/TFTP (dnsmasq) on the VLANs the region
|
||||
# assigns to it, keeps boot and OS images in a local cache, runs BMC commands
|
||||
# and passes the node API through to the region.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
[ -r /etc/rollcall/rack.json ] || { echo "write /etc/rollcall/rack.json first (see rack.example.json)" >&2; exit 1; }
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get install -y -q dnsmasq ipmitool pxelinux syslinux-common syslinux-efi jq >/dev/null
|
||||
|
||||
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
|
||||
[ -x ./rollcall ] && install -m 0755 rollcall /usr/local/bin/rollcall
|
||||
chgrp rollcall /etc/rollcall/rack.json && chmod 0640 /etc/rollcall/rack.json
|
||||
|
||||
TFTP=$(jq -r '.tftp_root // "/srv/tftp"' /etc/rollcall/rack.json)
|
||||
CACHE=$(jq -r '.cache_dir // "/var/cache/rollcall"' /etc/rollcall/rack.json)
|
||||
DNS=$(jq -r '.dnsmasq_dir // "/var/lib/rollcall/dnsmasq"' /etc/rollcall/rack.json)
|
||||
|
||||
# image cache: kernels/initrds are prefetched (TFTP cannot fetch on demand), the rest fills on first use
|
||||
install -d -o rollcall -g rollcall "$CACHE" "$CACHE/rollcall" "$CACHE/images" "$CACHE/os"
|
||||
|
||||
# TFTP root: syslinux for BIOS and UEFI, per-MAC files written by the rack controller
|
||||
install -d "$TFTP" "$TFTP/efi64"
|
||||
install -d -o rollcall -g rollcall -m 2775 "$TFTP/pxelinux.cfg"
|
||||
cp /usr/lib/PXELINUX/pxelinux.0 "$TFTP/"
|
||||
for f in ldlinux.c32 menu.c32 libutil.c32 libcom32.c32 poweroff.c32; do cp /usr/lib/syslinux/modules/bios/$f "$TFTP/"; done
|
||||
cp /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi "$TFTP/efi64/"
|
||||
for f in ldlinux.e64 menu.c32 libutil.c32 libcom32.c32; do cp /usr/lib/syslinux/modules/efi64/$f "$TFTP/efi64/"; done
|
||||
ln -sfn ../pxelinux.cfg "$TFTP/efi64/pxelinux.cfg"
|
||||
ln -sfn "$CACHE/rollcall" "$TFTP/rollcall"
|
||||
ln -sfn "$CACHE/images" "$TFTP/images"
|
||||
ln -sfn ../rollcall "$TFTP/efi64/rollcall"
|
||||
ln -sfn ../images "$TFTP/efi64/images"
|
||||
|
||||
# dnsmasq reads what the rack controller generates; until then it only answers local DNS
|
||||
install -d -o rollcall -g rollcall "$DNS" "$DNS/conf.d"
|
||||
for f in hosts hosts.dhcp; do [ -e "$DNS/$f" ] || install -m 0644 -o rollcall -g rollcall /dev/null "$DNS/$f"; done
|
||||
# bind-dynamic binds address by address, so systemd-resolved's 127.0.0.53 stub does not get in the way;
|
||||
# until the region assigns a VLAN only loopback is served
|
||||
printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=%s/conf.d/,*.conf\n' "$DNS" > /etc/dnsmasq.d/rollcall.conf
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service /etc/systemd/system/
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable dnsmasq.service rollcall-dnsmasq.path >/dev/null 2>&1
|
||||
systemctl restart dnsmasq.service || { journalctl -u dnsmasq -n 5 --no-pager >&2; exit 1; }
|
||||
systemctl start rollcall-dnsmasq.path
|
||||
systemctl enable --now rollcall-rack.service
|
||||
systemctl restart rollcall-rack.service
|
||||
sleep 2
|
||||
systemctl --no-pager status rollcall-rack.service | head -5
|
||||
+14
-1
@@ -34,8 +34,21 @@ install -m 0644 rollcall-os-sync.service rollcall-os-sync.timer /etc/systemd/sys
|
||||
command -v qemu-img >/dev/null || echo "WARNING: qemu-img missing (apt install qemu-utils) - rollcall os sync/import will fail" >&2
|
||||
# dnsmasq host export + reload trigger
|
||||
install -d -m 0755 /var/lib/rollcall
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq /var/lib/rollcall/dnsmasq/conf.d
|
||||
for f in hosts hosts.dhcp; do [ -e /var/lib/rollcall/dnsmasq/$f ] || install -m 0644 -o rollcall -g rollcall /dev/null /var/lib/rollcall/dnsmasq/$f; done
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service /etc/systemd/system/
|
||||
# the embedded rack controller writes conf.d/rollcall.conf once a VLAN with dhcp is assigned to it;
|
||||
# a hand-written dnsmasq configuration for the same interface has to go then (see README, Network)
|
||||
if [ -d /etc/dnsmasq.d ] && [ ! -e /etc/dnsmasq.d/rollcall.conf ]; then
|
||||
printf '# rollcall: generated from the network model (rollcall subnet / vlan)\nconf-dir=/var/lib/rollcall/dnsmasq/conf.d/,*.conf\n' > /etc/dnsmasq.d/rollcall.conf
|
||||
fi
|
||||
legacy=$(grep -lE '^(dhcp-range|bind-interfaces|enable-tftp)' /etc/dnsmasq.conf /etc/dnsmasq.d/*.conf 2>/dev/null | grep -v '/rollcall.conf$' || true)
|
||||
if [ -n "$legacy" ]; then
|
||||
echo "NOTE: hand-written dnsmasq DHCP/TFTP configuration in: $legacy" >&2
|
||||
echo " Once a VLAN with dhcp is assigned to this host (rollcall vlan add … -dhcp), rollcall generates" >&2
|
||||
echo " that configuration; move these files away then, or the generated one is rejected." >&2
|
||||
fi
|
||||
# optional: this host is also the Slurm controller -> keep its slurm.conf rendered from the profile
|
||||
install -m 0644 rollcall-controller-sync.service rollcall-controller-sync.timer /etc/systemd/system/
|
||||
install -m 0755 rollcall-controller-sync.sh /usr/local/bin/rollcall-controller-sync.sh
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "hall-b",
|
||||
"region": "http://10.0.0.234",
|
||||
"secret": "<rack_secret from the region's config.json>",
|
||||
"url": "http://10.213.78.1",
|
||||
"listen": "10.213.78.1:80",
|
||||
"tftp_root": "/srv/tftp",
|
||||
"dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
|
||||
"cache_dir": "/var/cache/rollcall",
|
||||
"ipmitool": "ipmitool",
|
||||
"dhcp_interfaces": []
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# Called by rollcall-dnsmasq.path when rollcall rewrites dnsmasq's files.
|
||||
#
|
||||
# Host files (leases, names) only need a HUP. When the generated configuration
|
||||
# changed, dnsmasq is restarted. If it does not come back (a conflicting
|
||||
# hand-written file in /etc/dnsmasq.d, for instance - `dnsmasq --test` misses
|
||||
# some of those), the last configuration that worked is put back and dnsmasq
|
||||
# is started with it, so DHCP stays up. The rejected content is kept next to it
|
||||
# and not tried again until it changes.
|
||||
set -u
|
||||
D=/var/lib/rollcall/dnsmasq
|
||||
CONF=$D/conf.d/rollcall.conf
|
||||
GOOD=$D/rollcall.conf.accepted
|
||||
BAD=$D/rollcall.conf.rejected
|
||||
STAMP=/run/rollcall-dnsmasq.sum
|
||||
|
||||
sum() { cat "$1" 2>/dev/null | sha256sum | cut -d' ' -f1; }
|
||||
restore() { # put the last working configuration back (or none)
|
||||
if [ -s "$GOOD" ]; then cp "$GOOD" "$CONF.tmp" && mv -f "$CONF.tmp" "$CONF"; else rm -f "$CONF"; fi
|
||||
sum "$CONF" > "$STAMP"
|
||||
}
|
||||
|
||||
new=$(sum "$CONF")
|
||||
if [ "$new" = "$(cat "$STAMP" 2>/dev/null)" ]; then
|
||||
systemctl kill -s HUP dnsmasq.service
|
||||
exit 0
|
||||
fi
|
||||
if [ -e "$CONF" ] && [ "$new" = "$(sum "$BAD")" ]; then
|
||||
restore
|
||||
echo "generated dnsmasq configuration was rejected before and has not changed; keeping the last working one" >&2
|
||||
exit 1
|
||||
fi
|
||||
if out=$(dnsmasq --test 2>&1) && systemctl restart dnsmasq.service; then
|
||||
echo "$new" > "$STAMP"
|
||||
if [ -e "$CONF" ]; then cp "$CONF" "$GOOD"; else rm -f "$GOOD"; fi
|
||||
rm -f "$BAD"
|
||||
echo "dnsmasq restarted: generated configuration changed"
|
||||
exit 0
|
||||
fi
|
||||
why=$(journalctl -u dnsmasq.service -n 5 --no-pager -o cat 2>/dev/null | grep -v '^$' | tail -3 | tr '\n' ' ')
|
||||
[ -e "$CONF" ] && cp "$CONF" "$BAD"
|
||||
restore
|
||||
if systemctl restart dnsmasq.service; then
|
||||
echo "generated dnsmasq configuration rejected ($out $why); dnsmasq runs with the last working one, the rejected file is $BAD" >&2
|
||||
exit 1
|
||||
fi
|
||||
# not even that starts: something outside rollcall's file conflicts. Run dnsmasq
|
||||
# without rollcall's file so whatever else is configured keeps working.
|
||||
rm -f "$CONF"
|
||||
sum "$CONF" > "$STAMP"
|
||||
systemctl restart dnsmasq.service || true
|
||||
echo "dnsmasq does not start with rollcall's configuration ($why); started without it - check /etc/dnsmasq.conf and /etc/dnsmasq.d for conflicting options" >&2
|
||||
exit 1
|
||||
@@ -1,9 +1,10 @@
|
||||
[Unit]
|
||||
Description=rollcall: reload dnsmasq when node host files change
|
||||
Description=rollcall: reload dnsmasq when its generated files change
|
||||
|
||||
[Path]
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts.dhcp
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/conf.d/rollcall.conf
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=rollcall: HUP dnsmasq to re-read dhcp-hostsfile / addn-hosts
|
||||
Description=rollcall: reload dnsmasq (HUP for host files, checked restart for configuration)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl kill -s HUP dnsmasq.service
|
||||
ExecStart=/usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=rollcall rack controller (DHCP/TFTP state, BMC commands, image cache, node API proxy)
|
||||
After=network-online.target dnsmasq.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/rollcall rack -config /etc/rollcall/rack.json
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
User=rollcall
|
||||
Group=rollcall
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/srv/tftp/pxelinux.cfg /var/lib/rollcall /var/cache/rollcall
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in new issue
Block a user