Files
werewulf/scripts/A60-node-overlays-ipa.sh
ilker 29bd39c135 Warewulf HPC test cluster: provisioning, identity, docs and overlay UI
A complete, verified HPC lab on Proxmox (hv01) with its own isolated subnet,
central identity and a web console for overlay management.

Infrastructure
- vmbr6: L2-only cluster fabric, deliberately without an address on the
  hypervisor so Warewulf owns DHCP/TFTP/PXE for the whole subnet.
- head01 (VM 300): Warewulf 4.7.0, dnsmasq-free DHCP/TFTP/iPXE chain,
  Slurm controller, MUNGE, NAT gateway, /opt exported read-only.
- wwnode1-4 (VM 301-304): diskless, PXE-booted, rootfs unpacked into RAM.
- idm1 (VM 305): FreeIPA (LDAP/Kerberos/Dogtag CA/DNS) plus the NFS home
  server. Dual-homed on purpose: idm1.hpc.local resolves to the cluster
  fabric so Kerberos and NFS never traverse head01's masquerade.

Stack
Rocky Linux 9.8, OpenHPC 3, Slurm 25.11.4, OpenMPI 5.0.7, FreeIPA 4.13.1.

Stateless domain join
Nodes are rebuilt in RAM every boot, so ipa-client-install cannot be used.
Enrolment happens once and is distributed through Warewulf overlays: shared
config as templates rendered per node via {{.Id}}, and a separate keytab per
node so wwnode1 cannot impersonate wwnode2.

Overlay web UI (webui/)
A Go service on the head node serving a browser interface over the Warewulf
REST API. It exists as a server rather than a static page because the API is
root-equivalent and bound to loopback; the service holds the credentials and
proxies same-origin, so the browser never sees one. Published behind Caddy
with bcrypt logins, server-side sessions and per-user/per-IP lockout.

Documentation
README plus docs/01-10 covering architecture, installation, Warewulf,
identity, storage, Slurm/MPI, operations, troubleshooting and reference,
with a 22-page PDF edition. Section 08 records twelve concrete failures hit
during the build, including a wwctl flag that fails silently.

No credentials are stored here. Every secret lives on the servers; the
documentation refers only to its path.
2026-08-29 09:07:51 +02:00

167 lines
5.3 KiB
Bash
Executable File

#!/bin/bash
# head01: IPA overlays for the compute nodes.
# ipa -> shared configuration (krb5.conf, ca.crt, templates)
# ipa.wwnodeN -> ONLY that node's host keytab
# sssd.conf and ipa/default.conf are written as Warewulf TEMPLATES; {{.Id}} gives
# each node its own FQDN. A keytab is binary and cannot be templated, hence the
# separate per-node overlay.
set -euo pipefail
DOMAIN=hpc.local
REALM=HPC.LOCAL
IPA_SRV=idm1.hpc.local
IPA_IP=10.10.10.5
KT=/root/node-keytabs
WORK=$(mktemp -d)
trap 'rm -rf "${WORK}"' EXIT
# ---------- shared overlay ----------
echo "==> 'ipa' overlay"
wwctl overlay create ipa 2>/dev/null || true
# krb5.conf - ipa-client-install never runs on the nodes, so the pkinit anchors
# (/var/lib/ipa-client/pki/...) do not exist; those lines are deliberately omitted.
cat > "${WORK}/krb5.conf" <<EOF
includedir /etc/krb5.conf.d/
[libdefaults]
default_realm = ${REALM}
dns_lookup_realm = true
dns_lookup_kdc = true
rdns = false
ticket_lifetime = 24h
forwardable = true
udp_preference_limit = 0
default_ccache_name = KEYRING:persistent:%{uid}
[realms]
${REALM} = {
kdc = ${IPA_SRV}:88
master_kdc = ${IPA_SRV}:88
admin_server = ${IPA_SRV}:749
kpasswd_server = ${IPA_SRV}:464
default_domain = ${DOMAIN}
auto_fast_armor = true
}
[domain_realm]
.${DOMAIN} = ${REALM}
${DOMAIN} = ${REALM}
EOF
# sssd.conf TEMPLATE: {{.Id}} is the node name (wwnode1, wwnode2, ...)
cat > "${WORK}/sssd.conf.ww" <<'EOF'
# This file is autogenerated by warewulf
[domain/hpc.local]
id_provider = ipa
auth_provider = ipa
chpass_provider = ipa
access_provider = ipa
ipa_domain = hpc.local
ipa_server = idm1.hpc.local
ipa_hostname = {{.Id}}.hpc.local
ldap_tls_cacert = /etc/ipa/ca.crt
cache_credentials = True
krb5_store_password_if_offline = True
# Stateless nodes: DNS records are static in IPA, dynamic updates disabled.
dyndns_update = False
[sssd]
services = nss, pam, ssh, sudo
domains = hpc.local
[nss]
homedir_substring = /home
EOF
cat > "${WORK}/default.conf.ww" <<'EOF'
# This file is autogenerated by warewulf
[global]
basedn = dc=hpc,dc=local
realm = HPC.LOCAL
domain = hpc.local
server = idm1.hpc.local
host = {{.Id}}.hpc.local
xmlrpc_uri = https://idm1.hpc.local/ipa/xml
enable_ra = True
EOF
wwctl overlay import -p -o ipa "${WORK}/krb5.conf" /etc/krb5.conf
wwctl overlay import -p -o ipa "${KT}/ca.crt" /etc/ipa/ca.crt
wwctl overlay import -p -o ipa "${WORK}/default.conf.ww" /etc/ipa/default.conf.ww
wwctl overlay import -p -o ipa "${WORK}/sssd.conf.ww" /etc/sssd/sssd.conf.ww
# Parent dirs created by "import -p" end up 0500 root:root -> fix them
wwctl overlay chmod ipa /etc 0755
wwctl overlay chmod ipa /etc/ipa 0755
wwctl overlay chmod ipa /etc/sssd 0711
wwctl overlay chmod ipa /etc/krb5.conf 0644
wwctl overlay chmod ipa /etc/ipa/ca.crt 0644
wwctl overlay chmod ipa /etc/ipa/default.conf.ww 0644
# SSSD refuses to start unless sssd.conf is 0600 root:root
wwctl overlay chmod ipa /etc/sssd/sssd.conf.ww 0600
# ---------- per-node keytab overlays ----------
for n in 1 2 3 4; do
OV="ipa.wwnode${n}"
echo "==> '${OV}' overlay (keytab)"
wwctl overlay create "${OV}" 2>/dev/null || true
wwctl overlay import -p -o "${OV}" "${KT}/wwnode${n}.keytab" /etc/krb5.keytab
wwctl overlay chmod "${OV}" /etc 0755
wwctl overlay chmod "${OV}" /etc/krb5.keytab 0600
done
# ---------- point node DNS at IPA ----------
echo "==> node DNS -> ${IPA_IP} (the resolv overlay reads netdev tags)"
# WARNING: "wwctl profile set --nettagadd" SILENTLY does nothing in this version
# (no error either). Write the tags straight into nodes.conf as YAML instead.
dnf install -y -q python3-pyyaml
python3 - <<'PY'
import yaml, shutil
p = "/etc/warewulf/nodes.conf"
shutil.copy(p, p + ".bak-dnstags")
d = yaml.safe_load(open(p))
for pname, prof in (d.get("nodeprofiles") or {}).items():
nd = prof.get("network devices") or {}
for name, dev in nd.items():
tags = dev.get("tags") or {}
tags["DNS1"] = "10.10.10.5"
tags["DNSSEARCH"] = "hpc.local"
dev["tags"] = tags
if nd:
prof["network devices"] = nd
yaml.safe_dump(d, open(p, "w"), default_flow_style=False, sort_keys=False)
print(" netdev DNS tags written")
PY
# ---------- /home now comes from idm1 ----------
echo "==> node fstab: /home -> ${IPA_SRV}:/export/home"
python3 - <<'PY'
import re
p = "/etc/warewulf/nodes.conf"
s = open(p).read()
s = s.replace('spec: warewulf:/home', 'spec: idm1.hpc.local:/export/home')
open(p, "w").write(s)
PY
grep -B2 -A3 "export/home" /etc/warewulf/nodes.conf | head -8
# ---------- attach overlays to the profile / nodes ----------
echo "==> attach overlays"
CUR=$(wwctl profile list -a default | awk '/SystemOverlay/ {print $NF}' | tail -1)
case ",${CUR}," in *,ipa,*) NEW="${CUR}" ;; *) NEW="${CUR},ipa" ;; esac
wwctl profile set --yes default -O "${NEW}"
# Node level gets ONLY its own keytab overlay: Warewulf MERGES the profile and
# node lists, so repeating the full list would apply every overlay twice.
for n in 1 2 3 4; do
wwctl node set --yes "wwnode${n}" -O "ipa.wwnode${n}"
done
echo "==> render check (wwnode2)"
wwctl overlay show --render wwnode2 ipa /etc/sssd/sssd.conf.ww | grep -E "ipa_hostname|Filename"
wwctl overlay show --render wwnode2 ipa /etc/ipa/default.conf.ww | grep -E "^host|Filename"
echo "==> overlay build"
wwctl overlay build
echo "NODE IPA OVERLAYS OK"