Files
werewulf/scripts/60-slurm-openhpc.sh
T
ilker 040574e8df Enable Slurm accounting and fix the default login environment
Two problems surfaced while using the cluster interactively.

MPI binaries failed at run time with "libucp.so.0: cannot open shared object
file" unless the user ran "module load gnu14 openmpi5" by hand. lmod-ohpc's
/etc/profile.d/lmod.sh already ends with "module try-add ohpc", but no such
modulefile existed: OpenHPC ships lmod-defaults-* only for gnu12 and gnu15
combinations, and this cluster is gnu14. Write the modulefile ourselves in
60-slurm-openhpc.sh. It lives on /opt/ohpc, which is exported read-only to the
nodes, so one copy serves the whole cluster and no image rebuild is needed.
A fresh login now has gnu14, openmpi5, ucx, hwloc, libfabric and prun loaded,
and srun/sbatch work without any manual module command.

sacct reported "accounting storage is disabled". A85-slurm-accounting.sh adds
slurmdbd against MariaDB on the head node, registers cluster wwtest with an
"hpc" account and associations for hpcuser and testuser, and pushes the
updated slurm.conf to the running nodes as well as into the hpc overlay, so
no reboot is required. Accounting is recorded but deliberately not enforced:
AccountingStorageEnforce stays unset so a user without an association can
still submit.

slurmdbd warns that innodb_buffer_pool_size is below its recommended 4G. That
is intentional and documented - head01 has 8G in total and also runs
slurmctld, warewulfd, NFS, dhcpd, tftp and the overlay UI, while a four-node
lab's accounting database is a few megabytes.

Docs: 06-slurm-mpi gains sections on the default login environment and on
accounting; 09-reference gains the new ports, paths and trade-offs.
2026-08-31 10:28:15 +02:00

195 lines
7.1 KiB
Bash
Executable File

#!/bin/bash
# Head node: OpenHPC 3 + Slurm controller + OpenMPI (shared /opt/ohpc)
# munge.key and slurm.conf reach the nodes through the Warewulf overlay named "hpc".
set -euo pipefail
IMG=rocky-9
# cloud-init writes a forced-command wrapper ("login as ilker") into
# /root/.ssh/authorized_keys; the ssh.authorized_keys overlay copies it verbatim
# to every node and locks out root SSH. Strip it and mint a cluster key pair.
echo "==> head root ssh keys"
if grep -q "command=" /root/.ssh/authorized_keys 2>/dev/null; then
cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.cloudinit.bak
grep -o "ssh-[a-z0-9-]* [A-Za-z0-9+/=]* .*$" /root/.ssh/authorized_keys.cloudinit.bak > /root/.ssh/authorized_keys
fi
[ -f /root/.ssh/cluster ] || ssh-keygen -t ed25519 -N "" -f /root/.ssh/cluster -q
grep -qf /root/.ssh/cluster.pub /root/.ssh/authorized_keys 2>/dev/null || \
cat /root/.ssh/cluster.pub >> /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
cat > /root/.ssh/config <<'EOF'
Host wwnode* 10.10.10.*
IdentityFile /root/.ssh/cluster
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
LogLevel ERROR
EOF
echo "==> OpenHPC repository + slurm server"
rpm -q ohpc-release >/dev/null 2>&1 || \
dnf install -y -q http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm
dnf install -y -q ohpc-base lmod-ohpc
dnf install -y -q ohpc-slurm-server munge
echo "==> compilers + OpenMPI (into /opt/ohpc/pub, shared over NFS)"
dnf install -y -q gnu14-compilers-ohpc openmpi5-gnu14-ohpc 2>/dev/null || \
dnf install -y -q gnu13-compilers-ohpc openmpi5-gnu13-ohpc
echo "==> default login environment (ohpc modulefile)"
# /etc/profile.d/lmod.sh (lmod-ohpc) ends with "module try-add ohpc", so this
# file is what gives every login shell the toolchain. OpenHPC ships
# lmod-defaults-* packages only for gnu12/gnu15, and this cluster is gnu14, so
# it is written by hand. Without it MPI binaries fail at run time with
# "libucp.so.0: cannot open shared object file" unless the user loads modules.
# It lives under /opt/ohpc, exported read-only to the nodes, so one copy serves
# the whole cluster and no image rebuild is needed.
install -d -m 0755 /opt/ohpc/pub/modulefiles
cat > /opt/ohpc/pub/modulefiles/ohpc <<'EOF'
#%Module1.0#####################################################################
# Default OpenHPC environment for this cluster (gnu14 + openmpi5)
#############################################################################
proc ModulesHelp { } {
puts stderr "Setup default login environment (gnu14 + openmpi5)"
}
prepend-path PATH /opt/ohpc/pub/bin
if { [ expr [module-info mode load] || [module-info mode display] ] } {
prepend-path MANPATH /usr/local/share/man:/usr/share/man/overrides:/usr/share/man/en:/usr/share/man
module try-add autotools
module try-add prun
module try-add gnu14
module try-add openmpi5
}
if [ module-info mode remove ] {
module del openmpi5
module del gnu14
module del prun
module del autotools
}
EOF
chmod 0644 /opt/ohpc/pub/modulefiles/ohpc
echo "==> munge key"
[ -f /etc/munge/munge.key ] || /usr/sbin/create-munge-key -f
chown munge:munge /etc/munge/munge.key
chmod 400 /etc/munge/munge.key
systemctl enable --now munge
echo "==> slurm.conf"
[ -f /etc/slurm/slurm.conf.orig ] || cp -a /etc/slurm/slurm.conf /etc/slurm/slurm.conf.orig 2>/dev/null || true
cat > /etc/slurm/slurm.conf <<'EOF'
# Warewulf test cluster
ClusterName=wwtest
SlurmctldHost=head01(10.10.10.1)
AuthType=auth/munge
CryptoType=crypto/munge
SlurmUser=slurm
SlurmdUser=root
SlurmctldPort=6817
SlurmdPort=6818
StateSaveLocation=/var/spool/slurm/ctld
SlurmdSpoolDir=/var/spool/slurm/d
SlurmctldPidFile=/var/run/slurmctld.pid
SlurmdPidFile=/var/run/slurmd.pid
ProctrackType=proctrack/cgroup
TaskPlugin=task/affinity,task/cgroup
MpiDefault=pmix
SchedulerType=sched/backfill
SelectType=select/cons_tres
SelectTypeParameters=CR_Core
SlurmctldTimeout=300
SlurmdTimeout=300
InactiveLimit=0
MinJobAge=300
KillWait=30
Waittime=0
ReturnToService=2
SlurmctldDebug=info
SlurmctldLogFile=/var/log/slurmctld.log
SlurmdDebug=info
SlurmdLogFile=/var/log/slurmd.log
NodeName=wwnode[1-4] CPUs=2 Sockets=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=7000 State=UNKNOWN
PartitionName=normal Nodes=wwnode[1-4] Default=YES MaxTime=INFINITE State=UP OverSubscribe=NO
EOF
cat > /etc/slurm/cgroup.conf <<'EOF'
CgroupPlugin=cgroup/v2
ConstrainCores=yes
ConstrainRAMSpace=yes
EOF
mkdir -p /var/spool/slurm/ctld /var/spool/slurm/d
chown -R slurm:slurm /var/spool/slurm/ctld
touch /var/log/slurmctld.log && chown slurm:slurm /var/log/slurmctld.log
echo "==> test user hpcuser"
id hpcuser &>/dev/null || useradd -m -s /bin/bash hpcuser
install -d -m 700 -o hpcuser -g hpcuser /home/hpcuser/.ssh
if [ ! -f /home/hpcuser/.ssh/id_ed25519 ]; then
sudo -u hpcuser ssh-keygen -t ed25519 -N "" -f /home/hpcuser/.ssh/id_ed25519 -q
cat /home/hpcuser/.ssh/id_ed25519.pub > /home/hpcuser/.ssh/authorized_keys
fi
cat > /home/hpcuser/.ssh/config <<'EOF'
Host wwnode* 10.10.10.*
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
LogLevel ERROR
EOF
chown -R hpcuser:hpcuser /home/hpcuser/.ssh
chmod 600 /home/hpcuser/.ssh/authorized_keys /home/hpcuser/.ssh/config
echo "==> 'hpc' overlay: munge.key + slurm.conf + cgroup.conf"
wwctl overlay create hpc 2>/dev/null || true
wwctl overlay import -p -o hpc /etc/munge/munge.key /etc/munge/munge.key
wwctl overlay import -p -o hpc /etc/slurm/slurm.conf /etc/slurm/slurm.conf
wwctl overlay import -p -o hpc /etc/slurm/cgroup.conf /etc/slurm/cgroup.conf
# Parent dirs created by "import -p" end up 0500 root:root. Left uncorrected:
# - /etc 0500 -> /etc becomes inaccessible to non-root on the node
# - /etc/munge root:root -> munged fails to start: "Permission denied"
wwctl overlay chown hpc /etc 0:0
wwctl overlay chmod hpc /etc 0755
wwctl overlay chown hpc /etc/munge "$(id -u munge):$(id -g munge)"
wwctl overlay chmod hpc /etc/munge 0700
wwctl overlay chown hpc /etc/munge/munge.key "$(id -u munge):$(id -g munge)"
wwctl overlay chmod hpc /etc/munge/munge.key 0400
wwctl overlay chmod hpc /etc/slurm 0755
echo "==> add the 'hpc' overlay to the default profile"
CUR=$(wwctl profile list -a default 2>/dev/null | awk '/SystemOverlay/ {print $NF}' | tail -1)
[ -z "${CUR}" ] && CUR="wwinit,wwclient,hosts,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,ifupdown,NetworkManager,wicked,ignition"
case ",${CUR}," in *,hpc,*) NEW="${CUR}" ;; *) NEW="${CUR},hpc" ;; esac
wwctl profile set --yes default -O "${NEW}"
echo "==> make the /opt NFS mount automatic (noauto -> auto)"
python3 - <<'PY'
import re
p = "/etc/warewulf/nodes.conf"
s = open(p).read()
s = s.replace("defaults,noauto,nofail,ro", "defaults,nofail,ro")
open(p, "w").write(s)
PY
grep -A3 "warewulf:/opt" /etc/warewulf/nodes.conf | head -6 || true
echo "==> user sync + overlay build"
wwctl image syncuser --write ${IMG} || true
wwctl overlay build
echo "==> NFS exports"
wwctl configure nfs
exportfs -ra
exportfs -v
echo "==> slurmctld"
systemctl enable --now slurmctld
systemctl is-active slurmctld && echo "slurmctld active" || journalctl -u slurmctld -n 20 --no-pager
echo "SLURM/OPENHPC HEAD OK"