Two problems surfaced while using the cluster interactively. MPI binaries failed at run time with "libucp.so.0: cannot open shared object file" unless the user ran "module load gnu14 openmpi5" by hand. lmod-ohpc's /etc/profile.d/lmod.sh already ends with "module try-add ohpc", but no such modulefile existed: OpenHPC ships lmod-defaults-* only for gnu12 and gnu15 combinations, and this cluster is gnu14. Write the modulefile ourselves in 60-slurm-openhpc.sh. It lives on /opt/ohpc, which is exported read-only to the nodes, so one copy serves the whole cluster and no image rebuild is needed. A fresh login now has gnu14, openmpi5, ucx, hwloc, libfabric and prun loaded, and srun/sbatch work without any manual module command. sacct reported "accounting storage is disabled". A85-slurm-accounting.sh adds slurmdbd against MariaDB on the head node, registers cluster wwtest with an "hpc" account and associations for hpcuser and testuser, and pushes the updated slurm.conf to the running nodes as well as into the hpc overlay, so no reboot is required. Accounting is recorded but deliberately not enforced: AccountingStorageEnforce stays unset so a user without an association can still submit. slurmdbd warns that innodb_buffer_pool_size is below its recommended 4G. That is intentional and documented - head01 has 8G in total and also runs slurmctld, warewulfd, NFS, dhcpd, tftp and the overlay UI, while a four-node lab's accounting database is a few megabytes. Docs: 06-slurm-mpi gains sections on the default login environment and on accounting; 09-reference gains the new ports, paths and trade-offs.
195 lines
7.1 KiB
Bash
Executable File
195 lines
7.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Head node: OpenHPC 3 + Slurm controller + OpenMPI (shared /opt/ohpc)
|
|
# munge.key and slurm.conf reach the nodes through the Warewulf overlay named "hpc".
|
|
set -euo pipefail
|
|
IMG=rocky-9
|
|
|
|
# cloud-init writes a forced-command wrapper ("login as ilker") into
|
|
# /root/.ssh/authorized_keys; the ssh.authorized_keys overlay copies it verbatim
|
|
# to every node and locks out root SSH. Strip it and mint a cluster key pair.
|
|
echo "==> head root ssh keys"
|
|
if grep -q "command=" /root/.ssh/authorized_keys 2>/dev/null; then
|
|
cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.cloudinit.bak
|
|
grep -o "ssh-[a-z0-9-]* [A-Za-z0-9+/=]* .*$" /root/.ssh/authorized_keys.cloudinit.bak > /root/.ssh/authorized_keys
|
|
fi
|
|
[ -f /root/.ssh/cluster ] || ssh-keygen -t ed25519 -N "" -f /root/.ssh/cluster -q
|
|
grep -qf /root/.ssh/cluster.pub /root/.ssh/authorized_keys 2>/dev/null || \
|
|
cat /root/.ssh/cluster.pub >> /root/.ssh/authorized_keys
|
|
chmod 600 /root/.ssh/authorized_keys
|
|
cat > /root/.ssh/config <<'EOF'
|
|
Host wwnode* 10.10.10.*
|
|
IdentityFile /root/.ssh/cluster
|
|
StrictHostKeyChecking no
|
|
UserKnownHostsFile /dev/null
|
|
LogLevel ERROR
|
|
EOF
|
|
|
|
echo "==> OpenHPC repository + slurm server"
|
|
rpm -q ohpc-release >/dev/null 2>&1 || \
|
|
dnf install -y -q http://repos.openhpc.community/OpenHPC/3/EL_9/x86_64/ohpc-release-3-1.el9.x86_64.rpm
|
|
dnf install -y -q ohpc-base lmod-ohpc
|
|
dnf install -y -q ohpc-slurm-server munge
|
|
|
|
echo "==> compilers + OpenMPI (into /opt/ohpc/pub, shared over NFS)"
|
|
dnf install -y -q gnu14-compilers-ohpc openmpi5-gnu14-ohpc 2>/dev/null || \
|
|
dnf install -y -q gnu13-compilers-ohpc openmpi5-gnu13-ohpc
|
|
|
|
echo "==> default login environment (ohpc modulefile)"
|
|
# /etc/profile.d/lmod.sh (lmod-ohpc) ends with "module try-add ohpc", so this
|
|
# file is what gives every login shell the toolchain. OpenHPC ships
|
|
# lmod-defaults-* packages only for gnu12/gnu15, and this cluster is gnu14, so
|
|
# it is written by hand. Without it MPI binaries fail at run time with
|
|
# "libucp.so.0: cannot open shared object file" unless the user loads modules.
|
|
# It lives under /opt/ohpc, exported read-only to the nodes, so one copy serves
|
|
# the whole cluster and no image rebuild is needed.
|
|
install -d -m 0755 /opt/ohpc/pub/modulefiles
|
|
cat > /opt/ohpc/pub/modulefiles/ohpc <<'EOF'
|
|
#%Module1.0#####################################################################
|
|
# Default OpenHPC environment for this cluster (gnu14 + openmpi5)
|
|
#############################################################################
|
|
|
|
proc ModulesHelp { } {
|
|
puts stderr "Setup default login environment (gnu14 + openmpi5)"
|
|
}
|
|
|
|
prepend-path PATH /opt/ohpc/pub/bin
|
|
|
|
if { [ expr [module-info mode load] || [module-info mode display] ] } {
|
|
prepend-path MANPATH /usr/local/share/man:/usr/share/man/overrides:/usr/share/man/en:/usr/share/man
|
|
module try-add autotools
|
|
module try-add prun
|
|
module try-add gnu14
|
|
module try-add openmpi5
|
|
}
|
|
|
|
if [ module-info mode remove ] {
|
|
module del openmpi5
|
|
module del gnu14
|
|
module del prun
|
|
module del autotools
|
|
}
|
|
EOF
|
|
chmod 0644 /opt/ohpc/pub/modulefiles/ohpc
|
|
|
|
echo "==> munge key"
|
|
[ -f /etc/munge/munge.key ] || /usr/sbin/create-munge-key -f
|
|
chown munge:munge /etc/munge/munge.key
|
|
chmod 400 /etc/munge/munge.key
|
|
systemctl enable --now munge
|
|
|
|
echo "==> slurm.conf"
|
|
[ -f /etc/slurm/slurm.conf.orig ] || cp -a /etc/slurm/slurm.conf /etc/slurm/slurm.conf.orig 2>/dev/null || true
|
|
cat > /etc/slurm/slurm.conf <<'EOF'
|
|
# Warewulf test cluster
|
|
ClusterName=wwtest
|
|
SlurmctldHost=head01(10.10.10.1)
|
|
|
|
AuthType=auth/munge
|
|
CryptoType=crypto/munge
|
|
SlurmUser=slurm
|
|
SlurmdUser=root
|
|
|
|
SlurmctldPort=6817
|
|
SlurmdPort=6818
|
|
StateSaveLocation=/var/spool/slurm/ctld
|
|
SlurmdSpoolDir=/var/spool/slurm/d
|
|
SlurmctldPidFile=/var/run/slurmctld.pid
|
|
SlurmdPidFile=/var/run/slurmd.pid
|
|
|
|
ProctrackType=proctrack/cgroup
|
|
TaskPlugin=task/affinity,task/cgroup
|
|
MpiDefault=pmix
|
|
|
|
SchedulerType=sched/backfill
|
|
SelectType=select/cons_tres
|
|
SelectTypeParameters=CR_Core
|
|
|
|
SlurmctldTimeout=300
|
|
SlurmdTimeout=300
|
|
InactiveLimit=0
|
|
MinJobAge=300
|
|
KillWait=30
|
|
Waittime=0
|
|
ReturnToService=2
|
|
|
|
SlurmctldDebug=info
|
|
SlurmctldLogFile=/var/log/slurmctld.log
|
|
SlurmdDebug=info
|
|
SlurmdLogFile=/var/log/slurmd.log
|
|
|
|
NodeName=wwnode[1-4] CPUs=2 Sockets=1 CoresPerSocket=2 ThreadsPerCore=1 RealMemory=7000 State=UNKNOWN
|
|
PartitionName=normal Nodes=wwnode[1-4] Default=YES MaxTime=INFINITE State=UP OverSubscribe=NO
|
|
EOF
|
|
|
|
cat > /etc/slurm/cgroup.conf <<'EOF'
|
|
CgroupPlugin=cgroup/v2
|
|
ConstrainCores=yes
|
|
ConstrainRAMSpace=yes
|
|
EOF
|
|
|
|
mkdir -p /var/spool/slurm/ctld /var/spool/slurm/d
|
|
chown -R slurm:slurm /var/spool/slurm/ctld
|
|
touch /var/log/slurmctld.log && chown slurm:slurm /var/log/slurmctld.log
|
|
|
|
echo "==> test user hpcuser"
|
|
id hpcuser &>/dev/null || useradd -m -s /bin/bash hpcuser
|
|
install -d -m 700 -o hpcuser -g hpcuser /home/hpcuser/.ssh
|
|
if [ ! -f /home/hpcuser/.ssh/id_ed25519 ]; then
|
|
sudo -u hpcuser ssh-keygen -t ed25519 -N "" -f /home/hpcuser/.ssh/id_ed25519 -q
|
|
cat /home/hpcuser/.ssh/id_ed25519.pub > /home/hpcuser/.ssh/authorized_keys
|
|
fi
|
|
cat > /home/hpcuser/.ssh/config <<'EOF'
|
|
Host wwnode* 10.10.10.*
|
|
StrictHostKeyChecking no
|
|
UserKnownHostsFile /dev/null
|
|
LogLevel ERROR
|
|
EOF
|
|
chown -R hpcuser:hpcuser /home/hpcuser/.ssh
|
|
chmod 600 /home/hpcuser/.ssh/authorized_keys /home/hpcuser/.ssh/config
|
|
|
|
echo "==> 'hpc' overlay: munge.key + slurm.conf + cgroup.conf"
|
|
wwctl overlay create hpc 2>/dev/null || true
|
|
wwctl overlay import -p -o hpc /etc/munge/munge.key /etc/munge/munge.key
|
|
wwctl overlay import -p -o hpc /etc/slurm/slurm.conf /etc/slurm/slurm.conf
|
|
wwctl overlay import -p -o hpc /etc/slurm/cgroup.conf /etc/slurm/cgroup.conf
|
|
# Parent dirs created by "import -p" end up 0500 root:root. Left uncorrected:
|
|
# - /etc 0500 -> /etc becomes inaccessible to non-root on the node
|
|
# - /etc/munge root:root -> munged fails to start: "Permission denied"
|
|
wwctl overlay chown hpc /etc 0:0
|
|
wwctl overlay chmod hpc /etc 0755
|
|
wwctl overlay chown hpc /etc/munge "$(id -u munge):$(id -g munge)"
|
|
wwctl overlay chmod hpc /etc/munge 0700
|
|
wwctl overlay chown hpc /etc/munge/munge.key "$(id -u munge):$(id -g munge)"
|
|
wwctl overlay chmod hpc /etc/munge/munge.key 0400
|
|
wwctl overlay chmod hpc /etc/slurm 0755
|
|
|
|
echo "==> add the 'hpc' overlay to the default profile"
|
|
CUR=$(wwctl profile list -a default 2>/dev/null | awk '/SystemOverlay/ {print $NF}' | tail -1)
|
|
[ -z "${CUR}" ] && CUR="wwinit,wwclient,hosts,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,ifupdown,NetworkManager,wicked,ignition"
|
|
case ",${CUR}," in *,hpc,*) NEW="${CUR}" ;; *) NEW="${CUR},hpc" ;; esac
|
|
wwctl profile set --yes default -O "${NEW}"
|
|
|
|
echo "==> make the /opt NFS mount automatic (noauto -> auto)"
|
|
python3 - <<'PY'
|
|
import re
|
|
p = "/etc/warewulf/nodes.conf"
|
|
s = open(p).read()
|
|
s = s.replace("defaults,noauto,nofail,ro", "defaults,nofail,ro")
|
|
open(p, "w").write(s)
|
|
PY
|
|
grep -A3 "warewulf:/opt" /etc/warewulf/nodes.conf | head -6 || true
|
|
|
|
echo "==> user sync + overlay build"
|
|
wwctl image syncuser --write ${IMG} || true
|
|
wwctl overlay build
|
|
|
|
echo "==> NFS exports"
|
|
wwctl configure nfs
|
|
exportfs -ra
|
|
exportfs -v
|
|
|
|
echo "==> slurmctld"
|
|
systemctl enable --now slurmctld
|
|
systemctl is-active slurmctld && echo "slurmctld active" || journalctl -u slurmctld -n 20 --no-pager
|
|
echo "SLURM/OPENHPC HEAD OK"
|