Two problems surfaced while using the cluster interactively. MPI binaries failed at run time with "libucp.so.0: cannot open shared object file" unless the user ran "module load gnu14 openmpi5" by hand. lmod-ohpc's /etc/profile.d/lmod.sh already ends with "module try-add ohpc", but no such modulefile existed: OpenHPC ships lmod-defaults-* only for gnu12 and gnu15 combinations, and this cluster is gnu14. Write the modulefile ourselves in 60-slurm-openhpc.sh. It lives on /opt/ohpc, which is exported read-only to the nodes, so one copy serves the whole cluster and no image rebuild is needed. A fresh login now has gnu14, openmpi5, ucx, hwloc, libfabric and prun loaded, and srun/sbatch work without any manual module command. sacct reported "accounting storage is disabled". A85-slurm-accounting.sh adds slurmdbd against MariaDB on the head node, registers cluster wwtest with an "hpc" account and associations for hpcuser and testuser, and pushes the updated slurm.conf to the running nodes as well as into the hpc overlay, so no reboot is required. Accounting is recorded but deliberately not enforced: AccountingStorageEnforce stays unset so a user without an association can still submit. slurmdbd warns that innodb_buffer_pool_size is below its recommended 4G. That is intentional and documented - head01 has 8G in total and also runs slurmctld, warewulfd, NFS, dhcpd, tftp and the overlay UI, while a four-node lab's accounting database is a few megabytes. Docs: 06-slurm-mpi gains sections on the default login environment and on accounting; 09-reference gains the new ports, paths and trade-offs.
157 lines
5.4 KiB
Bash
Executable File
157 lines
5.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Enable Slurm accounting (slurmdbd + MariaDB) on the head node.
|
|
#
|
|
# Without this, sacct reports "Slurm accounting storage is disabled" and job
|
|
# history exists only in slurmctld.log. With it you get sacct/sreport, and the
|
|
# associations that fair-share and per-user limits are built on.
|
|
#
|
|
# Accounting is recorded but NOT enforced: AccountingStorageEnforce is left
|
|
# unset so a user without an association can still submit. Turning enforcement
|
|
# on is a one-line change, noted at the end.
|
|
set -euo pipefail
|
|
|
|
DB=slurm_acct_db
|
|
DBUSER=slurm
|
|
CREDS=/root/.slurmdbd-credentials
|
|
CLUSTER=$(awk -F= '/^ClusterName=/ {print $2}' /etc/slurm/slurm.conf | tr -d ' ')
|
|
|
|
echo "==> packages"
|
|
dnf install -y -q mariadb-server slurm-slurmdbd-ohpc
|
|
|
|
echo "==> MariaDB tuning for slurmdbd"
|
|
# slurmdbd writes many small rows; the defaults cause lock timeouts on purge.
|
|
# slurmdbd logs "Database settings not recommended values: innodb_buffer_pool_size"
|
|
# because it wants >= 4G. That is deliberate here: head01 has 8G total and also
|
|
# runs slurmctld, warewulfd, NFS, dhcpd, tftp and the overlay UI. The accounting
|
|
# database for a four-node lab is a few megabytes, so 1G is ample and the
|
|
# warning is cosmetic.
|
|
cat > /etc/my.cnf.d/99-slurmdbd.cnf <<'EOF'
|
|
[mysqld]
|
|
innodb_buffer_pool_size=1024M
|
|
innodb_log_file_size=64M
|
|
innodb_lock_wait_timeout=900
|
|
EOF
|
|
systemctl enable --now mariadb
|
|
sleep 2
|
|
|
|
echo "==> database and user"
|
|
if [ ! -f ${CREDS} ]; then
|
|
DBPW=$(head -c 40 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | cut -c1-24)
|
|
printf 'SLURMDBD_DB=%s\nSLURMDBD_USER=%s\nSLURMDBD_PASS=%s\n' "${DB}" "${DBUSER}" "${DBPW}" > ${CREDS}
|
|
chmod 600 ${CREDS}
|
|
fi
|
|
. ${CREDS}
|
|
|
|
mysql -u root <<EOF
|
|
CREATE DATABASE IF NOT EXISTS ${SLURMDBD_DB};
|
|
CREATE USER IF NOT EXISTS '${SLURMDBD_USER}'@'localhost' IDENTIFIED BY '${SLURMDBD_PASS}';
|
|
ALTER USER '${SLURMDBD_USER}'@'localhost' IDENTIFIED BY '${SLURMDBD_PASS}';
|
|
GRANT ALL ON ${SLURMDBD_DB}.* TO '${SLURMDBD_USER}'@'localhost';
|
|
FLUSH PRIVILEGES;
|
|
EOF
|
|
echo " database ${SLURMDBD_DB} ready"
|
|
|
|
echo "==> slurmdbd.conf"
|
|
install -d -m 0755 -o slurm -g slurm /var/log/slurm
|
|
cat > /etc/slurm/slurmdbd.conf <<EOF
|
|
# Slurm database daemon. Contains the database password, hence mode 0600.
|
|
AuthType=auth/munge
|
|
DbdHost=localhost
|
|
DbdPort=6819
|
|
SlurmUser=slurm
|
|
|
|
StorageType=accounting_storage/mysql
|
|
StorageHost=localhost
|
|
StorageLoc=${SLURMDBD_DB}
|
|
StorageUser=${SLURMDBD_USER}
|
|
StoragePass=${SLURMDBD_PASS}
|
|
|
|
LogFile=/var/log/slurm/slurmdbd.log
|
|
PidFile=/var/run/slurmdbd/slurmdbd.pid
|
|
DebugLevel=info
|
|
|
|
# Keep a year of jobs; archive rather than delete outright.
|
|
ArchiveEvents=yes
|
|
ArchiveJobs=yes
|
|
ArchiveResvs=yes
|
|
ArchiveSteps=no
|
|
ArchiveSuspend=no
|
|
ArchiveTXN=no
|
|
ArchiveUsage=no
|
|
PurgeEventAfter=12month
|
|
PurgeJobAfter=12month
|
|
PurgeResvAfter=6month
|
|
PurgeStepAfter=6month
|
|
PurgeSuspendAfter=1month
|
|
PurgeTXNAfter=12month
|
|
PurgeUsageAfter=24month
|
|
EOF
|
|
chown slurm:slurm /etc/slurm/slurmdbd.conf
|
|
chmod 0600 /etc/slurm/slurmdbd.conf
|
|
install -d -m 0755 -o slurm -g slurm /var/run/slurmdbd
|
|
|
|
echo "==> slurmdbd"
|
|
systemctl enable --now slurmdbd
|
|
sleep 4
|
|
systemctl is-active slurmdbd || { journalctl -u slurmdbd -n 25 --no-pager; exit 1; }
|
|
|
|
echo "==> slurm.conf accounting directives"
|
|
cp -n /etc/slurm/slurm.conf /etc/slurm/slurm.conf.pre-accounting 2>/dev/null || true
|
|
python3 - <<'PY'
|
|
import re
|
|
p = "/etc/slurm/slurm.conf"
|
|
s = open(p).read()
|
|
# Drop any previous accounting block so this is idempotent.
|
|
s = re.sub(r'\n# --- Accounting ---\n(?:.*\n)*?# --- End accounting ---\n', '\n', s)
|
|
block = """
|
|
# --- Accounting ---
|
|
# Recorded, not enforced: no AccountingStorageEnforce, so a user without an
|
|
# association can still submit. Add "AccountingStorageEnforce=associations"
|
|
# (and limits/qos) once every user has one.
|
|
AccountingStorageType=accounting_storage/slurmdbd
|
|
AccountingStorageHost=head01
|
|
AccountingStoragePort=6819
|
|
JobAcctGatherType=jobacct_gather/cgroup
|
|
JobAcctGatherFrequency=30
|
|
# --- End accounting ---
|
|
"""
|
|
# Insert before the node definitions so the file stays readable.
|
|
marker = "\nNodeName="
|
|
s = s.replace(marker, block + marker, 1)
|
|
open(p, "w").write(s)
|
|
PY
|
|
grep -A9 "^# --- Accounting ---" /etc/slurm/slurm.conf | sed 's/^/ /'
|
|
|
|
echo "==> restart slurmctld"
|
|
systemctl restart slurmctld
|
|
sleep 4
|
|
systemctl is-active slurmctld || { journalctl -u slurmctld -n 25 --no-pager; exit 1; }
|
|
|
|
echo "==> register cluster and users in the accounting database"
|
|
sacctmgr -i add cluster "${CLUSTER}" 2>/dev/null || true
|
|
sleep 2
|
|
systemctl restart slurmctld; sleep 3
|
|
sacctmgr -i add account hpc Description="HPC cluster users" Organization="lab" 2>/dev/null || true
|
|
for u in hpcuser testuser; do
|
|
id "$u" >/dev/null 2>&1 && sacctmgr -i add user "$u" Account=hpc 2>/dev/null || true
|
|
done
|
|
|
|
echo "==> push slurm.conf to the nodes"
|
|
# slurm.conf must be identical cluster-wide. Update the overlay for future
|
|
# boots, and copy it to the running nodes now so no reboot is needed.
|
|
wwctl overlay import -p -o hpc /etc/slurm/slurm.conf /etc/slurm/slurm.conf >/dev/null
|
|
wwctl overlay chmod hpc /etc 0755 >/dev/null 2>&1 || true
|
|
wwctl overlay build >/dev/null 2>&1
|
|
for n in $(sinfo -h -N -o '%N' | sort -u); do
|
|
scp -q -o BatchMode=yes /etc/slurm/slurm.conf "root@${n}:/etc/slurm/slurm.conf" 2>/dev/null \
|
|
&& ssh -o BatchMode=yes "root@${n}" 'systemctl restart slurmd' 2>/dev/null \
|
|
&& echo " ${n}: slurm.conf updated, slurmd restarted" \
|
|
|| echo " ${n}: FAILED"
|
|
done
|
|
|
|
echo "==> verify"
|
|
sleep 3
|
|
sinfo
|
|
sacctmgr -n show cluster format=Cluster,ControlHost | sed 's/^/ /'
|
|
echo "ACCOUNTING OK (db credentials: ${CREDS})"
|