Files
ilker f29e8785ef Add a long-running MPI benchmark
mpi-ring finishes in about two seconds, so there is nothing to observe in
squeue, no meaningful sacct record and no load on the nodes.

heat_mpi solves 2D steady-state heat diffusion by Jacobi iteration over a
grid split into horizontal strips, one per rank. Every iteration performs a
five-point stencil, a halo exchange with the neighbouring ranks via
MPI_Sendrecv, and an MPI_Allreduce for the global residual - so it exercises
compute and both point-to-point and collective communication rather than just
starting up and exiting.

It runs to a wall-clock budget (-m minutes) instead of a fixed iteration
count, which is what makes it useful for watching the scheduler. Stopping is
collective: rank 0 decides and broadcasts, so every rank leaves on the same
iteration. The residual falls monotonically, which doubles as a check that the
halo exchange is correct.

Measured on this cluster: 6144 grid, 8 ranks over 4 nodes, ~12 GFLOP/s with
all nodes at ~95% CPU and ~72 MiB per rank.

heat-long.sbatch defaults to 20 minutes and takes GRID, MINUTES and REPORT
through --export.
2026-08-31 10:35:56 +02:00

177 lines
7.1 KiB
C

/* heat_mpi - a long-running MPI benchmark for the Warewulf test cluster.
*
* Solves 2D steady-state heat diffusion with Jacobi iteration on an N x N grid.
* The domain is split into horizontal strips, one per rank, so every iteration
* does real work of both kinds an HPC job does:
*
* - compute : a five-point stencil over the local strip
* - communication: halo exchange with the rank above and below (MPI_Sendrecv),
* plus a global MPI_Allreduce to test convergence
*
* Unlike hello_mpi, which finishes instantly, this runs for a wall-clock budget
* you choose, so there is something to watch in squeue, sacct and on the nodes.
*
* mpicc -O3 -march=native -o heat_mpi heat_mpi.c -lm
*
* -n <grid> grid points per side (default 4096)
* -m <minutes> wall-clock budget (default 10)
* -i <iters> stop after this many iters (default: unlimited)
* -r <every> report interval in iterations (default 200)
* -e <residual> stop early if converged below (default 0 = never)
*
* Stopping is collective: rank 0 decides and broadcasts, so every rank always
* leaves the loop on the same iteration.
*/
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <unistd.h>
#define HOT 100.0
static double now(void) { return MPI_Wtime(); }
int main(int argc, char **argv)
{
int rank, size;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
long N = 4096;
double minutes = 10.0;
long max_iter = 0; /* 0 = unlimited */
long report = 200;
double tol = 0.0; /* 0 = never stop early */
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-n") && i + 1 < argc) N = atol(argv[++i]);
else if (!strcmp(argv[i], "-m") && i + 1 < argc) minutes = atof(argv[++i]);
else if (!strcmp(argv[i], "-i") && i + 1 < argc) max_iter = atol(argv[++i]);
else if (!strcmp(argv[i], "-r") && i + 1 < argc) report = atol(argv[++i]);
else if (!strcmp(argv[i], "-e") && i + 1 < argc) tol = atof(argv[++i]);
else if (rank == 0) fprintf(stderr, "ignoring argument: %s\n", argv[i]);
}
if (N < (long)size * 3) { /* every rank needs interior rows */
if (rank == 0)
fprintf(stderr, "grid %ld too small for %d ranks; use -n >= %d\n",
N, size, size * 3);
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* Row decomposition, remainder spread over the first ranks so the strips
* differ by at most one row. */
long base = N / size, rem = N % size;
long rows = base + (rank < rem ? 1 : 0);
long row0 = rank * base + (rank < rem ? rank : rem);
/* Two ghost rows: index 0 is the neighbour above, rows+1 the one below. */
size_t stride = (size_t)N;
size_t cells = (size_t)(rows + 2) * stride;
double *cur = malloc(cells * sizeof *cur);
double *nxt = malloc(cells * sizeof *nxt);
if (!cur || !nxt) {
fprintf(stderr, "rank %d: out of memory (%.1f MiB per array)\n",
rank, (double)(cells * sizeof *cur) / (1024 * 1024));
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (size_t i = 0; i < cells; i++) cur[i] = nxt[i] = 0.0;
/* Boundary: the global top edge is held hot, everything else at zero. */
if (rank == 0)
for (long j = 0; j < N; j++) cur[stride + j] = nxt[stride + j] = HOT;
int up = (rank == 0) ? MPI_PROC_NULL : rank - 1;
int down = (rank == size - 1) ? MPI_PROC_NULL : rank + 1;
if (rank == 0) {
double mib = (double)(cells * sizeof *cur) * 2 / (1024 * 1024);
printf("heat_mpi: %ld x %ld grid, %d ranks, ~%.1f MiB/rank, budget %.1f min\n",
N, N, size, mib, minutes);
printf("%8s %14s %12s %12s %10s\n",
"iter", "residual", "elapsed(s)", "iter/s", "GFLOP/s");
fflush(stdout);
}
double deadline = now() + minutes * 60.0;
double t0 = now(), tlast = t0;
long iter = 0, last_iter = 0;
double gres = 0.0;
int stop = 0;
while (!stop) {
/* Halo exchange. Sendrecv pairs the sends and receives so this cannot
* deadlock however the ranks are scheduled. */
MPI_Sendrecv(&cur[1 * stride], N, MPI_DOUBLE, up, 0,
&cur[(rows + 1) * stride], N, MPI_DOUBLE, down, 0,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Sendrecv(&cur[rows * stride], N, MPI_DOUBLE, down, 1,
&cur[0], N, MPI_DOUBLE, up, 1,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
/* Five-point Jacobi stencil over the interior of the local strip. */
double res = 0.0;
for (long i = 1; i <= rows; i++) {
if (rank == 0 && i == 1) continue; /* hot edge is fixed */
if (rank == size - 1 && i == rows) continue; /* global bottom */
const double *up_r = &cur[(i - 1) * stride];
const double *me = &cur[i * stride];
const double *dn_r = &cur[(i + 1) * stride];
double *out = &nxt[i * stride];
for (long j = 1; j < N - 1; j++) {
double v = 0.25 * (up_r[j] + dn_r[j] + me[j - 1] + me[j + 1]);
double d = fabs(v - me[j]);
if (d > res) res = d;
out[j] = v;
}
}
double *tmp = cur; cur = nxt; nxt = tmp;
iter++;
/* One collective per iteration: global convergence measure. */
MPI_Allreduce(&res, &gres, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (iter % report == 0 || iter == 1) {
double t = now();
if (rank == 0) {
double dt = t - tlast;
long diter = iter - last_iter;
double ips = dt > 0 ? diter / dt : 0.0;
/* 6 flops per interior cell per iteration */
double gf = ips * 6.0 * (double)(N - 2) * (double)(N - 2) / 1e9;
printf("%8ld %14.6e %12.1f %12.2f %10.2f\n",
iter, gres, t - t0, ips, gf);
fflush(stdout);
tlast = t; last_iter = iter;
stop = (t >= deadline)
|| (max_iter > 0 && iter >= max_iter)
|| (tol > 0.0 && gres < tol);
}
/* Everyone leaves on the same iteration. */
MPI_Bcast(&stop, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
}
double total = now() - t0;
if (rank == 0) {
const char *why = (tol > 0.0 && gres < tol) ? "converged"
: (max_iter > 0 && iter >= max_iter) ? "iteration limit"
: "time budget reached";
printf("\nheat_mpi finished: %s\n", why);
printf(" iterations : %ld\n", iter);
printf(" residual : %.6e\n", gres);
printf(" elapsed : %.1f s (%.2f min)\n", total, total / 60.0);
printf(" mean : %.2f iter/s over %d ranks\n", iter / total, size);
fflush(stdout);
}
free(cur); free(nxt);
MPI_Finalize();
return 0;
}