mpi-ring finishes in about two seconds, so there is nothing to observe in squeue, no meaningful sacct record and no load on the nodes. heat_mpi solves 2D steady-state heat diffusion by Jacobi iteration over a grid split into horizontal strips, one per rank. Every iteration performs a five-point stencil, a halo exchange with the neighbouring ranks via MPI_Sendrecv, and an MPI_Allreduce for the global residual - so it exercises compute and both point-to-point and collective communication rather than just starting up and exiting. It runs to a wall-clock budget (-m minutes) instead of a fixed iteration count, which is what makes it useful for watching the scheduler. Stopping is collective: rank 0 decides and broadcasts, so every rank leaves on the same iteration. The residual falls monotonically, which doubles as a check that the halo exchange is correct. Measured on this cluster: 6144 grid, 8 ranks over 4 nodes, ~12 GFLOP/s with all nodes at ~95% CPU and ~72 MiB per rank. heat-long.sbatch defaults to 20 minutes and takes GRID, MINUTES and REPORT through --export.
177 lines
7.1 KiB
C
177 lines
7.1 KiB
C
/* heat_mpi - a long-running MPI benchmark for the Warewulf test cluster.
|
|
*
|
|
* Solves 2D steady-state heat diffusion with Jacobi iteration on an N x N grid.
|
|
* The domain is split into horizontal strips, one per rank, so every iteration
|
|
* does real work of both kinds an HPC job does:
|
|
*
|
|
* - compute : a five-point stencil over the local strip
|
|
* - communication: halo exchange with the rank above and below (MPI_Sendrecv),
|
|
* plus a global MPI_Allreduce to test convergence
|
|
*
|
|
* Unlike hello_mpi, which finishes instantly, this runs for a wall-clock budget
|
|
* you choose, so there is something to watch in squeue, sacct and on the nodes.
|
|
*
|
|
* mpicc -O3 -march=native -o heat_mpi heat_mpi.c -lm
|
|
*
|
|
* -n <grid> grid points per side (default 4096)
|
|
* -m <minutes> wall-clock budget (default 10)
|
|
* -i <iters> stop after this many iters (default: unlimited)
|
|
* -r <every> report interval in iterations (default 200)
|
|
* -e <residual> stop early if converged below (default 0 = never)
|
|
*
|
|
* Stopping is collective: rank 0 decides and broadcasts, so every rank always
|
|
* leaves the loop on the same iteration.
|
|
*/
|
|
|
|
#include <mpi.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <math.h>
|
|
#include <unistd.h>
|
|
|
|
#define HOT 100.0
|
|
|
|
static double now(void) { return MPI_Wtime(); }
|
|
|
|
int main(int argc, char **argv)
|
|
{
|
|
int rank, size;
|
|
MPI_Init(&argc, &argv);
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
|
|
MPI_Comm_size(MPI_COMM_WORLD, &size);
|
|
|
|
long N = 4096;
|
|
double minutes = 10.0;
|
|
long max_iter = 0; /* 0 = unlimited */
|
|
long report = 200;
|
|
double tol = 0.0; /* 0 = never stop early */
|
|
|
|
for (int i = 1; i < argc; i++) {
|
|
if (!strcmp(argv[i], "-n") && i + 1 < argc) N = atol(argv[++i]);
|
|
else if (!strcmp(argv[i], "-m") && i + 1 < argc) minutes = atof(argv[++i]);
|
|
else if (!strcmp(argv[i], "-i") && i + 1 < argc) max_iter = atol(argv[++i]);
|
|
else if (!strcmp(argv[i], "-r") && i + 1 < argc) report = atol(argv[++i]);
|
|
else if (!strcmp(argv[i], "-e") && i + 1 < argc) tol = atof(argv[++i]);
|
|
else if (rank == 0) fprintf(stderr, "ignoring argument: %s\n", argv[i]);
|
|
}
|
|
if (N < (long)size * 3) { /* every rank needs interior rows */
|
|
if (rank == 0)
|
|
fprintf(stderr, "grid %ld too small for %d ranks; use -n >= %d\n",
|
|
N, size, size * 3);
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
|
|
/* Row decomposition, remainder spread over the first ranks so the strips
|
|
* differ by at most one row. */
|
|
long base = N / size, rem = N % size;
|
|
long rows = base + (rank < rem ? 1 : 0);
|
|
long row0 = rank * base + (rank < rem ? rank : rem);
|
|
|
|
/* Two ghost rows: index 0 is the neighbour above, rows+1 the one below. */
|
|
size_t stride = (size_t)N;
|
|
size_t cells = (size_t)(rows + 2) * stride;
|
|
double *cur = malloc(cells * sizeof *cur);
|
|
double *nxt = malloc(cells * sizeof *nxt);
|
|
if (!cur || !nxt) {
|
|
fprintf(stderr, "rank %d: out of memory (%.1f MiB per array)\n",
|
|
rank, (double)(cells * sizeof *cur) / (1024 * 1024));
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
for (size_t i = 0; i < cells; i++) cur[i] = nxt[i] = 0.0;
|
|
|
|
/* Boundary: the global top edge is held hot, everything else at zero. */
|
|
if (rank == 0)
|
|
for (long j = 0; j < N; j++) cur[stride + j] = nxt[stride + j] = HOT;
|
|
|
|
int up = (rank == 0) ? MPI_PROC_NULL : rank - 1;
|
|
int down = (rank == size - 1) ? MPI_PROC_NULL : rank + 1;
|
|
|
|
if (rank == 0) {
|
|
double mib = (double)(cells * sizeof *cur) * 2 / (1024 * 1024);
|
|
printf("heat_mpi: %ld x %ld grid, %d ranks, ~%.1f MiB/rank, budget %.1f min\n",
|
|
N, N, size, mib, minutes);
|
|
printf("%8s %14s %12s %12s %10s\n",
|
|
"iter", "residual", "elapsed(s)", "iter/s", "GFLOP/s");
|
|
fflush(stdout);
|
|
}
|
|
|
|
double deadline = now() + minutes * 60.0;
|
|
double t0 = now(), tlast = t0;
|
|
long iter = 0, last_iter = 0;
|
|
double gres = 0.0;
|
|
int stop = 0;
|
|
|
|
while (!stop) {
|
|
/* Halo exchange. Sendrecv pairs the sends and receives so this cannot
|
|
* deadlock however the ranks are scheduled. */
|
|
MPI_Sendrecv(&cur[1 * stride], N, MPI_DOUBLE, up, 0,
|
|
&cur[(rows + 1) * stride], N, MPI_DOUBLE, down, 0,
|
|
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
|
|
MPI_Sendrecv(&cur[rows * stride], N, MPI_DOUBLE, down, 1,
|
|
&cur[0], N, MPI_DOUBLE, up, 1,
|
|
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
|
|
|
|
/* Five-point Jacobi stencil over the interior of the local strip. */
|
|
double res = 0.0;
|
|
for (long i = 1; i <= rows; i++) {
|
|
if (rank == 0 && i == 1) continue; /* hot edge is fixed */
|
|
if (rank == size - 1 && i == rows) continue; /* global bottom */
|
|
const double *up_r = &cur[(i - 1) * stride];
|
|
const double *me = &cur[i * stride];
|
|
const double *dn_r = &cur[(i + 1) * stride];
|
|
double *out = &nxt[i * stride];
|
|
for (long j = 1; j < N - 1; j++) {
|
|
double v = 0.25 * (up_r[j] + dn_r[j] + me[j - 1] + me[j + 1]);
|
|
double d = fabs(v - me[j]);
|
|
if (d > res) res = d;
|
|
out[j] = v;
|
|
}
|
|
}
|
|
|
|
double *tmp = cur; cur = nxt; nxt = tmp;
|
|
iter++;
|
|
|
|
/* One collective per iteration: global convergence measure. */
|
|
MPI_Allreduce(&res, &gres, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
|
|
|
|
if (iter % report == 0 || iter == 1) {
|
|
double t = now();
|
|
if (rank == 0) {
|
|
double dt = t - tlast;
|
|
long diter = iter - last_iter;
|
|
double ips = dt > 0 ? diter / dt : 0.0;
|
|
/* 6 flops per interior cell per iteration */
|
|
double gf = ips * 6.0 * (double)(N - 2) * (double)(N - 2) / 1e9;
|
|
printf("%8ld %14.6e %12.1f %12.2f %10.2f\n",
|
|
iter, gres, t - t0, ips, gf);
|
|
fflush(stdout);
|
|
tlast = t; last_iter = iter;
|
|
|
|
stop = (t >= deadline)
|
|
|| (max_iter > 0 && iter >= max_iter)
|
|
|| (tol > 0.0 && gres < tol);
|
|
}
|
|
/* Everyone leaves on the same iteration. */
|
|
MPI_Bcast(&stop, 1, MPI_INT, 0, MPI_COMM_WORLD);
|
|
}
|
|
}
|
|
|
|
double total = now() - t0;
|
|
if (rank == 0) {
|
|
const char *why = (tol > 0.0 && gres < tol) ? "converged"
|
|
: (max_iter > 0 && iter >= max_iter) ? "iteration limit"
|
|
: "time budget reached";
|
|
printf("\nheat_mpi finished: %s\n", why);
|
|
printf(" iterations : %ld\n", iter);
|
|
printf(" residual : %.6e\n", gres);
|
|
printf(" elapsed : %.1f s (%.2f min)\n", total, total / 60.0);
|
|
printf(" mean : %.2f iter/s over %d ranks\n", iter / total, size);
|
|
fflush(stdout);
|
|
}
|
|
|
|
free(cur); free(nxt);
|
|
MPI_Finalize();
|
|
return 0;
|
|
}
|