! Companion code for Chapter 15: Parallel Computing ! Computational Physics: From Equations to Simulation ! ! FIRST STEPS WITH MPI: two-dimensional heat diffusion in horizontal strips. ! ! Compile with a Fortran MPI wrapper (Open MPI or MPICH, for example): ! mpifort -O3 -std=f95 -Wall -Wextra ch15_mpi_heat_strips.f95 -o ch15_mpi_heat_strips ! Run, for example (from the directory containing the executable): ! mpiexec -n 1 ./ch15_mpi_heat_strips ! mpiexec -n 2 ./ch15_mpi_heat_strips ! mpiexec -n 4 ./ch15_mpi_heat_strips ! Use mpiexec/mpirun according to your MPI installation. This program uses ! 'use mpi' (not the Fortran 2008 mpi_f08 interface). Each process owns its ! own memory: an MPI installation and MPI-aware compiler are REQUIRED. ! ! Equation: u_t = alpha*(u_xx + u_yy) on the periodic unit square. ! Explicit five-point update: new = old + r*(four neighbors - 4*old), ! r = alpha*dt/dx**2, dx=dy=1/N, and stability requires 0 < r <= 1/4. ! ! Strips: rank k owns a consecutive range of global y-indices. Fortran ! arrays use a(i,j), so a full x-column at fixed j is contiguous in memory. ! Before EVERY update, MPI_Sendrecv exchanges the edge columns with the ! neighboring ranks; the received values are stored in ghost columns j=0 ! and j=local_n+1. All x boundaries remain periodic on each rank. ! Two arrays are essential: only OLD values are read in each step. ! The final field is gathered for a strict comparison with the serial solver. ! The diagnostic global heat is summed with MPI_Allreduce; floating-point ! reductions may differ slightly in rounding across process counts. ! ! Timing: each run starts from the same initial field; the MPI time is ! the MAXIMUM elapsed time across ranks, including halo communication and ! stencil updates, but excluding initialization, final gathering and checks. ! A serial reference is calculated for correctness on rank 0. Its timing ! is shown only with -n 1: in multi-process runs idle MPI ranks may spin ! and distort a serial timing on a small computer. Compare separate jobs ! with -n 1, 2, 4 for MPI scaling; do not oversubscribe for timing. ! On Open MPI, --oversubscribe permits extra processes but produces timings ! that are not valid hardware-scaling measurements. No speedup is promised. ! For a larger memory/communication experiment, try n_side=2048. ! The example divides the y direction into strips (horizontal in a plotted ! array); Figure 15.1 shows a similar one-dimensional strip decomposition ! in the other orientation. ! ! No random numbers, plots, GPU, or OpenMP. Editable settings follow. program ch15_mpi_heat_strips use mpi implicit none integer, parameter :: dp = selected_real_kind(15,300) ! ---- Editable example settings ---- integer, parameter :: n_side = 512 logical, parameter :: demonstrate_broken_exchange = .false. ! WARNING: true deliberately omits a halo message and MUST fail checks. integer, parameter :: time_steps = 500 integer, parameter :: repetitions = 3 real(dp), parameter :: alpha = 1.0_dp real(dp), parameter :: step_factor = 0.20_dp ! dt=step_factor*dx**2 real(dp), parameter :: amplitude = 0.25_dp integer, parameter :: mode_x = 8, mode_y = 6 integer :: ierr,rank,nproc,local_n,first_j,remainder,base integer :: i,j,k,rep,lower,upper,sendcount integer, allocatable :: counts(:),offsets(:) real(dp), allocatable :: a(:,:),b(:,:),packed(:),gathered(:) real(dp), allocatable :: global_field(:,:),serial_field(:,:) real(dp), allocatable :: serial_a(:,:),serial_b(:,:) real(dp) :: pi,dx,dt,r,final_time,decay_discrete,decay_pde real(dp) :: local_heat,global_heat,initial_mean,final_mean real(dp) :: max_field_diff,max_exact_err,max_pde_err,predicted_pde_err real(dp) :: start_clock,local_elapsed,mpi_elapsed,mpi_times(repetitions) real(dp) :: halo_elapsed,halo_max,halo_times(repetitions) real(dp) :: serial_times(repetitions),value_discrete,value_pde,serial_median call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD,rank,ierr) call MPI_Comm_size(MPI_COMM_WORLD,nproc,ierr) if (n_side < 16 .or. time_steps < 1 .or. repetitions < 1 .or. & alpha <= 0.0_dp .or. step_factor <= 0.0_dp .or. & abs(amplitude) > 1.0_dp .or. mode_x < 1 .or. mode_y < 1 .or. & 2*mode_x >= n_side .or. 2*mode_y >= n_side .or. nproc > n_side) then if (rank == 0) print '(a)', 'ERROR: invalid grid, mode, steps, or process count.' call MPI_Abort(MPI_COMM_WORLD,1,ierr) end if dx=1.0_dp/real(n_side,dp) dt=step_factor*dx*dx r=alpha*dt/(dx*dx) if (r > 0.25_dp) then if (rank == 0) print '(a,f12.6)', 'ERROR: unstable r (> 0.25): ',r call MPI_Abort(MPI_COMM_WORLD,1,ierr) end if pi=acos(-1.0_dp) final_time=real(time_steps,dp)*dt decay_discrete=(1.0_dp-4.0_dp*r*( & sin(pi*real(mode_x,dp)/real(n_side,dp))**2+ & sin(pi*real(mode_y,dp)/real(n_side,dp))**2))**time_steps decay_pde=exp(-4.0_dp*pi*pi*alpha* & real(mode_x*mode_x+mode_y*mode_y,dp)*final_time) predicted_pde_err=abs(amplitude)*abs(decay_discrete-decay_pde) base=n_side/nproc remainder=mod(n_side,nproc) local_n=base if (rank < remainder) local_n=local_n+1 first_j=rank*base+min(rank,remainder)+1 lower=mod(rank-1+nproc,nproc) upper=mod(rank+1,nproc) allocate(a(0:n_side+1,0:local_n+1),b(0:n_side+1,0:local_n+1)) allocate(packed(n_side*local_n)) a=0.0_dp b=0.0_dp do j=1,local_n do i=1,n_side a(i,j)=initial_value(i,first_j+j-1,n_side,pi) end do end do if (rank == 0) then allocate(counts(nproc),offsets(nproc)) do k=0,nproc-1 counts(k+1)=n_side*(base+merge(1,0,k < remainder)) offsets(k+1)=n_side*(k*base+min(k,remainder)) end do allocate(gathered(n_side*n_side),global_field(n_side,n_side)) allocate(serial_field(n_side,n_side)) allocate(serial_a(0:n_side+1,0:n_side+1)) allocate(serial_b(0:n_side+1,0:n_side+1)) print '(a)', 'Chapter 15: first steps with MPI heat-diffusion strips' print '(a,i0,a,i0)', 'Grid: ',n_side,' x ',n_side print '(a,i0)', 'Time steps: ',time_steps print '(a,i0)', 'MPI processes: ',nproc print '(a,f9.5,a)', 'Stencil r = ',r,' (must be <= 0.25)' print '(a,i0)', 'Timing repetitions: ',repetitions print '(a)', 'Each process owns consecutive y strips with two ghost columns.' print '(a)', 'Halos exchanged at every step; total heat uses MPI_Allreduce.' if (demonstrate_broken_exchange) print '(a)', & 'WARNING: BROKEN EXCHANGE DEMONSTRATION ENABLED (validation must fail).' else ! Nonroot MPI_Gatherv arguments are ignored but must be valid arrays. allocate(counts(1),offsets(1),gathered(1)) counts=0 offsets=0 end if ! Compute the initial mean from the DISTRIBUTED initial field. ! This remains valid when the initial condition is changed later. local_heat=sum(a(1:n_side,1:local_n)) call MPI_Allreduce(local_heat,global_heat,1,MPI_DOUBLE_PRECISION, & MPI_SUM,MPI_COMM_WORLD,ierr) initial_mean=global_heat/(real(n_side,dp)**2) ! Serial reference is used for verification at every process count. ! Timing it with other MPI ranks waiting may distort the CPU baseline; ! show its timing only for -n 1. if (rank == 0) then do rep=1,repetitions call initialize_full(serial_a,serial_b,n_side,pi) start_clock=MPI_Wtime() call evolve_serial(serial_a,serial_b,n_side,time_steps,r) serial_times(rep)=MPI_Wtime()-start_clock end do if (mod(time_steps,2) == 0) then serial_field=serial_a(1:n_side,1:n_side) else serial_field=serial_b(1:n_side,1:n_side) end if serial_median=median(serial_times) end if ! First untimed MPI warmup, then repeated runs from the same initial field. call initialize_local(a,b,n_side,local_n,first_j,pi) call evolve_mpi(a,b,n_side,local_n,time_steps,r,lower,upper,halo_elapsed) do rep=1,repetitions call initialize_local(a,b,n_side,local_n,first_j,pi) call MPI_Barrier(MPI_COMM_WORLD,ierr) start_clock=MPI_Wtime() call evolve_mpi(a,b,n_side,local_n,time_steps,r,lower,upper,halo_elapsed) local_elapsed=MPI_Wtime()-start_clock call MPI_Reduce(local_elapsed,mpi_elapsed,1,MPI_DOUBLE_PRECISION, & MPI_MAX,0,MPI_COMM_WORLD,ierr) call MPI_Reduce(halo_elapsed,halo_max,1,MPI_DOUBLE_PRECISION, & MPI_MAX,0,MPI_COMM_WORLD,ierr) if (rank == 0) then mpi_times(rep)=mpi_elapsed halo_times(rep)=halo_max end if end do ! Return local interior values to the root in global y order. k=0 do j=1,local_n do i=1,n_side k=k+1 if (mod(time_steps,2) == 0) then packed(k)=a(i,j) else packed(k)=b(i,j) end if end do end do sendcount=n_side*local_n call MPI_Gatherv(packed,sendcount,MPI_DOUBLE_PRECISION, & gathered,counts,offsets,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ierr) local_heat=sum(packed) call MPI_Allreduce(local_heat,global_heat,1,MPI_DOUBLE_PRECISION, & MPI_SUM,MPI_COMM_WORLD,ierr) final_mean=global_heat/(real(n_side,dp)**2) if (rank == 0) then global_field=reshape(gathered,(/n_side,n_side/)) max_field_diff=maxval(abs(global_field-serial_field)) max_exact_err=0.0_dp max_pde_err=0.0_dp do j=1,n_side do i=1,n_side value_discrete=1.0_dp+amplitude* & cos(2.0_dp*pi*real(mode_x*i,dp)/real(n_side,dp))* & cos(2.0_dp*pi*real(mode_y*j,dp)/real(n_side,dp))*decay_discrete value_pde=1.0_dp+amplitude* & cos(2.0_dp*pi*real(mode_x*i,dp)/real(n_side,dp))* & cos(2.0_dp*pi*real(mode_y*j,dp)/real(n_side,dp))*decay_pde max_exact_err=max(max_exact_err,abs(global_field(i,j)-value_discrete)) max_pde_err=max(max_pde_err,abs(global_field(i,j)-value_pde)) end do end do print '(a,es13.5)', 'Initial mean temperature: ',initial_mean print '(a,es13.5)', 'Mean temperature change: ', & abs(final_mean-initial_mean) print '(a,es13.5)', 'Maximum MPI vs serial difference: ',max_field_diff print '(a,es13.5)', 'Maximum exact discrete error: ',max_exact_err print '(a,es13.5)', 'Maximum PDE solution error: ',max_pde_err print '(a,es13.5)', 'Predicted PDE discretization error:',predicted_pde_err if (nproc == 1) then print '(a,f10.5)', 'Serial median (seconds): ',serial_median print '(a,f10.4)', 'Speedup vs serial (n=1): ', & serial_median/median(mpi_times) else print '(a)', 'Serial reference verified; serial timing omitted for n > 1.' print '(a)', 'Compare MPI times from separate runs to assess strong scaling.' end if print '(a,f10.5)', 'MPI median (seconds): ',median(mpi_times) print '(a,f10.5)', 'Max halo time (median s): ',median(halo_times) print '(a,f8.2,a)', 'Halo / total time (approx): ', & 100.0_dp*median(halo_times)/median(mpi_times),' percent' print '(a)', 'Maxima for halo and total time can occur on different ranks.' print '(a)', 'MPI timing includes halo communication, not final gathering.' print '(a)', 'For scaling, rerun with -n 1, -n 2, and -n 4.' if (max_field_diff > 1.0e-11_dp .or. & max_exact_err > 1.0e-10_dp .or. & abs(final_mean-initial_mean) > 1.0e-10_dp) then print '(a)', 'ERROR: field or conservation verification failed.' call MPI_Abort(MPI_COMM_WORLD,2,ierr) else print '(a)', 'Validation: PASS' end if end if deallocate(a,b,packed,counts,offsets,gathered) if (rank == 0) deallocate(global_field,serial_field,serial_a,serial_b) call MPI_Finalize(ierr) contains function initial_value(ix,jy,n,pi_value) result(value) integer, intent(in) :: ix,jy,n real(dp), intent(in) :: pi_value real(dp) :: value value=1.0_dp+amplitude* & cos(2.0_dp*pi_value*real(mode_x*ix,dp)/real(n,dp))* & cos(2.0_dp*pi_value*real(mode_y*jy,dp)/real(n,dp)) end function initial_value subroutine initialize_local(old,new,n,ny,jstart,pi_value) integer, intent(in) :: n,ny,jstart real(dp), intent(in) :: pi_value real(dp), intent(out) :: old(0:n+1,0:ny+1),new(0:n+1,0:ny+1) integer :: ix,jy old=0.0_dp new=0.0_dp do jy=1,ny do ix=1,n old(ix,jy)=initial_value(ix,jstart+jy-1,n,pi_value) end do end do end subroutine initialize_local subroutine exchange_halos(field,n,ny,below,above) integer, intent(in) :: n,ny,below,above real(dp), intent(inout) :: field(0:n+1,0:ny+1) integer :: stat(MPI_STATUS_SIZE),err ! Periodic boundary in x. Fixed-j slices are contiguous in Fortran. field(0,1:ny)=field(n,1:ny) field(n+1,1:ny)=field(1,1:ny) ! Send the first owned strip to BELOW; receive ABOVE into upper halo. call MPI_Sendrecv(field(1,1),n,MPI_DOUBLE_PRECISION,below,101, & field(1,ny+1),n,MPI_DOUBLE_PRECISION,above,101, & MPI_COMM_WORLD,stat,err) ! Send last owned strip to ABOVE; receive BELOW into lower halo. ! In the optional BROKEN mode, deliberately skip the second exchange. ! It should violate conservation and fail the serial-field check. if (.not. demonstrate_broken_exchange) then call MPI_Sendrecv(field(1,ny),n,MPI_DOUBLE_PRECISION,above,102, & field(1,0),n,MPI_DOUBLE_PRECISION,below,102, & MPI_COMM_WORLD,stat,err) end if end subroutine exchange_halos subroutine evolve_mpi(old,new,n,ny,steps,coefficient,below,above,halo_seconds) integer, intent(in) :: n,ny,steps,below,above real(dp), intent(in) :: coefficient real(dp), intent(inout) :: old(0:n+1,0:ny+1),new(0:n+1,0:ny+1) real(dp), intent(out) :: halo_seconds integer :: it,ix,jy real(dp) :: exchange_start halo_seconds=0.0_dp do it=1,steps exchange_start=MPI_Wtime() if (mod(it,2) == 1) then call exchange_halos(old,n,ny,below,above) else call exchange_halos(new,n,ny,below,above) end if halo_seconds=halo_seconds+(MPI_Wtime()-exchange_start) if (mod(it,2) == 1) then do jy=1,ny do ix=1,n new(ix,jy)=old(ix,jy)+coefficient*( & old(ix-1,jy)+old(ix+1,jy)+old(ix,jy-1)+ & old(ix,jy+1)-4.0_dp*old(ix,jy)) end do end do else do jy=1,ny do ix=1,n old(ix,jy)=new(ix,jy)+coefficient*( & new(ix-1,jy)+new(ix+1,jy)+new(ix,jy-1)+ & new(ix,jy+1)-4.0_dp*new(ix,jy)) end do end do end if end do end subroutine evolve_mpi subroutine initialize_full(old,new,n,pi_value) integer, intent(in) :: n real(dp), intent(in) :: pi_value real(dp), intent(out) :: old(0:n+1,0:n+1),new(0:n+1,0:n+1) integer :: ix,jy old=0.0_dp new=0.0_dp do jy=1,n do ix=1,n old(ix,jy)=initial_value(ix,jy,n,pi_value) end do end do end subroutine initialize_full subroutine serial_edges(field,n) integer, intent(in) :: n real(dp), intent(inout) :: field(0:n+1,0:n+1) field(0,1:n)=field(n,1:n) field(n+1,1:n)=field(1,1:n) field(1:n,0)=field(1:n,n) field(1:n,n+1)=field(1:n,1) end subroutine serial_edges subroutine evolve_serial(old,new,n,steps,coefficient) integer, intent(in) :: n,steps real(dp), intent(in) :: coefficient real(dp), intent(inout) :: old(0:n+1,0:n+1),new(0:n+1,0:n+1) integer :: it,ix,jy do it=1,steps if (mod(it,2) == 1) then call serial_edges(old,n) do jy=1,n do ix=1,n new(ix,jy)=old(ix,jy)+coefficient*( & old(ix-1,jy)+old(ix+1,jy)+old(ix,jy-1)+ & old(ix,jy+1)-4.0_dp*old(ix,jy)) end do end do else call serial_edges(new,n) do jy=1,n do ix=1,n old(ix,jy)=new(ix,jy)+coefficient*( & new(ix-1,jy)+new(ix+1,jy)+new(ix,jy-1)+ & new(ix,jy+1)-4.0_dp*new(ix,jy)) end do end do end if end do end subroutine evolve_serial function median(times) result(value) real(dp), intent(in) :: times(:) real(dp) :: value,buffer(size(times)),tmp integer :: ix,jy buffer=times do ix=2,size(buffer) tmp=buffer(ix) jy=ix-1 do while (jy >= 1) if (buffer(jy) <= tmp) exit buffer(jy+1)=buffer(jy) jy=jy-1 end do buffer(jy+1)=tmp end do if (mod(size(buffer),2) == 1) then value=buffer((size(buffer)+1)/2) else value=(buffer(size(buffer)/2)+buffer(size(buffer)/2+1))/2.0_dp end if end function median end program ch15_mpi_heat_strips