From f7b8f46205f4c431627f4452506c54d195b3fbf9 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 18:21:47 -0400 Subject: [PATCH] Updated coarray code so it can compile in the container. --- Dockerfile | 14 +++++------ cmake/Modules/SetFortranFlags.cmake | 4 ++++ docker/bin/swiftest | 2 +- src/coarray/coarray_clone.f90 | 8 +++---- src/rmvs/rmvs_coarray.f90 | 10 +------- src/swiftest/swiftest_coarray.f90 | 36 ++++++++++++++++++----------- src/swiftest/swiftest_module.f90 | 8 +++++++ 7 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0f2dcb88..35e450d7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14 wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ rm -rf /var/lib/apt/lists/* && \ cd hdf && \ ./HDF5-1.14.1-Linux.sh --skip-license && \ @@ -153,14 +153,13 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=ON -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ cmake --build build --verbose && \ cmake --install build # Driver container FROM ubuntu:20.04 as Driver -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -172,9 +171,8 @@ FROM continuumio/miniconda3 ENV LD_LIBRARY_PATH="/usr/local/lib" ENV SHELL="/bin/bash" -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ COPY ./python/ . +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /bin/ RUN apt-get update && apt-get upgrade -y && \ @@ -188,6 +186,8 @@ RUN apt-get update && apt-get upgrade -y && \ conda update --all -y && \ cd swiftest && conda develop . && \ mkdir -p /.astropy && \ - chmod -R 777 /.astropy + chmod -R 777 /.astropy && \ + mkdir -p /.config/matplotlib && \ + chmod -R 777 /.config/matplotlib ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index d1edd9ae9..47361de08 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -117,6 +117,10 @@ IF (CONTAINERIZE) SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" Fortran "-static-intel" # Intel ) + # Use static Intel MPI libraries + SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" + Fortran "-static_mpi" # Intel + ) IF (USE_OPENMP) SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" diff --git a/docker/bin/swiftest b/docker/bin/swiftest index bc94b2cb0..fead7f338 100755 --- a/docker/bin/swiftest +++ b/docker/bin/swiftest @@ -1,2 +1,2 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/src/coarray/coarray_clone.f90 b/src/coarray/coarray_clone.f90 index 9f7e1ea1a..893cff147 100644 --- a/src/coarray/coarray_clone.f90 +++ b/src/coarray/coarray_clone.f90 @@ -71,7 +71,7 @@ module subroutine coarray_component_clone_DP(var,src_img) sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -117,7 +117,7 @@ module subroutine coarray_component_clone_DP_arr1D(var,src_img) allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var + tmp(:)[img] = var end do sync images(*) else @@ -167,7 +167,7 @@ module subroutine coarray_component_clone_DP_arr2D(var,src_img) allocate(tmp(n1[si],n2[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:,:)[img] = var(:,:) + tmp(:,:)[img] = var(:,:) end do sync images(*) else @@ -252,7 +252,7 @@ module subroutine coarray_component_clone_DP_vec2D(var,src_img) allocate(tmp(NDIM,n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:,:)[img] = var(:,:) + tmp(:,:)[img] = var(:,:) end do sync images(*) else diff --git a/src/rmvs/rmvs_coarray.f90 b/src/rmvs/rmvs_coarray.f90 index 6f7467df8..3a7508159 100644 --- a/src/rmvs/rmvs_coarray.f90 +++ b/src/rmvs/rmvs_coarray.f90 @@ -68,7 +68,6 @@ module subroutine rmvs_coarray_coclone_pl(self) end subroutine rmvs_coarray_coclone_pl - module subroutine rmvs_coarray_coclone_system(self) !! author: David A. Minton !! @@ -141,14 +140,7 @@ module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) do i = 1, n[si] call tmp(i)%coclone() end do - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var(:) - end do - - sync images(*) - else - sync images(si) + if (this_image() /= si) then if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 7dbd1a816..8329856b9 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -54,6 +54,7 @@ module subroutine swiftest_coarray_balance_system(nbody_system, param) return end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_coclone_body(self) !! author: David A. Minton !! @@ -94,6 +95,21 @@ module subroutine swiftest_coarray_coclone_body(self) return end subroutine swiftest_coarray_coclone_body + module subroutine swiftest_coarray_coclone_kin(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_kinship),intent(inout),codimension[*] :: self !! Swiftest kinship object + + call coclone(self%parent) + call coclone(self%nchild) + call coclone(self%child) + + return + end subroutine swiftest_coarray_coclone_kin + module subroutine swiftest_coarray_coclone_nc(self) !! author: David A. Minton !! @@ -327,10 +343,6 @@ module subroutine swiftest_coarray_coclone_system(self) ! Internals integer(I4B) :: i - call self%cb%coclone() - call self%pl%coclone() - call self%tp%coclone() - call coclone(self%maxid) call coclone(self%t) call coclone(self%GMtot) @@ -474,7 +486,7 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) integer(I4B), intent(in),optional :: src_img ! Internals type(swiftest_kinship), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si + integer(I4B) :: i, img, si integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] @@ -484,21 +496,17 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) si = 1 end if - allocate(isalloc[*]) - allocate(n[*]) + sync all isalloc = allocated(var) if (isalloc) n = size(var) sync all if (.not. isalloc[si]) return allocate(tmp(n[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) - else - sync images(si) + do i = 1, n[si] + call tmp(i)%coclone() + end do + if (this_image() /= si) then if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index a54e2351b..b7bbd109c 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -93,6 +93,9 @@ module swiftest integer(I4B), dimension(:), allocatable :: child !! Index of children particles contains procedure :: dealloc => swiftest_util_dealloc_kin !! Deallocates all allocatable arrays +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_kin !! Clones the image 1 body object to all other images in the coarray structure. +#endif final :: swiftest_final_kin !! Finalizes the Swiftest kinship object - deallocates all allocatables end type swiftest_kinship @@ -1776,6 +1779,11 @@ module subroutine swiftest_coarray_coclone_cb(self) class(swiftest_cb),intent(inout),codimension[*] :: self !! Swiftest cb object end subroutine swiftest_coarray_coclone_cb + module subroutine swiftest_coarray_coclone_kin(self) + implicit none + class(swiftest_kinship),intent(inout),codimension[*] :: self !! Swiftest kinship object + end subroutine swiftest_coarray_coclone_kin + module subroutine swiftest_coarray_coclone_nc(self) implicit none class(swiftest_netcdf_parameters),intent(inout),codimension[*] :: self !! Swiftest body object