diff --git a/cmake/Modules/FindCoarray_Fortran.cmake b/cmake/Modules/FindCoarray_Fortran.cmake new file mode 100644 index 000000000..6a05818b8 --- /dev/null +++ b/cmake/Modules/FindCoarray_Fortran.cmake @@ -0,0 +1,90 @@ +# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. + +# - Finds Coarray support +# This module can be used to detect Coarray support in a compiler. +# If the compiler supports Coarray, the flags required to compile with +# coarray support are set. +# +# This module was modified from the standard FindOpenMP module to find Fortran +# flags. +# +# The following variables are set: +# Coarray_Fortran_FLAGS - flags to add to the Fortran compiler for Coarray +# support. In general, you must use these at both +# compile- and link-time. +# OMP_NUM_PROCS - the max number of processors available to Coarray + +#============================================================================= + +INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) + + +SET (Coarray_Fortran_FLAG_CANDIDATES + #Intel + "-coarray=distributed" + #Intel windows + "/Qcoarray:distributed" + #Gnu + "-fcorray=lib -lcaf_mpi" + #Empty, if compiler automatically accepts coarray + " " +) + +IF (DEFINED Coarray_Fortran_FLAGS) + SET (Coarray_Fortran_FLAG_CANDIDATES) +ENDIF (DEFINED Coarray_Fortran_FLAGS) + +# check fortran compiler. also determine number of processors +FOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES}) + SET (SAFE_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + SET (CMAKE_REQUIRED_FLAGS "${FLAG}") + UNSET (Coarray_FLAG_DETECTED CACHE) + MESSAGE (STATUS "Try Coarray Fortran flag = [${FLAG}]") + FILE (WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90" +" +program TestCoarray + integer, codimension[*] :: i + write(*,'(I2)',ADVANCE='NO') num_images() +end program TestCoarray +") + SET (MACRO_CHECK_FUNCTION_DEFINITIONS + "-DCoarray_FLAG_DETECTED ${CMAKE_REQUIRED_FLAGS}") + TRY_RUN (Coarray_RUN_FAILED Coarray_FLAG_DETECTED ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90 + COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} + CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} + COMPILE_OUTPUT_VARIABLE OUTPUT + RUN_OUTPUT_VARIABLE OMP_NUM_PROCS_INTERNAL) + IF (Coarray_FLAG_DETECTED) + FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log + "Determining if the Fortran compiler supports Coarray passed with " + "the following output:\n${OUTPUT}\n\n") + SET (Coarray_FLAG_DETECTED 1) + IF (Coarray_RUN_FAILED) + MESSAGE (FATAL_ERROR "Coarray found, but test code did not run") + ENDIF (Coarray_RUN_FAILED) + SET (Coarray_Fortran_FLAGS_INTERNAL "${FLAG}") + BREAK () + ELSE () + FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Determining if the Fortran compiler supports Coarray failed with " + "the following output:\n${OUTPUT}\n\n") + SET (Coarray_FLAG_DETECTED 0) + ENDIF (Coarray_FLAG_DETECTED) +ENDFOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES}) + +SET (Coarray_Fortran_FLAGS "${Coarray_Fortran_FLAGS_INTERNAL}" + CACHE STRING "Fortran compiler flags for Coarray parallization") + +# handle the standard arguments for FIND_PACKAGE +FIND_PACKAGE_HANDLE_STANDARD_ARGS (Coarray_Fortran DEFAULT_MSG + Coarray_Fortran_FLAGS) + +MARK_AS_ADVANCED(Coarray_Fortran_FLAGS) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index d869e89b6..64554ac22 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -79,9 +79,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-xhost" # Intel "/QxHost" # Intel Windows ${GNUNATIVE} # GNU - "-ta=host" # Portland Group ) + ################### ### DEBUG FLAGS ### ################### @@ -99,12 +99,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-warn all" # Intel "/warn:all" # Intel Windows "-Wall" # GNU - # Portland Group (on by default) ) # Traceback SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-traceback" # Intel/Portland Group + Fortran "-traceback" # Intel Group "/traceback" # Intel Windows "-fbacktrace" # GNU (gfortran) "-ftrace=full" # GNU (g95) @@ -116,7 +115,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" "/check:bounds" # Intel Windows "-fcheck=bounds" # GNU (New style) "-fbounds-check" # GNU (Old style) - "-Mbounds" # Portland Group ) # Initializes matrices/arrays with NaN values @@ -225,7 +223,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-unroll" # Intel "/unroll" # Intel Windows "-funroll-loops" # GNU - "-Munroll" # Portland Group ) # Inline functions @@ -233,7 +230,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-inline" # Intel "/Qinline" # Intel Windows "-finline-functions" # GNU - "-Minline" # Portland Group ) diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake index 224806406..d809cc00b 100644 --- a/cmake/Modules/SetParallelizationLibrary.cmake +++ b/cmake/Modules/SetParallelizationLibrary.cmake @@ -27,6 +27,10 @@ IF (USE_MPI) # Find MPI IF (NOT MPI_Fortran_FOUND) FIND_PACKAGE (MPI REQUIRED) + FIND_PACKAGE (Coarray_Fortran) + IF (NOT Coarray_Fortran_FLAGS) + MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays") + ENDIF (NOT Coarray_Fortran_FLAGS) ENDIF (NOT MPI_Fortran_FOUND) ENDIF (USE_MPI) @@ -35,6 +39,7 @@ IF (NOT USE_OPENMP AND NOT USE_MPI) SET (OMP_NUM_PROCS 0 CACHE STRING "Number of processors OpenMP may use" FORCE) UNSET (OpenMP_Fortran_FLAGS CACHE) + UNSET (Coarray_Fortran_FLAGS CACHE) UNSET (GOMP_Fortran_LINK_FLAGS CACHE) UNSET (MPI_FOUND CACHE) UNSET (MPI_COMPILER CACHE) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4fd3a1cce..ad2a6a7af 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,13 +100,15 @@ IF(USE_OPENMP) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}" LINK_FLAGS "${OpenMP_Fortran_FLAGS}") -ELSEIF(USE_MPI) +ENDIF(USE_OPENMP) + +IF(USE_MPI) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES - COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS}" - LINK_FLAGS "${MPI_Fortran_LINK_FLAGS}") + COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS} ${Coarray_Fortran_FLAGS}" + LINK_FLAGS "${MPI_Fortran_LINK_FLAGS} ${Coarray_Fortran_FLAGS}") INCLUDE_DIRECTORIES(${MPI_Fortran_INCLUDE_PATH}) TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${MPI_Fortran_LIBRARIES}) -ENDIF(USE_OPENMP) +ENDIF(USE_MPI) ##################################### diff --git a/src/collision/collision_regime.f90 b/src/collision/collision_regime.f90 index a1ae47fe0..fb0f2f7da 100644 --- a/src/collision/collision_regime.f90 +++ b/src/collision/collision_regime.f90 @@ -150,7 +150,7 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, real(DP), parameter :: SUPERCAT_QRATIO = 1.8_DP ! See Section 4.1 of LS12 ! Internals real(DP) :: a1, alpha, aint, b, bcrit, c_star, egy, zeta, l, lint, mu, phi, theta - real(DP) :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat + real(DP) :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat, Qmerge real(DP) :: Vhr, Verosion, Vescp, Vhill, Vimp, Vsupercat real(DP) :: Mint, Mtot, Mtmp real(DP) :: Rp, rhill @@ -215,6 +215,8 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, Qloss = 0.0_DP U_binding = (3.0_DP * Mtot) / (5.0_DP * Rp) ! LS12 eq. 27 + Qmerge = Gc * m1 * m2 / (.mag.(rh2 - rh1)) + 3*m1 / (5*rad1) + 3*m2 / (5*rad1) - U_binding ! Change in energy due to a pure merger + if ((m1 < min_mfrag).or.(m2 < min_mfrag)) then regime = COLLRESOLVE_REGIME_MERGE !perfect merging regime Mlr = Mtot diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index a745f7b3e..21dacc918 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -543,7 +543,6 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) call collision_history%take_snapshot(param,nbody_system, t, "before") call nbody_system%get_energy_and_momentum(param) - collider%pe(1) = nbody_system%pe call collider%generate(nbody_system, param, t) @@ -576,7 +575,7 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) if (.not.lplpl_collision) exit if (loop == MAXCASCADE) then write(*,*) - write(*,*) "An runaway collisional cascade has been detected in collision_resolve_plpl." + write(*,*) "A runaway collisional cascade has been detected in collision_resolve_plpl." write(*,*) "Consider reducing the step size or changing the parameters in the collisional model to reduce the number of fragments." call util_exit(FAILURE) end if diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index beb883abe..019768166 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -312,6 +312,7 @@ module swiftest class(swiftest_cb), allocatable :: cb !! Central body data structure class(swiftest_pl), allocatable :: pl !! Massive body data structure class(swiftest_tp), allocatable :: tp !! Test particle data structure + class(swiftest_tp), dimension(:),codimension[:], allocatable :: cotp !! Co-array test particle data structure class(swiftest_tp), allocatable :: tp_discards !! Discarded test particle data structure class(swiftest_pl), allocatable :: pl_discards !! Discarded massive body particle data structure