Added infrastructure for coarray support

cmake/Modules/FindCoarray_Fortran.cmake

@@ -0,0 +1,90 @@
+# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+
+# - Finds Coarray support
+# This module can be used to detect Coarray support in a compiler.
+# If the compiler supports Coarray, the flags required to compile with
+# coarray support are set.  
+#
+# This module was modified from the standard FindOpenMP module to find Fortran
+# flags.
+#
+# The following variables are set:
+#   Coarray_Fortran_FLAGS - flags to add to the Fortran compiler for Coarray
+#                          support.  In general, you must use these at both
+#                          compile- and link-time.
+#   OMP_NUM_PROCS - the max number of processors available to Coarray
+
+#=============================================================================
+
+INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
+
+
+SET (Coarray_Fortran_FLAG_CANDIDATES
+     #Intel
+     "-coarray=distributed"
+     #Intel windows
+     "/Qcoarray:distributed" 
+     #Gnu
+     "-fcorray=lib -lcaf_mpi"
+     #Empty, if compiler automatically accepts coarray
+     " "
+)
+
+IF (DEFINED Coarray_Fortran_FLAGS)
+    SET (Coarray_Fortran_FLAG_CANDIDATES)
+ENDIF (DEFINED Coarray_Fortran_FLAGS)
+
+# check fortran compiler. also determine number of processors
+FOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES})
+    SET (SAFE_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
+    SET (CMAKE_REQUIRED_FLAGS "${FLAG}")
+    UNSET (Coarray_FLAG_DETECTED CACHE)
+    MESSAGE (STATUS "Try Coarray Fortran flag = [${FLAG}]")
+    FILE (WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90" 
+"
+program TestCoarray
+ integer, codimension[*] :: i
+ write(*,'(I2)',ADVANCE='NO') num_images()
+end program TestCoarray
+")
+    SET (MACRO_CHECK_FUNCTION_DEFINITIONS
+         "-DCoarray_FLAG_DETECTED ${CMAKE_REQUIRED_FLAGS}")
+    TRY_RUN (Coarray_RUN_FAILED Coarray_FLAG_DETECTED ${CMAKE_BINARY_DIR}
+        ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90
+        COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
+        CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
+        COMPILE_OUTPUT_VARIABLE OUTPUT
+        RUN_OUTPUT_VARIABLE OMP_NUM_PROCS_INTERNAL)
+    IF (Coarray_FLAG_DETECTED)
+        FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
+             "Determining if the Fortran compiler supports Coarray passed with "
+             "the following output:\n${OUTPUT}\n\n")
+        SET (Coarray_FLAG_DETECTED 1)
+        IF (Coarray_RUN_FAILED)
+            MESSAGE (FATAL_ERROR "Coarray found, but test code did not run")
+        ENDIF (Coarray_RUN_FAILED)
+        SET (Coarray_Fortran_FLAGS_INTERNAL "${FLAG}")
+        BREAK ()
+    ELSE ()
+        FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
+             "Determining if the Fortran compiler supports Coarray failed with "
+             "the following output:\n${OUTPUT}\n\n")
+        SET (Coarray_FLAG_DETECTED 0)
+    ENDIF (Coarray_FLAG_DETECTED)
+ENDFOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES})
+
+SET (Coarray_Fortran_FLAGS "${Coarray_Fortran_FLAGS_INTERNAL}"
+     CACHE STRING "Fortran compiler flags for Coarray parallization")
+
+# handle the standard arguments for FIND_PACKAGE
+FIND_PACKAGE_HANDLE_STANDARD_ARGS (Coarray_Fortran DEFAULT_MSG 
+    Coarray_Fortran_FLAGS)
+
+MARK_AS_ADVANCED(Coarray_Fortran_FLAGS)

cmake/Modules/SetFortranFlags.cmake

@@ -79,9 +79,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
                  Fortran "-xhost"        # Intel
                          "/QxHost"       # Intel Windows
                          ${GNUNATIVE}    # GNU
-                         "-ta=host"      # Portland Group
                 )
 
+
 ###################
 ### DEBUG FLAGS ###
 ###################
@@ -99,12 +99,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
                  Fortran "-warn all" # Intel
                          "/warn:all" # Intel Windows
                          "-Wall"     # GNU
-                                     # Portland Group (on by default)
                 )
 
 # Traceback
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
-                 Fortran "-traceback"   # Intel/Portland Group
+                 Fortran "-traceback"   # Intel Group
                          "/traceback"   # Intel Windows
                          "-fbacktrace"  # GNU (gfortran)
                          "-ftrace=full" # GNU (g95)
@@ -116,7 +115,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
                          "/check:bounds"  # Intel Windows
                          "-fcheck=bounds" # GNU (New style)
                          "-fbounds-check" # GNU (Old style)
-                         "-Mbounds"       # Portland Group
                 )
 
 # Initializes matrices/arrays with NaN values
@@ -225,15 +223,13 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
                  Fortran "-unroll"        # Intel
                          "/unroll"        # Intel Windows
                          "-funroll-loops" # GNU
-                         "-Munroll"       # Portland Group
                 )
 
 # Inline functions
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
                  Fortran "-inline"            # Intel
                          "/Qinline"           # Intel Windows
                          "-finline-functions" # GNU
-                         "-Minline"           # Portland Group
                 )
 
 

cmake/Modules/SetParallelizationLibrary.cmake

@@ -27,6 +27,10 @@ IF (USE_MPI)
     # Find MPI
     IF (NOT MPI_Fortran_FOUND)
         FIND_PACKAGE (MPI REQUIRED)
+        FIND_PACKAGE (Coarray_Fortran)
+        IF (NOT Coarray_Fortran_FLAGS)
+            MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays")
+        ENDIF (NOT Coarray_Fortran_FLAGS)
     ENDIF (NOT MPI_Fortran_FOUND)
 ENDIF (USE_MPI)
 
@@ -35,6 +39,7 @@ IF (NOT USE_OPENMP AND NOT USE_MPI)
     SET (OMP_NUM_PROCS 0 CACHE
          STRING "Number of processors OpenMP may use" FORCE)
     UNSET (OpenMP_Fortran_FLAGS CACHE)
+    UNSET (Coarray_Fortran_FLAGS CACHE)
     UNSET (GOMP_Fortran_LINK_FLAGS CACHE)
     UNSET (MPI_FOUND CACHE)
     UNSET (MPI_COMPILER CACHE)

src/CMakeLists.txt

@@ -100,13 +100,15 @@ IF(USE_OPENMP)
     SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
                           COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}"
                           LINK_FLAGS "${OpenMP_Fortran_FLAGS}")
-ELSEIF(USE_MPI)
+ENDIF(USE_OPENMP)
+
+IF(USE_MPI)
     SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
-                          COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS}"
-                          LINK_FLAGS "${MPI_Fortran_LINK_FLAGS}")
+                          COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS} ${Coarray_Fortran_FLAGS}"
+                          LINK_FLAGS "${MPI_Fortran_LINK_FLAGS} ${Coarray_Fortran_FLAGS}")
     INCLUDE_DIRECTORIES(${MPI_Fortran_INCLUDE_PATH})
     TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${MPI_Fortran_LIBRARIES})
-ENDIF(USE_OPENMP)
+ENDIF(USE_MPI)
 
 
 #####################################

src/collision/collision_regime.f90

@@ -150,7 +150,7 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2,
       real(DP), parameter   :: SUPERCAT_QRATIO = 1.8_DP ! See Section 4.1 of LS12
       ! Internals
       real(DP)           :: a1, alpha, aint, b, bcrit, c_star, egy, zeta, l, lint, mu, phi, theta
-      real(DP)           :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat
+      real(DP)           :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat, Qmerge
       real(DP)           :: Vhr, Verosion, Vescp, Vhill, Vimp, Vsupercat
       real(DP)           :: Mint, Mtot, Mtmp
       real(DP)           :: Rp, rhill 
@@ -215,6 +215,8 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2,
       Qloss = 0.0_DP
       U_binding = (3.0_DP * Mtot) / (5.0_DP * Rp) ! LS12 eq. 27
 
+      Qmerge = Gc * m1 * m2 / (.mag.(rh2 - rh1)) + 3*m1 / (5*rad1) + 3*m2 / (5*rad1) - U_binding ! Change in energy due to a pure merger
+
       if ((m1 < min_mfrag).or.(m2 < min_mfrag)) then 
          regime = COLLRESOLVE_REGIME_MERGE !perfect merging regime
          Mlr = Mtot

src/collision/collision_resolve.f90

@@ -543,7 +543,6 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec)
                      call collision_history%take_snapshot(param,nbody_system, t, "before") 
 
                      call nbody_system%get_energy_and_momentum(param)
-                     collider%pe(1)  = nbody_system%pe
 
                      call collider%generate(nbody_system, param, t)
 
@@ -576,7 +575,7 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec)
                   if (.not.lplpl_collision) exit
                   if (loop == MAXCASCADE) then
                      write(*,*) 
-                     write(*,*) "An runaway collisional cascade has been detected in collision_resolve_plpl."
+                     write(*,*) "A runaway collisional cascade has been detected in collision_resolve_plpl."
                      write(*,*) "Consider reducing the step size or changing the parameters in the collisional model to reduce the number of fragments."
                      call util_exit(FAILURE)
                   end if

src/swiftest/swiftest_module.f90

@@ -312,6 +312,7 @@ module swiftest
       class(swiftest_cb),                  allocatable :: cb                !! Central body data structure
       class(swiftest_pl),                  allocatable :: pl                !! Massive body data structure
       class(swiftest_tp),                  allocatable :: tp                !! Test particle data structure
+      class(swiftest_tp), dimension(:),codimension[:], allocatable :: cotp  !! Co-array test particle data structure
 
       class(swiftest_tp),                  allocatable :: tp_discards       !! Discarded test particle data structure
       class(swiftest_pl),                  allocatable :: pl_discards       !! Discarded massive body particle data structure