Skip to content
This repository was archived by the owner on Aug 28, 2024. It is now read-only.

Commit

Permalink
Merge branch 'coarray_test_particles' into debug
Browse files Browse the repository at this point in the history
  • Loading branch information
daminton committed May 11, 2023
2 parents d7e4d14 + 704ce2d commit df48d5e
Show file tree
Hide file tree
Showing 39 changed files with 5,329 additions and 2,808 deletions.
6 changes: 4 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ IF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)

# Set some options the user may choose
# Uncomment the below if you want the user to choose a parallelization library
OPTION(USE_MPI "Use the MPI library for parallelization" ON)
OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF)
OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)

IF (USE_COARRAY)
ADD_DEFINITIONS(-DCOARRAY)
ENDIF()

# Locate and set parallelization libraries. There are some CMake peculiarities
# taken care of here, such as the fact that the FindOpenMP routine doesn't know
Expand Down
104 changes: 104 additions & 0 deletions cmake/Modules/FindCoarray_Fortran.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
# This file is part of Swiftest.
# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along with Swiftest.
# If not, see: https://www.gnu.org/licenses.

# - Finds Coarray support
# This module can be used to detect Coarray support in a compiler.
# If the compiler supports Coarray, the flags required to compile with
# coarray support are set.
#
# This module was modified from the standard FindOpenMP module to find Fortran
# flags.
#
# The following variables are set:
# Coarray_Fortran_FLAGS - flags to add to the Fortran compiler for Coarray
# support. In general, you must use these at both
# compile- and link-time.
# OMP_NUM_PROCS - the max number of processors available to Coarray

#=============================================================================

INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)

STRING(TOUPPER "${CMAKE_BUILD_TYPE}" BT)
IF(BT STREQUAL "DEBUG")
SET (Coarray_Fortran_FLAG_CANDIDATES
#Intel
"-coarray=single"
#Intel windows
"/Qcoarray:single"
#Gnu
"-fcoarray=single"
#Empty, if compiler automatically accepts coarray
" "
)
ELSE()
SET (Coarray_Fortran_FLAG_CANDIDATES
#Intel
"-coarray=distributed"
#Intel windows
"/Qcoarray:distributed"
#Gnu
"-fcoarray=lib -lcaf_mpi"
#Empty, if compiler automatically accepts coarray
" "
)
ENDIF()


IF (DEFINED Coarray_Fortran_FLAGS)
SET (Coarray_Fortran_FLAG_CANDIDATES)
ENDIF (DEFINED Coarray_Fortran_FLAGS)

# check fortran compiler. also determine number of processors
FOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES})
SET (SAFE_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
SET (CMAKE_REQUIRED_FLAGS "${FLAG}")
UNSET (Coarray_FLAG_DETECTED CACHE)
MESSAGE (STATUS "Try Coarray Fortran flag = [${FLAG}]")
FILE (WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90"
"
program TestCoarray
integer, codimension[*] :: i
write(*,'(I2)',ADVANCE='NO') num_images()
end program TestCoarray
")
SET (MACRO_CHECK_FUNCTION_DEFINITIONS
"-DCoarray_FLAG_DETECTED ${CMAKE_REQUIRED_FLAGS}")
TRY_RUN (Coarray_RUN_FAILED Coarray_FLAG_DETECTED ${CMAKE_BINARY_DIR}
${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90
COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS}
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
COMPILE_OUTPUT_VARIABLE OUTPUT
RUN_OUTPUT_VARIABLE OMP_NUM_PROCS_INTERNAL)
IF (Coarray_FLAG_DETECTED)
FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Determining if the Fortran compiler supports Coarray passed with "
"the following output:\n${OUTPUT}\n\n")
SET (Coarray_FLAG_DETECTED 1)
IF (Coarray_RUN_FAILED)
MESSAGE (FATAL_ERROR "Coarray found, but test code did not run")
ENDIF (Coarray_RUN_FAILED)
SET (Coarray_Fortran_FLAGS_INTERNAL "${FLAG}")
BREAK ()
ELSE ()
FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Determining if the Fortran compiler supports Coarray failed with "
"the following output:\n${OUTPUT}\n\n")
SET (Coarray_FLAG_DETECTED 0)
ENDIF (Coarray_FLAG_DETECTED)
ENDFOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES})

SET (Coarray_Fortran_FLAGS "${Coarray_Fortran_FLAGS_INTERNAL}"
CACHE STRING "Fortran compiler flags for Coarray parallization")

# handle the standard arguments for FIND_PACKAGE
FIND_PACKAGE_HANDLE_STANDARD_ARGS (Coarray_Fortran DEFAULT_MSG
Coarray_Fortran_FLAGS)

MARK_AS_ADVANCED(Coarray_Fortran_FLAGS)
13 changes: 5 additions & 8 deletions cmake/Modules/SetFortranFlags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
Fortran "-xhost" # Intel
"/QxHost" # Intel Windows
${GNUNATIVE} # GNU
"-ta=host" # Portland Group
)


###################
### DEBUG FLAGS ###
###################
Expand All @@ -99,12 +99,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
Fortran "-warn all" # Intel
"/warn:all" # Intel Windows
"-Wall" # GNU
# Portland Group (on by default)
)

# Traceback
SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
Fortran "-traceback" # Intel/Portland Group
Fortran "-traceback" # Intel Group
"/traceback" # Intel Windows
"-fbacktrace" # GNU (gfortran)
"-ftrace=full" # GNU (g95)
Expand Down Expand Up @@ -233,15 +232,13 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
Fortran "-unroll" # Intel
"/unroll" # Intel Windows
"-funroll-loops" # GNU
"-Munroll" # Portland Group
)

# Inline functions
SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
Fortran "-inline" # Intel
"/Qinline" # Intel Windows
"-finline-functions" # GNU
"-Minline" # Portland Group
)


Expand Down Expand Up @@ -323,9 +320,9 @@ SET_COMPILE_FLAG(FASTMATH_FLAGS "${FASTMATH_FLAGS}"
#####################
# Enables the optimization reports to be generated
SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_RELEASE}"
Fortran "-pg -qopt-report=5 -traceback -p -g3" # Intel
"/Qopt-report:5 /traceback -g3" # Windows Intel
"-pg -fbacktrace"
Fortran "-O2 -pg -qopt-report=5 -traceback -p -g3" # Intel
"/O2 /Qopt-report:5 /traceback -g3" # Windows Intel
"-O2 -pg -fbacktrace"
)

# Sanitize
Expand Down
30 changes: 12 additions & 18 deletions cmake/Modules/SetParallelizationLibrary.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,6 @@
# You should have received a copy of the GNU General Public License along with Swiftest.
# If not, see: https://www.gnu.org/licenses.

# Turns on either OpenMP or MPI
# If both are requested, the other is disabled
# When one is turned on, the other is turned off
# If both are off, we explicitly disable them just in case

IF (USE_OPENMP)
# Find OpenMP
IF (NOT OpenMP_Fortran_FLAGS)
Expand All @@ -20,23 +15,22 @@ IF (USE_OPENMP)
MESSAGE (FATAL_ERROR "Fortran compiler does not support OpenMP")
ENDIF (NOT OpenMP_Fortran_FLAGS)
ENDIF (NOT OpenMP_Fortran_FLAGS)
# Turn of MPI
ENDIF (USE_OPENMP)

IF (USE_MPI)
# Find MPI
IF (NOT MPI_Fortran_FOUND)
FIND_PACKAGE (MPI REQUIRED)
ENDIF (NOT MPI_Fortran_FOUND)
ENDIF (USE_MPI)
IF (USE_COARRAY)
IF (NOT Coarray_Fortran_FLAGS)
FIND_PACKAGE (Coarray_Fortran)
IF (NOT Coarray_Fortran_FLAGS)
MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays")
ENDIF (NOT Coarray_Fortran_FLAGS)
ENDIF (NOT Coarray_Fortran_FLAGS)
ENDIF (USE_COARRAY)

IF (NOT USE_OPENMP AND NOT USE_MPI)
# Turn off both OpenMP and MPI
IF (NOT USE_OPENMP AND NOT USE_COARRAY)
# Turn off both OpenMP and CAF
SET (OMP_NUM_PROCS 0 CACHE
STRING "Number of processors OpenMP may use" FORCE)
UNSET (OpenMP_Fortran_FLAGS CACHE)
UNSET (Coarray_Fortran_FLAGS CACHE)
UNSET (GOMP_Fortran_LINK_FLAGS CACHE)
UNSET (MPI_FOUND CACHE)
UNSET (MPI_COMPILER CACHE)
UNSET (MPI_LIBRARY CACHE)
ENDIF (NOT USE_OPENMP AND NOT USE_MPI)
ENDIF (NOT USE_OPENMP AND NOT USE_COARRAY)
8 changes: 5 additions & 3 deletions python/swiftest/swiftest/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
"ENCOUNTER_CHECK",
"TSTART",
"DUMP_CADENCE",
"ENCOUNTER_SAVE")
"ENCOUNTER_SAVE",
"COARRAY")

# This list defines features that are booleans, so must be converted to/from string when writing/reading from file
bool_param = ["RESTART",
Expand All @@ -46,7 +47,8 @@
"ENERGY",
"GR",
"YARKOVSKY",
"YORP"]
"YORP",
"COARRAY"]

int_param = ["ISTEP_OUT", "DUMP_CADENCE"]
float_param = ["T0", "TSTART", "TSTOP", "DT", "CHK_RMIN", "CHK_RMAX", "CHK_EJECT", "CHK_QMIN", "DU2M", "MU2KG",
Expand All @@ -59,7 +61,7 @@

# This defines Xarray Dataset variables that are strings, which must be processed due to quirks in how NetCDF-Fortran
# handles strings differently than Python's Xarray.
string_varnames = ["name", "particle_type", "status", "origin_type", "stage", "regime"]
string_varnames = ["name", "particle_type", "origin_type", "stage", "regime"]
char_varnames = ["space"]
int_varnames = ["id", "ntp", "npl", "nplm", "discard_body_id", "collision_id", "status"]

Expand Down
20 changes: 20 additions & 0 deletions python/swiftest/swiftest/simulation_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,11 @@ def __init__(self,read_param: bool = False,
Parameter input file equivalent: `ENCOUNTER_CHECK`
dask : bool, default False
Use Dask to lazily load data (useful for very large datasets)
coarray : bool, default False
If true, will employ Coarrays on test particle structures to run in single program/multiple data parallel mode.
*Note" In order to use this capability, Swiftest must be compiled for Coarray support. Only certain integrators
can use Coarrays: RMVS, WHM, Helio are all compatible, but SyMBA is not, due to the way tp-pl close encounters
are handeled.
verbose : bool, default True
If set to True, then more information is printed by Simulation methods as they are executed. Setting to
False suppresses most messages other than errors.
Expand Down Expand Up @@ -824,6 +829,7 @@ def set_parameter(self, verbose: bool = True, **kwargs):
"ephemeris_date": "MBCL",
"restart": False,
"encounter_save" : "NONE",
"coarray" : False,
"simdir" : self.simdir
}
param_file = kwargs.pop("param_file",None)
Expand Down Expand Up @@ -1065,6 +1071,7 @@ def set_feature(self,
interaction_loops: Literal["TRIANGULAR", "FLAT"] | None = None,
encounter_check_loops: Literal["TRIANGULAR", "SORTSWEEP"] | None = None,
encounter_save: Literal["NONE", "TRAJECTORY", "CLOSEST", "BOTH"] | None = None,
coarray: bool | None = None,
verbose: bool | None = None,
simdir: str | os.PathLike = None,
**kwargs: Any
Expand Down Expand Up @@ -1130,6 +1137,11 @@ def set_feature(self,
* "SORTSWEEP" : A Sort-Sweep algorithm is used to reduce the population of potential close encounter bodies.
This algorithm is still in development, and does not necessarily speed up the encounter checking.
Use with caution.
coarray : bool, default False
If true, will employ Coarrays on test particle structures to run in single program/multiple data parallel mode.
*Note" In order to use this capability, Swiftest must be compiled for Coarray support. Only certain integrators
can use Coarrays: RMVS, WHM, Helio are all compatible, but SyMBA is not, due to the way tp-pl close encounters
are handeled.
tides: bool, optional
Turns on tidal model (IN DEVELOPMENT - IGNORED)
Yarkovsky: bool, optional
Expand Down Expand Up @@ -1279,7 +1291,14 @@ def set_feature(self,
self.driver_executable = self.binary_path / "swiftest_driver"
self.param_file = Path(kwargs.pop("param_file","param.in"))

if coarray is not None:
if self.codename == "Swiftest":
self.param["COARRAY"] = coarray
update_list.append("coarray")


self.param["TIDES"] = False


feature_dict = self.get_feature(update_list, verbose)
return feature_dict
Expand Down Expand Up @@ -1321,6 +1340,7 @@ def get_feature(self, arg_list: str | List[str] | None = None, verbose: bool | N
"big_discard": "BIG_DISCARD",
"interaction_loops": "INTERACTION_LOOPS",
"encounter_check_loops": "ENCOUNTER_CHECK",
"coarray" : "COARRAY",
"restart": "RESTART"
}

Expand Down
35 changes: 28 additions & 7 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,27 @@ SET(FAST_MATH_FILES
${SRC}/swiftest/swiftest_driver.f90
)

set(SWIFTEST_src ${FAST_MATH_FILES} ${STRICT_MATH_FILES})
SET(COARRAY_FILES
${SRC}/coarray/coarray_module.f90
${SRC}/coarray/coarray_clone.f90
${SRC}/coarray/coarray_collect.f90
${SRC}/swiftest/swiftest_coarray.f90
${SRC}/whm/whm_coarray.f90
${SRC}/rmvs/rmvs_coarray.f90
)

IF(USE_COARRAY)
set(SWIFTEST_src ${COARRAY_FILES} ${FAST_MATH_FILES} ${STRICT_MATH_FILES})
ELSE()
set(SWIFTEST_src ${FAST_MATH_FILES} ${STRICT_MATH_FILES})
ENDIF(USE_COARRAY)

# Define the executable in terms of the source files
ADD_EXECUTABLE(${SWIFTEST_DRIVER} ${SWIFTEST_src})

# Turn preprocessor on for all files
SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON)

#####################################################
# Add the needed libraries and special compiler flags
#####################################################
Expand All @@ -99,14 +115,14 @@ IF(USE_OPENMP)
SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}"
LINK_FLAGS "${OpenMP_Fortran_FLAGS}")
ELSEIF(USE_MPI)
SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS}"
LINK_FLAGS "${MPI_Fortran_LINK_FLAGS}")
INCLUDE_DIRECTORIES(${MPI_Fortran_INCLUDE_PATH})
TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${MPI_Fortran_LIBRARIES})
ENDIF(USE_OPENMP)

IF(USE_COARRAY)
SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
COMPILE_FLAGS "${Coarray_Fortran_FLAGS}"
LINK_FLAGS "${Coarray_Fortran_FLAGS}")
ENDIF(USE_COARRAY)


#####################################
# Tell how to install this executable
Expand All @@ -127,3 +143,8 @@ IF(BT STREQUAL "RELEASE" OR BT STREQUAL "PROFILE")
SET_PROPERTY(SOURCE ${FAST_MATH_FILES} APPEND_STRING PROPERTY COMPILE_FLAGS "${FASTMATH_FLAGS}")
ENDIF()

IF(BT STREQUAL "DEBUG")
ADD_DEFINITIONS(-DDEBUG)
ELSEIF(BT STREQUAL "PROFILE")
ADD_DEFINITIONS(-DPROFILE)
ENDIF()
Loading

0 comments on commit df48d5e

Please sign in to comment.