Merge branch 'debug' of github.itap.purdue.edu:MintonGroup/swiftest i…

…nto debug
MintonGroup · Jun 12, 2023 · 381b20a · 381b20a
2 parents 2da4a93 + 9a60cee
commit 381b20a
Show file tree

Hide file tree

Showing 37 changed files with 682 additions and 604 deletions.
diff --git a/.gitignore b/.gitignore
@@ -34,7 +34,7 @@ dump*
 !docker/
 !singularity/
 !Dockerfile
-!swiftest.def
+!environment.yml
 
 bin/
 build/*

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -34,13 +34,6 @@ OPTION(USE_SIMD "Use SIMD vectorization" ON)
 OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF)
 OPTION(BUILD_SHARED_LIBS "Build using shared libraries" ON)
 
-# Locate and set external libraries.  There are some CMake peculiarities
-# taken care of here, such as the fact that the FindOpenMP routine doesn't know
-# about Fortran.
-IF (NOT BUILD_SHARED_LIBS)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
-ENDIF ()
-
 INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake)
@@ -51,7 +44,6 @@ INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake)
 
 INCLUDE_DIRECTORIES($ENV{NETCDF_FORTRAN_HOME}/include;$ENV{NETCDF_HOME}/include)
 
-
 # There is an error in CMAKE with this flag for pgf90.  Unset it
 GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME)
 IF(FCNAME STREQUAL "pgf90")

diff --git a/Dockerfile b/Dockerfile
diff --git a/apptainer/.gitignore b/apptainer/.gitignore
@@ -0,0 +1,4 @@
+!bin/
+!bin/swiftest_driver
+!bin/swiftest
+!install.sh
diff --git a/apptainer/bin/swiftest b/apptainer/bin/swiftest
@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+apptainer run --bind $(pwd):$(pwd) --cleanenv --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@"
diff --git a/apptainer/bin/swiftest_driver b/apptainer/bin/swiftest_driver
@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+apptainer exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@"
diff --git a/apptainer/install.sh b/apptainer/install.sh
@@ -0,0 +1,21 @@
+#!/bin/sh --
+# This will install the Apptainer version of the swiftest_driver in place of the native compiled version into ../bin as 
+# well as the swiftest_python script that is used to execute a Python input file.
+# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.apptainer. 
+# To change this, just set environment variable SIF_DIR prior to running this script.
+# 
+# The script takes an optional argument "tag" if you want to pull a container other than "latest".
+# 
+# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, 
+# which requires this script to be called via source:
+# $ source ./install.sh
+# or 
+# $ . ./install.sh
+TAG=${1:-latest}
+
+SIF_DIR=${SIF_DIR:-${HOME}/.apptainer}
+echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container"
+apptainer pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG}
+cp -rf bin/swiftest ../bin/
+cp -rf bin/swiftest_driver ../bin/
+export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif
diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake
@@ -46,6 +46,16 @@ ELSE()
     MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE not valid! ${BUILD_TYPE_MSG}")
 ENDIF(BT STREQUAL "RELEASE")
 
+IF (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+    IF (APPLE)
+        SET(MACHINE_CODE_VALUE "tune=native" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.")
+    ELSE ()
+        SET(MACHINE_CODE_VALUE "arch=native" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.")
+    ENDIF ()
+ELSE ()
+    SET(MACHINE_CODE_VALUE "host" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.")
+ENDIF ()
+
 #########################################################
 # If the compiler flags have already been set, return now
 #########################################################
@@ -81,20 +91,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
                 Fortran "-fsignaling-nans " # GNU
                 ) 
 
-
-# Determines whether the current Fortran Standard behavior of the compiler is fully implemented. 
-SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}"
-                Fortran "-standard-semantics" # Intel
-                        "/standard-semantics" # Intel Windows
-                )
-
-# Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018).                
-SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}"
-                Fortran "-stand f18"  # Intel
-                        "/stand:f18"  # Intel Windows
-                        "-fstd=f2018" # GNU
-                )  
-
 # Allows for lines longer than 80 characters without truncation
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
                 Fortran "-ffree-line-length-none" # GNU (gfortran)
@@ -119,35 +115,22 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
 
                 )
 
-IF (CONTAINERIZE)
-        # There is some bug where -march=native doesn't work on Mac
-        IF(APPLE)
-                SET(GNUNATIVE "-mtune=generic")
-        ELSE()
-                SET(GNUNATIVE "-march=generic")
-        ENDIF()
-
+IF (NOT BUILD_SHARED_LIBS)
         # Use static Intel libraries
         SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}"
                 Fortran "-static-intel"  # Intel
         )
+        # Use static Intel MPI libraries
+        SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}"
+                Fortran "-static_mpi"  # Intel
+        )
 
         IF (USE_OPENMP)
                 SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}"
                         Fortran "-qopenmp-link=static"  # Intel
                 )
         ENDIF (USE_OPENMP)
-
-ELSE ()
-        # There is some bug where -march=native doesn't work on Mac
-        IF(APPLE)
-                SET(GNUNATIVE "-mtune=native")
-        ELSE()
-                SET(GNUNATIVE "-march=native")
-        ENDIF()
-ENDIF (CONTAINERIZE)
-
-
+ENDIF ()
 
 IF (USE_SIMD)
         # Enables OpenMP SIMD compilation when OpenMP parallelization is disabled. 
@@ -158,21 +141,12 @@ IF (USE_SIMD)
                         )     
         ENDIF (NOT USE_OPENMP)
 
-        IF (CONTAINERIZE)
-                # Optimize for an old enough processor that it should run on most computers
-                SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
-                                Fortran "-xSSE2" # Intel
-                                        "/QxSSE2" # Intel Windows
-                                        ${GNUNATIVE}    # GNU
-                                )
-        ELSE ()
-                # Optimize for the host's architecture
-                SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
-                                Fortran "-xhost"        # Intel
-                                        "/QxHost"       # Intel Windows
-                                        ${GNUNATIVE}    # GNU
-                                )
-        ENDIF (CONTAINERIZE)
+        # Optimize for an old enough processor that it should run on most computers
+        SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
+                        Fortran "-x${MACHINE_CODE_VALUE}" # Intel
+                                "/Qx${MACHINE_CODE_VALUE}" # Intel Windows
+                                "-m${MACHINE_CODE_VALUE}"    # GNU
+                        )
 
         # Generate an extended set of vector functions
         SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
@@ -215,6 +189,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
                  Fortran "-Wno-unused-dummy-argument" # GNU
                 )
 
+# Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018).                
+SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"
+                Fortran "-stand f18"  # Intel
+                        "/stand:f18"  # Intel Windows
+                        "-fstd=f2018" # GNU
+                )  
 
 # Traceback
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}"

diff --git a/docker/.gitignore b/docker/.gitignore
@@ -3,3 +3,4 @@
 !install.sh
 !bin
 !bin/swiftest
+!bin/swiftest_driver
diff --git a/docker/bin/swiftest b/docker/bin/swiftest
@@ -1,2 +1,4 @@
 #!/bin/sh --
-docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0  "$@"
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES mintongroup/swiftest "$@"
diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver
@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES MintonGroup/swiftest  "$@"
diff --git a/docker/install.sh b/docker/install.sh
@@ -1,5 +1,6 @@
 #!/bin/sh --
 tag=${1:-latest}
 echo "Installing swiftest:${tag} Docker container and executable script"
-docker pull daminton/swiftest:${tag}
-cp -rf bin/swiftest ../bin/
+docker pull mintongroup/swiftest:${tag}
+cp -rf bin/swiftest ../bin/
+cp -rf bin/swiftest_driver ../bin/
diff --git a/environment.yml b/environment.yml
@@ -0,0 +1,22 @@
+name: swiftest-env
+
+channels:
+  - conda-forge
+  - defaults
+
+dependencies:
+  - numpy
+  - scipy
+  - matplotlib
+  - pandas
+  - xarray
+  - h5netcdf
+  - netcdf4
+  - dask
+  - bottleneck
+  - astropy
+  - astroquery
+  - tqdm
+  - x264
+  - ffmpeg
+  - conda-build
diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py
@@ -470,6 +470,9 @@ def _type_scrub(output_data):
                     sys.exit()
         except:
             warnings.warn(f"Error executing main swiftest_driver program", stacklevel=2)
+            res = p.communicate()
+            for line in res[1]:
+                print(line, end='')
             sys.exit()
 
         pbar.close()
@@ -502,11 +505,9 @@ def run(self,dask: bool = False, **kwargs):
 
         if not self.binary_source.exists():
             msg = "Path to swiftest_driver has not been set!"
-            msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}"
+            msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}"
             warnings.warn(msg,stacklevel=2)
             return
-        else:
-            shutil.copy2(self.binary_source, self.driver_executable) 
 
         if not self.restart:
             self.clean()
@@ -942,16 +943,15 @@ def set_integrator(self,
             update_list.append("codename")
             if self.codename == "Swiftest":
                 self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver"
-                self.binary_path = self.simdir.resolve()
-                self.driver_executable = self.binary_path / "swiftest_driver"
+                self.driver_executable = self.binary_source
                 if not self.binary_source.exists():
-                    warnings.warn(f"Cannot find the Swiftest driver in {str(self.binary_path)}",stacklevel=2)
+                    warnings.warn(f"Cannot find the Swiftest driver at {str(self.binary_source)}",stacklevel=2)
                     self.driver_executable = None
                 else:
-                    if self.binary_path.exists():
+                    if self.binary_source.exists():
                         self.driver_executable.resolve()
             else:
-                self.binary_path = "NOT IMPLEMENTED FOR THIS CODE"
+                self.binary_source = "NOT IMPLEMENTED FOR THIS CODE"
                 self.driver_executable = None
             update_list.append("driver_executable")
 
@@ -1200,8 +1200,6 @@ def set_feature(self,
                     msg = f"Cannot create the {self.simdir.resolve()} directory: File exists."
                     msg += "\nDelete the file or change the location of param_file"
                     raise NotADirectoryError(msg)
-            self.binary_path = self.simdir.resolve()
-            self.driver_executable = self.binary_path / "swiftest_driver"
             self.param_file = Path(kwargs.pop("param_file","param.in"))
 
         if self.codename == "Swiftest": 
@@ -2754,7 +2752,6 @@ def write_param(self,
         self.driver_script = os.path.join(self.simdir, "swiftest_driver.sh")
         with open(self.driver_script, 'w') as f:
             f.write(f"#{self._shell_full}\n")
-            #f.write(f"source ~/.{self._shell}rc\n")
             f.write(f"cd {self.simdir}\n")
             f.write(f"{str(self.driver_executable)} {self.integrator} {str(self.param_file)} compact\n")
 
@@ -2991,11 +2988,9 @@ def save(self,
             self.write_param(param_file=param_file,**kwargs)
             if not self.binary_source.exists():
                 msg = "Path to swiftest_driver has not been set!"
-                msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}"
+                msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}"
                 warnings.warn(msg,stacklevel=2)
                 return
-            else:
-                shutil.copy2(self.binary_source, self.driver_executable)  
         elif codename == "Swifter":
             swifter_param = io.swiftest2swifter_param(param)
             if "rhill" in self.data:

diff --git a/singularity/.gitignore b/singularity/.gitignore
@@ -1,4 +1,4 @@
 !bin/
 !bin/swiftest_driver
+!bin/swiftest
 !install.sh
-!setenv.sh
diff --git a/singularity/bin/swiftest b/singularity/bin/swiftest
@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+singularity run --bind $(pwd):$(pwd) --cleanenv --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@"
diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver
@@ -1,2 +1,4 @@
 #!/bin/sh --
-singularity run --bind $(pwd):$(pwd)  --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@"
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+singularity exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@"
diff --git a/singularity/install.sh b/singularity/install.sh
@@ -1,10 +1,21 @@
 #!/bin/sh --
-# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. 
-# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source:
-# $ . ./install.sh
+# This will install the Singularity version of the swiftest_driver in place of the native compiled version into ../bin as 
+# well as the swiftest_python script that is used to execute a Python input file.
+# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.singularity. 
+# To change this, just set environment variable SIF_DIR prior to running this script.
+# 
+# The script takes an optional argument "tag" if you want to pull a container other than "latest".
 # 
-tag=${1:-latest}
-echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container"
-singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag}
+# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, 
+# which requires this script to be called via source:
+# $ source ./install.sh
+# or 
+# $ . ./install.sh
+TAG=${1:-latest}
+
+SIF_DIR=${SIF_DIR:-${HOME}/.singularity}
+echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container"
+singularity pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG}
+cp -rf bin/swiftest ../bin/
 cp -rf bin/swiftest_driver ../bin/
-source ./setenv.sh
+export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif
diff --git a/singularity/setenv.sh b/singularity/setenv.sh
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -108,7 +108,7 @@ SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON)
 # Add the needed libraries and special compiler flags
 #####################################################
 
-TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES})
+TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES} $ENV{LIBS})
 
 IF(USE_OPENMP)
     SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS  "${OpenMP_Fortran_FLAGS} ")