Merge branch 'master' of https://github.com/carlislewishard/swiftest

.gitignore

@@ -30,6 +30,13 @@ dump*
 !docs/*/*/*
 !README_figs/*
 
+#Docker and Singularity files
+!docker/
+!singularity/
+!Dockerfile
+!environment.yml
+
 bin/
 build/*
-disruption_headon/swiftest_driver.sh
+
+

CMakeLists.txt

@@ -30,14 +30,10 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
 # Set some options the user may choose
 OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF)
 OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)
+OPTION(USE_SIMD "Use SIMD vectorization" ON)
+OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF)
+OPTION(BUILD_SHARED_LIBS "Build using shared libraries" ON)
 
-IF (USE_COARRAY)
-  ADD_DEFINITIONS(-DCOARRAY)
-ENDIF()
-
-# Locate and set parallelization libraries.  There are some CMake peculiarities
-# taken care of here, such as the fact that the FindOpenMP routine doesn't know
-# about Fortran.
 INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake)
@@ -48,7 +44,6 @@ INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake)
 
 INCLUDE_DIRECTORIES($ENV{NETCDF_FORTRAN_HOME}/include;$ENV{NETCDF_HOME}/include)
 
-
 # There is an error in CMAKE with this flag for pgf90.  Unset it
 GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME)
 IF(FCNAME STREQUAL "pgf90")
@@ -78,3 +73,17 @@ ADD_SUBDIRECTORY(${SRC} ${BIN})
 ADD_CUSTOM_TARGET(distclean 
     COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/distclean.cmake
 )
+
+# Add an automatic test
+ADD_CUSTOM_TARGET(Run_Test ALL 
+    COMMAND echo "***Running Python script examples/Basic_Simulation/initial_conditions.py***" 
+    COMMAND python ${CMAKE_SOURCE_DIR}/examples/Basic_Simulation/initial_conditions.py
+    COMMAND echo "***Saving data to directory examples/Basic_Simulation/simdata***"
+    COMMAND echo "***Running Python script examples/Basic_Simulation/output_reader.py***"
+    COMMAND python ${CMAKE_SOURCE_DIR}/examples/Basic_Simulation/output_reader.py
+    COMMAND echo "***Plotting results as examples/Basic_Simulation/output.eps***"
+    COMMAND echo "***Calculating errors with Python script examples/Basic_Simulation/errors.py***"
+    COMMAND python ${CMAKE_SOURCE_DIR}/examples/Basic_Simulation/errors.py
+    COMMAND echo "***Test Complete***"
+)
+

Dockerfile

@@ -0,0 +1,158 @@
+# Copyright 2023 - David Minton
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+#
+# This Dockerfile will build the Swiftest driver program with minimal external dependencies using the Intel Oneapi toolkit. 
+# This is done by building static versions of a minimal set of libraries that NetCDF-Fortran needs (Netcdf-C, HDF5, and Zlib). 
+# These, along with the Intel runtime libraries, are linked statically to the executable. Only the OS-specific libraries are linked
+# dynamically. 
+
+# This build target compiles all dependencies and the swiftest driver itself
+FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build_deps
+
+ENV INSTALL_DIR="/usr/local"
+ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx"
+ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx"
+ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx"
+ENV F77="${FC}"
+
+# Get the HDF5, NetCDF-C, and NetCDF-Fortran libraries
+RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz | tar xvz && \
+    wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \
+    wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \
+    wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz
+
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  m4 && \
+  rm -rf /var/lib/apt/lists/* 
+
+RUN cd zlib-1.2.13 && \
+  ./configure --prefix=${INSTALL_DIR} --static && \
+  make && \
+  make install
+
+RUN cd hdf5-1.14.1-2 && \
+  ./configure --disable-shared \
+              --enable-build-mode=production \
+              --disable-fortran \
+              --disable-java \
+              --disable-cxx \
+              --prefix=${INSTALL_DIR} \
+              --with-zlib=${INSTALL_DIR} && \
+              make && \
+              make install
+
+RUN cd netcdf-c-4.9.2 && \
+  ./configure --disable-shared \
+              --disable-dap \
+              --disable-libxml2 \
+              --disable-byterange \
+              --prefix=${INSTALL_DIR} && \
+              make && \
+              make install
+
+ENV NCDIR="${INSTALL_DIR}"
+ENV NFDIR="${INSTALL_DIR}"
+ENV HDF5_ROOT="${INSTALL_DIR}"
+ENV HDF5_LIBDIR="${HDF5_ROOT}/lib"
+ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include"
+ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin"
+
+# NetCDF-Fortran library
+ENV CFLAGS="-fPIC"
+ENV FCFLAGS="${CFLAGS} -standard-semantics"
+ENV FFLAGS=${CFLAGS}
+ENV CPPFLAGS="-I${INSTALL_DIR}/include"
+ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lm -lz" 
+RUN cd netcdf-fortran-4.6.1 && \
+  ./configure --disable-shared --prefix=${NFDIR} && \
+  make && \
+  make install  
+
+FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build_driver
+COPY --from=build_deps /usr/local/. /usr/local/
+ENV INSTALL_DIR="/usr/local"
+ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx"
+ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx"
+ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx"
+ENV F77="${FC}"
+
+# The MACHINE_CODE_VALUE argument is a string that is used when compiling the swiftest_driver. It is appended to the "-x" compiler 
+# option: (-x${MACHINE_CODE_VALUE}). The default value is set to "sse2" which allows for certain SIMD instructions to be used while 
+# remaining # compatible with a wide range of CPUs. To get the highest performance, you can pass "host" as an argument, but the 
+# compiled binary # would only run on a CPU with an architecture compatible with the one that the build was performed on. 
+# For more details and other options, see:
+# https://www.intel.com/content/www/us/en/docs/fortran-compiler/developer-guide-reference/2023-1/x-qx.html
+ARG MACHINE_CODE_VALUE="sse2"
+
+# Build type options are DEBUG, RELEASE, PROFILE, or TESTING.
+ARG BUILD_TYPE="RELEASE"  
+
+# Swiftest
+ENV NETCDF_HOME=${INSTALL_DIR}
+ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME}
+ENV NETCDF_LIBRARY=${NETCDF_HOME}
+ENV FOR_COARRAY_NUM_IMAGES=1
+ENV OMP_NUM_THREADS=1
+ENV FC="${ONEAPI_ROOT}/mpi/latest/bin/mpiifort"
+ENV FFLAGS="-fPIC -standard-semantics"
+ENV LDFLAGS="-L/usr/local/lib"
+ENV LIBS="-lhdf5_hl -lhdf5 -lz"
+COPY ./cmake/ /swiftest/cmake/
+COPY ./src/ /swiftest/src/
+COPY ./CMakeLists.txt /swiftest/
+RUN cd swiftest && \
+  cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  -DMACHINE_CODE_VALUE=${MACHINE_CODE} \
+  -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+  -DUSE_COARRAY=OFF \
+  -DBUILD_SHARED_LIBS=OFF && \
+  cmake --build build && \
+  cmake --install build
+
+# This build target creates a container that executes just the driver program
+FROM ubuntu:20.04 as driver
+COPY --from=build_driver /usr/local/bin/swiftest_driver /usr/local/bin/
+ENTRYPOINT ["/usr/local/bin/swiftest_driver"]
+
+# This build target exports the binary to the host
+FROM scratch AS export_driver
+COPY --from=build_driver /usr/local/bin/swiftest_driver /
+
+# This build target creates a container with a conda environment with all dependencies needed to run the Python front end and 
+# analysis tools
+FROM continuumio/miniconda3 as python
+SHELL ["/bin/bash", "--login", "-c"]
+ENV SHELL="/bin/bash"
+ENV PATH="/opt/conda/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/usr/local/lib"
+
+RUN conda update --all -y && \
+  conda install conda-libmamba-solver -y && \
+  conda config --set solver libmamba 
+
+COPY environment.yml .
+RUN conda env create -f environment.yml && \
+  conda init bash && \
+  echo "conda activate swiftest-env" >> ~/.bashrc 
+
+COPY ./python/. /opt/conda/pkgs/
+COPY --from=build_driver /usr/local/bin/swiftest_driver /opt/conda/bin/swiftest_driver
+
+# Start new shell to activate the environment and install Swiftest
+RUN cd /opt/conda/pkgs/swiftest && conda develop . && \
+  conda clean --all -y && \
+  mkdir -p /.astropy && \
+  chmod -R 777 /.astropy && \
+  mkdir -p /.cache/matplotlib && \
+  mkdir -p /.config/matplotlib && \
+  chmod -R 777 /.cache/matplotlib && \
+  chmod -R 777 /.config/matplotlib
+
+ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "swiftest-env"]

README.md

@@ -17,7 +17,9 @@ Swiftest also includes the collisional fragmentation algorithm **Fraggle**, an a
 
 #### Installation
 
-**System Requirements**
+In order to use Swiftest, you need to have a working `swiftest_driver` executable. Currently, this can be obtained by either compiling the source code on the system you plan to run simulations on (fastest), or by running it from a Docker/Singularity container compiled for an x86_64 CPU using the Intel Fortran compiler (slower) or compiled using the GNU/gfortran compiler (slowest). 
+
+**Building the `swiftest_driver` executable**
 
 Swiftest is designed to be downloaded, compiled, and run on a Linux based system. It is untested on Windows systems.
 
@@ -44,7 +46,7 @@ Parallelization in Swiftest is done with OpenMP. Version 3.1.4 or higher is nece
 The easiest way to get Swiftest on your machine is to clone the GitHub repository. To do so, open a terminal window and type the following:
 
 ```
-$ git clone https://github.itap.purdue.edu/MintonGroup/swiftest.git
+$ git clone https://github.com/carlislewishard/swiftest.git
 ```
 
 If your cloned version is not already set to the master branch:
@@ -83,7 +85,7 @@ CMake allows the user to specify a set of compiler flags to use during compilati
 
 As a general rule, the release flags are fully optimized and best used when running Swiftest with the goal of generating results. This is the default set of flags. When making changes to the Swiftest source code, it is best to compile Swiftest using the debug set of flags. You may also define your own set of compiler flags. 
 
-To build Swiftest with the release flags (default), type the following:
+To build Swiftest with the release flags (default) using the Intel fortran compiler (ifort), type the following:
 ```
 $ cmake ..
 ```
@@ -95,14 +97,45 @@ To build with another set of flags, simply replace ```DEBUG``` in the above line
 
 Add ```-CMAKE_PREFIX_PATH=/path/to/netcdf/``` to these commands as needed.
 
-After building Swiftest, make the executable using: 
+If using the GCC fortran compiler (gfortran), add the following flags:
+```
+-DCMAKE_Fortran_FLAGS="-I/usr/lib64/gfortran/modules/ -ffree-line-length-512"
+```
+You can manually specify the compiler you wish to use with the following flag:
+```
+c-DCMAKE_Fortran_COMPILER=$(which ifort)
+```
 
+After building Swiftest, make the executable using: 
 ```
 $ make
 ```
 
 The Swiftest executable, called ```swiftest_driver```, should now be created in the ```/swiftest/bin/``` directory.
 
+**Download the `swiftest_driver` as a Docker or Singularity container.**
+
+The Swiftest driver is available as a Docker container on DockerHub in two versions: Intel and GNU. The Intel version was compiled for the x86_64 CPU using the Intel classic Fortran compiler. The GNU version was compliled for the x86_64 CPU using gfortran. The Intel version is faster than the GNU version (though not as fast as a native compile to the target CPU that you wish to run it on due to vectorization optimizations that Swiftest takes advantage of), however it is much larger: The Intel version is ~2.7GB while the GNU version is ~300MB. The Singularity container pulls from the same DockerHub container.
+
+To facilitate installation of the container, we provide a set of shell scripts to help automate the process of installing container versions of the executable. To install the default Intel version of the docker container from within the `swiftest\` project directory
+
+```
+$ cd docker
+$ . ./install.sh
+```
+
+To install the GNU version:
+
+```
+$ cd docker
+$ . ./install.sh gnu
+```
+
+The Singularity versions are installed the same way, just replace `cd docker` with `cd singularity` above. 
+
+Whether installing either the Docker or Singularity containers, the install script will copy an executable shell script `swiftest_driver` into `swiftest/bin/`. Not that when installing the Singularity container, the install script will set an environment variable called `SWIFTEST_SIF` that must point to the aboslute path of the container file called `swiftest_driver.sif`. To use the driver script in a future shell, rather than running the install script again, we suggest adding the environment variable definition to your shell startup script (e.g. add `export SWIFTEST_SIF="/path/to/swiftest/singularity/swiftest.sif"` to your `.zshrc`)
+
+
 **Swiftest Python Package**
 
 Included with Swiftest, in the ```/swiftest/python/swiftest/``` directory, is a Python package designed to facilitate seamless data processing and analysis. The Python package, also called Swiftest, can be used to generate input files, run Swiftest simulations, and process output files in the NetCDF file format.

README_tables/add_body_kwargs.md

@@ -17,5 +17,5 @@
 | ```rhill```     | Hill Radius value(s) of bodies. Only for massive bodies.                                                                                | float or array-like of floats
 | ```rot```       | Rotation rate vector(s) of bodies in radians/sec. Only for massive bodies. Only used if ```rotation``` is set to ```True```.                           | (n,3) array-like of floats
 | ```Ip```        | Principal axes moments of inertia vector(s) of bodies. Only for massive bodies. Only used if ```rotation``` is set to ```True```.       | (n,3) array-like of floats
-| ```J2```        | The J2 term of the central body.                                                                                                        | float or array-like of floats
-| ```J4```        | The J4 term of the central body.                                                                                                        | float or array-like of floats
+| ```J2```        | The unitless value of the spherical harmonic term equal to J2*R^2 where R is the radius of the central body.                                                                                                     | float or array-like of floats
+| ```J4```        | The unitless value of the spherical harmonic term equal to J4*R^4 where R is the radius of the central body.                                                                                                     | float or array-like of floats

apptainer/.gitignore

@@ -0,0 +1,4 @@
+!bin/
+!bin/swiftest_driver
+!bin/swiftest
+!install.sh

apptainer/bin/swiftest

@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+apptainer run --bind $(pwd):$(pwd) --cleanenv --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@"

apptainer/bin/swiftest_driver

@@ -0,0 +1,4 @@
+#!/bin/sh --
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`}
+FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1}
+apptainer exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@"

apptainer/install.sh

@@ -0,0 +1,21 @@
+#!/bin/sh --
+# This will install the Apptainer version of the swiftest_driver in place of the native compiled version into ../bin as 
+# well as the swiftest_python script that is used to execute a Python input file.
+# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.apptainer. 
+# To change this, just set environment variable SIF_DIR prior to running this script.
+# 
+# The script takes an optional argument "tag" if you want to pull a container other than "latest".
+# 
+# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, 
+# which requires this script to be called via source:
+# $ source ./install.sh
+# or 
+# $ . ./install.sh
+TAG=${1:-latest}
+
+SIF_DIR=${SIF_DIR:-${HOME}/.apptainer}
+echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container"
+apptainer pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG}
+cp -rf bin/swiftest ../bin/
+cp -rf bin/swiftest_driver ../bin/
+export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif

cmake/Modules/FindNETCDF.cmake

@@ -8,11 +8,14 @@
 # If not, see: https://www.gnu.org/licenses. 
 
 # - Finds the NetCDF libraries 
-find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)
-find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME)
-find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME)
+
+find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME ENV CPATH)
+find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH)
+find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH)
 
 set(NETCDF_FOUND TRUE)
 set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})
-set(NETCDF_LIBRARIES ${NETCDF_LIBRARY} ${NETCDF_FORTRAN_LIBRARY})
+# Note for posterity: When building static libraries, NETCDF_FORTRAN_LIBRARY must come *before* NETCDF_LIBRARY. Otherwise you get a bunch of "undefined reference to" errors
+set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY})
+
 mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)