Merge branch 'debug'

MintonGroup · Nov 29, 2022 · 2d0f6cb · 2d0f6cb
2 parents dc9f2c6 + 1b8a91f
commit 2d0f6cb
Show file tree

Hide file tree

Showing 45 changed files with 1,316 additions and 1,074 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -29,18 +29,21 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
 
 # Set some options the user may choose
 # Uncomment the below if you want the user to choose a parallelization library
-OPTION(USE_MPI "Use the MPI library for parallelization" OFF)
+OPTION(USE_MPI "Use the MPI library for parallelization" ON)
 OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)
 
-# This INCLUDE statement executes code that sets the compile flags for DEBUG,
-# RELEASE, and TESTING.  You should  review this file and make sure the flags 
-# are to your liking.
-INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) 
+
 # Locate and set parallelization libraries.  There are some CMake peculiarities
 # taken care of here, such as the fact that the FindOpenMP routine doesn't know
 # about Fortran.
 INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake)
+INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake)
+
+# This INCLUDE statement executes code that sets the compile flags for DEBUG,
+# RELEASE, PROFILING, and TESTING.  
+INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) 
+
 
 # There is an error in CMAKE with this flag for pgf90.  Unset it
 GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME)
@@ -59,9 +62,10 @@ SET(SWIFTEST_DRIVER swiftest_driver)
 SET(SRC ${CMAKE_SOURCE_DIR}/src)
 SET(LIB ${CMAKE_SOURCE_DIR}/lib)
 SET(BIN ${CMAKE_SOURCE_DIR}/bin)
+SET(MOD ${CMAKE_SOURCE_DIR}/include)
 
 # Have the .mod files placed in the lib folder
-SET(CMAKE_Fortran_MODULE_DIRECTORY ${LIB})
+SET(CMAKE_Fortran_MODULE_DIRECTORY ${MOD})
 
 # The source for the SWIFTEST binary and have it placed in the bin folder
 ADD_SUBDIRECTORY(${SRC} ${BIN})

diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake
@@ -0,0 +1,17 @@
+# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+
+# - Finds the Intel MKL libraries 
+find_path(MKL_INCLUDE_DIR NAMES mkl.h HINTS ENV MKLROOT PATH_SUFFIXES include)
+find_library(MKL_LIBRARY NAMES libmkl_core.a  HINTS ENV MKLROOT PATH_SUFFIXES lib lib/intel64 )
+
+set(MKL_FOUND TRUE)
+set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR})
+set(MKL_LIBRARIES ${MKL_LIBRARY})
+mark_as_advanced(MKL_LIBRARY MKL_INCLUDE_DIR)
diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake
@@ -222,9 +222,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_TESTING "${CMAKE_Fortran_FLAGS_TESTING}"
 
 # Unroll loops
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
-                 Fortran "-funroll-loops" # GNU
-                         "-unroll"        # Intel
+                 Fortran "-unroll"        # Intel
                          "/unroll"        # Intel Windows
+                         "-funroll-loops" # GNU
                          "-Munroll"       # Portland Group
                 )
 
@@ -288,6 +288,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
                  Fortran "-fma" # Intel
                 )
 
+# Generate fused multiply-add instructions
+SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
+                 Fortran "-qmkl=cluster" # Intel
+                 Fortran "-qmkl" # Intel
+                 Fortran "-mkl" # Old Intel
+                ) 
 
 #####################
 ### MATH FLAGS ###

diff --git a/cmake/Modules/SetMKL.cmake b/cmake/Modules/SetMKL.cmake
@@ -0,0 +1,14 @@
+# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+
+# Find MKL if not already found
+IF(NOT MKL_FOUND)
+    ENABLE_LANGUAGE(C) # Some libraries need a C compiler to find 
+    FIND_PACKAGE(MKL REQUIRED)
+ENDIF(NOT MKL_FOUND)
diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake
@@ -12,9 +12,7 @@
 # When one is turned on, the other is turned off
 # If both are off, we explicitly disable them just in case
 
-IF (USE_OPENMP AND USE_MPI)
-    MESSAGE (FATAL_ERROR "Cannot use both OpenMP and MPI")
-ELSEIF (USE_OPENMP)
+IF (USE_OPENMP)
     # Find OpenMP
     IF (NOT OpenMP_Fortran_FLAGS)
         FIND_PACKAGE (OpenMP_Fortran)
@@ -23,20 +21,16 @@ ELSEIF (USE_OPENMP)
         ENDIF (NOT OpenMP_Fortran_FLAGS)
     ENDIF (NOT OpenMP_Fortran_FLAGS)
     # Turn of MPI
-    UNSET (MPI_FOUND CACHE)
-    UNSET (MPI_COMPILER CACHE)
-    UNSET (MPI_LIBRARY CACHE)
-ELSEIF (USE_MPI)
+ENDIF (USE_OPENMP)
+
+IF (USE_MPI)
     # Find MPI
     IF (NOT MPI_Fortran_FOUND)
         FIND_PACKAGE (MPI REQUIRED)
     ENDIF (NOT MPI_Fortran_FOUND)
-    # Turn off OpenMP
-    SET (OMP_NUM_PROCS 0 CACHE
-         STRING "Number of processors OpenMP may use" FORCE)
-    UNSET (OpenMP_C_FLAGS CACHE)
-    UNSET (GOMP_Fortran_LINK_FLAGS CACHE)
-ELSE ()
+ENDIF (USE_MPI)
+
+IF (NOT USE_OPENMP AND NOT USE_MPI)
     # Turn off both OpenMP and MPI
     SET (OMP_NUM_PROCS 0 CACHE
          STRING "Number of processors OpenMP may use" FORCE)
@@ -45,4 +39,4 @@ ELSE ()
     UNSET (MPI_FOUND CACHE)
     UNSET (MPI_COMPILER CACHE)
     UNSET (MPI_LIBRARY CACHE)
-ENDIF (USE_OPENMP AND USE_MPI)
+ENDIF (NOT USE_OPENMP AND NOT USE_MPI)
diff --git a/examples/.gitignore b/examples/.gitignore
@@ -1 +1,4 @@
 */param.*
+*/simdata/*
+*/*/simdata/*
+
diff --git a/examples/Basic_Simulation/initial_conditions.ipynb b/examples/Basic_Simulation/initial_conditions.ipynb
@@ -9,7 +9,8 @@
    "source": [
     "import swiftest\n",
     "import numpy as np\n",
-    "from numpy.random import default_rng"
+    "from numpy.random import default_rng\n",
+    "%env OMP_NUM_THREADS=4"
    ]
   },
   {
@@ -20,8 +21,7 @@
    "outputs": [],
    "source": [
     "# Initialize the simulation object as a variable\n",
-    "sim = swiftest.Simulation(tstart=0.0, tstop=10.0, dt=0.005, tstep_out=1.0, fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8)\n",
-    "sim.get_parameter()"
+    "sim = swiftest.Simulation(tstart=0.0, tstop=1.0e6, dt=0.005, tstep_out=1.0e3, fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8)"
    ]
   },
   {
@@ -119,6 +119,17 @@
     "sim.add_body(name_tp, a_tp, e_tp, inc_tp, capom_tp, omega_tp, capm_tp)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "135bd3f8-0a56-4d62-b728-f150debc1a76",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Display the run configuration parameters\n",
+    "sim.get_parameter()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

diff --git a/examples/Basic_Simulation/initial_conditions.py b/examples/Basic_Simulation/initial_conditions.py
@@ -23,8 +23,7 @@
 from numpy.random import default_rng
 
 # Initialize the simulation object as a variable
-sim = swiftest.Simulation(tstart=0.0, tstop=10.0, dt=0.005, tstep_out=1.0, fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8)
-sim.get_parameter()
+sim = swiftest.Simulation(tstart=0.0, tstop=1.0e3, dt=0.005, tstep_out=1.0e0, fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8)
 
 # Add the modern planets and the Sun using the JPL Horizons Database
 sim.add_solar_system_body(["Sun","Mercury","Venus","Earth","Mars","Jupiter","Saturn","Uranus","Neptune","Pluto"])
@@ -64,6 +63,8 @@
 capm_tp     = default_rng().uniform(0.0, 360.0, ntp)
 
 sim.add_body(name_tp, a_tp, e_tp, inc_tp, capom_tp, omega_tp, capm_tp)
+# Display the run configuration parameters
+sim.get_parameter()
 
 # Run the simulation
 sim.run()
diff --git a/examples/Basic_Simulation/run_from_file.ipynb b/examples/Basic_Simulation/run_from_file.ipynb
@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "aa57e957-f141-4373-9a62-a91845203aa3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "env: OMP_NUM_THREADS=8\n"
+     ]
+    }
+   ],
+   "source": [
+    "import swiftest\n",
+    "import xarray as xr\n",
+    "import numpy as np\n",
+    "import os\n",
+    "%env OMP_NUM_THREADS=8"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a9c020aa-0a87-4fa9-9b11-62213edb370c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reading Swiftest file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n",
+      "\n",
+      "Creating Dataset from NetCDF file\n",
+      "Successfully converted 1 output frames.\n"
+     ]
+    }
+   ],
+   "source": [
+    "sim = swiftest.Simulation(read_param=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "209523b6-d7a8-46f0-8687-54f199015c2d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Writing parameter inputs to file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n",
+      "Running a Swiftest symba run from tstart=0.0 y to tstop=1000.0 y\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ca7e9529651b46209bc86174955f7a01",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Time:   0.0 / 1000.0 y :   0%|          , npl: 14 ntp: 10 nplm: 13"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/daminton/git_debug/swiftest/python/swiftest/swiftest/simulation_class.py:465: UserWarning: Error executing main swiftest_driver program\n",
+      "  self._run_swiftest_driver()\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Creating Dataset from NetCDF file\n",
+      "Successfully converted 19 output frames.\n",
+      "Swiftest simulation data stored as xarray DataSet .data\n"
+     ]
+    }
+   ],
+   "source": [
+    "sim.run()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (My debug_env Kernel)",
+   "language": "python",
+   "name": "debug_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/Basic_Simulation/run_from_file.py b/examples/Basic_Simulation/run_from_file.py
@@ -1,3 +1,3 @@
 import swiftest
-sim = swiftest.Simulation()
-sim.run(tstop=20.0)
+sim = swiftest.Simulation(read_param=True)
+sim.run()
diff --git a/examples/Basic_Simulation/run_simulation.ipynb b/examples/Basic_Simulation/run_simulation.ipynb
@@ -21,7 +21,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sim = swiftest.Simulation()"
+    "sim = swiftest.Simulation(read_param=True)"
    ]
   },
   {
@@ -31,7 +31,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sim.run(tstop=10.0)"
+    "sim.run()"
    ]
   },
   {