Merge branch 'Simulation_API_improvements' into debug

CMakeLists.txt

@@ -29,18 +29,21 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
 
 # Set some options the user may choose
 # Uncomment the below if you want the user to choose a parallelization library
-OPTION(USE_MPI "Use the MPI library for parallelization" OFF)
+OPTION(USE_MPI "Use the MPI library for parallelization" ON)
 OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)
 
-# This INCLUDE statement executes code that sets the compile flags for DEBUG,
-# RELEASE, and TESTING.  You should  review this file and make sure the flags 
-# are to your liking.
-INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) 
+
 # Locate and set parallelization libraries.  There are some CMake peculiarities
 # taken care of here, such as the fact that the FindOpenMP routine doesn't know
 # about Fortran.
 INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake)
+INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake)
+
+# This INCLUDE statement executes code that sets the compile flags for DEBUG,
+# RELEASE, PROFILING, and TESTING.  
+INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) 
+
 
 # There is an error in CMAKE with this flag for pgf90.  Unset it
 GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME)

cmake/Modules/FindMKL.cmake

@@ -0,0 +1,17 @@
+# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+
+# - Finds the Intel MKL libraries 
+find_path(MKL_INCLUDE_DIR NAMES mkl.h HINTS ENV MKLROOT PATH_SUFFIXES include)
+find_library(MKL_LIBRARY NAMES libmkl_core.a  HINTS ENV MKLROOT PATH_SUFFIXES lib lib/intel64 )
+
+set(MKL_FOUND TRUE)
+set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR})
+set(MKL_LIBRARIES ${MKL_LIBRARY})
+mark_as_advanced(MKL_LIBRARY MKL_INCLUDE_DIR)

cmake/Modules/SetFortranFlags.cmake

@@ -222,9 +222,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_TESTING "${CMAKE_Fortran_FLAGS_TESTING}"
 
 # Unroll loops
 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
-                 Fortran "-funroll-loops" # GNU
-                         "-unroll"        # Intel
+                 Fortran "-unroll"        # Intel
                          "/unroll"        # Intel Windows
+                         "-funroll-loops" # GNU
                          "-Munroll"       # Portland Group
                 )
 
@@ -288,6 +288,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
                  Fortran "-fma" # Intel
                 )
 
+# Generate fused multiply-add instructions
+SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}"
+                 Fortran "-qmkl=cluster" # Intel
+                 Fortran "-qmkl" # Intel
+                 Fortran "-mkl" # Old Intel
+                ) 
 
 #####################
 ### MATH FLAGS ###

cmake/Modules/SetMKL.cmake

@@ -0,0 +1,14 @@
+# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh
+# This file is part of Swiftest.
+# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License 
+# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along with Swiftest. 
+# If not, see: https://www.gnu.org/licenses. 
+
+# Find MKL if not already found
+IF(NOT MKL_FOUND)
+    ENABLE_LANGUAGE(C) # Some libraries need a C compiler to find 
+    FIND_PACKAGE(MKL REQUIRED)
+ENDIF(NOT MKL_FOUND)

cmake/Modules/SetParallelizationLibrary.cmake

@@ -12,9 +12,7 @@
 # When one is turned on, the other is turned off
 # If both are off, we explicitly disable them just in case
 
-IF (USE_OPENMP AND USE_MPI)
-    MESSAGE (FATAL_ERROR "Cannot use both OpenMP and MPI")
-ELSEIF (USE_OPENMP)
+IF (USE_OPENMP)
     # Find OpenMP
     IF (NOT OpenMP_Fortran_FLAGS)
         FIND_PACKAGE (OpenMP_Fortran)
@@ -23,20 +21,16 @@ ELSEIF (USE_OPENMP)
         ENDIF (NOT OpenMP_Fortran_FLAGS)
     ENDIF (NOT OpenMP_Fortran_FLAGS)
     # Turn of MPI
-    UNSET (MPI_FOUND CACHE)
-    UNSET (MPI_COMPILER CACHE)
-    UNSET (MPI_LIBRARY CACHE)
-ELSEIF (USE_MPI)
+ENDIF (USE_OPENMP)
+
+IF (USE_MPI)
     # Find MPI
     IF (NOT MPI_Fortran_FOUND)
         FIND_PACKAGE (MPI REQUIRED)
     ENDIF (NOT MPI_Fortran_FOUND)
-    # Turn off OpenMP
-    SET (OMP_NUM_PROCS 0 CACHE
-         STRING "Number of processors OpenMP may use" FORCE)
-    UNSET (OpenMP_C_FLAGS CACHE)
-    UNSET (GOMP_Fortran_LINK_FLAGS CACHE)
-ELSE ()
+ENDIF (USE_MPI)
+
+IF (NOT USE_OPENMP AND NOT USE_MPI)
     # Turn off both OpenMP and MPI
     SET (OMP_NUM_PROCS 0 CACHE
          STRING "Number of processors OpenMP may use" FORCE)
@@ -45,4 +39,4 @@ ELSE ()
     UNSET (MPI_FOUND CACHE)
     UNSET (MPI_COMPILER CACHE)
     UNSET (MPI_LIBRARY CACHE)
-ENDIF (USE_OPENMP AND USE_MPI)
+ENDIF (NOT USE_OPENMP AND NOT USE_MPI)

examples/Basic_Simulation/run_from_file.ipynb

@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "aa57e957-f141-4373-9a62-a91845203aa3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "env: OMP_NUM_THREADS=8\n"
+     ]
+    }
+   ],
+   "source": [
+    "import swiftest\n",
+    "import xarray as xr\n",
+    "import numpy as np\n",
+    "import os\n",
+    "%env OMP_NUM_THREADS=8"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a9c020aa-0a87-4fa9-9b11-62213edb370c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reading Swiftest file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n",
+      "\n",
+      "Creating Dataset from NetCDF file\n",
+      "Successfully converted 1 output frames.\n"
+     ]
+    }
+   ],
+   "source": [
+    "sim = swiftest.Simulation(read_param=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "209523b6-d7a8-46f0-8687-54f199015c2d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Writing parameter inputs to file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n",
+      "Running a Swiftest symba run from tstart=0.0 y to tstop=1000.0 y\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ca7e9529651b46209bc86174955f7a01",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Time:   0.0 / 1000.0 y :   0%|          , npl: 14 ntp: 10 nplm: 13"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/daminton/git_debug/swiftest/python/swiftest/swiftest/simulation_class.py:465: UserWarning: Error executing main swiftest_driver program\n",
+      "  self._run_swiftest_driver()\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Creating Dataset from NetCDF file\n",
+      "Successfully converted 19 output frames.\n",
+      "Swiftest simulation data stored as xarray DataSet .data\n"
+     ]
+    }
+   ],
+   "source": [
+    "sim.run()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (My debug_env Kernel)",
+   "language": "python",
+   "name": "debug_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

python/swiftest/swiftest/simulation_class.py

@@ -383,11 +383,12 @@ def _type_scrub(output_data):
         process_output = False
         noutput = int((self.param['TSTOP'] - self.param['T0']) / self.param['DT'])
         iloop = int((self.param['TSTART'] - self.param['T0']) / self.param['DT'])
-        post_message = f"Time: {self.param['TSTART']} / {self.param['TSTOP']} {self.TU_name} "
-        post_message += f"npl: {self.data['npl'].values[0]} ntp: {self.data['ntp'].values[0]}"
+        if self.param['TSTOP'] < 1e4:
+           pre_message = f"Time: {self.param['TSTART']:>5} / {self.param['TSTOP']:>5} {self.TU_name} "
+        post_message = f"npl: {self.data['npl'].values[0]} ntp: {self.data['ntp'].values[0]}"
         if "nplm" in self.data:
             post_message += f" nplm: {self.data['nplm'].values[0]}"
-        pbar = tqdm(total=noutput, postfix=post_message, bar_format='{l_bar}{bar}{postfix}')
+        pbar = tqdm(total=noutput, desc=pre_message, postfix=post_message, bar_format='{l_bar}{bar}{postfix}')
         try:
             with subprocess.Popen(shlex.split(cmd),
                                   stdout=subprocess.PIPE,
@@ -402,13 +403,15 @@ def _type_scrub(output_data):
                     if process_output:
                         kvstream=line.replace('\n','').strip().split(';') # Removes the newline character,
                         output_data = _type_scrub({kv.split()[0]: kv.split()[1] for kv in kvstream[:-1]})
-                        post_message = f"Time: {output_data['T']} / {self.param['TSTOP']} {self.TU_name}"
-                        post_message += f" npl: {output_data['NPL']} ntp: {output_data['NTP']}"
+                        if self.param['TSTOP'] < 1e4:
+                           pre_message = f"Time: {output_data['T']:>5} / {self.param['TSTOP']:>5} {self.TU_name}"
+                        post_message = f" npl: {output_data['NPL']} ntp: {output_data['NTP']}"
                         if "NPLM" in output_data:
                             post_message += f" nplm: {output_data['NPLM']}"
                         interval = output_data['ILOOP'] - iloop
                         if interval > 0:
                            pbar.update(interval)
+                           pbar.set_description_str(pre_message)
                            pbar.set_postfix_str(post_message)
                         iloop = output_data['ILOOP']