diff --git a/CMakeLists.txt b/CMakeLists.txt index 828a37bb2..5d4c8637f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,18 +29,21 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) # Set some options the user may choose # Uncomment the below if you want the user to choose a parallelization library -OPTION(USE_MPI "Use the MPI library for parallelization" OFF) +OPTION(USE_MPI "Use the MPI library for parallelization" ON) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) -# This INCLUDE statement executes code that sets the compile flags for DEBUG, -# RELEASE, and TESTING. You should review this file and make sure the flags -# are to your liking. -INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) + # Locate and set parallelization libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know # about Fortran. INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake) INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake) +INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake) + +# This INCLUDE statement executes code that sets the compile flags for DEBUG, +# RELEASE, PROFILING, and TESTING. +INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) + # There is an error in CMAKE with this flag for pgf90. Unset it GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME) diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake new file mode 100644 index 000000000..9e48932c3 --- /dev/null +++ b/cmake/Modules/FindMKL.cmake @@ -0,0 +1,17 @@ +# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. + +# - Finds the Intel MKL libraries +find_path(MKL_INCLUDE_DIR NAMES mkl.h HINTS ENV MKLROOT PATH_SUFFIXES include) +find_library(MKL_LIBRARY NAMES libmkl_core.a HINTS ENV MKLROOT PATH_SUFFIXES lib lib/intel64 ) + +set(MKL_FOUND TRUE) +set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) +set(MKL_LIBRARIES ${MKL_LIBRARY}) +mark_as_advanced(MKL_LIBRARY MKL_INCLUDE_DIR) \ No newline at end of file diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 7850fbdb8..d869e89b6 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -222,9 +222,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_TESTING "${CMAKE_Fortran_FLAGS_TESTING}" # Unroll loops SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-funroll-loops" # GNU - "-unroll" # Intel + Fortran "-unroll" # Intel "/unroll" # Intel Windows + "-funroll-loops" # GNU "-Munroll" # Portland Group ) @@ -288,6 +288,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-fma" # Intel ) +# Generate fused multiply-add instructions +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" + Fortran "-qmkl=cluster" # Intel + Fortran "-qmkl" # Intel + Fortran "-mkl" # Old Intel + ) ##################### ### MATH FLAGS ### diff --git a/cmake/Modules/SetMKL.cmake b/cmake/Modules/SetMKL.cmake new file mode 100644 index 000000000..e58c9f51a --- /dev/null +++ b/cmake/Modules/SetMKL.cmake @@ -0,0 +1,14 @@ +# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. + +# Find MKL if not already found +IF(NOT MKL_FOUND) + ENABLE_LANGUAGE(C) # Some libraries need a C compiler to find + FIND_PACKAGE(MKL REQUIRED) +ENDIF(NOT MKL_FOUND) diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake index 03ab970e6..224806406 100644 --- a/cmake/Modules/SetParallelizationLibrary.cmake +++ b/cmake/Modules/SetParallelizationLibrary.cmake @@ -12,9 +12,7 @@ # When one is turned on, the other is turned off # If both are off, we explicitly disable them just in case -IF (USE_OPENMP AND USE_MPI) - MESSAGE (FATAL_ERROR "Cannot use both OpenMP and MPI") -ELSEIF (USE_OPENMP) +IF (USE_OPENMP) # Find OpenMP IF (NOT OpenMP_Fortran_FLAGS) FIND_PACKAGE (OpenMP_Fortran) @@ -23,20 +21,16 @@ ELSEIF (USE_OPENMP) ENDIF (NOT OpenMP_Fortran_FLAGS) ENDIF (NOT OpenMP_Fortran_FLAGS) # Turn of MPI - UNSET (MPI_FOUND CACHE) - UNSET (MPI_COMPILER CACHE) - UNSET (MPI_LIBRARY CACHE) -ELSEIF (USE_MPI) +ENDIF (USE_OPENMP) + +IF (USE_MPI) # Find MPI IF (NOT MPI_Fortran_FOUND) FIND_PACKAGE (MPI REQUIRED) ENDIF (NOT MPI_Fortran_FOUND) - # Turn off OpenMP - SET (OMP_NUM_PROCS 0 CACHE - STRING "Number of processors OpenMP may use" FORCE) - UNSET (OpenMP_C_FLAGS CACHE) - UNSET (GOMP_Fortran_LINK_FLAGS CACHE) -ELSE () +ENDIF (USE_MPI) + +IF (NOT USE_OPENMP AND NOT USE_MPI) # Turn off both OpenMP and MPI SET (OMP_NUM_PROCS 0 CACHE STRING "Number of processors OpenMP may use" FORCE) @@ -45,4 +39,4 @@ ELSE () UNSET (MPI_FOUND CACHE) UNSET (MPI_COMPILER CACHE) UNSET (MPI_LIBRARY CACHE) -ENDIF (USE_OPENMP AND USE_MPI) +ENDIF (NOT USE_OPENMP AND NOT USE_MPI) diff --git a/examples/Basic_Simulation/run_from_file.ipynb b/examples/Basic_Simulation/run_from_file.ipynb new file mode 100644 index 000000000..d2f4fdeb1 --- /dev/null +++ b/examples/Basic_Simulation/run_from_file.ipynb @@ -0,0 +1,119 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "aa57e957-f141-4373-9a62-a91845203aa3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: OMP_NUM_THREADS=8\n" + ] + } + ], + "source": [ + "import swiftest\n", + "import xarray as xr\n", + "import numpy as np\n", + "import os\n", + "%env OMP_NUM_THREADS=8" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a9c020aa-0a87-4fa9-9b11-62213edb370c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Swiftest file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n", + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 1 output frames.\n" + ] + } + ], + "source": [ + "sim = swiftest.Simulation(read_param=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "209523b6-d7a8-46f0-8687-54f199015c2d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing parameter inputs to file /home/daminton/git_debug/swiftest/examples/Basic_Simulation/simdata/param.in\n", + "Running a Swiftest symba run from tstart=0.0 y to tstop=1000.0 y\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ca7e9529651b46209bc86174955f7a01", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Time: 0.0 / 1000.0 y : 0%| , npl: 14 ntp: 10 nplm: 13" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/daminton/git_debug/swiftest/python/swiftest/swiftest/simulation_class.py:465: UserWarning: Error executing main swiftest_driver program\n", + " self._run_swiftest_driver()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 19 output frames.\n", + "Swiftest simulation data stored as xarray DataSet .data\n" + ] + } + ], + "source": [ + "sim.run()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (My debug_env Kernel)", + "language": "python", + "name": "debug_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index f5ca2df8b..f4f244b73 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -383,11 +383,12 @@ def _type_scrub(output_data): process_output = False noutput = int((self.param['TSTOP'] - self.param['T0']) / self.param['DT']) iloop = int((self.param['TSTART'] - self.param['T0']) / self.param['DT']) - post_message = f"Time: {self.param['TSTART']} / {self.param['TSTOP']} {self.TU_name} " - post_message += f"npl: {self.data['npl'].values[0]} ntp: {self.data['ntp'].values[0]}" + if self.param['TSTOP'] < 1e4: + pre_message = f"Time: {self.param['TSTART']:>5} / {self.param['TSTOP']:>5} {self.TU_name} " + post_message = f"npl: {self.data['npl'].values[0]} ntp: {self.data['ntp'].values[0]}" if "nplm" in self.data: post_message += f" nplm: {self.data['nplm'].values[0]}" - pbar = tqdm(total=noutput, postfix=post_message, bar_format='{l_bar}{bar}{postfix}') + pbar = tqdm(total=noutput, desc=pre_message, postfix=post_message, bar_format='{l_bar}{bar}{postfix}') try: with subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, @@ -402,13 +403,15 @@ def _type_scrub(output_data): if process_output: kvstream=line.replace('\n','').strip().split(';') # Removes the newline character, output_data = _type_scrub({kv.split()[0]: kv.split()[1] for kv in kvstream[:-1]}) - post_message = f"Time: {output_data['T']} / {self.param['TSTOP']} {self.TU_name}" - post_message += f" npl: {output_data['NPL']} ntp: {output_data['NTP']}" + if self.param['TSTOP'] < 1e4: + pre_message = f"Time: {output_data['T']:>5} / {self.param['TSTOP']:>5} {self.TU_name}" + post_message = f" npl: {output_data['NPL']} ntp: {output_data['NTP']}" if "NPLM" in output_data: post_message += f" nplm: {output_data['NPLM']}" interval = output_data['ILOOP'] - iloop if interval > 0: pbar.update(interval) + pbar.set_description_str(pre_message) pbar.set_postfix_str(post_message) iloop = output_data['ILOOP']