From 78d9d77e5d834e0f13afd2c83f251b596819eb10 Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 15 Feb 2024 09:07:29 -0500 Subject: [PATCH] Replaced OneAPI with v19 in the Intel build scripts on Negishi. Also fixed a bug in the Fortran Cmake that was preventing preprocessor flags being set properly in the library build --- buildscripts/_build_getopts.sh | 4 +++- buildscripts/build_rcac_intel.sh | 21 ++++++++++----------- buildscripts/get_platform.sh | 3 +++ buildscripts/set_compilers.sh | 12 +++++++++--- cmake/Modules/SetSwiftestFlags.cmake | 6 ------ src/CMakeLists.txt | 2 ++ src/swiftest/swiftest_util.f90 | 2 +- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/buildscripts/_build_getopts.sh b/buildscripts/_build_getopts.sh index e98f2fb9a..b2ace2e02 100755 --- a/buildscripts/_build_getopts.sh +++ b/buildscripts/_build_getopts.sh @@ -13,8 +13,10 @@ set -a SCRIPT_DIR=$(realpath $(dirname $0)) ROOT_DIR=$(realpath ${SCRIPT_DIR}/..) +echo "Getting the OS and ARCH values" # Get platform and architecture read -r OS ARCH < <($SCRIPT_DIR/get_platform.sh) +echo "Gotem! OS: $OS, ARCH: $ARCH" # Parse arguments USTMT="Usage: ${0} [-d /path/to/dependency/source] [-p /prefix/path] [-m MACOSX_DEPLOYMENT_TARGET]" @@ -55,7 +57,7 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-${BUILD_DIR}} case $OS in - Linux-gnu|Linux-ifx|Linux-ifort) + Linux-gnu|Linux-ifx|Linux-ifort|Linux-mpiifort|MacOSX) . ${SCRIPT_DIR}/set_environment_linux.sh ;; MacOSX) diff --git a/buildscripts/build_rcac_intel.sh b/buildscripts/build_rcac_intel.sh index 08e4bae2e..b12631fce 100755 --- a/buildscripts/build_rcac_intel.sh +++ b/buildscripts/build_rcac_intel.sh @@ -43,14 +43,15 @@ if [[ $MACHINE_NAME == "bell" ]]; then elif [[ $MACHINE_NAME == "negishi" ]]; then module purge module use /depot/daminton/etc/modules/negishi - module load intel-oneapi-compilers/2023.0.0 - module load intel-oneapi-mkl/2023.0.0 - module load intel-oneapi-mpi/2021.8.0 - source ${INTEL_ONEAPI_COMPILERS_HOME}/setvars.sh > /dev/null 2>&1 - module load netcdf-fortran/intel-oneapi/4.6.1 - module load shtools/intel-oneapi/4.11.10 + module load intel/19.1.3.304 + module load intel-mkl/2019.9.304 + module load impi/2019.9.304 + module load shtools/intel19/4.11.10 module load cmake/3.24.3 module load ninja/1.11.1 + module load hdf5/1.13.2 + module load netcdf-c/4.9.0 + module load netcdf-fortran/4.6.0 module load use.own module load conda-env/swiftest-env-py3.9.13 MACHINE_CODE_VALUE="SSE2" @@ -63,18 +64,16 @@ if [[ $BUILD_TYPE == "Release" ]]; then --config-settings=cmake.build-type="${BUILD_TYPE}" \ --config-settings=cmake.args="-DUSE_SIMD=ON" \ --config-settings=cmake.args="-DUSE_OPENMP=ON" \ - --config-settings=cmake.args="-DCMAKE_Fortran_COMPILER=mpiifort" \ - --config-settings=cmake.args="-DCMAKE_Fortran_FLAGS=\"-f90=ifort\"" \ + --config-settings=cmake.args="-DCMAKE_Fortran_COMPILER=${FC}" \ --config-settings=cmake.args="-DMACHINE_CODE_VALUE=$MACHINE_CODE_VALUE" \ --config-settings=install.strip=false \ --no-build-isolation \ -ve . else cmake -B ${ROOT_DIR}/build -S . -G Ninja \ - -DMACHINE_CODE_VALUE=$MACHINE_CODE_VALUE \ + -DMACHINE_CODE_VALUE=${MACHINE_CODE_VALUE} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DCMAKE_Fortran_COMPILER=mpiifort \ - -DCMAKE_Fortran_FLAGS="-f90=ifort" + -DCMAKE_Fortran_COMPILER=${FC} \ cmake --build ${ROOT_DIR}/build -j${OMP_NUM_THREADS} -v fi \ No newline at end of file diff --git a/buildscripts/get_platform.sh b/buildscripts/get_platform.sh index 6aaaf21bf..70aa0210d 100755 --- a/buildscripts/get_platform.sh +++ b/buildscripts/get_platform.sh @@ -50,6 +50,9 @@ case $OS in *ifx) OS="Linux-ifx" ;; + *mpiifort) + OS="Linux-mpiifort" + ;; *ifort) OS="Linux-ifort" ;; diff --git a/buildscripts/set_compilers.sh b/buildscripts/set_compilers.sh index b04b02702..85bfa2762 100755 --- a/buildscripts/set_compilers.sh +++ b/buildscripts/set_compilers.sh @@ -16,7 +16,7 @@ SCRIPT_DIR=$(realpath $(dirname $0)) ROOT_DIR=$(realpath ${SCRIPT_DIR}/..) case "$OS" in - Linux-gnu|Linux-ifx|Linux-ifort|MacOSX) + Linux-gnu|Linux-ifx|Linux-ifort|Linux-mpiifort|MacOSX) ;; *) echo "Unknown compiler type: $OS" @@ -42,8 +42,14 @@ case $OS in ;; Linux-ifort) FC=$(command -v ifort) - CC=$(command -v icx) - CXX=$(command -v icpx) + CC=$(command -v icc) + CXX=$(command -v icpc) + CPP=$(command -v cpp) + ;; + Linux-mpiifort) + FC=$(command -v mpiifort) + CC=$(command -v mpiicc) + CXX=$(command -v mpiicpc) CPP=$(command -v cpp) ;; MacOSX) diff --git a/cmake/Modules/SetSwiftestFlags.cmake b/cmake/Modules/SetSwiftestFlags.cmake index bcd73a0ba..f4be6a6dc 100644 --- a/cmake/Modules/SetSwiftestFlags.cmake +++ b/cmake/Modules/SetSwiftestFlags.cmake @@ -340,9 +340,6 @@ IF (CMAKE_BUILD_TYPE STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE STREQUAL "TESTING" ) SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-check all" # Intel ) - SET_COMPILE_FLAG(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" - C "-fcheck=conversions,stack,uninit" # Intel - ) # Initializes matrices/arrays with NaN values SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-init=snan,arrays" # Intel @@ -560,9 +557,6 @@ IF (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "PROFILE") SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-mkl" ) - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-qmkl" - ) # Enables additional interprocedural optimizations for a single file compilation SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-ip" # Intel diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45bc1257d..33612029f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -174,6 +174,7 @@ end program TestDoConcurrentLoc TRY_COMPILE(DOCONLOC_WORKS ${CMAKE_BINARY_DIR} ${TESTFILE} COMPILE_DEFINITIONS "${CMAKE_Fortran_FLAGS}" OUTPUT_VARIABLE OUTPUT) IF (DOCONLOC_WORKS) MESSAGE(STATUS "DO CONCURRENT supports locality-spec") + TARGET_COMPILE_DEFINITIONS(${SWIFTEST_LIBRARY} PRIVATE -DDOCONLOC) TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DDOCONLOC) ELSE () MESSAGE(STATUS "DO CONCURRENT does not support locality-spec") @@ -192,6 +193,7 @@ end program TestQuadPrecisionReal TRY_COMPILE(QUADPREC ${CMAKE_BINARY_DIR} ${TESTFILE} COMPILE_DEFINITIONS "${CMAKE_Fortran_FLAGS}" OUTPUT_VARIABLE OUTPUT) IF (QUADPREC) MESSAGE(STATUS "Quad precision real is supported") + TARGET_COMPILE_DEFINITIONS(${SWIFTEST_LIBRARY} PRIVATE -DQUADPREC) TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DQUADPREC) ELSE () MESSAGE(STATUS "Quad precision real is not supported") diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 7c6ac1964..6b6b00b65 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1225,7 +1225,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) Lcborbit(:) = cb%mass * (cb%rb(:) .cross. cb%vb(:)) #ifdef DOCONLOC - do concurrent (i = 1:npl, pl%lmask(i)) shared(pl,Lplorbit,kepl) local(h) + do concurrent (i = 1:npl, pl%lmask(i)) shared(pl,Lplorbit,kepl,npl) local(h) #else do concurrent (i = 1:npl, pl%lmask(i)) #endif