From 63684b0f3416125815dcb06503f0a1b0ddd9ee4b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 5 Nov 2021 22:41:00 -0400 Subject: [PATCH 1/7] Updated Makefiles on debug branch --- Makefile | 18 +++++++++--------- Makefile.Defines | 20 +++++++++++++++----- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index d1da4b19f..7ac420ee3 100644 --- a/Makefile +++ b/Makefile @@ -65,8 +65,8 @@ MODULES = $(SWIFTEST_MODULES) $(USER_MODULES) .PHONY : all mod fast strict drivers bin clean force % : %.f90 force - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) $< -o $@ \ - -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff $(LMKL) $(LADVIXE) + $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $< -o $@ \ + -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff $(LMKL) $(INSTALL_PROGRAM) $@ $(SWIFTEST_HOME)/bin rm -f $@ @@ -79,7 +79,7 @@ all: mod: cd $(SWIFTEST_HOME)/src/modules/; \ - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c $(MODULES); \ + $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c $(MODULES); \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o; \ $(INSTALL_DATA) *.mod *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.mod *.smod @@ -188,34 +188,34 @@ strict: ln -s $(SWIFTEST_HOME)/Makefile .; \ make strictdir cd $(SWIFTEST_HOME)/src/helio; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c helio_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c helio_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/symba; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c symba_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c symba_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/rmvs; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c rmvs_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c rmvs_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/whm; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c whm_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c whm_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod fastdir: - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c *.f90; \ + $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod strictdir: - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c *.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smo diff --git a/Makefile.Defines b/Makefile.Defines index 4bb4e55a8..c943d55d5 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -45,12 +45,13 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ # DO NOT include in FFLAGS the "-c" option to compile object only # this is done explicitly as needed in the Makefile -ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vec -shared-intel -ldl -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -traceback +ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune +IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma -PAR = -qopenmp -parallel +PAR = -qopenmp -parallel HEAPARR = -heap-arrays 4194304 OPTREPORT = -qopt-report=5 IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -sox $(PAR) $(SIMDVEC) #$(HEAPARR) @@ -62,17 +63,26 @@ GMEM = -fsanitize-address-use-after-scope -fstack-check -fsanitize=bounds-stri GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporaries GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) + MKL_ROOT = /apps/spack/bell/apps/intel-parallel-studio/cluster.2019.5-intel-19.0.5-4brgqlf/mkl/lib IMKL = -I$(MKLROOT)/include LMKL = -L$(MKLROOT)/lib/intel64 -qopt-matmul -FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -FFLAGS = $(IPRODUCTION) -fp-model=fast +FSTRICTFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) +FFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) FORTRAN = ifort AR = xiar CC = icc -CC = icc +#FORTRAN = gfortran +#FFLAGS = $(GDEBUG) $(GMEM) #$(GPAR) +#FSTRICTFLAGS = $(GDEBUG) $(GMEM) #$(GPAR) +#AR = ar +#CC = cc + +# DO NOT include in CFLAGS the "-c" option to compile object only +# this is done explicitly as needed in the Makefile + CFLAGS = -O3 -w -m64 -std=c99 64_BIT_REALS = -r8 From e6deb35cf0bb5e016f9ffe51e1a74526d7daa4cc Mon Sep 17 00:00:00 2001 From: Carlisle Wishard Date: Thu, 11 Nov 2021 14:49:51 -0500 Subject: [PATCH 2/7] added check to look for BIN_OUT in directory that param is in --- python/swiftest/swiftest/simulation_class.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 8c9ecbc7b..47be96e75 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -55,15 +55,18 @@ def __init__(self, codename="Swiftest", param_file="", readbin=True): } self.codename = codename if param_file != "" : + dir_path = os.path.dirname(os.path.realpath(param_file)) self.read_param(param_file, codename) if readbin: if os.path.exists(self.param['BIN_OUT']): self.bin2xr() + elif os.path.exists(dir_path + '/' + self.param['BIN_OUT']): + self.param['BIN_OUT'] = dir_path + '/' + self.param['BIN_OUT'] + self.bin2xr() else: print(f"BIN_OUT file {self.param['BIN_OUT']} not found.") return - def add(self, plname, date=date.today().isoformat(), idval=None): """ Adds a solar system body to an existing simulation DataSet. From 5c615652f6f6690bfec63032ef8852bcd055e918 Mon Sep 17 00:00:00 2001 From: Carlisle Wishard Date: Thu, 11 Nov 2021 15:31:55 -0500 Subject: [PATCH 3/7] removed IF statement checking current directory for BIN_OUT --- python/swiftest/swiftest/simulation_class.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 47be96e75..465ef9c30 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -58,9 +58,7 @@ def __init__(self, codename="Swiftest", param_file="", readbin=True): dir_path = os.path.dirname(os.path.realpath(param_file)) self.read_param(param_file, codename) if readbin: - if os.path.exists(self.param['BIN_OUT']): - self.bin2xr() - elif os.path.exists(dir_path + '/' + self.param['BIN_OUT']): + if os.path.exists(dir_path + '/' + self.param['BIN_OUT']): self.param['BIN_OUT'] = dir_path + '/' + self.param['BIN_OUT'] self.bin2xr() else: From da07db948efaf92e968553ca8b0b44c4c8175e69 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 Nov 2021 09:11:36 -0500 Subject: [PATCH 4/7] Improved debugger compiler flags --- Makefile | 18 +++++++++--------- Makefile.Defines | 18 ++++++++++-------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7ac420ee3..09fcfcc0b 100644 --- a/Makefile +++ b/Makefile @@ -65,8 +65,8 @@ MODULES = $(SWIFTEST_MODULES) $(USER_MODULES) .PHONY : all mod fast strict drivers bin clean force % : %.f90 force - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $< -o $@ \ - -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff $(LMKL) + $(FORTRAN) $(FFLAGS) $(INCLUDES) $< -o $@ \ + -L$(SWIFTEST_HOME)/lib $(LINKS) $(INSTALL_PROGRAM) $@ $(SWIFTEST_HOME)/bin rm -f $@ @@ -79,7 +79,7 @@ all: mod: cd $(SWIFTEST_HOME)/src/modules/; \ - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c $(MODULES); \ + $(FORTRAN) $(FFLAGS) $(INCLUDES) -c $(MODULES); \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o; \ $(INSTALL_DATA) *.mod *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.mod *.smod @@ -188,34 +188,34 @@ strict: ln -s $(SWIFTEST_HOME)/Makefile .; \ make strictdir cd $(SWIFTEST_HOME)/src/helio; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c helio_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c helio_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/symba; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c symba_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c symba_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/rmvs; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c rmvs_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c rmvs_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod cd $(SWIFTEST_HOME)/src/whm; \ - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c whm_kick.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c whm_kick.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod fastdir: - $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \ + $(FORTRAN) $(FFLAGS) $(INCLUDES) -c *.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smod strictdir: - $(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \ + $(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c *.f90; \ $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \ $(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \ rm -f *.o *.smo diff --git a/Makefile.Defines b/Makefile.Defines index c943d55d5..8339f5617 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -41,6 +41,9 @@ SWIFTEST_HOME = $(ROOT_DIR) USER_MODULES = COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ +#NETCDF_FORTRAN_HOME = /home/daminton/local + + # Compiler definitions # DO NOT include in FFLAGS the "-c" option to compile object only @@ -48,13 +51,13 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune -IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin +IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma -PAR = -qopenmp -parallel +PAR = -qopenmp -parallel -parallel-source-info=2 HEAPARR = -heap-arrays 4194304 OPTREPORT = -qopt-report=5 -IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -sox $(PAR) $(SIMDVEC) #$(HEAPARR) +IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -qopt-matmul -sox $(PAR) $(SIMDVEC) #$(HEAPARR) #gfortran flags GDEBUG = -g -Og -fbacktrace -fbounds-check -ffree-line-length-none @@ -63,13 +66,12 @@ GMEM = -fsanitize-address-use-after-scope -fstack-check -fsanitize=bounds-stri GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporaries GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) +INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include -I$(ADVISOR_2019_DIR)/include/intel64 +LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -ladvisor -qopt-matmul $(PAR) -MKL_ROOT = /apps/spack/bell/apps/intel-parallel-studio/cluster.2019.5-intel-19.0.5-4brgqlf/mkl/lib -IMKL = -I$(MKLROOT)/include -LMKL = -L$(MKLROOT)/lib/intel64 -qopt-matmul +FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) +FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) -FSTRICTFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) -FFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) FORTRAN = ifort AR = xiar CC = icc From 145d5e5a2f21350442e98f81dc5f7ed67902a977 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 Nov 2021 09:13:05 -0500 Subject: [PATCH 5/7] Changes to OpenMP directives --- src/kick/kick.f90 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/kick/kick.f90 b/src/kick/kick.f90 index d739427c4..891936f20 100644 --- a/src/kick/kick.f90 +++ b/src/kick/kick.f90 @@ -106,7 +106,7 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad if (present(radius)) then !$omp parallel do default(private) schedule(static)& !$omp shared(nplpl, k_plpl, x, Gmass, radius) & - !$omp lastprivate(rji2, rlim2, xr, yr, zr) & + !$omp lastprivate(i, j, rji2, rlim2, xr, yr, zr) & !$omp reduction(+:ahi) & !$omp reduction(-:ahj) do k = 1_I8B, nplpl @@ -124,7 +124,7 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad else !$omp parallel do default(private) schedule(static)& !$omp shared(nplpl, k_plpl, x, Gmass, radius) & - !$omp lastprivate(rji2, xr, yr, zr) & + !$omp lastprivate(i, j, rji2, xr, yr, zr) & !$omp reduction(+:ahi) & !$omp reduction(-:ahj) do k = 1_I8B, nplpl @@ -137,12 +137,10 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad call kick_getacch_int_one_pl(rji2, xr, yr, zr, Gmass(i), Gmass(j), & ahi(1,i), ahi(2,i), ahi(3,i), ahj(1,j), ahj(2,j), ahj(3,j)) end do - !$omp end parallel do + !$omp end parallel do end if - do concurrent(i = 1:npl) - acc(:,i) = acc(:,i) + ahi(:,i) + ahj(:,i) - end do + acc(:,:) = acc(:,:) + ahi(:,:) + ahj(:,:) return end subroutine kick_getacch_int_all_flat_pl From 4502dab5cecdc104b09c45d9b0761dc989d90deb Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 Nov 2021 15:25:00 -0500 Subject: [PATCH 6/7] Changed the types of npl/nplm to be 8-byte integers in order to handle situations in which nplpl/nplplm are higher than the maximum 4-byte integer value --- src/symba/symba_util.f90 | 10 +++++----- src/util/util_flatten.f90 | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/symba/symba_util.f90 b/src/symba/symba_util.f90 index d38106705..5a985faea 100644 --- a/src/symba/symba_util.f90 +++ b/src/symba/symba_util.f90 @@ -400,8 +400,8 @@ module subroutine symba_util_flatten_eucl_plpl(self, param) class(symba_pl), intent(inout) :: self !! SyMBA massive body object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I8B) :: k - integer(I4B) :: i, j, npl, nplm, err + integer(I8B) :: k, npl, nplm + integer(I4B) :: i, j, err associate(pl => self, nplpl => self%nplpl, nplplm => self%nplplm) npl = int(self%nbody, kind=I8B) @@ -411,8 +411,8 @@ module subroutine symba_util_flatten_eucl_plpl(self, param) end select nplm = count(.not. pl%lmtiny(1:npl)) pl%nplm = int(nplm, kind=I4B) - nplpl = (npl * (npl - 1) / 2) ! number of entries in a strict lower triangle, npl x npl, minus first column - nplplm = nplm * npl - nplm * (nplm + 1) / 2 ! number of entries in a strict lower triangle, npl x npl, minus first column including only mutually interacting bodies + nplpl = (npl * (npl - 1_I8B)) / 2_I8B ! number of entries in a strict lower triangle, npl x npl, minus first column + nplplm = nplm * npl - nplm * (nplm + 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl, minus first column including only mutually interacting bodies if (param%lflatten_interactions) then if (allocated(self%k_plpl)) deallocate(self%k_plpl) ! Reset the index array if it's been set previously allocate(self%k_plpl(2, nplpl), stat=err) @@ -420,7 +420,7 @@ module subroutine symba_util_flatten_eucl_plpl(self, param) param%lflatten_interactions = .false. else do concurrent (i=1:npl, j=1:npl, j>i) - call util_flatten_eucl_ij_to_k(npl, i, j, k) + call util_flatten_eucl_ij_to_k(self%nbody, i, j, k) self%k_plpl(1, k) = i self%k_plpl(2, k) = j end do diff --git a/src/util/util_flatten.f90 b/src/util/util_flatten.f90 index b124d031d..1f6f60fe6 100644 --- a/src/util/util_flatten.f90 +++ b/src/util/util_flatten.f90 @@ -75,12 +75,12 @@ module subroutine util_flatten_eucl_plpl(self, param) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, npl, err - integer(I8B) :: k + integer(I4B) :: i, j, err + integer(I8B) :: k, npl npl = int(self%nbody, kind=I8B) associate(nplpl => self%nplpl) - nplpl = (npl * (npl - 1) / 2) ! number of entries in a strict lower triangle, npl x npl + nplpl = npl * (npl - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl if (param%lflatten_interactions) then if (allocated(self%k_plpl)) deallocate(self%k_plpl) ! Reset the index array if it's been set previously allocate(self%k_plpl(2, nplpl), stat=err) @@ -88,7 +88,7 @@ module subroutine util_flatten_eucl_plpl(self, param) param%lflatten_interactions = .false. else do concurrent (i=1:npl, j=1:npl, j>i) - call util_flatten_eucl_ij_to_k(npl, i, j, k) + call util_flatten_eucl_ij_to_k(self%nbody, i, j, k) self%k_plpl(1, k) = i self%k_plpl(2, k) = j end do From 971ea3e91e6723af159b2bf6b516c05529581edb Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 12 Nov 2021 15:32:10 -0500 Subject: [PATCH 7/7] Updated compiler flags for production on Intel --- Makefile | 3 +-- Makefile.Defines | 13 ++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 09fcfcc0b..1f5a9828b 100644 --- a/Makefile +++ b/Makefile @@ -259,5 +259,4 @@ clean: cd $(SWIFTEST_HOME)/include; rm -f *.mod *.smod cd $(COLLRESOLVE_HOME); rm -rf autom4te.cache aux Makefile stamp-h1 configure config.status config.h config.log aclocal.m4 lib* *.in *.o *.lo cambioni2019/*.o cambioni2019/*.lo - -force: \ No newline at end of file +force: diff --git a/Makefile.Defines b/Makefile.Defines index 8339f5617..a5417a18d 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -48,13 +48,13 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ # DO NOT include in FFLAGS the "-c" option to compile object only # this is done explicitly as needed in the Makefile -ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback +ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -I$(ADVISOR_2019_DIR)/include/intel64 -parallel-source-info=2 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma -PAR = -qopenmp -parallel -parallel-source-info=2 +PAR = -qopenmp -parallel HEAPARR = -heap-arrays 4194304 OPTREPORT = -qopt-report=5 IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -qopt-matmul -sox $(PAR) $(SIMDVEC) #$(HEAPARR) @@ -66,11 +66,11 @@ GMEM = -fsanitize-address-use-after-scope -fstack-check -fsanitize=bounds-stri GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporaries GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) -INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include -I$(ADVISOR_2019_DIR)/include/intel64 -LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -ladvisor -qopt-matmul $(PAR) +INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include +LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) -FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) -FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) +FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback +FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback FORTRAN = ifort AR = xiar @@ -88,4 +88,3 @@ CC = icc CFLAGS = -O3 -w -m64 -std=c99 64_BIT_REALS = -r8 -