Merge branch 'debug'

MintonGroup · Nov 12, 2021 · f91a35c · f91a35c
2 parents 80818a1 + 971ea3e
commit f91a35c
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 37 deletions.
diff --git a/Makefile b/Makefile
@@ -65,8 +65,8 @@ MODULES         = $(SWIFTEST_MODULES) $(USER_MODULES)
 .PHONY : all mod fast strict drivers bin clean force 
 
 % : %.f90 force
-	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) $< -o $@ \
-	  -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff $(LMKL) $(LADVIXE)
+	$(FORTRAN) $(FFLAGS) $(INCLUDES) $< -o $@ \
+	  -L$(SWIFTEST_HOME)/lib $(LINKS) 
 	$(INSTALL_PROGRAM) $@ $(SWIFTEST_HOME)/bin
 	rm -f $@
 
@@ -79,7 +79,7 @@ all:
 
 mod:
 	cd $(SWIFTEST_HOME)/src/modules/; \
-	  $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c $(MODULES); \
+	  $(FORTRAN) $(FFLAGS) $(INCLUDES) -c $(MODULES); \
 	  $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o; \
 	  $(INSTALL_DATA) *.mod *.smod $(SWIFTEST_HOME)/include; \
 	  rm -f *.o *.mod  *.smod
@@ -188,34 +188,34 @@ strict:
 	  ln -s $(SWIFTEST_HOME)/Makefile .; \
 	  make strictdir
 	cd $(SWIFTEST_HOME)/src/helio; \
-		$(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c helio_kick.f90; \
+		$(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c helio_kick.f90; \
 		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 		rm -f *.o *.smod	
 	cd $(SWIFTEST_HOME)/src/symba; \
-		$(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c symba_kick.f90; \
+		$(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c symba_kick.f90; \
 		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 		rm -f *.o *.smod
 	cd $(SWIFTEST_HOME)/src/rmvs; \
-		$(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c rmvs_kick.f90; \
+		$(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c rmvs_kick.f90; \
 		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 		rm -f *.o *.smod	
 	cd $(SWIFTEST_HOME)/src/whm; \
-		$(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c whm_kick.f90; \
+		$(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c whm_kick.f90; \
 		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 		rm -f *.o *.smod
 
 fastdir:
-	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c *.f90; \
+	$(FORTRAN) $(FFLAGS) $(INCLUDES) -c *.f90; \
 	$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 	$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 	rm -f *.o *.smod
 
 strictdir:
-	$(FORTRAN) $(FSTRICTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $(IADVIXE) -c *.f90; \
+	$(FORTRAN) $(FSTRICTFLAGS) $(INCLUDES) -c *.f90; \
 	$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 	$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 	rm -f *.o *.smo	
@@ -259,5 +259,4 @@ clean:
 	cd $(SWIFTEST_HOME)/include; rm -f *.mod *.smod 
 	cd $(COLLRESOLVE_HOME); rm -rf autom4te.cache aux Makefile stamp-h1 configure config.status config.h config.log aclocal.m4 lib* *.in *.o *.lo cambioni2019/*.o cambioni2019/*.lo
 
-
-force:
+force:
diff --git a/Makefile.Defines b/Makefile.Defines
@@ -41,19 +41,23 @@ SWIFTEST_HOME    = $(ROOT_DIR)
 USER_MODULES    =
 COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/
 
+#NETCDF_FORTRAN_HOME = /home/daminton/local
+
+
 # Compiler definitions
 
 # DO NOT include in FFLAGS the "-c" option to compile object only
 # this is done explicitly as needed in the Makefile
-ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vec -shared-intel -ldl -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -traceback 
+ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -I$(ADVISOR_2019_DIR)/include/intel64 -parallel-source-info=2 
 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune
 
+IDEBUG = -O0 -init=snan,arrays  -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin
 STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens 
 SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma
 PAR = -qopenmp -parallel
 HEAPARR = -heap-arrays 4194304 
 OPTREPORT = -qopt-report=5 
-IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -sox $(PAR) $(SIMDVEC) #$(HEAPARR)
+IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -qopt-matmul -sox $(PAR) $(SIMDVEC) #$(HEAPARR)
 
 #gfortran flags
 GDEBUG = -g -Og -fbacktrace -fbounds-check -ffree-line-length-none
@@ -62,18 +66,25 @@ GMEM =  -fsanitize-address-use-after-scope -fstack-check  -fsanitize=bounds-stri
 GWARNINGS = -Wall -Warray-bounds  -Wimplicit-interface -Wextra -Warray-temporaries
 GPRODUCTION = -O3 -ffree-line-length-none $(GPAR)
 
-MKL_ROOT = /apps/spack/bell/apps/intel-parallel-studio/cluster.2019.5-intel-19.0.5-4brgqlf/mkl/lib
-IMKL = -I$(MKLROOT)/include
-LMKL = -L$(MKLROOT)/lib/intel64 -qopt-matmul 
+INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include 
+LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff  -qopt-matmul $(PAR)
+
+FSTRICTFLAGS  = $(IPRODUCTION) $(STRICTREAL) -g -traceback
+FFLAGS  = $(IPRODUCTION) -fp-model=fast -g -traceback
 
-FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL)
-FFLAGS = $(IPRODUCTION) -fp-model=fast
 FORTRAN = ifort
 AR     = xiar
 CC    = icc
 
-CC              = icc
+#FORTRAN = gfortran
+#FFLAGS =  $(GDEBUG)  $(GMEM) #$(GPAR)
+#FSTRICTFLAGS =  $(GDEBUG)  $(GMEM) #$(GPAR)
+#AR     = ar 
+#CC     = cc
+
+# DO NOT include in CFLAGS the "-c" option to compile object only
+# this is done explicitly as needed in the Makefile
+
 CFLAGS          = -O3 -w -m64 -std=c99
 
 64_BIT_REALS    = -r8
-
diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py
@@ -55,15 +55,16 @@ def __init__(self, codename="Swiftest", param_file="", readbin=True):
         }
         self.codename = codename
         if param_file != "" :
+            dir_path = os.path.dirname(os.path.realpath(param_file))
             self.read_param(param_file, codename)
             if readbin:
-                if os.path.exists(self.param['BIN_OUT']):
+                if os.path.exists(dir_path + '/' + self.param['BIN_OUT']):
+                    self.param['BIN_OUT'] = dir_path + '/' + self.param['BIN_OUT']
                     self.bin2xr()
                 else:
                     print(f"BIN_OUT file {self.param['BIN_OUT']} not found.")
         return
 
-
     def add(self, plname, date=date.today().isoformat(), idval=None):
         """
         Adds a solar system body to an existing simulation DataSet.

diff --git a/src/kick/kick.f90 b/src/kick/kick.f90
@@ -106,7 +106,7 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad
       if (present(radius)) then
          !$omp parallel do default(private) schedule(static)&
          !$omp shared(nplpl, k_plpl, x, Gmass, radius) &
-         !$omp lastprivate(rji2, rlim2, xr, yr, zr) &
+         !$omp lastprivate(i, j, rji2, rlim2, xr, yr, zr) &
          !$omp reduction(+:ahi) &
          !$omp reduction(-:ahj) 
          do k = 1_I8B, nplpl
@@ -124,7 +124,7 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad
       else
          !$omp parallel do default(private) schedule(static)&
          !$omp shared(nplpl, k_plpl, x, Gmass, radius) &
-         !$omp lastprivate(rji2, xr, yr, zr) &
+         !$omp lastprivate(i, j, rji2, xr, yr, zr) &
          !$omp reduction(+:ahi) &
          !$omp reduction(-:ahj) 
          do k = 1_I8B, nplpl
@@ -137,12 +137,10 @@ module subroutine kick_getacch_int_all_flat_pl(npl, nplpl, k_plpl, x, Gmass, rad
             call kick_getacch_int_one_pl(rji2, xr, yr, zr, Gmass(i), Gmass(j), &
                                          ahi(1,i), ahi(2,i), ahi(3,i), ahj(1,j), ahj(2,j), ahj(3,j))
          end do
-         !$omp end parallel do 
+         !$omp end parallel do
       end if
 
-      do concurrent(i = 1:npl)
-         acc(:,i) = acc(:,i) + ahi(:,i) + ahj(:,i)
-      end do
+      acc(:,:) = acc(:,:) + ahi(:,:) + ahj(:,:)
 
       return
    end subroutine kick_getacch_int_all_flat_pl

diff --git a/src/symba/symba_util.f90 b/src/symba/symba_util.f90
@@ -400,8 +400,8 @@ module subroutine symba_util_flatten_eucl_plpl(self, param)
       class(symba_pl),            intent(inout) :: self  !! SyMBA massive body object
       class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters
       ! Internals
-      integer(I8B) :: k
-      integer(I4B) :: i, j, npl, nplm, err
+      integer(I8B) :: k, npl, nplm
+      integer(I4B) :: i, j, err
 
       associate(pl => self, nplpl => self%nplpl, nplplm => self%nplplm)
          npl = int(self%nbody, kind=I8B)
@@ -411,16 +411,16 @@ module subroutine symba_util_flatten_eucl_plpl(self, param)
          end select
          nplm = count(.not. pl%lmtiny(1:npl))
          pl%nplm = int(nplm, kind=I4B)
-         nplpl = (npl * (npl - 1) / 2) ! number of entries in a strict lower triangle, npl x npl, minus first column
-         nplplm = nplm * npl - nplm * (nplm + 1) / 2 ! number of entries in a strict lower triangle, npl x npl, minus first column including only mutually interacting bodies
+         nplpl = (npl * (npl - 1_I8B)) / 2_I8B ! number of entries in a strict lower triangle, npl x npl, minus first column
+         nplplm = nplm * npl - nplm * (nplm + 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl, minus first column including only mutually interacting bodies
          if (param%lflatten_interactions) then
             if (allocated(self%k_plpl)) deallocate(self%k_plpl) ! Reset the index array if it's been set previously
             allocate(self%k_plpl(2, nplpl), stat=err)
             if (err /= 0) then ! An error occurred trying to allocate this big array. This probably means it's too big to fit in memory, and so we will force the run back into triangular mode
                param%lflatten_interactions = .false.
             else
                do concurrent (i=1:npl, j=1:npl, j>i)
-                  call util_flatten_eucl_ij_to_k(npl, i, j, k)
+                  call util_flatten_eucl_ij_to_k(self%nbody, i, j, k)
                   self%k_plpl(1, k) = i
                   self%k_plpl(2, k) = j
                end do

diff --git a/src/util/util_flatten.f90 b/src/util/util_flatten.f90
@@ -75,20 +75,20 @@ module subroutine util_flatten_eucl_plpl(self, param)
       class(swiftest_pl),         intent(inout) :: self  !! Swiftest massive body object
       class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters
       ! Internals
-      integer(I4B) :: i, j, npl, err
-      integer(I8B) :: k
+      integer(I4B) :: i, j, err
+      integer(I8B) :: k, npl
 
       npl = int(self%nbody, kind=I8B)
       associate(nplpl => self%nplpl)
-         nplpl = (npl * (npl - 1) / 2) ! number of entries in a strict lower triangle, npl x npl
+         nplpl = npl * (npl - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl
          if (param%lflatten_interactions) then
             if (allocated(self%k_plpl)) deallocate(self%k_plpl) ! Reset the index array if it's been set previously
             allocate(self%k_plpl(2, nplpl), stat=err)
             if (err /=0) then ! An error occurred trying to allocate this big array. This probably means it's too big to fit in memory, and so we will force the run back into triangular mode
                param%lflatten_interactions = .false.
             else
                do concurrent (i=1:npl, j=1:npl, j>i)
-                  call util_flatten_eucl_ij_to_k(npl, i, j, k)
+                  call util_flatten_eucl_ij_to_k(self%nbody, i, j, k)
                   self%k_plpl(1, k) = i
                   self%k_plpl(2, k) = j
                end do