From a1e541ca973866c174cb673cd9ccdfd9032225ed Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 Nov 2021 15:57:47 -0500 Subject: [PATCH 1/7] Updated debug compiler flags --- Makefile.Defines | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index 8339f5617..cdfae9240 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -66,11 +66,13 @@ GMEM = -fsanitize-address-use-after-scope -fstack-check -fsanitize=bounds-stri GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporaries GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) -INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include -I$(ADVISOR_2019_DIR)/include/intel64 -LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -ladvisor -qopt-matmul $(PAR) +INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include +LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) +#FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback +#FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback FORTRAN = ifort AR = xiar From 7b848c287399977ccd1ff0d3e723a662bdda9956 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 09:41:46 -0500 Subject: [PATCH 2/7] Added check for OUT_STAT=='APPEND' in symba_io_param_reader in order to correctly set the param%lrestart flag in SyMBA runs prior to calling the main io_param_reader subroutine --- src/symba/symba_io.f90 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/symba/symba_io.f90 b/src/symba/symba_io.f90 index 80354e064..3e217bc6a 100644 --- a/src/symba/symba_io.f90 +++ b/src/symba/symba_io.f90 @@ -44,6 +44,9 @@ module subroutine symba_io_param_reader(self, unit, iotype, v_list, iostat, ioms ifirst = ilast + 1 param_value = io_get_token(line_trim, ifirst, ilast, iostat) select case (param_name) + case ("OUT_STAT") ! We need to duplicate this from the standard io_param_reader in order to make sure that the restart flag gets set properly in SyMBA + call io_toupper(param_value) + param%out_stat = param_value case ("FRAGMENTATION") call io_toupper(param_value) if (param_value == "YES" .or. param_value == "T") self%lfragmentation = .true. @@ -80,6 +83,8 @@ module subroutine symba_io_param_reader(self, unit, iotype, v_list, iostat, ioms 1 continue close(unit) + param%lrestart = (param%out_stat == "APPEND") + if (self%GMTINY < 0.0_DP) then write(iomsg,*) "GMTINY invalid or not set: ", self%GMTINY iostat = -1 From 60d06d0e9ba60de05c3b1e4c15f3ef0e5cbbe401 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 09:44:09 -0500 Subject: [PATCH 3/7] Fixed problem with debug compiler flags --- Makefile.Defines | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index 846bb0fc3..849f205b1 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -51,7 +51,8 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -parallel-source-info=2 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune -IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect -parens -CB -no-wrap-margi-parallel-source-info=2 TRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens +IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margi-parallel-source-info=2 +STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma PAR = -qopenmp -parallel HEAPARR = -heap-arrays 4194304 @@ -70,8 +71,8 @@ LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcd FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) -#FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback -#FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback +# FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback +# FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback FORTRAN = ifort AR = xiar From 17438ebd38b28a1518401c9d71bd2b277de5a76a Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 12:18:19 -0500 Subject: [PATCH 4/7] Fixed typo in compiler flags --- Makefile.Defines | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index 849f205b1..d7244a886 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -51,7 +51,7 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -parallel-source-info=2 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune -IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margi-parallel-source-info=2 +IDEBUG = -O2 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin -parallel-source-info=2 STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma PAR = -qopenmp -parallel @@ -69,8 +69,8 @@ GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) -FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) -FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) +FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) $(HEAPARR) +FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) $(HEAPARR) # FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback # FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback From e44c0d9236a5096f9805869025c9855a7c8cfe86 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 15:23:19 -0500 Subject: [PATCH 5/7] Correctly read in initial values of central body mass, radius, and angular momentum when restarting from NetCDF. --- src/netcdf/netcdf.f90 | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/netcdf/netcdf.f90 b/src/netcdf/netcdf.f90 index b99188855..fd2406720 100644 --- a/src/netcdf/netcdf.f90 +++ b/src/netcdf/netcdf.f90 @@ -66,6 +66,7 @@ module function netcdf_get_old_t_final_system(self, param) result(old_t_final) integer(I4B) :: itmax, idmax real(DP), dimension(:), allocatable :: vals real(DP), dimension(1) :: val + real(DP), dimension(NDIM) :: rot0 real(DP) :: KE_orb_orig, KE_spin_orig, PE_orig, Ltmp call param%nciu%open(param) @@ -74,7 +75,8 @@ module function netcdf_get_old_t_final_system(self, param) result(old_t_final) allocate(vals(idmax)) call check( nf90_get_var(param%nciu%ncid, param%nciu%time_varid, val, start=[1], count=[1]) ) - old_t_final = val(1) + !old_t_final = val(1) + old_t_final = param%t0 ! For NetCDF it is safe to overwrite the final t value on a restart if (param%lenergy) then call check( nf90_get_var(param%nciu%ncid, param%nciu%KE_orb_varid, val, start=[1], count=[1]) ) @@ -107,6 +109,24 @@ module function netcdf_get_old_t_final_system(self, param) result(old_t_final) call check( nf90_get_var(param%nciu%ncid, param%nciu%Gmass_varid, vals, start=[1,1], count=[idmax,1]) ) self%GMtot_orig = vals(1) + sum(vals(2:idmax), vals(2:idmax) == vals(2:idmax)) + select type(cb => self%cb) + class is (symba_cb) + cb%GM0 = vals(1) + cb%dGM = cb%Gmass - cb%GM0 + + call check( nf90_get_var(param%nciu%ncid, param%nciu%radius_varid, val, start=[1,1], count=[1,1]) ) + cb%R0 = val(1) + + call check( nf90_get_var(param%nciu%ncid, param%nciu%rotx_varid, val, start=[1,1], count=[1,1]) ) + rot0(1) = val(1) + call check( nf90_get_var(param%nciu%ncid, param%nciu%roty_varid, val, start=[1,1], count=[1,1]) ) + rot0(2) = val(1) + call check( nf90_get_var(param%nciu%ncid, param%nciu%rotz_varid, val, start=[1,1], count=[1,1]) ) + rot0(3) = val(1) + + cb%L0(:) = cb%Ip(3) * cb%GM0 * cb%R0**2 * rot0(:) + end select + end if deallocate(vals) From 001247ba2f8d25a72a25404fc1eacb8abc5d76d2 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 15:23:47 -0500 Subject: [PATCH 6/7] Misc. formatting fixes --- src/io/io.f90 | 1 - src/kick/kick.f90 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/io/io.f90 b/src/io/io.f90 index ebdd68531..0266af0bc 100644 --- a/src/io/io.f90 +++ b/src/io/io.f90 @@ -1351,7 +1351,6 @@ subroutine io_read_in_cb(self, param) close(iu, err = 667, iomsg = errmsg) if (ierr == 0) then - if (self%j2rp2 /= 0.0_DP) param%loblatecb = .true. if (param%rmin < 0.0) param%rmin = self%radius diff --git a/src/kick/kick.f90 b/src/kick/kick.f90 index 891936f20..c3e37d927 100644 --- a/src/kick/kick.f90 +++ b/src/kick/kick.f90 @@ -189,7 +189,7 @@ module subroutine kick_getacch_int_all_triangular_pl(npl, nplm, x, Gmass, radius end do !$omp end parallel do else - !$omp parallel do default(private) schedule(static)& + !$omp parallel do default(private) schedule(static)& !$omp shared(npl, nplm, x, Gmass, radius) & !$omp lastprivate(rji2, xr, yr, zr) & !$omp reduction(+:ahi) & From 4260a05c99ea75c490be16b019f4c2d34da3c217 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 15:24:53 -0500 Subject: [PATCH 7/7] Fixed debug flags for non-parallel debugging --- Makefile.Defines | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index d7244a886..90a2ceb58 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -51,10 +51,10 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -parallel-source-info=2 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune -IDEBUG = -O2 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin -parallel-source-info=2 +IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma -PAR = -qopenmp -parallel +PAR = -qopenmp -parallel -parallel-source-info=2 HEAPARR = -heap-arrays 4194304 OPTREPORT = -qopt-report=5 IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -qopt-matmul -sox $(PAR) $(SIMDVEC) #$(HEAPARR) @@ -67,12 +67,12 @@ GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporari GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include -LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) +LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcdf -lnetcdff -qopt-matmul #$(PAR) -FSTRICTFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) $(HEAPARR) -FFLAGS = $(IDEBUG) $(SIMDVEC) $(PAR) $(HEAPARR) -# FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback -# FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback +FSTRICTFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) $(HEAPARR) +FFLAGS = $(IDEBUG) #$(SIMDVEC) $(PAR) $(HEAPARR) +#FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback +#FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback FORTRAN = ifort AR = xiar