From c2ddf3e2988deda56aa0965396ea71f0c1963f59 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 Nov 2021 15:35:52 -0500 Subject: [PATCH 1/5] Tweaks to compiler flags --- Makefile.Defines | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index a5417a18d..046d9d3f9 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -41,9 +41,6 @@ SWIFTEST_HOME = $(ROOT_DIR) USER_MODULES = COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ -#NETCDF_FORTRAN_HOME = /home/daminton/local - - # Compiler definitions # DO NOT include in FFLAGS the "-c" option to compile object only @@ -69,8 +66,8 @@ GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) -FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) -g -traceback -FFLAGS = $(IPRODUCTION) -fp-model=fast -g -traceback +FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) +FFLAGS = $(IPRODUCTION) -fp-model=fast FORTRAN = ifort AR = xiar From 125946f35741fb0655ec112f56a37c46b91058af Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 Nov 2021 15:26:58 -0500 Subject: [PATCH 2/5] Added more diagnostic info to the mass conservation failure --- src/io/io.f90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/io/io.f90 b/src/io/io.f90 index ebdd68531..c657ec058 100644 --- a/src/io/io.f90 +++ b/src/io/io.f90 @@ -72,6 +72,10 @@ module subroutine io_conservation_report(self, param, lterminal) if (lterminal) write(*, EGYTERMFMT) Lerror, Ecoll_error, Etotal_error, Merror if (abs(Merror) > 100 * epsilon(Merror)) then write(*,*) "Severe error! Mass not conserved! Halting!" + write(*,*) "Merror = ", Merror + write(*,*) "GMtot_now : ",GMtot_now + write(*,*) "GMtot_orig: ",system%GMtot_orig + write(*,*) "Difference: ",GMtot_now - system%GMtot_orig call pl%xv2el(cb) call self%write_hdr(param%nciu, param) call cb%write_frame(param%nciu, param) From 8eac6a0f3d1f9b904f357ac48ba721e707aadd89 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 24 Nov 2021 11:14:46 -0500 Subject: [PATCH 3/5] Updated NetCDF reads from the profiling branch --- src/netcdf/netcdf.f90 | 87 +++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/src/netcdf/netcdf.f90 b/src/netcdf/netcdf.f90 index d28fcc788..6c1e05e7a 100644 --- a/src/netcdf/netcdf.f90 +++ b/src/netcdf/netcdf.f90 @@ -250,7 +250,8 @@ module subroutine netcdf_initialize_output(self, param) call check( nf90_def_var(self%ncid, ORIGIN_TIME_VARNAME, self%out_type, self%id_dimid, self%origin_time_varid) ) !call check( nf90_def_var_chunking(self%ncid, self%origin_time_varid, NF90_CHUNKED, [self%id_chunk]) ) - call check( nf90_def_var(self%ncid, ORIGIN_TYPE_VARNAME, NF90_CHAR, [self%str_dimid, self%id_dimid], self%origin_type_varid) ) + call check( nf90_def_var(self%ncid, ORIGIN_TYPE_VARNAME, NF90_CHAR, [self%str_dimid, self%id_dimid], & + self%origin_type_varid) ) !call check( nf90_def_var_chunking(self%ncid, self%origin_type_varid, NF90_CHUNKED, [NAMELEN, self%id_chunk]) ) call check( nf90_def_var(self%ncid, ORIGIN_XHX_VARNAME, self%out_type, self%id_dimid, self%origin_xhx_varid) ) !call check( nf90_def_var_chunking(self%ncid, self%origin_xhx_varid, NF90_CHUNKED, [self%id_chunk]) ) @@ -543,55 +544,55 @@ module function netcdf_read_frame_system(self, iu, param) result(ierr) ! Now read in each variable and split the outputs by body type if ((param%in_form == XV) .or. (param%in_form == XVEL)) then call check( nf90_get_var(iu%ncid, iu%xhx_varid, rtemp, start=[1, tslot]) ) - pl%xh(1,:) = pack(rtemp, plmask) - tp%xh(1,:) = pack(rtemp, tpmask) + if (npl > 0) pl%xh(1,:) = pack(rtemp, plmask) + if (ntp > 0) tp%xh(1,:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%xhy_varid, rtemp, start=[1, tslot]) ) - pl%xh(2,:) = pack(rtemp, plmask) - tp%xh(2,:) = pack(rtemp, tpmask) + if (npl > 0) pl%xh(2,:) = pack(rtemp, plmask) + if (ntp > 0) tp%xh(2,:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%xhz_varid, rtemp, start=[1, tslot]) ) - pl%xh(3,:) = pack(rtemp, plmask) - tp%xh(3,:) = pack(rtemp, tpmask) + if (npl > 0) pl%xh(3,:) = pack(rtemp, plmask) + if (ntp > 0) tp%xh(3,:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%vhx_varid, rtemp, start=[1, tslot]) ) - pl%vh(1,:) = pack(rtemp, plmask) - tp%vh(1,:) = pack(rtemp, tpmask) + if (npl > 0) pl%vh(1,:) = pack(rtemp, plmask) + if (ntp > 0) tp%vh(1,:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%vhy_varid, rtemp, start=[1, tslot]) ) - pl%vh(2,:) = pack(rtemp, plmask) - tp%vh(2,:) = pack(rtemp, tpmask) + if (npl > 0) pl%vh(2,:) = pack(rtemp, plmask) + if (ntp > 0) tp%vh(2,:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%vhz_varid, rtemp, start=[1, tslot]) ) - pl%vh(3,:) = pack(rtemp, plmask) - tp%vh(3,:) = pack(rtemp, tpmask) + if (npl > 0) pl%vh(3,:) = pack(rtemp, plmask) + if (ntp > 0) tp%vh(3,:) = pack(rtemp, tpmask) end if if ((param%in_form == EL) .or. (param%in_form == XVEL)) then call check( nf90_get_var(iu%ncid, iu%a_varid, rtemp, start=[1, tslot]) ) - pl%a(:) = pack(rtemp, plmask) - tp%a(:) = pack(rtemp, tpmask) + if (npl > 0) pl%a(:) = pack(rtemp, plmask) + if (ntp > 0) tp%a(:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%e_varid, rtemp, start=[1, tslot]) ) - pl%e(:) = pack(rtemp, plmask) - tp%e(:) = pack(rtemp, tpmask) + if (npl > 0) pl%e(:) = pack(rtemp, plmask) + if (ntp > 0) tp%e(:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%inc_varid, rtemp, start=[1, tslot]) ) - pl%inc(:) = pack(rtemp, plmask) - tp%inc(:) = pack(rtemp, tpmask) + if (npl > 0) pl%inc(:) = pack(rtemp, plmask) + if (ntp > 0) tp%inc(:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%capom_varid, rtemp, start=[1, tslot]) ) - pl%capom(:) = pack(rtemp, plmask) - tp%capom(:) = pack(rtemp, tpmask) + if (npl > 0) pl%capom(:) = pack(rtemp, plmask) + if (ntp > 0) tp%capom(:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%omega_varid, rtemp, start=[1, tslot]) ) - pl%omega(:) = pack(rtemp, plmask) - tp%omega(:) = pack(rtemp, tpmask) + if (npl > 0) pl%omega(:) = pack(rtemp, plmask) + if (ntp > 0) tp%omega(:) = pack(rtemp, tpmask) call check( nf90_get_var(iu%ncid, iu%capm_varid, rtemp, start=[1, tslot]) ) - pl%capm(:) = pack(rtemp, plmask) - tp%capm(:) = pack(rtemp, tpmask) + if (npl > 0) pl%capm(:) = pack(rtemp, plmask) + if (ntp > 0) tp%capm(:) = pack(rtemp, tpmask) end if @@ -599,57 +600,59 @@ module function netcdf_read_frame_system(self, iu, param) result(ierr) cb%Gmass = rtemp(1) cb%mass = cb%Gmass / param%GU - pl%Gmass(:) = pack(rtemp, plmask) - pl%mass(:) = pl%Gmass(:) / param%GU + if (npl > 0) then + pl%Gmass(:) = pack(rtemp, plmask) + pl%mass(:) = pl%Gmass(:) / param%GU - if (param%lrhill_present) then - call check( nf90_get_var(iu%ncid, iu%rhill_varid, rtemp, start=[1, tslot]) ) - pl%rhill(:) = pack(rtemp, plmask) + if (param%lrhill_present) then + call check( nf90_get_var(iu%ncid, iu%rhill_varid, rtemp, start=[1, tslot]) ) + pl%rhill(:) = pack(rtemp, plmask) + end if end if if (param%lclose) then call check( nf90_get_var(iu%ncid, iu%radius_varid, rtemp, start=[1, tslot]) ) cb%radius = rtemp(1) - pl%radius(:) = pack(rtemp, plmask) + if (npl > 0) pl%radius(:) = pack(rtemp, plmask) else cb%radius = param%rmin - pl%radius(:) = 0.0_DP + if (npl > 0) pl%radius(:) = 0.0_DP end if if (param%lrotation) then call check( nf90_get_var(iu%ncid, iu%Ip1_varid, rtemp, start=[1, tslot]) ) cb%Ip(1) = rtemp(1) - pl%Ip(1,:) = pack(rtemp, plmask) + if (npl > 0) pl%Ip(1,:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%Ip2_varid, rtemp, start=[1, tslot]) ) cb%Ip(2) = rtemp(1) - pl%Ip(2,:) = pack(rtemp, plmask) + if (npl > 0) pl%Ip(2,:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%Ip3_varid, rtemp, start=[1, tslot]) ) cb%Ip(3) = rtemp(1) - pl%Ip(3,:) = pack(rtemp, plmask) + if (npl > 0) pl%Ip(3,:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%rotx_varid, rtemp, start=[1, tslot]) ) cb%rot(1) = rtemp(1) - pl%rot(1,:) = pack(rtemp, plmask) + if (npl > 0) pl%rot(1,:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%roty_varid, rtemp, start=[1, tslot]) ) cb%rot(2) = rtemp(1) - pl%rot(2,:) = pack(rtemp, plmask) + if (npl > 0) pl%rot(2,:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%rotz_varid, rtemp, start=[1, tslot]) ) cb%rot(3) = rtemp(1) - pl%rot(3,:) = pack(rtemp, plmask) + if (npl > 0) pl%rot(3,:) = pack(rtemp, plmask) end if if (param%ltides) then call check( nf90_get_var(iu%ncid, iu%k2_varid, rtemp, start=[1, tslot]) ) cb%k2 = rtemp(1) - pl%k2(:) = pack(rtemp, plmask) + if (npl > 0) pl%k2(:) = pack(rtemp, plmask) call check( nf90_get_var(iu%ncid, iu%Q_varid, rtemp, start=[1, tslot]) ) cb%Q = rtemp(1) - pl%Q(:) = pack(rtemp, plmask) + if (npl > 0) pl%Q(:) = pack(rtemp, plmask) end if call self%read_particle_info(iu, param, plmask, tpmask) @@ -936,7 +939,9 @@ module subroutine netcdf_write_frame_base(self, iu, param) select type(self) class is (swiftest_pl) ! Additional output if the passed polymorphic object is a massive body call check( nf90_put_var(iu%ncid, iu%Gmass_varid, self%Gmass(j), start=[idslot, tslot]) ) - if (param%lrhill_present) call check( nf90_put_var(iu%ncid, iu%rhill_varid, self%rhill(j), start=[idslot, tslot]) ) + if (param%lrhill_present) then + call check( nf90_put_var(iu%ncid, iu%rhill_varid, self%rhill(j), start=[idslot, tslot]) ) + end if if (param%lclose) call check( nf90_put_var(iu%ncid, iu%radius_varid, self%radius(j), start=[idslot, tslot]) ) if (param%lrotation) then call check( nf90_put_var(iu%ncid, iu%Ip1_varid, self%Ip(1, j), start=[idslot, tslot]) ) From 638c507611a76fe78106924c9e4e5a06b52b2aa3 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 24 Nov 2021 11:40:18 -0500 Subject: [PATCH 4/5] Removed Intel Advisor stuff from the Makefile.Defines --- Makefile.Defines | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile.Defines b/Makefile.Defines index 9f002eeda..fcc43b14a 100644 --- a/Makefile.Defines +++ b/Makefile.Defines @@ -45,13 +45,10 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/ # DO NOT include in FFLAGS the "-c" option to compile object only # this is done explicitly as needed in the Makefile -ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback -I$(ADVISOR_2019_DIR)/include/intel64 -parallel-source-info=2 -#Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune - IDEBUG = -O0 -init=snan,arrays -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -qopt-matmul -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin STRICTREAL = -fp-model=precise -prec-div -prec-sqrt -assume protect-parens SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except -fma -PAR = -qopenmp -parallel -parallel-source-info=2 +PAR = -qopenmp -parallel HEAPARR = -heap-arrays 4194304 OPTREPORT = -qopt-report=5 IPRODUCTION = -no-wrap-margin -O3 -qopt-prefetch=0 -qopt-matmul -sox $(PAR) $(SIMDVEC) #$(HEAPARR) @@ -64,7 +61,7 @@ GWARNINGS = -Wall -Warray-bounds -Wimplicit-interface -Wextra -Warray-temporari GPRODUCTION = -O3 -ffree-line-length-none $(GPAR) INCLUDES = -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -I$(MKLROOT)/include -LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -L$(ADVISOR_2019_DIR)/lib64 -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) +LINKS = -L$(MKLROOT)/lib/intel64 -L$(NETCDF_FORTRAN_HOME)/lib -lswiftest -lnetcdf -lnetcdff -qopt-matmul $(PAR) FSTRICTFLAGS = $(IPRODUCTION) $(STRICTREAL) FFLAGS = $(IPRODUCTION) -fp-model=fast From 522cfd669a24151bb62e7a90839d14c91a085eb4 Mon Sep 17 00:00:00 2001 From: Carlisle Wishard Date: Sun, 5 Dec 2021 11:47:02 -0500 Subject: [PATCH 5/5] fixed index of massive body names --- src/discard/discard.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/discard/discard.f90 b/src/discard/discard.f90 index ad5b73426..d791274ef 100644 --- a/src/discard/discard.f90 +++ b/src/discard/discard.f90 @@ -247,7 +247,7 @@ subroutine discard_pl_tp(tp, system, param) write(idstrj, *) pl%id(j) write(timestr, *) param%t write(*, *) "Test particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstri)) // ")" & - // " too close to massive body " // trim(adjustl(pl%info(i)%name)) // " (" // trim(adjustl(idstrj)) & + // " too close to massive body " // trim(adjustl(pl%info(j)%name)) // " (" // trim(adjustl(idstrj)) // ")" & // " at t = " // trim(adjustl(timestr)) tp%ldiscard(i) = .true. call tp%info(i)%set_value(status="DISCARDED_PLR", discard_time=param%t, discard_xh=tp%xh(:,i), discard_vh=tp%vh(:,i), discard_body_id=pl%id(j))