Merge branch 'debug' into DocumentationTesting

Makefile

@@ -59,25 +59,30 @@ SWIFTEST_MODULES =   swiftest_globals.f90 \
 
 include Makefile.Defines
 
+MKL_ROOT = /apps/spack/bell/apps/intel-parallel-studio/cluster.2019.5-intel-19.0.5-4brgqlf/mkl/lib
+IMKL = -I$(MKLROOT)/include
+LMKL = -L$(MKLROOT)/lib/intel64 -qopt-matmul 
+
 MODULES         = $(SWIFTEST_MODULES) $(USER_MODULES) 
 
-.PHONY : all mod lib libdir drivers bin clean force 
+.PHONY : all mod lib libdir fast drivers bin clean force 
 
 % : %.f90 force
-	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include  $< -o $@ \
-	  -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff
+	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) $< -o $@ \
+	  -L$(SWIFTEST_HOME)/lib -lswiftest -L$(NETCDF_FORTRAN_HOME)/lib -lnetcdf -lnetcdff $(LMKL)
 	$(INSTALL_PROGRAM) $@ $(SWIFTEST_HOME)/bin
 	rm -f $@
 
 all:
 	cd $(SWIFTEST_HOME); \
 	  make mod; \
 	  make lib; \
+	  make fast; \
 	  make drivers; \
 
 mod:
 	cd $(SWIFTEST_HOME)/src/modules/; \
-	  $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -c $(MODULES); \
+	  $(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c $(MODULES); \
 	  $(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o; \
 	  $(INSTALL_DATA) *.mod *.smod $(SWIFTEST_HOME)/include; \
 	  rm -f *.o *.mod  *.smod
@@ -93,11 +98,6 @@ lib:
 	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
 	  ln -s $(SWIFTEST_HOME)/Makefile .; \
 	  make libdir
-	cd $(SWIFTEST_HOME)/src/fraggle; \
-	  rm -f Makefile.Defines Makefile; \
-	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
-	  ln -s $(SWIFTEST_HOME)/Makefile .; \
-	  make libdir
 	cd $(SWIFTEST_HOME)/src/gr; \
 	  rm -f Makefile.Defines Makefile; \
 	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
@@ -143,11 +143,6 @@ lib:
 	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
 	  ln -s $(SWIFTEST_HOME)/Makefile .; \
 	  make libdir
-	cd $(SWIFTEST_HOME)/src/util; \
-	  rm -f Makefile.Defines Makefile; \
-	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
-	  ln -s $(SWIFTEST_HOME)/Makefile .; \
-	  make libdir
 	cd $(SWIFTEST_HOME)/src/whm; \
 	  rm -f Makefile.Defines Makefile; \
 	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
@@ -174,8 +169,39 @@ lib:
 	  ln -s $(SWIFTEST_HOME)/Makefile .; \
 	  make libdir
 
+fast:
+	cd $(SWIFTEST_HOME)/src/fraggle; \
+	  rm -f Makefile.Defines Makefile; \
+	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
+	  ln -s $(SWIFTEST_HOME)/Makefile .; \
+	  make fastdir
+
+	cd $(SWIFTEST_HOME)/src/util; \
+	  rm -f Makefile.Defines Makefile; \
+	  ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \
+	  ln -s $(SWIFTEST_HOME)/Makefile .; \
+	  make fastdir
+
+	cd $(SWIFTEST_HOME)/src/rmvs; \
+		$(FORTRAN) $(FFASTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c rmvs_encounter_check.f90; \
+		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
+		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
+		rm -f *.o *.smod	
+
+	cd $(SWIFTEST_HOME)/src/symba; \
+		$(FORTRAN) $(FFASTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c symba_encounter_check.f90; \
+		$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
+		$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
+		rm -f *.o *.smod
+
 libdir:
-	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include -c *.f90; \
+	$(FORTRAN) $(FFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \
+	$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
+	$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
+	rm -f *.o *.smod
+
+fastdir:
+	$(FORTRAN) $(FFASTFLAGS) -I$(SWIFTEST_HOME)/include -I$(NETCDF_FORTRAN_HOME)/include $(IMKL) -c *.f90; \
 	$(AR) rv $(SWIFTEST_HOME)/lib/libswiftest.a *.o *.smod; \
 	$(INSTALL_DATA) *.smod $(SWIFTEST_HOME)/include; \
 	rm -f *.o *.smod

Makefile.Defines

@@ -46,37 +46,36 @@ COLLRESOLVE_HOME = $(ROOT_DIR)/collresolve/
 # DO NOT include in FFLAGS the "-c" option to compile object only
 # this is done explicitly as needed in the Makefile
 ADVIXE_DIR = /apps/cent7/intel/advisor_2019
-ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vec -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -fp-model no-except -mp1 -xhost -traceback 
+ADVIXE_FLAGS = -g -O2 -qopt-report=5 -vecabi=cmdtarget -simd -shared-intel -debug inline-debug-info -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -xhost -traceback 
 
-VTUNE_FLAGS = -g -O2 -vec -simd -shared-intel -qopenmp -debug inline-debug-info -parallel-source-info=2 -parallel -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -qopenmp -fp-model no-except -mp1 -xhost -traceback 
+VTUNE_FLAGS = -g -O2 -qopt-report=5 -simd -shared-intel -qopenmp -debug inline-debug-info -parallel-source-info=2 -parallel -DTBB_DEBUG -DTBB_USE_THREADING_TOOLS -qopenmp -fp-model no-except -mp1 -xhost -traceback 
 #Be sure to set the environment variable KMP_FORKJOIN_FRAMES=1 for OpenMP debuging in vtune
 
 IDEBUG = -O0 -init=snan,arrays  -nogen-interfaces -no-pie -no-ftz -fpe-all=0 -g -traceback -mp1 -fp-model strict -fpe0 -debug all -align all -pad -ip -prec-div -prec-sqrt -assume protect-parens -CB -no-wrap-margin
-STRICTREAL = -fp-model strict -fp-model no-except -prec-div -prec-sqrt -assume protect-parens 
-SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -prec-div -prec-sqrt -assume protect-parens
-PAR = -qopenmp #-parallel  #Something goes wrong in SyMBA at the moment with auto-paralellization enabled
-HEAPARR = -heap-arrays 1048576 
+STRICTREAL = -fp-model strict -prec-div -prec-sqrt -assume protect-parens 
+SIMDVEC = -simd -xhost -align all -assume contiguous_assumed_shape -vecabi=cmdtarget -fp-model no-except 
+PAR = -qopenmp -parallel  
+HEAPARR = -heap-arrays 4194304 
 OPTREPORT = -qopt-report=5 
-IPRODUCTION = -init=snan,arrays  -no-wrap-margin -O3 $(STRICTREAL) $(PAR) $(SIMDVEC) $(HEAPARR) 
+IPRODUCTION = -no-wrap-margin -O3 $(PAR) $(SIMDVEC) #$(HEAPARR)
 
 #gfortran flags
 GDEBUG = -g -Og -fbacktrace -fbounds-check -ffree-line-length-none
 GPAR = -fopenmp #-ftree-parallelize-loops=4
 GMEM =  -fsanitize-address-use-after-scope -fstack-check  -fsanitize=bounds-strict -fsanitize=undefined -fsanitize=signed-integer-overflow -fsanitize=object-size -fstack-protector-all
 GWARNINGS = -Wall -Warray-bounds  -Wimplicit-interface -Wextra -Warray-temporaries
-GPRODUCTION = -O3 -ffree-line-length-none $(GPAR)
+GPRODUCTION = -O2 -ffree-line-length-none $(GPAR)
 
-#FFLAGS  = $(IDEBUG) $(HEAPARR) $(SIMDVEC) $(PAR)
-FFLAGS  = $(IPRODUCTION) $(OPTREPORT)
+
+#FFLAGS  = $(IDEBUG) $(SIMDVEC) $(PAR)
+FFLAGS = $(IPRODUCTION) $(STRICTREAL) $(OPTREPORT) 
+FFASTFLAGS = $(IPRODUCTION) -fp-model fast $(OPTREPORT)
 FORTRAN = ifort
 #AR     = xiar
 
 #FORTRAN = gfortran
 #FFLAGS =  $(GDEBUG) $(GMEM) $(GPAR)
-#FFLAGS =  $(GPRODUCTION) -g -fbacktrace #-fcheck=all #-Wall 
-AR     = ar
-
-# DO NOT include in CFLAGS the "-c" option to compile object only
+#FFLAGS =  $(GPRODUCTION) -g -fbacktrace #-fcheck=all #-Wall AR     = ar # DO NOT include in CFLAGS the "-c" option to compile object only
 # this is done explicitly as needed in the Makefile
 
 CC              = icc

docs/src/rmvs_encounter_check.f90

@@ -36,10 +36,7 @@ module function rmvs_encounter_check_tp(self, system, dt) result(lencounter)
                   if ((.not.tp%lmask(i)).or.(tp%plencP(i) /= 0)) cycle
                   xr(:) = tp%xh(:, i) - pl%xbeg(:, j)
                   vr(:) = tp%vh(:, i) - pl%vbeg(:, j)
-                  r2 = dot_product(xr(:), xr(:))
-                  v2 = dot_product(vr(:), vr(:))
-                  vdotr = dot_product(vr(:), xr(:))
-                  lflag = rmvs_chk_ind(r2, v2, vdotr, dt, r2crit(j))
+                  lflag = rmvs_chk_ind(xr(1), xr(2), xr(3), vr(1), vr(2), vr(3), dt, r2crit(j))
                   if (lflag) tp%plencP(i) = j
                end do
                pl%nenc(j) = count(tp%plencP(1:ntp) == j)

docs/src/symba_encounter_check.f90

@@ -261,20 +261,20 @@ module pure elemental subroutine symba_encounter_check_one(xr, yr, zr, vxr, vyr,
       !! Adapted from Hal Levison's Swift routine symba5_chk.f
       implicit none
       ! Arguments
-      real(DP),     intent(in)  :: xr, yr, zr, vxr, vyr, vzr
-      real(DP),     intent(in)  :: rhill1, rhill2, dt
-      integer(I4B), intent(in)  :: irec
-      logical,      intent(out) :: lencounter, lvdotr
+      real(DP),     intent(in)  :: xr, yr, zr     !! Relative distance vector components
+      real(DP),     intent(in)  :: vxr, vyr, vzr  !! Relative velocity vector components
+      real(DP),     intent(in)  :: rhill1, rhill2 !! Hill spheres of the two bodies
+      real(DP),     intent(in)  :: dt             !! Step size
+      integer(I4B), intent(in)  :: irec           !! Current SyMBA recursion level
+      real(DP),     intent(in)  :: r2crit         !! Square of the critical encounter distance
+      logical,      intent(out) :: lencounter     !! Flag indicating that an encounter has occurred
+      logical,      intent(out) :: lvdotr         !! Logical flag indicating the direction of the v .dot. r vector
       ! Internals
-      real(DP)     :: r2, v2, rcrit, r2crit, vdotr
+      real(DP)     :: r2crit
 
-      rcrit = (rhill1 + rhill2)*RHSCALE*(RSHELL**(irec))
-      r2crit = rcrit**2
-      r2 = xr**2 + yr**2 + zr**2
-      v2 = vxr**2 + vyr**2 + vzr**2
-      vdotr = xr * vxr + yr * vyr + zr * vzr
-      lencounter = rmvs_chk_ind(r2, v2, vdotr, dt, r2crit)
-      lvdotr = (vdotr < 0.0_DP)
+      r2crit = (rhill1 + rhill2)*RHSCALE*(RSHELL**(irec))
+      r2crit = r2crit**2
+      call rmvs_chk_ind(xr, yr, zr, vxr, vyr, vzr, dt, r2crit, lencounter, lvdotr)
 
       return
    end subroutine symba_encounter_check_one

examples/symba_mars_disk/aescattermovie.py

@@ -79,8 +79,7 @@ def setup_plot(self):
     def data_stream(self, frame=0):
         while True:
             d = self.ds.isel(time=frame)
-
-            d = d.where(d['radius'] < self.Rcb, drop=True)
+            d = d.where(np.invert(np.isnan(d['a'])), drop=True)
             d['radmarker'] = (d['radius'] / self.Rcb) * radscale
             radius = d['radmarker'].values
 

examples/symba_mars_disk/param.in

@@ -1,17 +1,17 @@
 !Parameter file for the SyMBA-RINGMOONS test
 T0             0.0 
-TSTOP          1.0e12 
+TSTOP          60000.0
 DT             600.0
 CB_IN          cb.in
 PL_IN          mars.in
 TP_IN          tp.in
 IN_TYPE        ASCII
-ISTEP_OUT      1
-ISTEP_DUMP     1
+ISTEP_OUT      100
+ISTEP_DUMP     100
 BIN_OUT        bin.nc 
 PARTICLE_OUT   particle.dat
-OUT_TYPE       NETCDF_DOUBLE
-OUT_FORM       XVEL
+OUT_TYPE       REAL8
+OUT_FORM       XV
 OUT_STAT       REPLACE
 CHK_CLOSE      yes
 CHK_RMIN       3389500.0

examples/symba_swifter_comparison/8pl_16tp_encounters/cb.in

@@ -1,5 +1,5 @@
 Sun
-0.0002959122081920778
+0.00029591220819207774
 0.004650467260962157
 4.7535806948127355e-12
 -2.2473967953572827e-18

examples/symba_swifter_comparison/8pl_16tp_encounters/cb.swiftest.in

@@ -1,5 +1,5 @@
 Sun
-0.0002959122081920778
+0.00029591220819207774
 0.004650467260962157
 4.7535806948127355e-12
 -2.2473967953572827e-18

examples/symba_swifter_comparison/8pl_16tp_encounters/init_cond.py

@@ -19,7 +19,7 @@
 swiftest_pl    = "pl.swiftest.in"
 swiftest_tp    = "tp.swiftest.in"
 swiftest_cb    = "cb.swiftest.in"
-swiftest_bin   = "bin.swiftest.dat"
+swiftest_bin   = "bin.swiftest.nc"
 swiftest_enc   = "enc.swiftest.dat"
 swiftest_dis  = "discard.swiftest.dat"
 
@@ -28,8 +28,8 @@
 sim.param['T0'] = 0.0
 sim.param['DT'] = 1.0 
 sim.param['TSTOP'] = 365.25e1
-sim.param['ISTEP_OUT']  = 11
-sim.param['ISTEP_DUMP'] = 1
+sim.param['ISTEP_OUT']  = 10
+sim.param['ISTEP_DUMP'] = 10
 sim.param['CHK_QMIN_COORD'] = "HELIO"
 sim.param['CHK_QMIN'] = swiftest.RSun / swiftest.AU2M
 sim.param['CHK_QMIN_RANGE'] = f"{swiftest.RSun / swiftest.AU2M} 1000.0"
@@ -141,4 +141,5 @@
 sim.param['TP_IN'] = swifter_tp
 sim.param['BIN_OUT'] = swifter_bin
 sim.param['ENC_OUT'] = swifter_enc
+sim.param['OUT_TYPE'] = "REAL8"
 sim.save(swifter_input, codename="Swifter")

examples/symba_swifter_comparison/8pl_16tp_encounters/param.swifter.in

@@ -2,10 +2,10 @@
 T0               0.0
 TSTOP            3652.5
 DT               1.0
-ISTEP_OUT        11
-ISTEP_DUMP       1
+ISTEP_OUT        10
+ISTEP_DUMP       10
 OUT_FORM         XV
-OUT_TYPE         NETCDF_DOUBLE
+OUT_TYPE         REAL8
 OUT_STAT         UNKNOWN
 IN_TYPE          ASCII
 PL_IN            pl.swifter.in

examples/symba_swifter_comparison/8pl_16tp_encounters/param.swiftest.in

@@ -2,16 +2,16 @@
 T0               0.0
 TSTOP            3652.5
 DT               1.0
-ISTEP_OUT        11
-ISTEP_DUMP       1
+ISTEP_OUT        10
+ISTEP_DUMP       10
 OUT_FORM         XV
 OUT_TYPE         NETCDF_DOUBLE
 OUT_STAT         UNKNOWN
 IN_TYPE          ASCII
 PL_IN            pl.swiftest.in
 TP_IN            tp.swiftest.in
 CB_IN            cb.swiftest.in
-BIN_OUT          bin.swiftest.dat
+BIN_OUT          bin.swiftest.nc
 CHK_QMIN         0.004650467260962157
 CHK_RMIN         0.004650467260962157
 CHK_RMAX         1000.0

examples/symba_swifter_comparison/8pl_16tp_encounters/pl.in

@@ -1,33 +1,33 @@
 8
-Mercury 4.9125474498983625056e-11 0.0014751323154597007903
+Mercury 4.9125474498983623693e-11 0.0014751323154597003982
 1.6306381826061645943e-05
 0.053584775529809842987 -0.4548355025417368247 -0.04208301187261995896
 0.022298358665237189014 0.0047355207618514265702 -0.0016584224113858070382
-Venus 7.243452483873647106e-10 0.0067590814914530454873
+Venus 7.243452483873646905e-10 0.006759081491453044288
 4.0453784346544178454e-05
 0.12681182092868958922 -0.7161485778943049718 -0.017146261752773749032
 0.01978070713081106144 0.0034557070729633850362 -0.00109402215681010293
-Earth 8.997011382166018993e-10 0.010044922299157372164
+Earth 8.9970113821660187435e-10 0.010044922299157369357
 4.25875607065040958e-05
 0.9913796310092216624 -0.17236385208280941006 4.574442303609438109e-06
 0.0026673818939059660942 0.016885702625202340249 -8.2074388361713082097e-07
-Mars 9.549535102761465872e-11 0.007246507286611460043
+Mars 9.549535102761465607e-11 0.0072465072866114584993
 2.265740805092889601e-05
 -1.6436878725691590475 -0.09931688681832298582 0.038237939117251117105
 0.0013642455487206960919 -0.0127728951275482699446 -0.00030115173687901287654
-Jupiter 2.8253459086313549713e-07 0.3552710784524730732
+Jupiter 2.825345908631354893e-07 0.35527107845247299128
 0.00046732617030490929307
 4.304060110247122317 -2.579516473452256875 -0.08558202993848706974
 0.003792902202341501966 0.0068350794283332117623 -0.00011324814038141340017
-Saturn 8.45971518300641563e-08 0.4376659135625219704
+Saturn 8.459715183006415395e-08 0.43766591356252188504
 0.00038925687730393611812
 6.54409134618183419 -7.483470470167333133 -0.1303290586096018111
 0.003893524262024787054 0.0036668581511023591937 -0.00021865564058601801348
-Uranus 1.2920249163736673984e-08 0.46971473227488383167
+Uranus 1.2920249163736673626e-08 0.46971473227488373932
 0.00016953449859497231466
 14.692788408572690528 13.179130291284799625 -0.14143429698462339772
 -0.0026516826407085368304 0.0027513763836455209892 4.4427867883713361775e-05
-Neptune 1.5243589003230834746e-08 0.78149679568494038567
+Neptune 1.5243589003230834323e-08 0.7814967956849401736
 0.000164587904124493665
 29.58593540166936009 -4.435365846939811618 -0.5905556302070252839
 0.0004489890080502080224 0.003131021601122137201 -7.4728898269552307757e-05

examples/symba_swifter_comparison/8pl_16tp_encounters/pl.swifter.in

@@ -1,36 +1,36 @@
 9
-0 0.0002959122081920778
+0 0.00029591220819207774
 0.0 0.0 0.0
 0.0 0.0 0.0
-1 4.9125474498983625056e-11 0.0014751323154597007903
+1 4.9125474498983623693e-11 0.0014751323154597003982
 1.6306381826061645943e-05
 0.053584775529809842987 -0.4548355025417368247 -0.04208301187261995896
 0.022298358665237189014 0.0047355207618514265702 -0.0016584224113858070382
-2 7.243452483873647106e-10 0.0067590814914530454873
+2 7.243452483873646905e-10 0.006759081491453044288
 4.0453784346544178454e-05
 0.12681182092868958922 -0.7161485778943049718 -0.017146261752773749032
 0.01978070713081106144 0.0034557070729633850362 -0.00109402215681010293
-3 8.997011382166018993e-10 0.010044922299157372164
+3 8.9970113821660187435e-10 0.010044922299157369357
 4.25875607065040958e-05
 0.9913796310092216624 -0.17236385208280941006 4.574442303609438109e-06
 0.0026673818939059660942 0.016885702625202340249 -8.2074388361713082097e-07
-4 9.549535102761465872e-11 0.007246507286611460043
+4 9.549535102761465607e-11 0.0072465072866114584993
 2.265740805092889601e-05
 -1.6436878725691590475 -0.09931688681832298582 0.038237939117251117105
 0.0013642455487206960919 -0.0127728951275482699446 -0.00030115173687901287654
-5 2.8253459086313549713e-07 0.3552710784524730732
+5 2.825345908631354893e-07 0.35527107845247299128
 0.00046732617030490929307
 4.304060110247122317 -2.579516473452256875 -0.08558202993848706974
 0.003792902202341501966 0.0068350794283332117623 -0.00011324814038141340017
-6 8.45971518300641563e-08 0.4376659135625219704
+6 8.459715183006415395e-08 0.43766591356252188504
 0.00038925687730393611812
 6.54409134618183419 -7.483470470167333133 -0.1303290586096018111
 0.003893524262024787054 0.0036668581511023591937 -0.00021865564058601801348
-7 1.2920249163736673984e-08 0.46971473227488383167
+7 1.2920249163736673626e-08 0.46971473227488373932
 0.00016953449859497231466
 14.692788408572690528 13.179130291284799625 -0.14143429698462339772
 -0.0026516826407085368304 0.0027513763836455209892 4.4427867883713361775e-05
-8 1.5243589003230834746e-08 0.78149679568494038567
+8 1.5243589003230834323e-08 0.7814967956849401736
 0.000164587904124493665
 29.58593540166936009 -4.435365846939811618 -0.5905556302070252839
 0.0004489890080502080224 0.003131021601122137201 -7.4728898269552307757e-05