From 24e4be48418721627d5fff3658485e191b2b65a0 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 4 Jan 2023 20:59:16 -0500 Subject: [PATCH 001/149] Added infrastructure for coarray support --- cmake/Modules/FindCoarray_Fortran.cmake | 90 +++++++++++++++++++ cmake/Modules/SetFortranFlags.cmake | 8 +- cmake/Modules/SetParallelizationLibrary.cmake | 5 ++ src/CMakeLists.txt | 10 ++- src/collision/collision_regime.f90 | 4 +- src/collision/collision_resolve.f90 | 3 +- src/swiftest/swiftest_module.f90 | 1 + 7 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 cmake/Modules/FindCoarray_Fortran.cmake diff --git a/cmake/Modules/FindCoarray_Fortran.cmake b/cmake/Modules/FindCoarray_Fortran.cmake new file mode 100644 index 000000000..6a05818b8 --- /dev/null +++ b/cmake/Modules/FindCoarray_Fortran.cmake @@ -0,0 +1,90 @@ +# Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. + +# - Finds Coarray support +# This module can be used to detect Coarray support in a compiler. +# If the compiler supports Coarray, the flags required to compile with +# coarray support are set. +# +# This module was modified from the standard FindOpenMP module to find Fortran +# flags. +# +# The following variables are set: +# Coarray_Fortran_FLAGS - flags to add to the Fortran compiler for Coarray +# support. In general, you must use these at both +# compile- and link-time. +# OMP_NUM_PROCS - the max number of processors available to Coarray + +#============================================================================= + +INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) + + +SET (Coarray_Fortran_FLAG_CANDIDATES + #Intel + "-coarray=distributed" + #Intel windows + "/Qcoarray:distributed" + #Gnu + "-fcorray=lib -lcaf_mpi" + #Empty, if compiler automatically accepts coarray + " " +) + +IF (DEFINED Coarray_Fortran_FLAGS) + SET (Coarray_Fortran_FLAG_CANDIDATES) +ENDIF (DEFINED Coarray_Fortran_FLAGS) + +# check fortran compiler. also determine number of processors +FOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES}) + SET (SAFE_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + SET (CMAKE_REQUIRED_FLAGS "${FLAG}") + UNSET (Coarray_FLAG_DETECTED CACHE) + MESSAGE (STATUS "Try Coarray Fortran flag = [${FLAG}]") + FILE (WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90" +" +program TestCoarray + integer, codimension[*] :: i + write(*,'(I2)',ADVANCE='NO') num_images() +end program TestCoarray +") + SET (MACRO_CHECK_FUNCTION_DEFINITIONS + "-DCoarray_FLAG_DETECTED ${CMAKE_REQUIRED_FLAGS}") + TRY_RUN (Coarray_RUN_FAILED Coarray_FLAG_DETECTED ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCoarray.f90 + COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} + CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} + COMPILE_OUTPUT_VARIABLE OUTPUT + RUN_OUTPUT_VARIABLE OMP_NUM_PROCS_INTERNAL) + IF (Coarray_FLAG_DETECTED) + FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log + "Determining if the Fortran compiler supports Coarray passed with " + "the following output:\n${OUTPUT}\n\n") + SET (Coarray_FLAG_DETECTED 1) + IF (Coarray_RUN_FAILED) + MESSAGE (FATAL_ERROR "Coarray found, but test code did not run") + ENDIF (Coarray_RUN_FAILED) + SET (Coarray_Fortran_FLAGS_INTERNAL "${FLAG}") + BREAK () + ELSE () + FILE (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Determining if the Fortran compiler supports Coarray failed with " + "the following output:\n${OUTPUT}\n\n") + SET (Coarray_FLAG_DETECTED 0) + ENDIF (Coarray_FLAG_DETECTED) +ENDFOREACH (FLAG ${Coarray_Fortran_FLAG_CANDIDATES}) + +SET (Coarray_Fortran_FLAGS "${Coarray_Fortran_FLAGS_INTERNAL}" + CACHE STRING "Fortran compiler flags for Coarray parallization") + +# handle the standard arguments for FIND_PACKAGE +FIND_PACKAGE_HANDLE_STANDARD_ARGS (Coarray_Fortran DEFAULT_MSG + Coarray_Fortran_FLAGS) + +MARK_AS_ADVANCED(Coarray_Fortran_FLAGS) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index d869e89b6..64554ac22 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -79,9 +79,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-xhost" # Intel "/QxHost" # Intel Windows ${GNUNATIVE} # GNU - "-ta=host" # Portland Group ) + ################### ### DEBUG FLAGS ### ################### @@ -99,12 +99,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-warn all" # Intel "/warn:all" # Intel Windows "-Wall" # GNU - # Portland Group (on by default) ) # Traceback SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-traceback" # Intel/Portland Group + Fortran "-traceback" # Intel Group "/traceback" # Intel Windows "-fbacktrace" # GNU (gfortran) "-ftrace=full" # GNU (g95) @@ -116,7 +115,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" "/check:bounds" # Intel Windows "-fcheck=bounds" # GNU (New style) "-fbounds-check" # GNU (Old style) - "-Mbounds" # Portland Group ) # Initializes matrices/arrays with NaN values @@ -225,7 +223,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-unroll" # Intel "/unroll" # Intel Windows "-funroll-loops" # GNU - "-Munroll" # Portland Group ) # Inline functions @@ -233,7 +230,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-inline" # Intel "/Qinline" # Intel Windows "-finline-functions" # GNU - "-Minline" # Portland Group ) diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake index 224806406..d809cc00b 100644 --- a/cmake/Modules/SetParallelizationLibrary.cmake +++ b/cmake/Modules/SetParallelizationLibrary.cmake @@ -27,6 +27,10 @@ IF (USE_MPI) # Find MPI IF (NOT MPI_Fortran_FOUND) FIND_PACKAGE (MPI REQUIRED) + FIND_PACKAGE (Coarray_Fortran) + IF (NOT Coarray_Fortran_FLAGS) + MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays") + ENDIF (NOT Coarray_Fortran_FLAGS) ENDIF (NOT MPI_Fortran_FOUND) ENDIF (USE_MPI) @@ -35,6 +39,7 @@ IF (NOT USE_OPENMP AND NOT USE_MPI) SET (OMP_NUM_PROCS 0 CACHE STRING "Number of processors OpenMP may use" FORCE) UNSET (OpenMP_Fortran_FLAGS CACHE) + UNSET (Coarray_Fortran_FLAGS CACHE) UNSET (GOMP_Fortran_LINK_FLAGS CACHE) UNSET (MPI_FOUND CACHE) UNSET (MPI_COMPILER CACHE) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4fd3a1cce..ad2a6a7af 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,13 +100,15 @@ IF(USE_OPENMP) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}" LINK_FLAGS "${OpenMP_Fortran_FLAGS}") -ELSEIF(USE_MPI) +ENDIF(USE_OPENMP) + +IF(USE_MPI) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES - COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS}" - LINK_FLAGS "${MPI_Fortran_LINK_FLAGS}") + COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS} ${Coarray_Fortran_FLAGS}" + LINK_FLAGS "${MPI_Fortran_LINK_FLAGS} ${Coarray_Fortran_FLAGS}") INCLUDE_DIRECTORIES(${MPI_Fortran_INCLUDE_PATH}) TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${MPI_Fortran_LIBRARIES}) -ENDIF(USE_OPENMP) +ENDIF(USE_MPI) ##################################### diff --git a/src/collision/collision_regime.f90 b/src/collision/collision_regime.f90 index a1ae47fe0..fb0f2f7da 100644 --- a/src/collision/collision_regime.f90 +++ b/src/collision/collision_regime.f90 @@ -150,7 +150,7 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, real(DP), parameter :: SUPERCAT_QRATIO = 1.8_DP ! See Section 4.1 of LS12 ! Internals real(DP) :: a1, alpha, aint, b, bcrit, c_star, egy, zeta, l, lint, mu, phi, theta - real(DP) :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat + real(DP) :: Qr, Qrd_pstar, Qr_erosion, Qr_supercat, Qmerge real(DP) :: Vhr, Verosion, Vescp, Vhill, Vimp, Vsupercat real(DP) :: Mint, Mtot, Mtmp real(DP) :: Rp, rhill @@ -215,6 +215,8 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, Qloss = 0.0_DP U_binding = (3.0_DP * Mtot) / (5.0_DP * Rp) ! LS12 eq. 27 + Qmerge = Gc * m1 * m2 / (.mag.(rh2 - rh1)) + 3*m1 / (5*rad1) + 3*m2 / (5*rad1) - U_binding ! Change in energy due to a pure merger + if ((m1 < min_mfrag).or.(m2 < min_mfrag)) then regime = COLLRESOLVE_REGIME_MERGE !perfect merging regime Mlr = Mtot diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index a745f7b3e..21dacc918 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -543,7 +543,6 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) call collision_history%take_snapshot(param,nbody_system, t, "before") call nbody_system%get_energy_and_momentum(param) - collider%pe(1) = nbody_system%pe call collider%generate(nbody_system, param, t) @@ -576,7 +575,7 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) if (.not.lplpl_collision) exit if (loop == MAXCASCADE) then write(*,*) - write(*,*) "An runaway collisional cascade has been detected in collision_resolve_plpl." + write(*,*) "A runaway collisional cascade has been detected in collision_resolve_plpl." write(*,*) "Consider reducing the step size or changing the parameters in the collisional model to reduce the number of fragments." call util_exit(FAILURE) end if diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index beb883abe..019768166 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -312,6 +312,7 @@ module swiftest class(swiftest_cb), allocatable :: cb !! Central body data structure class(swiftest_pl), allocatable :: pl !! Massive body data structure class(swiftest_tp), allocatable :: tp !! Test particle data structure + class(swiftest_tp), dimension(:),codimension[:], allocatable :: cotp !! Co-array test particle data structure class(swiftest_tp), allocatable :: tp_discards !! Discarded test particle data structure class(swiftest_pl), allocatable :: pl_discards !! Discarded massive body particle data structure From b8391ef7d1a6980cf0dd992b5b63c6e961991179 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 6 Mar 2023 15:57:39 -0500 Subject: [PATCH 002/149] Restrucuted coarray test particles to not be attached to an abstract class --- src/misc/solver_module.f90 | 2 +- src/rmvs/rmvs_module.f90 | 2 +- src/rmvs/rmvs_util.f90 | 67 +++++++++++++++++--------------- src/swiftest/swiftest_module.f90 | 2 - src/whm/whm_module.f90 | 1 + 5 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/misc/solver_module.f90 b/src/misc/solver_module.f90 index 410116b2a..504e9215e 100644 --- a/src/misc/solver_module.f90 +++ b/src/misc/solver_module.f90 @@ -16,7 +16,7 @@ module solver use lambda_function use, intrinsic :: ieee_exceptions private - public :: solve_linear_system, solve_rkf45, solve_roots + public :: solve_linear_system, solve_roots interface solve_linear_system module procedure solve_linear_system_dp diff --git a/src/rmvs/rmvs_module.f90 b/src/rmvs/rmvs_module.f90 index 733a81e60..3a950bc8d 100644 --- a/src/rmvs/rmvs_module.f90 +++ b/src/rmvs/rmvs_module.f90 @@ -98,7 +98,7 @@ module rmvs integer(I4B), dimension(:), allocatable :: plind !! Connects the planetocentric indices back to the heliocentric planet list type(rmvs_interp), dimension(:), allocatable :: outer !! interpolated heliocentric central body position for outer encounters type(rmvs_interp), dimension(:), allocatable :: inner !! interpolated heliocentric central body position for inner encounters - class(rmvs_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) + class(swiftest_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) logical :: lplanetocentric = .false. !! Flag that indicates that the object is a planetocentric set of masive bodies used for close encounter calculations contains procedure :: setup => rmvs_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index b9462840b..d4d486da7 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -356,39 +356,42 @@ module subroutine rmvs_util_setup_initialize_system(self, param) tp%lplanetocentric = .false. cb%lplanetocentric = .false. associate(npl => pl%nbody) - allocate(pl%planetocentric(npl)) - pl%planetocentric(:)%lplanetocentric = .true. - do i = 1, npl - allocate(pl%planetocentric(i)%cb, source=cb) - allocate(rmvs_pl :: pl%planetocentric(i)%pl) - select type(cbenci => pl%planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => pl%planetocentric(i)%pl) - class is (rmvs_pl) - cbenci%lplanetocentric = .true. - plenci%lplanetocentric = .true. - call plenci%setup(npl, param) - plenci%status(:) = ACTIVE - plenci%lmask(:) = .true. - ! plind stores the heliocentric index value of a planetocentric planet - ! e.g. Consider an encounter with planet 3. - ! Then the following will be the values of plind: - ! pl%planetocentric(3)%pl%plind(1) = 0 (central body - never used) - ! pl%planetocentric(3)%pl%plind(2) = 1 - ! pl%planetocentric(3)%pl%plind(3) = 2 - ! pl%planetocentric(3)%pl%plind(4) = 4 - ! pl%planetocentric(3)%pl%plind(5) = 5 - ! etc. - allocate(plenci%plind(npl)) - plenci%plind(1:npl) = [(j,j=1,npl)] - plenci%plind(2:npl) = pack(plenci%plind(1:npl), plenci%plind(1:npl) /= i) - plenci%plind(1) = 0 - plenci%Gmass(1) = cb%Gmass - plenci%Gmass(2:npl) = pl%Gmass(plenci%plind(2:npl)) - cbenci%Gmass = pl%Gmass(i) + allocate(rmvs_nbody_system :: pl%planetocentric(npl)) + select type(planetocentric => pl%planetocentric) + class is (rmvs_nbody_system) + planetocentric(:)%lplanetocentric = .true. + do i = 1, npl + allocate(planetocentric(i)%cb, source=cb) + allocate(rmvs_pl :: planetocentric(i)%pl) + select type(cbenci => planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => planetocentric(i)%pl) + class is (rmvs_pl) + cbenci%lplanetocentric = .true. + plenci%lplanetocentric = .true. + call plenci%setup(npl, param) + plenci%status(:) = ACTIVE + plenci%lmask(:) = .true. + ! plind stores the heliocentric index value of a planetocentric planet + ! e.g. Consider an encounter with planet 3. + ! Then the following will be the values of plind: + ! pl%planetocentric(3)%pl%plind(1) = 0 (central body - never used) + ! pl%planetocentric(3)%pl%plind(2) = 1 + ! pl%planetocentric(3)%pl%plind(3) = 2 + ! pl%planetocentric(3)%pl%plind(4) = 4 + ! pl%planetocentric(3)%pl%plind(5) = 5 + ! etc. + allocate(plenci%plind(npl)) + plenci%plind(1:npl) = [(j,j=1,npl)] + plenci%plind(2:npl) = pack(plenci%plind(1:npl), plenci%plind(1:npl) /= i) + plenci%plind(1) = 0 + plenci%Gmass(1) = cb%Gmass + plenci%Gmass(2:npl) = pl%Gmass(plenci%plind(2:npl)) + cbenci%Gmass = pl%Gmass(i) + end select end select - end select - end do + end do + end select end associate end select end select diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 270247e28..a033e6618 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -311,8 +311,6 @@ module swiftest class(swiftest_cb), allocatable :: cb !! Central body data structure class(swiftest_pl), allocatable :: pl !! Massive body data structure class(swiftest_tp), allocatable :: tp !! Test particle data structure - class(swiftest_tp), dimension(:),codimension[:], allocatable :: cotp !! Co-array test particle data structure - class(swiftest_tp), allocatable :: tp_discards !! Discarded test particle data structure class(swiftest_pl), allocatable :: pl_discards !! Discarded massive body particle data structure class(swiftest_pl), allocatable :: pl_adds !! List of added bodies in mergers or collisions diff --git a/src/whm/whm_module.f90 b/src/whm/whm_module.f90 index d2d10f971..fc97eee3b 100644 --- a/src/whm/whm_module.f90 +++ b/src/whm/whm_module.f90 @@ -70,6 +70,7 @@ module whm !> An abstract class for the WHM integrator nbody system type, extends(swiftest_nbody_system) :: whm_nbody_system + class(whm_tp), dimension(:),codimension[:], allocatable :: cotp !! Co-array test particle data structure contains !> Replace the abstract procedures with concrete ones procedure :: initialize => whm_util_setup_initialize_system !! Performs WHM-specific initilization steps, like calculating the Jacobi masses From dcbf6e7b4b600e59cb8dea0ddadf349977c871b1 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 9 Mar 2023 10:22:00 -0500 Subject: [PATCH 003/149] Fixed typo --- cmake/Modules/FindCoarray_Fortran.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/FindCoarray_Fortran.cmake b/cmake/Modules/FindCoarray_Fortran.cmake index 6a05818b8..05f28ef0a 100644 --- a/cmake/Modules/FindCoarray_Fortran.cmake +++ b/cmake/Modules/FindCoarray_Fortran.cmake @@ -32,7 +32,7 @@ SET (Coarray_Fortran_FLAG_CANDIDATES #Intel windows "/Qcoarray:distributed" #Gnu - "-fcorray=lib -lcaf_mpi" + "-fcoarray=lib -lcaf_mpi" #Empty, if compiler automatically accepts coarray " " ) From 5df51a6229c65aca9e7b190ef4a7baa37a6398c2 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 20 Mar 2023 10:00:28 -0400 Subject: [PATCH 004/149] Updated coarray branch with modification that will help make coarrays work (eventually) --- CMakeLists.txt | 2 +- cmake/Modules/SetParallelizationLibrary.cmake | 24 +-- src/CMakeLists.txt | 10 +- src/rmvs/rmvs_module.f90 | 2 +- src/rmvs/rmvs_step.f90 | 186 +++++++++--------- src/swiftest/swiftest_driver.f90 | 133 +++++++------ src/swiftest/swiftest_module.f90 | 28 +-- src/swiftest/swiftest_util.f90 | 127 ++++++------ src/whm/whm_module.f90 | 1 - 9 files changed, 257 insertions(+), 256 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ce5af9cf..eb709cbe0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) # Set some options the user may choose # Uncomment the below if you want the user to choose a parallelization library -OPTION(USE_MPI "Use the MPI library for parallelization" ON) +OPTION(USE_CAF "Use Coarray Fortran for parallelization" ON) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake index d809cc00b..3f3153f4d 100644 --- a/cmake/Modules/SetParallelizationLibrary.cmake +++ b/cmake/Modules/SetParallelizationLibrary.cmake @@ -7,11 +7,6 @@ # You should have received a copy of the GNU General Public License along with Swiftest. # If not, see: https://www.gnu.org/licenses. -# Turns on either OpenMP or MPI -# If both are requested, the other is disabled -# When one is turned on, the other is turned off -# If both are off, we explicitly disable them just in case - IF (USE_OPENMP) # Find OpenMP IF (NOT OpenMP_Fortran_FLAGS) @@ -20,28 +15,23 @@ IF (USE_OPENMP) MESSAGE (FATAL_ERROR "Fortran compiler does not support OpenMP") ENDIF (NOT OpenMP_Fortran_FLAGS) ENDIF (NOT OpenMP_Fortran_FLAGS) - # Turn of MPI ENDIF (USE_OPENMP) -IF (USE_MPI) +IF (USE_CAF) # Find MPI - IF (NOT MPI_Fortran_FOUND) - FIND_PACKAGE (MPI REQUIRED) + IF (NOT Coarray_Fortran_FLAGS) FIND_PACKAGE (Coarray_Fortran) IF (NOT Coarray_Fortran_FLAGS) MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays") ENDIF (NOT Coarray_Fortran_FLAGS) - ENDIF (NOT MPI_Fortran_FOUND) -ENDIF (USE_MPI) + ENDIF (NOT Coarray_Fortran_FLAGS) +ENDIF (USE_CAF) -IF (NOT USE_OPENMP AND NOT USE_MPI) - # Turn off both OpenMP and MPI +IF (NOT USE_OPENMP AND NOT USE_CAF) + # Turn off both OpenMP and CAF SET (OMP_NUM_PROCS 0 CACHE STRING "Number of processors OpenMP may use" FORCE) UNSET (OpenMP_Fortran_FLAGS CACHE) UNSET (Coarray_Fortran_FLAGS CACHE) UNSET (GOMP_Fortran_LINK_FLAGS CACHE) - UNSET (MPI_FOUND CACHE) - UNSET (MPI_COMPILER CACHE) - UNSET (MPI_LIBRARY CACHE) -ENDIF (NOT USE_OPENMP AND NOT USE_MPI) +ENDIF (NOT USE_OPENMP AND NOT USE_CAF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c41433ede..b8cc5be2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -101,13 +101,11 @@ IF(USE_OPENMP) LINK_FLAGS "${OpenMP_Fortran_FLAGS}") ENDIF(USE_OPENMP) -IF(USE_MPI) +IF(USE_CAF) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES - COMPILE_FLAGS "${MPI_Fortran_COMPILE_FLAGS} ${Coarray_Fortran_FLAGS}" - LINK_FLAGS "${MPI_Fortran_LINK_FLAGS} ${Coarray_Fortran_FLAGS}") - INCLUDE_DIRECTORIES(${MPI_Fortran_INCLUDE_PATH}) - TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${MPI_Fortran_LIBRARIES}) -ENDIF(USE_MPI) + COMPILE_FLAGS "${Coarray_Fortran_FLAGS}" + LINK_FLAGS "${Coarray_Fortran_FLAGS}") +ENDIF(USE_CAF) ##################################### diff --git a/src/rmvs/rmvs_module.f90 b/src/rmvs/rmvs_module.f90 index 3a950bc8d..74910d7d3 100644 --- a/src/rmvs/rmvs_module.f90 +++ b/src/rmvs/rmvs_module.f90 @@ -98,7 +98,7 @@ module rmvs integer(I4B), dimension(:), allocatable :: plind !! Connects the planetocentric indices back to the heliocentric planet list type(rmvs_interp), dimension(:), allocatable :: outer !! interpolated heliocentric central body position for outer encounters type(rmvs_interp), dimension(:), allocatable :: inner !! interpolated heliocentric central body position for inner encounters - class(swiftest_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) + class(base_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) logical :: lplanetocentric = .false. !! Flag that indicates that the object is a planetocentric set of masive bodies used for close encounter calculations contains procedure :: setup => rmvs_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess diff --git a/src/rmvs/rmvs_step.f90 b/src/rmvs/rmvs_step.f90 index bcb38797a..18f9bd644 100644 --- a/src/rmvs/rmvs_step.f90 +++ b/src/rmvs/rmvs_step.f90 @@ -451,64 +451,67 @@ subroutine rmvs_make_planetocentric(param, cb, pl, tp) if (allocated(encmask)) deallocate(encmask) allocate(encmask(ntp)) encmask(1:ntp) = tp%plencP(1:ntp) == i - allocate(rmvs_tp :: pl%planetocentric(i)%tp) - ! Create encountering test particle structure - select type(cbenci => pl%planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => pl%planetocentric(i)%pl) - class is (rmvs_pl) - select type(tpenci => pl%planetocentric(i)%tp) - class is (rmvs_tp) - tpenci%lplanetocentric = .true. - associate(nenci => pl%nenc(i)) - call tpenci%setup(nenci, param) - tpenci%cb_heliocentric = cb - tpenci%ipleP = i - tpenci%lmask(1:nenci) = .true. - tpenci%status(1:nenci) = ACTIVE - ! Grab all the encountering test particles and convert them to a planetocentric frame - tpenci%id(1:nenci) = pack(tp%id(1:ntp), encmask(1:ntp)) - do j = 1, NDIM - tpenci%rheliocentric(j, 1:nenci) = pack(tp%rh(j,1:ntp), encmask(:)) - tpenci%rh(j, 1:nenci) = tpenci%rheliocentric(j, 1:nenci) - pl%inner(0)%x(j, i) - tpenci%vh(j, 1:nenci) = pack(tp%vh(j, 1:ntp), encmask(1:ntp)) - pl%inner(0)%v(j, i) - end do - tpenci%lperi(1:nenci) = pack(tp%lperi(1:ntp), encmask(1:ntp)) - tpenci%plperP(1:nenci) = pack(tp%plperP(1:ntp), encmask(1:ntp)) - ! Make sure that the test particles get the planetocentric value of mu - allocate(cbenci%inner(0:NTPHENC)) - do inner_index = 0, NTPHENC - allocate(plenci%inner(inner_index)%x, mold=pl%inner(inner_index)%x) - allocate(plenci%inner(inner_index)%v, mold=pl%inner(inner_index)%x) - allocate(cbenci%inner(inner_index)%x(NDIM,1)) - allocate(cbenci%inner(inner_index)%v(NDIM,1)) - cbenci%inner(inner_index)%x(:,1) = pl%inner(inner_index)%x(:, i) - cbenci%inner(inner_index)%v(:,1) = pl%inner(inner_index)%v(:, i) - plenci%inner(inner_index)%x(:,1) = -cbenci%inner(inner_index)%x(:,1) - plenci%inner(inner_index)%v(:,1) = -cbenci%inner(inner_index)%v(:,1) - - if (param%loblatecb) then - allocate(plenci%inner(inner_index)%aobl, mold=pl%inner(inner_index)%aobl) - allocate(cbenci%inner(inner_index)%aobl(NDIM,1)) - cbenci%inner(inner_index)%aobl(:,1) = pl%inner(inner_index)%aobl(:, i) - end if - - if (param%ltides) then - allocate(plenci%inner(inner_index)%atide, mold=pl%inner(inner_index)%atide) - allocate(cbenci%inner(inner_index)%atide(NDIM,1)) - cbenci%inner(inner_index)%atide(:,1) = pl%inner(inner_index)%atide(:, i) - end if - - do j = 2, npl - ipc2hc = plenci%plind(j) - plenci%inner(inner_index)%x(:,j) = pl%inner(inner_index)%x(:, ipc2hc) & - - cbenci%inner(inner_index)%x(:,1) - plenci%inner(inner_index)%v(:,j) = pl%inner(inner_index)%v(:, ipc2hc) & - - cbenci%inner(inner_index)%v(:,1) + select type (planetocentric => pl%planetocentric) + class is(rmvs_nbody_system) + allocate(rmvs_tp :: planetocentric(i)%tp) + ! Create encountering test particle structure + select type(cbenci => planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => planetocentric(i)%pl) + class is (rmvs_pl) + select type(tpenci => planetocentric(i)%tp) + class is (rmvs_tp) + tpenci%lplanetocentric = .true. + associate(nenci => pl%nenc(i)) + call tpenci%setup(nenci, param) + tpenci%cb_heliocentric = cb + tpenci%ipleP = i + tpenci%lmask(1:nenci) = .true. + tpenci%status(1:nenci) = ACTIVE + ! Grab all the encountering test particles and convert them to a planetocentric frame + tpenci%id(1:nenci) = pack(tp%id(1:ntp), encmask(1:ntp)) + do j = 1, NDIM + tpenci%rheliocentric(j, 1:nenci) = pack(tp%rh(j,1:ntp), encmask(:)) + tpenci%rh(j, 1:nenci) = tpenci%rheliocentric(j, 1:nenci) - pl%inner(0)%x(j, i) + tpenci%vh(j, 1:nenci) = pack(tp%vh(j, 1:ntp), encmask(1:ntp)) - pl%inner(0)%v(j, i) + end do + tpenci%lperi(1:nenci) = pack(tp%lperi(1:ntp), encmask(1:ntp)) + tpenci%plperP(1:nenci) = pack(tp%plperP(1:ntp), encmask(1:ntp)) + ! Make sure that the test particles get the planetocentric value of mu + allocate(cbenci%inner(0:NTPHENC)) + do inner_index = 0, NTPHENC + allocate(plenci%inner(inner_index)%x, mold=pl%inner(inner_index)%x) + allocate(plenci%inner(inner_index)%v, mold=pl%inner(inner_index)%x) + allocate(cbenci%inner(inner_index)%x(NDIM,1)) + allocate(cbenci%inner(inner_index)%v(NDIM,1)) + cbenci%inner(inner_index)%x(:,1) = pl%inner(inner_index)%x(:, i) + cbenci%inner(inner_index)%v(:,1) = pl%inner(inner_index)%v(:, i) + plenci%inner(inner_index)%x(:,1) = -cbenci%inner(inner_index)%x(:,1) + plenci%inner(inner_index)%v(:,1) = -cbenci%inner(inner_index)%v(:,1) + + if (param%loblatecb) then + allocate(plenci%inner(inner_index)%aobl, mold=pl%inner(inner_index)%aobl) + allocate(cbenci%inner(inner_index)%aobl(NDIM,1)) + cbenci%inner(inner_index)%aobl(:,1) = pl%inner(inner_index)%aobl(:, i) + end if + + if (param%ltides) then + allocate(plenci%inner(inner_index)%atide, mold=pl%inner(inner_index)%atide) + allocate(cbenci%inner(inner_index)%atide(NDIM,1)) + cbenci%inner(inner_index)%atide(:,1) = pl%inner(inner_index)%atide(:, i) + end if + + do j = 2, npl + ipc2hc = plenci%plind(j) + plenci%inner(inner_index)%x(:,j) = pl%inner(inner_index)%x(:, ipc2hc) & + - cbenci%inner(inner_index)%x(:,1) + plenci%inner(inner_index)%v(:,j) = pl%inner(inner_index)%v(:, ipc2hc) & + - cbenci%inner(inner_index)%v(:,1) + end do end do - end do - call tpenci%set_mu(cbenci) - end associate + call tpenci%set_mu(cbenci) + end associate + end select end select end select end select @@ -611,39 +614,42 @@ subroutine rmvs_end_planetocentric(pl, tp) associate (npl => pl%nbody, ntp => tp%nbody) do i = 1, npl if (pl%nenc(i) == 0) cycle - select type(cbenci => pl%planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => pl%planetocentric(i)%pl) - class is (rmvs_pl) - select type(tpenci => pl%planetocentric(i)%tp) - class is (rmvs_tp) - associate(nenci => pl%nenc(i)) - if (allocated(tpind)) deallocate(tpind) - allocate(tpind(nenci)) - ! Index array of encountering test particles - if (allocated(encmask)) deallocate(encmask) - allocate(encmask(ntp)) - encmask(1:ntp) = tp%plencP(1:ntp) == i - tpind(1:nenci) = pack([(j,j=1,ntp)], encmask(1:ntp)) - - ! Copy the results of the integration back over and shift back to heliocentric reference - tp%status(tpind(1:nenci)) = tpenci%status(1:nenci) - tp%lmask(tpind(1:nenci)) = tpenci%lmask(1:nenci) - do j = 1, NDIM - tp%rh(j, tpind(1:nenci)) = tpenci%rh(j,1:nenci) + pl%inner(NTPHENC)%x(j, i) - tp%vh(j, tpind(1:nenci)) = tpenci%vh(j,1:nenci) + pl%inner(NTPHENC)%v(j, i) - end do - tp%lperi(tpind(1:nenci)) = tpenci%lperi(1:nenci) - tp%plperP(tpind(1:nenci)) = tpenci%plperP(1:nenci) - deallocate(pl%planetocentric(i)%tp) - deallocate(cbenci%inner) - do inner_index = 0, NTPHENC - deallocate(plenci%inner(inner_index)%x) - deallocate(plenci%inner(inner_index)%v) - if (allocated(plenci%inner(inner_index)%aobl)) deallocate(plenci%inner(inner_index)%aobl) - if (allocated(plenci%inner(inner_index)%atide)) deallocate(plenci%inner(inner_index)%atide) - end do - end associate + select type(planetocentric => pl%planetocentric) + class is(rmvs_nbody_system) + select type(cbenci => planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => planetocentric(i)%pl) + class is (rmvs_pl) + select type(tpenci => planetocentric(i)%tp) + class is (rmvs_tp) + associate(nenci => pl%nenc(i)) + if (allocated(tpind)) deallocate(tpind) + allocate(tpind(nenci)) + ! Index array of encountering test particles + if (allocated(encmask)) deallocate(encmask) + allocate(encmask(ntp)) + encmask(1:ntp) = tp%plencP(1:ntp) == i + tpind(1:nenci) = pack([(j,j=1,ntp)], encmask(1:ntp)) + + ! Copy the results of the integration back over and shift back to heliocentric reference + tp%status(tpind(1:nenci)) = tpenci%status(1:nenci) + tp%lmask(tpind(1:nenci)) = tpenci%lmask(1:nenci) + do j = 1, NDIM + tp%rh(j, tpind(1:nenci)) = tpenci%rh(j,1:nenci) + pl%inner(NTPHENC)%x(j, i) + tp%vh(j, tpind(1:nenci)) = tpenci%vh(j,1:nenci) + pl%inner(NTPHENC)%v(j, i) + end do + tp%lperi(tpind(1:nenci)) = tpenci%lperi(1:nenci) + tp%plperP(tpind(1:nenci)) = tpenci%plperP(1:nenci) + deallocate(planetocentric(i)%tp) + deallocate(cbenci%inner) + do inner_index = 0, NTPHENC + deallocate(plenci%inner(inner_index)%x) + deallocate(plenci%inner(inner_index)%v) + if (allocated(plenci%inner(inner_index)%aobl)) deallocate(plenci%inner(inner_index)%aobl) + if (allocated(plenci%inner(inner_index)%atide)) deallocate(plenci%inner(inner_index)%atide) + end do + end associate + end select end select end select end select diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 2da264bea..336578eb2 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -19,7 +19,7 @@ program swiftest_driver use swiftest implicit none - class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated + class(base_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated class(swiftest_parameters), allocatable :: param !! Run configuration parameters character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters @@ -70,75 +70,78 @@ program swiftest_driver ! Construct the main n-body nbody_system using the user-input integrator to choose the type of nbody_system call swiftest_util_setup_construct_system(nbody_system, param) - - !> Define the maximum number of threads - nthreads = 1 ! In the *serial* case - !$ nthreads = omp_get_max_threads() ! In the *parallel* case - !$ write(param%display_unit,'(a)') ' OpenMP parameters:' - !$ write(param%display_unit,'(a)') ' ------------------' - !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads - !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads - - call nbody_system%initialize(param) - - associate (system_history => nbody_system%system_history) - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. - call nbody_system%display_run_information(param, integration_timer, phase="first") - if (param%lenergy) then - if (param%lrestart) then - call nbody_system%get_t0_values(param) - else - call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum + select type(nbody_system) + class is (swiftest_nbody_system) + + !> Define the maximum number of threads + nthreads = 1 ! In the *serial* case + !$ nthreads = omp_get_max_threads() ! In the *parallel* case + !$ write(param%display_unit,'(a)') ' OpenMP parameters:' + !$ write(param%display_unit,'(a)') ' ------------------' + !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads + !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads + + call nbody_system%initialize(param) + + associate (system_history => nbody_system%system_history) + ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. + call nbody_system%display_run_information(param, integration_timer, phase="first") + if (param%lenergy) then + if (param%lrestart) then + call nbody_system%get_t0_values(param) + else + call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum + end if + call nbody_system%conservation_report(param, lterminal=.true.) end if - call nbody_system%conservation_report(param, lterminal=.true.) - end if - call system_history%take_snapshot(param,nbody_system) - call nbody_system%dump(param) - - do iloop = istart, nloops - !> Step the nbody_system forward in time - call integration_timer%start() - call nbody_system%step(param, nbody_system%t, dt) - call integration_timer%stop() - - nbody_system%t = t0 + iloop * dt - - !> Evaluate any discards or collisional outcomes - call nbody_system%discard(param) - - !> If the loop counter is at the output cadence value, append the data file with a single frame - if (istep_out > 0) then - iout = iout + 1 - if ((iout == istep) .or. (iloop == nloops)) then - iout = 0 - idump = idump + 1 - if (ltstretch) then - nout = nout + 1 - istep = floor(istep_out * fstep_out**nout, kind=I4B) - end if - - call system_history%take_snapshot(param,nbody_system) + call system_history%take_snapshot(param,nbody_system) + call nbody_system%dump(param) + + do iloop = istart, nloops + !> Step the nbody_system forward in time + call integration_timer%start() + call nbody_system%step(param, nbody_system%t, dt) + call integration_timer%stop() + + nbody_system%t = t0 + iloop * dt + + !> Evaluate any discards or collisional outcomes + call nbody_system%discard(param) + + !> If the loop counter is at the output cadence value, append the data file with a single frame + if (istep_out > 0) then + iout = iout + 1 + if ((iout == istep) .or. (iloop == nloops)) then + iout = 0 + idump = idump + 1 + if (ltstretch) then + nout = nout + 1 + istep = floor(istep_out * fstep_out**nout, kind=I4B) + end if + + call system_history%take_snapshot(param,nbody_system) + + if (idump == dump_cadence) then + idump = 0 + call nbody_system%dump(param) + end if + + call integration_timer%report(message="Integration steps:", unit=display_unit, nsubsteps=istep_out) + call nbody_system%display_run_information(param, integration_timer) + call integration_timer%reset() + if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) - if (idump == dump_cadence) then - idump = 0 - call nbody_system%dump(param) end if - - call integration_timer%report(message="Integration steps:", unit=display_unit, nsubsteps=istep_out) - call nbody_system%display_run_information(param, integration_timer) - call integration_timer%reset() - if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) - end if - end if - end do - ! Dump any remaining history if it exists - call nbody_system%dump(param) - call system_history%dump(param) - call nbody_system%display_run_information(param, integration_timer, phase="last") - - end associate + end do + ! Dump any remaining history if it exists + call nbody_system%dump(param) + call system_history%dump(param) + call nbody_system%display_run_information(param, integration_timer, phase="last") + + end associate + end select end associate call base_util_exit(SUCCESS) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 9f8b7e0c1..92e1a4006 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -304,7 +304,7 @@ module swiftest !> An abstract class for a basic Swiftest nbody system - type, abstract, extends(base_nbody_system) :: swiftest_nbody_system + type, extends(base_nbody_system) :: swiftest_nbody_system !! This superclass contains a minimial nbody_system of a set of test particles (tp), massive bodies (pl), and a central body (cb) !! The full swiftest_nbody_system type that is used as the parent class of all integrators is defined in collision @@ -373,7 +373,7 @@ module swiftest !! the test particles contains !> Each integrator will have its own version of the step - procedure(abstract_step_system), deferred :: step + ! Concrete classes that are common to the basic integrator (only test particles considered for discard) procedure :: discard => swiftest_discard_system !! Perform a discard step on the nbody_system @@ -390,7 +390,8 @@ module swiftest procedure :: read_in => swiftest_io_read_in_system !! Reads the initial conditions for an nbody system procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body - procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer + procedure :: step => swiftest_step_system !! Placeholder step method. Each integrator will override + procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum procedure :: get_idvals => swiftest_util_get_idvalues_system !! Returns an array of all id values in use in the nbody_system procedure :: rescale => swiftest_util_rescale_system !! Rescales the nbody_system into a new set of units @@ -447,15 +448,6 @@ subroutine abstract_step_body(self, nbody_system, param, t, dt) real(DP), intent(in) :: t !! Simulation time real(DP), intent(in) :: dt !! Current stepsize end subroutine abstract_step_body - - subroutine abstract_step_system(self, param, t, dt) - import DP, swiftest_nbody_system, swiftest_parameters - implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - real(DP), intent(in) :: t !! Simulation time - real(DP), intent(in) :: dt !! Current stepsize - end subroutine abstract_step_system end interface @@ -1057,7 +1049,7 @@ end subroutine swiftest_util_setup_body module subroutine swiftest_util_setup_construct_system(nbody_system, param) implicit none - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(base_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_construct_system @@ -1951,6 +1943,16 @@ subroutine swiftest_final_kin(self) end subroutine swiftest_final_kin + subroutine swiftest_step_system(self, param, t, dt) + implicit none + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + real(DP), intent(in) :: t !! Simulation time + real(DP), intent(in) :: dt !! Current stepsize + return + end subroutine swiftest_step_system + + subroutine swiftest_final_storage(self) !! author: David A. Minton !! diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index cc1b9e398..c68dd1ac3 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2703,8 +2703,8 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) !! implicit none ! Arguments - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(base_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters select case(param%integrator) case (INT_BS) @@ -2767,11 +2767,11 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) call base_util_exit(FAILURE) end select - allocate(swiftest_particle_info :: nbody_system%cb%info) - - - nbody_system%t = param%tstart - + select type(nbody_system) + class is (swiftest_nbody_system) + allocate(swiftest_particle_info :: nbody_system%cb%info) + nbody_system%t = param%tstart + end select return end subroutine swiftest_util_setup_construct_system @@ -3055,7 +3055,7 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) ! Internals - class(swiftest_nbody_system), allocatable :: snapshot + class(base_nbody_system), allocatable :: snapshot ! To allow for runs to be restarted in a bit-identical way, we'll need to run the same coordinate conversion routines we would run upon restarting select type(pl => nbody_system%pl) @@ -3077,60 +3077,63 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar ! Take a minimal snapshot wihout all of the extra storage objects allocate(snapshot, mold=nbody_system) - allocate(snapshot%cb, source=nbody_system%cb ) - allocate(snapshot%pl, source=nbody_system%pl ) - allocate(snapshot%tp, source=nbody_system%tp ) - allocate(snapshot%system_history) - allocate(snapshot%system_history%nc, source=nbody_system%system_history%nc) - snapshot%system_history%nc%lfile_is_open = .true. - - snapshot%t = nbody_system%t - snapshot%GMtot = nbody_system%GMtot - snapshot%ke_orbit = nbody_system%ke_orbit - snapshot%ke_spin = nbody_system%ke_spin - snapshot%pe = nbody_system%pe - snapshot%be = nbody_system%be - snapshot%te = nbody_system%te - snapshot%oblpot = nbody_system%oblpot - snapshot%L_orbit = nbody_system%L_orbit - snapshot%L_spin = nbody_system%L_spin - snapshot%L_total = nbody_system%L_total - snapshot%ke_orbit_orig = nbody_system%ke_orbit_orig - snapshot%ke_spin_orig = nbody_system%ke_spin_orig - snapshot%pe_orig = nbody_system%pe_orig - snapshot%be_orig = nbody_system%be_orig - snapshot%E_orbit_orig = nbody_system%E_orbit_orig - snapshot%GMtot_orig = nbody_system%GMtot_orig - snapshot%L_total_orig = nbody_system%L_total_orig - snapshot%L_orbit_orig = nbody_system%L_orbit_orig - snapshot%L_spin_orig = nbody_system%L_spin_orig - snapshot%L_escape = nbody_system%L_escape - snapshot%GMescape = nbody_system%GMescape - snapshot%E_collisions = nbody_system%E_collisions - snapshot%E_untracked = nbody_system%E_untracked - snapshot%ke_orbit_error = nbody_system%ke_orbit_error - snapshot%ke_spin_error = nbody_system%ke_spin_error - snapshot%pe_error = nbody_system%pe_error - snapshot%be_error = nbody_system%be_error - snapshot%E_orbit_error = nbody_system%E_orbit_error - snapshot%Ecoll_error = nbody_system%Ecoll_error - snapshot%E_untracked_error = nbody_system%E_untracked_error - snapshot%te_error = nbody_system%te_error - snapshot%L_orbit_error = nbody_system%L_orbit_error - snapshot%L_spin_error = nbody_system%L_spin_error - snapshot%L_escape_error = nbody_system%L_escape_error - snapshot%L_total_error = nbody_system%L_total_error - snapshot%Mtot_error = nbody_system%Mtot_error - snapshot%Mescape_error = nbody_system%Mescape_error - snapshot%lbeg = nbody_system%lbeg - - - ! Store a snapshot of the nbody_system for posterity - call base_util_snapshot_save(self, snapshot) - self%nt = self%iframe - self%nid = self%nid + 1 ! Central body - if (allocated(nbody_system%pl)) self%nid = self%nid + nbody_system%pl%nbody - if (allocated(nbody_system%tp)) self%nid = self%nid + nbody_system%tp%nbody + select type(snapshot) + class is (swiftest_nbody_system) + allocate(snapshot%cb, source=nbody_system%cb ) + allocate(snapshot%pl, source=nbody_system%pl ) + allocate(snapshot%tp, source=nbody_system%tp ) + allocate(snapshot%system_history) + allocate(snapshot%system_history%nc, source=nbody_system%system_history%nc) + snapshot%system_history%nc%lfile_is_open = .true. + + snapshot%t = nbody_system%t + snapshot%GMtot = nbody_system%GMtot + snapshot%ke_orbit = nbody_system%ke_orbit + snapshot%ke_spin = nbody_system%ke_spin + snapshot%pe = nbody_system%pe + snapshot%be = nbody_system%be + snapshot%te = nbody_system%te + snapshot%oblpot = nbody_system%oblpot + snapshot%L_orbit = nbody_system%L_orbit + snapshot%L_spin = nbody_system%L_spin + snapshot%L_total = nbody_system%L_total + snapshot%ke_orbit_orig = nbody_system%ke_orbit_orig + snapshot%ke_spin_orig = nbody_system%ke_spin_orig + snapshot%pe_orig = nbody_system%pe_orig + snapshot%be_orig = nbody_system%be_orig + snapshot%E_orbit_orig = nbody_system%E_orbit_orig + snapshot%GMtot_orig = nbody_system%GMtot_orig + snapshot%L_total_orig = nbody_system%L_total_orig + snapshot%L_orbit_orig = nbody_system%L_orbit_orig + snapshot%L_spin_orig = nbody_system%L_spin_orig + snapshot%L_escape = nbody_system%L_escape + snapshot%GMescape = nbody_system%GMescape + snapshot%E_collisions = nbody_system%E_collisions + snapshot%E_untracked = nbody_system%E_untracked + snapshot%ke_orbit_error = nbody_system%ke_orbit_error + snapshot%ke_spin_error = nbody_system%ke_spin_error + snapshot%pe_error = nbody_system%pe_error + snapshot%be_error = nbody_system%be_error + snapshot%E_orbit_error = nbody_system%E_orbit_error + snapshot%Ecoll_error = nbody_system%Ecoll_error + snapshot%E_untracked_error = nbody_system%E_untracked_error + snapshot%te_error = nbody_system%te_error + snapshot%L_orbit_error = nbody_system%L_orbit_error + snapshot%L_spin_error = nbody_system%L_spin_error + snapshot%L_escape_error = nbody_system%L_escape_error + snapshot%L_total_error = nbody_system%L_total_error + snapshot%Mtot_error = nbody_system%Mtot_error + snapshot%Mescape_error = nbody_system%Mescape_error + snapshot%lbeg = nbody_system%lbeg + + + ! Store a snapshot of the nbody_system for posterity + call base_util_snapshot_save(self, snapshot) + self%nt = self%iframe + self%nid = self%nid + 1 ! Central body + if (allocated(nbody_system%pl)) self%nid = self%nid + nbody_system%pl%nbody + if (allocated(nbody_system%tp)) self%nid = self%nid + nbody_system%tp%nbody + end select return end subroutine swiftest_util_snapshot_system diff --git a/src/whm/whm_module.f90 b/src/whm/whm_module.f90 index fc97eee3b..d2d10f971 100644 --- a/src/whm/whm_module.f90 +++ b/src/whm/whm_module.f90 @@ -70,7 +70,6 @@ module whm !> An abstract class for the WHM integrator nbody system type, extends(swiftest_nbody_system) :: whm_nbody_system - class(whm_tp), dimension(:),codimension[:], allocatable :: cotp !! Co-array test particle data structure contains !> Replace the abstract procedures with concrete ones procedure :: initialize => whm_util_setup_initialize_system !! Performs WHM-specific initilization steps, like calculating the Jacobi masses From 00d2ff5171928bf7a0532a7d338a63f766ebe291 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 20 Mar 2023 13:25:41 -0400 Subject: [PATCH 005/149] Updated class of nbody_system --- src/swiftest/swiftest_driver.f90 | 133 ++++++++++++++++--------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 2da264bea..efc50e7c0 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -19,7 +19,7 @@ program swiftest_driver use swiftest implicit none - class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated + class(base_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated class(swiftest_parameters), allocatable :: param !! Run configuration parameters character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters @@ -70,75 +70,78 @@ program swiftest_driver ! Construct the main n-body nbody_system using the user-input integrator to choose the type of nbody_system call swiftest_util_setup_construct_system(nbody_system, param) - - !> Define the maximum number of threads - nthreads = 1 ! In the *serial* case - !$ nthreads = omp_get_max_threads() ! In the *parallel* case - !$ write(param%display_unit,'(a)') ' OpenMP parameters:' - !$ write(param%display_unit,'(a)') ' ------------------' - !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads - !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads - - call nbody_system%initialize(param) - - associate (system_history => nbody_system%system_history) - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. - call nbody_system%display_run_information(param, integration_timer, phase="first") - if (param%lenergy) then - if (param%lrestart) then - call nbody_system%get_t0_values(param) - else - call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum + select type(nbody_system) + class is (swiftest_nbody_system) + + !> Define the maximum number of threads + nthreads = 1 ! In the *serial* case + !$ nthreads = omp_get_max_threads() ! In the *parallel* case + !$ write(param%display_unit,'(a)') ' OpenMP parameters:' + !$ write(param%display_unit,'(a)') ' ------------------' + !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads + !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads + + call nbody_system%initialize(param) + + associate (system_history => nbody_system%system_history) + ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. + call nbody_system%display_run_information(param, integration_timer, phase="first") + if (param%lenergy) then + if (param%lrestart) then + call nbody_system%get_t0_values(param) + else + call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum + end if + call nbody_system%conservation_report(param, lterminal=.true.) end if - call nbody_system%conservation_report(param, lterminal=.true.) - end if - call system_history%take_snapshot(param,nbody_system) - call nbody_system%dump(param) - - do iloop = istart, nloops - !> Step the nbody_system forward in time - call integration_timer%start() - call nbody_system%step(param, nbody_system%t, dt) - call integration_timer%stop() - - nbody_system%t = t0 + iloop * dt - - !> Evaluate any discards or collisional outcomes - call nbody_system%discard(param) - - !> If the loop counter is at the output cadence value, append the data file with a single frame - if (istep_out > 0) then - iout = iout + 1 - if ((iout == istep) .or. (iloop == nloops)) then - iout = 0 - idump = idump + 1 - if (ltstretch) then - nout = nout + 1 - istep = floor(istep_out * fstep_out**nout, kind=I4B) - end if - - call system_history%take_snapshot(param,nbody_system) + call system_history%take_snapshot(param,nbody_system) + call nbody_system%dump(param) + + do iloop = istart, nloops + !> Step the nbody_system forward in time + call integration_timer%start() + call nbody_system%step(param, nbody_system%t, dt) + call integration_timer%stop() + + nbody_system%t = t0 + iloop * dt + + !> Evaluate any discards or collisional outcomes + call nbody_system%discard(param) + + !> If the loop counter is at the output cadence value, append the data file with a single frame + if (istep_out > 0) then + iout = iout + 1 + if ((iout == istep) .or. (iloop == nloops)) then + iout = 0 + idump = idump + 1 + if (ltstretch) then + nout = nout + 1 + istep = floor(istep_out * fstep_out**nout, kind=I4B) + end if + + call system_history%take_snapshot(param,nbody_system) + + if (idump == dump_cadence) then + idump = 0 + call nbody_system%dump(param) + end if + + call integration_timer%report(message="Integration steps:", unit=display_unit) + call nbody_system%display_run_information(param, integration_timer) + call integration_timer%reset() + if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) - if (idump == dump_cadence) then - idump = 0 - call nbody_system%dump(param) end if - - call integration_timer%report(message="Integration steps:", unit=display_unit, nsubsteps=istep_out) - call nbody_system%display_run_information(param, integration_timer) - call integration_timer%reset() - if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) - end if - end if - end do - ! Dump any remaining history if it exists - call nbody_system%dump(param) - call system_history%dump(param) - call nbody_system%display_run_information(param, integration_timer, phase="last") - - end associate + end do + ! Dump any remaining history if it exists + call nbody_system%dump(param) + call system_history%dump(param) + call nbody_system%display_run_information(param, integration_timer, phase="last") + + end associate + end select end associate call base_util_exit(SUCCESS) From a03a27e59bdcc0625e242bf0d13b823f75731323 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 21 Mar 2023 11:12:48 -0400 Subject: [PATCH 006/149] Updated CMAKE files to better handle coarrays and preprocessor variables --- CMakeLists.txt | 5 ++- cmake/Modules/FindCoarray_Fortran.cmake | 34 +++++++++++++------ cmake/Modules/SetParallelizationLibrary.cmake | 9 +++-- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb709cbe0..5ffa15283 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,9 +29,12 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) # Set some options the user may choose # Uncomment the below if you want the user to choose a parallelization library -OPTION(USE_CAF "Use Coarray Fortran for parallelization" ON) +OPTION(USE_COARRAY "Use Coarray Fortran for parallelization" OFF) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) +IF (USE_COARRAY) + ADD_DEFINITIONS(-DCOARRAY) +ENDIF() # Locate and set parallelization libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know diff --git a/cmake/Modules/FindCoarray_Fortran.cmake b/cmake/Modules/FindCoarray_Fortran.cmake index 05f28ef0a..314371326 100644 --- a/cmake/Modules/FindCoarray_Fortran.cmake +++ b/cmake/Modules/FindCoarray_Fortran.cmake @@ -25,17 +25,31 @@ INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) +STRING(TOUPPER "${CMAKE_BUILD_TYPE}" BT) +IF(BT STREQUAL "DEBUG") + SET (Coarray_Fortran_FLAG_CANDIDATES + #Intel + "-coarray=single" + #Intel windows + "/Qcoarray:single" + #Gnu + "-fcoarray=single" + #Empty, if compiler automatically accepts coarray + " " + ) +ELSE() + SET (Coarray_Fortran_FLAG_CANDIDATES + #Intel + "-coarray=distributed" + #Intel windows + "/Qcoarray:distributed" + #Gnu + "-fcoarray=lib -lcaf_mpi" + #Empty, if compiler automatically accepts coarray + " " + ) +ENDIF() -SET (Coarray_Fortran_FLAG_CANDIDATES - #Intel - "-coarray=distributed" - #Intel windows - "/Qcoarray:distributed" - #Gnu - "-fcoarray=lib -lcaf_mpi" - #Empty, if compiler automatically accepts coarray - " " -) IF (DEFINED Coarray_Fortran_FLAGS) SET (Coarray_Fortran_FLAG_CANDIDATES) diff --git a/cmake/Modules/SetParallelizationLibrary.cmake b/cmake/Modules/SetParallelizationLibrary.cmake index 3f3153f4d..505a77d62 100644 --- a/cmake/Modules/SetParallelizationLibrary.cmake +++ b/cmake/Modules/SetParallelizationLibrary.cmake @@ -17,21 +17,20 @@ IF (USE_OPENMP) ENDIF (NOT OpenMP_Fortran_FLAGS) ENDIF (USE_OPENMP) -IF (USE_CAF) - # Find MPI +IF (USE_COARRAY) IF (NOT Coarray_Fortran_FLAGS) FIND_PACKAGE (Coarray_Fortran) IF (NOT Coarray_Fortran_FLAGS) MESSAGE (FATAL_ERROR "Fortran compiler does not support Coarrays") ENDIF (NOT Coarray_Fortran_FLAGS) ENDIF (NOT Coarray_Fortran_FLAGS) -ENDIF (USE_CAF) +ENDIF (USE_COARRAY) -IF (NOT USE_OPENMP AND NOT USE_CAF) +IF (NOT USE_OPENMP AND NOT USE_COARRAY) # Turn off both OpenMP and CAF SET (OMP_NUM_PROCS 0 CACHE STRING "Number of processors OpenMP may use" FORCE) UNSET (OpenMP_Fortran_FLAGS CACHE) UNSET (Coarray_Fortran_FLAGS CACHE) UNSET (GOMP_Fortran_LINK_FLAGS CACHE) -ENDIF (NOT USE_OPENMP AND NOT USE_CAF) +ENDIF (NOT USE_OPENMP AND NOT USE_COARRAY) From 0df70cb956d14e02ba4a606c307765327f9e7831 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 21 Mar 2023 12:40:45 -0400 Subject: [PATCH 007/149] More updates to coarray CMAKE options --- src/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b8cc5be2c..b63b7148a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -88,6 +88,9 @@ set(SWIFTEST_src ${FAST_MATH_FILES} ${STRICT_MATH_FILES}) # Define the executable in terms of the source files ADD_EXECUTABLE(${SWIFTEST_DRIVER} ${SWIFTEST_src}) +# Turn preprocessor on for all files +SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON) + ##################################################### # Add the needed libraries and special compiler flags ##################################################### @@ -101,11 +104,11 @@ IF(USE_OPENMP) LINK_FLAGS "${OpenMP_Fortran_FLAGS}") ENDIF(USE_OPENMP) -IF(USE_CAF) +IF(USE_COARRAY) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES COMPILE_FLAGS "${Coarray_Fortran_FLAGS}" LINK_FLAGS "${Coarray_Fortran_FLAGS}") -ENDIF(USE_CAF) +ENDIF(USE_COARRAY) ##################################### @@ -127,3 +130,7 @@ IF(BT STREQUAL "RELEASE" OR BT STREQUAL "PROFILE") SET_PROPERTY(SOURCE ${FAST_MATH_FILES} APPEND_STRING PROPERTY COMPILE_FLAGS "${FASTMATH_FLAGS}") ENDIF() +IF(BT STREQUAL "DEBUG") + ADD_DEFINITIONS(-DDEBUG) +ENDIF() + From 1963fa7e5be5d274da58188901499b7e4da8e595 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 21 Mar 2023 12:41:07 -0400 Subject: [PATCH 008/149] More updates to coarray CMAKE options --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ffa15283..1a9ed99c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,8 +28,7 @@ IF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) # Set some options the user may choose -# Uncomment the below if you want the user to choose a parallelization library -OPTION(USE_COARRAY "Use Coarray Fortran for parallelization" OFF) +OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) IF (USE_COARRAY) From 622502d1dc3b30f1b343697ef607a07aae54b248 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 21 Mar 2023 14:04:43 -0400 Subject: [PATCH 009/149] Reverted coarray branch source code in order to take a different approach --- src/collision/collision_regime.f90 | 2 - src/misc/solver_module.f90 | 2 +- src/rmvs/rmvs_module.f90 | 2 +- src/rmvs/rmvs_step.f90 | 186 ++++++++++++++--------------- src/rmvs/rmvs_util.f90 | 67 +++++------ src/swiftest/swiftest_driver.f90 | 133 ++++++++++----------- src/swiftest/swiftest_module.f90 | 29 +++-- src/swiftest/swiftest_util.f90 | 127 ++++++++++---------- 8 files changed, 265 insertions(+), 283 deletions(-) diff --git a/src/collision/collision_regime.f90 b/src/collision/collision_regime.f90 index 5bb77099a..7480c6a0d 100644 --- a/src/collision/collision_regime.f90 +++ b/src/collision/collision_regime.f90 @@ -249,8 +249,6 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, ke = 0.5_DP * Vimp**2 pe = - GC * m1 * m2 / (Mtot * norm2(rh2 - rh1)) - Qmerge = Gc * m1 * m2 / (.mag.(rh2 - rh1)) + 3*m1 / (5*rad1) + 3*m2 / (5*rad1) - U_binding ! Change in energy due to a pure merger - if ((m1 < min_mfrag).or.(m2 < min_mfrag)) then regime = COLLRESOLVE_REGIME_MERGE !perfect merging regime call swiftest_io_log_one_message(COLLISION_LOG_OUT, & diff --git a/src/misc/solver_module.f90 b/src/misc/solver_module.f90 index 504e9215e..410116b2a 100644 --- a/src/misc/solver_module.f90 +++ b/src/misc/solver_module.f90 @@ -16,7 +16,7 @@ module solver use lambda_function use, intrinsic :: ieee_exceptions private - public :: solve_linear_system, solve_roots + public :: solve_linear_system, solve_rkf45, solve_roots interface solve_linear_system module procedure solve_linear_system_dp diff --git a/src/rmvs/rmvs_module.f90 b/src/rmvs/rmvs_module.f90 index 74910d7d3..733a81e60 100644 --- a/src/rmvs/rmvs_module.f90 +++ b/src/rmvs/rmvs_module.f90 @@ -98,7 +98,7 @@ module rmvs integer(I4B), dimension(:), allocatable :: plind !! Connects the planetocentric indices back to the heliocentric planet list type(rmvs_interp), dimension(:), allocatable :: outer !! interpolated heliocentric central body position for outer encounters type(rmvs_interp), dimension(:), allocatable :: inner !! interpolated heliocentric central body position for inner encounters - class(base_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) + class(rmvs_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) logical :: lplanetocentric = .false. !! Flag that indicates that the object is a planetocentric set of masive bodies used for close encounter calculations contains procedure :: setup => rmvs_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess diff --git a/src/rmvs/rmvs_step.f90 b/src/rmvs/rmvs_step.f90 index 18f9bd644..bcb38797a 100644 --- a/src/rmvs/rmvs_step.f90 +++ b/src/rmvs/rmvs_step.f90 @@ -451,67 +451,64 @@ subroutine rmvs_make_planetocentric(param, cb, pl, tp) if (allocated(encmask)) deallocate(encmask) allocate(encmask(ntp)) encmask(1:ntp) = tp%plencP(1:ntp) == i - select type (planetocentric => pl%planetocentric) - class is(rmvs_nbody_system) - allocate(rmvs_tp :: planetocentric(i)%tp) - ! Create encountering test particle structure - select type(cbenci => planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => planetocentric(i)%pl) - class is (rmvs_pl) - select type(tpenci => planetocentric(i)%tp) - class is (rmvs_tp) - tpenci%lplanetocentric = .true. - associate(nenci => pl%nenc(i)) - call tpenci%setup(nenci, param) - tpenci%cb_heliocentric = cb - tpenci%ipleP = i - tpenci%lmask(1:nenci) = .true. - tpenci%status(1:nenci) = ACTIVE - ! Grab all the encountering test particles and convert them to a planetocentric frame - tpenci%id(1:nenci) = pack(tp%id(1:ntp), encmask(1:ntp)) - do j = 1, NDIM - tpenci%rheliocentric(j, 1:nenci) = pack(tp%rh(j,1:ntp), encmask(:)) - tpenci%rh(j, 1:nenci) = tpenci%rheliocentric(j, 1:nenci) - pl%inner(0)%x(j, i) - tpenci%vh(j, 1:nenci) = pack(tp%vh(j, 1:ntp), encmask(1:ntp)) - pl%inner(0)%v(j, i) - end do - tpenci%lperi(1:nenci) = pack(tp%lperi(1:ntp), encmask(1:ntp)) - tpenci%plperP(1:nenci) = pack(tp%plperP(1:ntp), encmask(1:ntp)) - ! Make sure that the test particles get the planetocentric value of mu - allocate(cbenci%inner(0:NTPHENC)) - do inner_index = 0, NTPHENC - allocate(plenci%inner(inner_index)%x, mold=pl%inner(inner_index)%x) - allocate(plenci%inner(inner_index)%v, mold=pl%inner(inner_index)%x) - allocate(cbenci%inner(inner_index)%x(NDIM,1)) - allocate(cbenci%inner(inner_index)%v(NDIM,1)) - cbenci%inner(inner_index)%x(:,1) = pl%inner(inner_index)%x(:, i) - cbenci%inner(inner_index)%v(:,1) = pl%inner(inner_index)%v(:, i) - plenci%inner(inner_index)%x(:,1) = -cbenci%inner(inner_index)%x(:,1) - plenci%inner(inner_index)%v(:,1) = -cbenci%inner(inner_index)%v(:,1) - - if (param%loblatecb) then - allocate(plenci%inner(inner_index)%aobl, mold=pl%inner(inner_index)%aobl) - allocate(cbenci%inner(inner_index)%aobl(NDIM,1)) - cbenci%inner(inner_index)%aobl(:,1) = pl%inner(inner_index)%aobl(:, i) - end if - - if (param%ltides) then - allocate(plenci%inner(inner_index)%atide, mold=pl%inner(inner_index)%atide) - allocate(cbenci%inner(inner_index)%atide(NDIM,1)) - cbenci%inner(inner_index)%atide(:,1) = pl%inner(inner_index)%atide(:, i) - end if - - do j = 2, npl - ipc2hc = plenci%plind(j) - plenci%inner(inner_index)%x(:,j) = pl%inner(inner_index)%x(:, ipc2hc) & - - cbenci%inner(inner_index)%x(:,1) - plenci%inner(inner_index)%v(:,j) = pl%inner(inner_index)%v(:, ipc2hc) & - - cbenci%inner(inner_index)%v(:,1) - end do + allocate(rmvs_tp :: pl%planetocentric(i)%tp) + ! Create encountering test particle structure + select type(cbenci => pl%planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => pl%planetocentric(i)%pl) + class is (rmvs_pl) + select type(tpenci => pl%planetocentric(i)%tp) + class is (rmvs_tp) + tpenci%lplanetocentric = .true. + associate(nenci => pl%nenc(i)) + call tpenci%setup(nenci, param) + tpenci%cb_heliocentric = cb + tpenci%ipleP = i + tpenci%lmask(1:nenci) = .true. + tpenci%status(1:nenci) = ACTIVE + ! Grab all the encountering test particles and convert them to a planetocentric frame + tpenci%id(1:nenci) = pack(tp%id(1:ntp), encmask(1:ntp)) + do j = 1, NDIM + tpenci%rheliocentric(j, 1:nenci) = pack(tp%rh(j,1:ntp), encmask(:)) + tpenci%rh(j, 1:nenci) = tpenci%rheliocentric(j, 1:nenci) - pl%inner(0)%x(j, i) + tpenci%vh(j, 1:nenci) = pack(tp%vh(j, 1:ntp), encmask(1:ntp)) - pl%inner(0)%v(j, i) + end do + tpenci%lperi(1:nenci) = pack(tp%lperi(1:ntp), encmask(1:ntp)) + tpenci%plperP(1:nenci) = pack(tp%plperP(1:ntp), encmask(1:ntp)) + ! Make sure that the test particles get the planetocentric value of mu + allocate(cbenci%inner(0:NTPHENC)) + do inner_index = 0, NTPHENC + allocate(plenci%inner(inner_index)%x, mold=pl%inner(inner_index)%x) + allocate(plenci%inner(inner_index)%v, mold=pl%inner(inner_index)%x) + allocate(cbenci%inner(inner_index)%x(NDIM,1)) + allocate(cbenci%inner(inner_index)%v(NDIM,1)) + cbenci%inner(inner_index)%x(:,1) = pl%inner(inner_index)%x(:, i) + cbenci%inner(inner_index)%v(:,1) = pl%inner(inner_index)%v(:, i) + plenci%inner(inner_index)%x(:,1) = -cbenci%inner(inner_index)%x(:,1) + plenci%inner(inner_index)%v(:,1) = -cbenci%inner(inner_index)%v(:,1) + + if (param%loblatecb) then + allocate(plenci%inner(inner_index)%aobl, mold=pl%inner(inner_index)%aobl) + allocate(cbenci%inner(inner_index)%aobl(NDIM,1)) + cbenci%inner(inner_index)%aobl(:,1) = pl%inner(inner_index)%aobl(:, i) + end if + + if (param%ltides) then + allocate(plenci%inner(inner_index)%atide, mold=pl%inner(inner_index)%atide) + allocate(cbenci%inner(inner_index)%atide(NDIM,1)) + cbenci%inner(inner_index)%atide(:,1) = pl%inner(inner_index)%atide(:, i) + end if + + do j = 2, npl + ipc2hc = plenci%plind(j) + plenci%inner(inner_index)%x(:,j) = pl%inner(inner_index)%x(:, ipc2hc) & + - cbenci%inner(inner_index)%x(:,1) + plenci%inner(inner_index)%v(:,j) = pl%inner(inner_index)%v(:, ipc2hc) & + - cbenci%inner(inner_index)%v(:,1) end do - call tpenci%set_mu(cbenci) - end associate - end select + end do + call tpenci%set_mu(cbenci) + end associate end select end select end select @@ -614,42 +611,39 @@ subroutine rmvs_end_planetocentric(pl, tp) associate (npl => pl%nbody, ntp => tp%nbody) do i = 1, npl if (pl%nenc(i) == 0) cycle - select type(planetocentric => pl%planetocentric) - class is(rmvs_nbody_system) - select type(cbenci => planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => planetocentric(i)%pl) - class is (rmvs_pl) - select type(tpenci => planetocentric(i)%tp) - class is (rmvs_tp) - associate(nenci => pl%nenc(i)) - if (allocated(tpind)) deallocate(tpind) - allocate(tpind(nenci)) - ! Index array of encountering test particles - if (allocated(encmask)) deallocate(encmask) - allocate(encmask(ntp)) - encmask(1:ntp) = tp%plencP(1:ntp) == i - tpind(1:nenci) = pack([(j,j=1,ntp)], encmask(1:ntp)) - - ! Copy the results of the integration back over and shift back to heliocentric reference - tp%status(tpind(1:nenci)) = tpenci%status(1:nenci) - tp%lmask(tpind(1:nenci)) = tpenci%lmask(1:nenci) - do j = 1, NDIM - tp%rh(j, tpind(1:nenci)) = tpenci%rh(j,1:nenci) + pl%inner(NTPHENC)%x(j, i) - tp%vh(j, tpind(1:nenci)) = tpenci%vh(j,1:nenci) + pl%inner(NTPHENC)%v(j, i) - end do - tp%lperi(tpind(1:nenci)) = tpenci%lperi(1:nenci) - tp%plperP(tpind(1:nenci)) = tpenci%plperP(1:nenci) - deallocate(planetocentric(i)%tp) - deallocate(cbenci%inner) - do inner_index = 0, NTPHENC - deallocate(plenci%inner(inner_index)%x) - deallocate(plenci%inner(inner_index)%v) - if (allocated(plenci%inner(inner_index)%aobl)) deallocate(plenci%inner(inner_index)%aobl) - if (allocated(plenci%inner(inner_index)%atide)) deallocate(plenci%inner(inner_index)%atide) - end do - end associate - end select + select type(cbenci => pl%planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => pl%planetocentric(i)%pl) + class is (rmvs_pl) + select type(tpenci => pl%planetocentric(i)%tp) + class is (rmvs_tp) + associate(nenci => pl%nenc(i)) + if (allocated(tpind)) deallocate(tpind) + allocate(tpind(nenci)) + ! Index array of encountering test particles + if (allocated(encmask)) deallocate(encmask) + allocate(encmask(ntp)) + encmask(1:ntp) = tp%plencP(1:ntp) == i + tpind(1:nenci) = pack([(j,j=1,ntp)], encmask(1:ntp)) + + ! Copy the results of the integration back over and shift back to heliocentric reference + tp%status(tpind(1:nenci)) = tpenci%status(1:nenci) + tp%lmask(tpind(1:nenci)) = tpenci%lmask(1:nenci) + do j = 1, NDIM + tp%rh(j, tpind(1:nenci)) = tpenci%rh(j,1:nenci) + pl%inner(NTPHENC)%x(j, i) + tp%vh(j, tpind(1:nenci)) = tpenci%vh(j,1:nenci) + pl%inner(NTPHENC)%v(j, i) + end do + tp%lperi(tpind(1:nenci)) = tpenci%lperi(1:nenci) + tp%plperP(tpind(1:nenci)) = tpenci%plperP(1:nenci) + deallocate(pl%planetocentric(i)%tp) + deallocate(cbenci%inner) + do inner_index = 0, NTPHENC + deallocate(plenci%inner(inner_index)%x) + deallocate(plenci%inner(inner_index)%v) + if (allocated(plenci%inner(inner_index)%aobl)) deallocate(plenci%inner(inner_index)%aobl) + if (allocated(plenci%inner(inner_index)%atide)) deallocate(plenci%inner(inner_index)%atide) + end do + end associate end select end select end select diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index d4d486da7..b9462840b 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -356,42 +356,39 @@ module subroutine rmvs_util_setup_initialize_system(self, param) tp%lplanetocentric = .false. cb%lplanetocentric = .false. associate(npl => pl%nbody) - allocate(rmvs_nbody_system :: pl%planetocentric(npl)) - select type(planetocentric => pl%planetocentric) - class is (rmvs_nbody_system) - planetocentric(:)%lplanetocentric = .true. - do i = 1, npl - allocate(planetocentric(i)%cb, source=cb) - allocate(rmvs_pl :: planetocentric(i)%pl) - select type(cbenci => planetocentric(i)%cb) - class is (rmvs_cb) - select type(plenci => planetocentric(i)%pl) - class is (rmvs_pl) - cbenci%lplanetocentric = .true. - plenci%lplanetocentric = .true. - call plenci%setup(npl, param) - plenci%status(:) = ACTIVE - plenci%lmask(:) = .true. - ! plind stores the heliocentric index value of a planetocentric planet - ! e.g. Consider an encounter with planet 3. - ! Then the following will be the values of plind: - ! pl%planetocentric(3)%pl%plind(1) = 0 (central body - never used) - ! pl%planetocentric(3)%pl%plind(2) = 1 - ! pl%planetocentric(3)%pl%plind(3) = 2 - ! pl%planetocentric(3)%pl%plind(4) = 4 - ! pl%planetocentric(3)%pl%plind(5) = 5 - ! etc. - allocate(plenci%plind(npl)) - plenci%plind(1:npl) = [(j,j=1,npl)] - plenci%plind(2:npl) = pack(plenci%plind(1:npl), plenci%plind(1:npl) /= i) - plenci%plind(1) = 0 - plenci%Gmass(1) = cb%Gmass - plenci%Gmass(2:npl) = pl%Gmass(plenci%plind(2:npl)) - cbenci%Gmass = pl%Gmass(i) - end select + allocate(pl%planetocentric(npl)) + pl%planetocentric(:)%lplanetocentric = .true. + do i = 1, npl + allocate(pl%planetocentric(i)%cb, source=cb) + allocate(rmvs_pl :: pl%planetocentric(i)%pl) + select type(cbenci => pl%planetocentric(i)%cb) + class is (rmvs_cb) + select type(plenci => pl%planetocentric(i)%pl) + class is (rmvs_pl) + cbenci%lplanetocentric = .true. + plenci%lplanetocentric = .true. + call plenci%setup(npl, param) + plenci%status(:) = ACTIVE + plenci%lmask(:) = .true. + ! plind stores the heliocentric index value of a planetocentric planet + ! e.g. Consider an encounter with planet 3. + ! Then the following will be the values of plind: + ! pl%planetocentric(3)%pl%plind(1) = 0 (central body - never used) + ! pl%planetocentric(3)%pl%plind(2) = 1 + ! pl%planetocentric(3)%pl%plind(3) = 2 + ! pl%planetocentric(3)%pl%plind(4) = 4 + ! pl%planetocentric(3)%pl%plind(5) = 5 + ! etc. + allocate(plenci%plind(npl)) + plenci%plind(1:npl) = [(j,j=1,npl)] + plenci%plind(2:npl) = pack(plenci%plind(1:npl), plenci%plind(1:npl) /= i) + plenci%plind(1) = 0 + plenci%Gmass(1) = cb%Gmass + plenci%Gmass(2:npl) = pl%Gmass(plenci%plind(2:npl)) + cbenci%Gmass = pl%Gmass(i) end select - end do - end select + end select + end do end associate end select end select diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index efc50e7c0..bfa950222 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -19,7 +19,7 @@ program swiftest_driver use swiftest implicit none - class(base_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated + class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated class(swiftest_parameters), allocatable :: param !! Run configuration parameters character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters @@ -70,78 +70,75 @@ program swiftest_driver ! Construct the main n-body nbody_system using the user-input integrator to choose the type of nbody_system call swiftest_util_setup_construct_system(nbody_system, param) - select type(nbody_system) - class is (swiftest_nbody_system) - - !> Define the maximum number of threads - nthreads = 1 ! In the *serial* case - !$ nthreads = omp_get_max_threads() ! In the *parallel* case - !$ write(param%display_unit,'(a)') ' OpenMP parameters:' - !$ write(param%display_unit,'(a)') ' ------------------' - !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads - !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads - - call nbody_system%initialize(param) - - associate (system_history => nbody_system%system_history) - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. - call nbody_system%display_run_information(param, integration_timer, phase="first") - if (param%lenergy) then - if (param%lrestart) then - call nbody_system%get_t0_values(param) - else - call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum - end if - call nbody_system%conservation_report(param, lterminal=.true.) + + !> Define the maximum number of threads + nthreads = 1 ! In the *serial* case + !$ nthreads = omp_get_max_threads() ! In the *parallel* case + !$ write(param%display_unit,'(a)') ' OpenMP parameters:' + !$ write(param%display_unit,'(a)') ' ------------------' + !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads + !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads + + call nbody_system%initialize(param) + + associate (system_history => nbody_system%system_history) + ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. + call nbody_system%display_run_information(param, integration_timer, phase="first") + if (param%lenergy) then + if (param%lrestart) then + call nbody_system%get_t0_values(param) + else + call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum end if - call system_history%take_snapshot(param,nbody_system) - call nbody_system%dump(param) - - do iloop = istart, nloops - !> Step the nbody_system forward in time - call integration_timer%start() - call nbody_system%step(param, nbody_system%t, dt) - call integration_timer%stop() - - nbody_system%t = t0 + iloop * dt - - !> Evaluate any discards or collisional outcomes - call nbody_system%discard(param) - - !> If the loop counter is at the output cadence value, append the data file with a single frame - if (istep_out > 0) then - iout = iout + 1 - if ((iout == istep) .or. (iloop == nloops)) then - iout = 0 - idump = idump + 1 - if (ltstretch) then - nout = nout + 1 - istep = floor(istep_out * fstep_out**nout, kind=I4B) - end if - - call system_history%take_snapshot(param,nbody_system) - - if (idump == dump_cadence) then - idump = 0 - call nbody_system%dump(param) - end if - - call integration_timer%report(message="Integration steps:", unit=display_unit) - call nbody_system%display_run_information(param, integration_timer) - call integration_timer%reset() - if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) + call nbody_system%conservation_report(param, lterminal=.true.) + end if + call system_history%take_snapshot(param,nbody_system) + call nbody_system%dump(param) + + do iloop = istart, nloops + !> Step the nbody_system forward in time + call integration_timer%start() + call nbody_system%step(param, nbody_system%t, dt) + call integration_timer%stop() + + nbody_system%t = t0 + iloop * dt + + !> Evaluate any discards or collisional outcomes + call nbody_system%discard(param) + + !> If the loop counter is at the output cadence value, append the data file with a single frame + if (istep_out > 0) then + iout = iout + 1 + if ((iout == istep) .or. (iloop == nloops)) then + iout = 0 + idump = idump + 1 + if (ltstretch) then + nout = nout + 1 + istep = floor(istep_out * fstep_out**nout, kind=I4B) + end if + + call system_history%take_snapshot(param,nbody_system) + if (idump == dump_cadence) then + idump = 0 + call nbody_system%dump(param) end if + + call integration_timer%report(message="Integration steps:", unit=display_unit) + call nbody_system%display_run_information(param, integration_timer) + call integration_timer%reset() + if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) + end if + end if - end do - ! Dump any remaining history if it exists - call nbody_system%dump(param) - call system_history%dump(param) - call nbody_system%display_run_information(param, integration_timer, phase="last") - - end associate - end select + end do + ! Dump any remaining history if it exists + call nbody_system%dump(param) + call system_history%dump(param) + call nbody_system%display_run_information(param, integration_timer, phase="last") + + end associate end associate call base_util_exit(SUCCESS) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 00c5ba228..4ac6cb3c3 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -304,13 +304,14 @@ module swiftest !> An abstract class for a basic Swiftest nbody system - type, extends(base_nbody_system) :: swiftest_nbody_system + type, abstract, extends(base_nbody_system) :: swiftest_nbody_system !! This superclass contains a minimial nbody_system of a set of test particles (tp), massive bodies (pl), and a central body (cb) !! The full swiftest_nbody_system type that is used as the parent class of all integrators is defined in collision class(swiftest_cb), allocatable :: cb !! Central body data structure class(swiftest_pl), allocatable :: pl !! Massive body data structure class(swiftest_tp), allocatable :: tp !! Test particle data structure + class(swiftest_tp), allocatable :: tp_discards !! Discarded test particle data structure class(swiftest_pl), allocatable :: pl_discards !! Discarded massive body particle data structure class(swiftest_pl), allocatable :: pl_adds !! List of added bodies in mergers or collisions @@ -373,7 +374,7 @@ module swiftest !! the test particles contains !> Each integrator will have its own version of the step - + procedure(abstract_step_system), deferred :: step ! Concrete classes that are common to the basic integrator (only test particles considered for discard) procedure :: discard => swiftest_discard_system !! Perform a discard step on the nbody_system @@ -390,8 +391,7 @@ module swiftest procedure :: read_in => swiftest_io_read_in_system !! Reads the initial conditions for an nbody system procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body - procedure :: step => swiftest_step_system !! Placeholder step method. Each integrator will override - procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer + procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum procedure :: get_idvals => swiftest_util_get_idvalues_system !! Returns an array of all id values in use in the nbody_system procedure :: rescale => swiftest_util_rescale_system !! Rescales the nbody_system into a new set of units @@ -448,6 +448,15 @@ subroutine abstract_step_body(self, nbody_system, param, t, dt) real(DP), intent(in) :: t !! Simulation time real(DP), intent(in) :: dt !! Current stepsize end subroutine abstract_step_body + + subroutine abstract_step_system(self, param, t, dt) + import DP, swiftest_nbody_system, swiftest_parameters + implicit none + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + real(DP), intent(in) :: t !! Simulation time + real(DP), intent(in) :: dt !! Current stepsize + end subroutine abstract_step_system end interface @@ -1049,7 +1058,7 @@ end subroutine swiftest_util_setup_body module subroutine swiftest_util_setup_construct_system(nbody_system, param) implicit none - class(base_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_construct_system @@ -1943,16 +1952,6 @@ subroutine swiftest_final_kin(self) end subroutine swiftest_final_kin - subroutine swiftest_step_system(self, param, t, dt) - implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - real(DP), intent(in) :: t !! Simulation time - real(DP), intent(in) :: dt !! Current stepsize - return - end subroutine swiftest_step_system - - subroutine swiftest_final_storage(self) !! author: David A. Minton !! diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index bf6a30087..fcf130f83 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2703,8 +2703,8 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) !! implicit none ! Arguments - class(base_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters select case(param%integrator) case (INT_BS) @@ -2767,11 +2767,11 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) call base_util_exit(FAILURE) end select - select type(nbody_system) - class is (swiftest_nbody_system) - allocate(swiftest_particle_info :: nbody_system%cb%info) - nbody_system%t = param%tstart - end select + allocate(swiftest_particle_info :: nbody_system%cb%info) + + + nbody_system%t = param%tstart + return end subroutine swiftest_util_setup_construct_system @@ -3055,7 +3055,7 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) ! Internals - class(base_nbody_system), allocatable :: snapshot + class(swiftest_nbody_system), allocatable :: snapshot ! To allow for runs to be restarted in a bit-identical way, we'll need to run the same coordinate conversion routines we would run upon restarting select type(pl => nbody_system%pl) @@ -3077,63 +3077,60 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar ! Take a minimal snapshot wihout all of the extra storage objects allocate(snapshot, mold=nbody_system) - select type(snapshot) - class is (swiftest_nbody_system) - allocate(snapshot%cb, source=nbody_system%cb ) - allocate(snapshot%pl, source=nbody_system%pl ) - allocate(snapshot%tp, source=nbody_system%tp ) - allocate(snapshot%system_history) - allocate(snapshot%system_history%nc, source=nbody_system%system_history%nc) - snapshot%system_history%nc%lfile_is_open = .true. - - snapshot%t = nbody_system%t - snapshot%GMtot = nbody_system%GMtot - snapshot%ke_orbit = nbody_system%ke_orbit - snapshot%ke_spin = nbody_system%ke_spin - snapshot%pe = nbody_system%pe - snapshot%be = nbody_system%be - snapshot%te = nbody_system%te - snapshot%oblpot = nbody_system%oblpot - snapshot%L_orbit = nbody_system%L_orbit - snapshot%L_spin = nbody_system%L_spin - snapshot%L_total = nbody_system%L_total - snapshot%ke_orbit_orig = nbody_system%ke_orbit_orig - snapshot%ke_spin_orig = nbody_system%ke_spin_orig - snapshot%pe_orig = nbody_system%pe_orig - snapshot%be_orig = nbody_system%be_orig - snapshot%E_orbit_orig = nbody_system%E_orbit_orig - snapshot%GMtot_orig = nbody_system%GMtot_orig - snapshot%L_total_orig = nbody_system%L_total_orig - snapshot%L_orbit_orig = nbody_system%L_orbit_orig - snapshot%L_spin_orig = nbody_system%L_spin_orig - snapshot%L_escape = nbody_system%L_escape - snapshot%GMescape = nbody_system%GMescape - snapshot%E_collisions = nbody_system%E_collisions - snapshot%E_untracked = nbody_system%E_untracked - snapshot%ke_orbit_error = nbody_system%ke_orbit_error - snapshot%ke_spin_error = nbody_system%ke_spin_error - snapshot%pe_error = nbody_system%pe_error - snapshot%be_error = nbody_system%be_error - snapshot%E_orbit_error = nbody_system%E_orbit_error - snapshot%Ecoll_error = nbody_system%Ecoll_error - snapshot%E_untracked_error = nbody_system%E_untracked_error - snapshot%te_error = nbody_system%te_error - snapshot%L_orbit_error = nbody_system%L_orbit_error - snapshot%L_spin_error = nbody_system%L_spin_error - snapshot%L_escape_error = nbody_system%L_escape_error - snapshot%L_total_error = nbody_system%L_total_error - snapshot%Mtot_error = nbody_system%Mtot_error - snapshot%Mescape_error = nbody_system%Mescape_error - snapshot%lbeg = nbody_system%lbeg - - - ! Store a snapshot of the nbody_system for posterity - call base_util_snapshot_save(self, snapshot) - self%nt = self%iframe - self%nid = self%nid + 1 ! Central body - if (allocated(nbody_system%pl)) self%nid = self%nid + nbody_system%pl%nbody - if (allocated(nbody_system%tp)) self%nid = self%nid + nbody_system%tp%nbody - end select + allocate(snapshot%cb, source=nbody_system%cb ) + allocate(snapshot%pl, source=nbody_system%pl ) + allocate(snapshot%tp, source=nbody_system%tp ) + allocate(snapshot%system_history) + allocate(snapshot%system_history%nc, source=nbody_system%system_history%nc) + snapshot%system_history%nc%lfile_is_open = .true. + + snapshot%t = nbody_system%t + snapshot%GMtot = nbody_system%GMtot + snapshot%ke_orbit = nbody_system%ke_orbit + snapshot%ke_spin = nbody_system%ke_spin + snapshot%pe = nbody_system%pe + snapshot%be = nbody_system%be + snapshot%te = nbody_system%te + snapshot%oblpot = nbody_system%oblpot + snapshot%L_orbit = nbody_system%L_orbit + snapshot%L_spin = nbody_system%L_spin + snapshot%L_total = nbody_system%L_total + snapshot%ke_orbit_orig = nbody_system%ke_orbit_orig + snapshot%ke_spin_orig = nbody_system%ke_spin_orig + snapshot%pe_orig = nbody_system%pe_orig + snapshot%be_orig = nbody_system%be_orig + snapshot%E_orbit_orig = nbody_system%E_orbit_orig + snapshot%GMtot_orig = nbody_system%GMtot_orig + snapshot%L_total_orig = nbody_system%L_total_orig + snapshot%L_orbit_orig = nbody_system%L_orbit_orig + snapshot%L_spin_orig = nbody_system%L_spin_orig + snapshot%L_escape = nbody_system%L_escape + snapshot%GMescape = nbody_system%GMescape + snapshot%E_collisions = nbody_system%E_collisions + snapshot%E_untracked = nbody_system%E_untracked + snapshot%ke_orbit_error = nbody_system%ke_orbit_error + snapshot%ke_spin_error = nbody_system%ke_spin_error + snapshot%pe_error = nbody_system%pe_error + snapshot%be_error = nbody_system%be_error + snapshot%E_orbit_error = nbody_system%E_orbit_error + snapshot%Ecoll_error = nbody_system%Ecoll_error + snapshot%E_untracked_error = nbody_system%E_untracked_error + snapshot%te_error = nbody_system%te_error + snapshot%L_orbit_error = nbody_system%L_orbit_error + snapshot%L_spin_error = nbody_system%L_spin_error + snapshot%L_escape_error = nbody_system%L_escape_error + snapshot%L_total_error = nbody_system%L_total_error + snapshot%Mtot_error = nbody_system%Mtot_error + snapshot%Mescape_error = nbody_system%Mescape_error + snapshot%lbeg = nbody_system%lbeg + + + ! Store a snapshot of the nbody_system for posterity + call base_util_snapshot_save(self, snapshot) + self%nt = self%iframe + self%nid = self%nid + 1 ! Central body + if (allocated(nbody_system%pl)) self%nid = self%nid + nbody_system%pl%nbody + if (allocated(nbody_system%tp)) self%nid = self%nid + nbody_system%tp%nbody return end subroutine swiftest_util_snapshot_system From 36fa56c1c2be974ec38246b8ad72a2811d6c4ced Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 24 Mar 2023 08:41:30 -0400 Subject: [PATCH 010/149] Added profiling preprocessor flags and a bit of test code --- src/CMakeLists.txt | 5 +++-- src/swiftest/swiftest_kick.f90 | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b63b7148a..0ad615170 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,5 +132,6 @@ ENDIF() IF(BT STREQUAL "DEBUG") ADD_DEFINITIONS(-DDEBUG) -ENDIF() - +ELSEIF(BT STREQUAL "PROFILE") + ADD_DEFINITIONS(-DPROFILE) +ENDIF() \ No newline at end of file diff --git a/src/swiftest/swiftest_kick.f90 b/src/swiftest/swiftest_kick.f90 index 54da2f82a..430679b43 100644 --- a/src/swiftest/swiftest_kick.f90 +++ b/src/swiftest/swiftest_kick.f90 @@ -22,7 +22,9 @@ module subroutine swiftest_kick_getacch_int_pl(self, param) class(swiftest_parameters), intent(inout) :: param !! Current swiftest run configuration parameters ! Internals logical, save :: lfirst = .true. - +#ifdef PROFILE + type(walltimer), save :: timer +#endif if (param%lflatten_interactions) then if (param%lclose) then From d7e4d14397e8c559877ae3f33b9f2329bc0d677c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 24 Mar 2023 08:43:03 -0400 Subject: [PATCH 011/149] Pulled in collision regime fix from the master branch --- src/collision/collision_regime.f90 | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/collision/collision_regime.f90 b/src/collision/collision_regime.f90 index 7480c6a0d..f41f9490a 100644 --- a/src/collision/collision_regime.f90 +++ b/src/collision/collision_regime.f90 @@ -285,19 +285,12 @@ subroutine collision_regime_LS12_SI(Mcb, m1, m2, rad1, rad2, rh1, rh2, vb1, vb2, Mlr = max((1.0_DP - Qr / Qrd_pstar / 2.0_DP) * Mtot, min_mfrag) ! [kg] # LS12 eq (5) end if Mbig = max(m1,Mlr) - Msmall = Mtot - Mbig - if (Msmall < min_mfrag) then - regime = COLLRESOLVE_REGIME_MERGE - Mbig = Mtot - Mslr = 0.0_DP - Mslr_hitandrun = 0.0_DP + Msmall = mtot - Mbig + Mslr_hitandrun = max(calc_Qrd_rev(Msmall, Mbig, Mint, den1, den2, Vimp, c_star), min_mfrag) + if (regime == COLLRESOLVE_REGIME_HIT_AND_RUN ) then + Mslr = Mslr_hitandrun else - Mslr_hitandrun = max(calc_Qrd_rev(Msmall, Mbig, Mint, den1, den2, Vimp, c_star), min_mfrag) - if (regime == COLLRESOLVE_REGIME_HIT_AND_RUN ) then - Mslr = Mslr_hitandrun - else - Mslr = max(Mtot * (3.0_DP - BETA) * (1.0_DP - N1 * Mlr / Mtot) / (N2 * BETA), min_mfrag) !LS12 eq (37) - end if + Mslr = max(Mtot * (3.0_DP - BETA) * (1.0_DP - N1 * Mlr / Mtot) / (N2 * BETA), min_mfrag) !LS12 eq (37) end if Mresidual = Mtot - Mlr - Mslr From 346ff32f252414e04e08ead40f237b0b5d943f52 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 24 Mar 2023 15:33:43 -0400 Subject: [PATCH 012/149] Rearranged objects to put id into the swiftest_body type out of the base --- src/base/base_module.f90 | 23 ----------------------- src/collision/collision_module.f90 | 4 +++- src/collision/collision_util.f90 | 2 ++ src/swiftest/swiftest_module.f90 | 4 +++- src/swiftest/swiftest_util.f90 | 5 ++--- 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 0a9429dbc..042c53909 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -206,14 +206,6 @@ end subroutine abstract_util_dealloc_object end interface - type, abstract, extends(base_object) :: base_multibody - integer(I4B) :: nbody = 0 !! Number of bodies - integer(I4B), dimension(:), allocatable :: id !! Identifier - contains - procedure :: dealloc => base_util_dealloc_multibody - end type base_multibody - - !> Class definition for the kinship relationships used in bookkeeping multiple collisions bodies in a single time step. type, abstract :: base_kinship end type base_kinship @@ -240,21 +232,6 @@ subroutine base_util_copy_store(self, source) end subroutine base_util_copy_store - subroutine base_util_dealloc_multibody(self) - !! author: David A. Minton - !! - !! Finalize the multibody body object - deallocates all allocatables - implicit none - ! Argument - class(base_multibody), intent(inout) :: self - - self%nbody = 0 - if (allocated(self%id)) deallocate(self%id) - - return - end subroutine base_util_dealloc_multibody - - subroutine base_util_dealloc_param(self) !! author: David A. Minton !! diff --git a/src/collision/collision_module.f90 b/src/collision/collision_module.f90 index 042c67af7..86b3ff76c 100644 --- a/src/collision/collision_module.f90 +++ b/src/collision/collision_module.f90 @@ -97,8 +97,10 @@ module collision !> Class definition for the variables that describe a collection of fragments in barycentric coordinates - type, extends(base_multibody) :: collision_fragments + type, extends(base_object) :: collision_fragments + integer(I4B) :: nbody = 0 !! Number of bodies real(DP) :: mtot !! Total mass of fragments + integer(I4B), dimension(:), allocatable :: id !! Identifier class(base_particle_info), dimension(:), allocatable :: info !! Particle metadata information integer(I4B), dimension(:), allocatable :: status !! An integrator-specific status indicator real(DP), dimension(:,:), allocatable :: rh !! Heliocentric position diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 19a688c8a..9eeb1746f 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -299,6 +299,8 @@ module subroutine collision_util_dealloc_fragments(self) ! Arguments class(collision_fragments), intent(inout) :: self + self%nbody = 0 + if (allocated(self%id)) deallocate(self%id) if (allocated(self%info)) deallocate(self%info) if (allocated(self%status)) deallocate(self%status) if (allocated(self%rh)) deallocate(self%rh) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 4ac6cb3c3..b4b43b526 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -92,9 +92,11 @@ module swiftest !> An abstract class for a generic collection of Swiftest bodies - type, abstract, extends(base_multibody) :: swiftest_body + type, abstract, extends(base_object) :: swiftest_body !! Superclass that defines the generic elements of a Swiftest particle logical :: lfirst = .true. !! Run the current step as a first + integer(I4B) :: nbody = 0 !! Number of bodies + integer(I4B), dimension(:), allocatable :: id !! Identifier type(swiftest_particle_info), dimension(:), allocatable :: info !! Particle metadata information logical, dimension(:), allocatable :: lmask !! Logical mask used to select a subset of bodies when performing certain operations (drift, kick, accel, etc.) integer(I4B), dimension(:), allocatable :: status !! An integrator-specific status indicator diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index fcf130f83..18b6d22df 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -727,7 +727,8 @@ module subroutine swiftest_util_dealloc_body(self) class(swiftest_body), intent(inout) :: self self%lfirst = .true. - + self%nbody = 0 + if (allocated(self%id)) deallocate(self%id) if (allocated(self%info)) deallocate(self%info) if (allocated(self%status)) deallocate(self%status) if (allocated(self%lmask)) deallocate(self%lmask) @@ -754,8 +755,6 @@ module subroutine swiftest_util_dealloc_body(self) if (allocated(self%omega)) deallocate(self%omega) if (allocated(self%capm)) deallocate(self%capm) - call base_util_dealloc_multibody(self) - return end subroutine swiftest_util_dealloc_body From 27fcda6faa512dfb9f314512009aa5fd41132688 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 28 Mar 2023 10:44:42 -0400 Subject: [PATCH 013/149] Changed param variable from allocatable class to fixed type. This was a hold over from when SyMBA had its own parameter type --- src/swiftest/swiftest_driver.f90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index bfa950222..321ed2487 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -20,7 +20,7 @@ program swiftest_driver implicit none class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated - class(swiftest_parameters), allocatable :: param !! Run configuration parameters + type(swiftest_parameters) :: param !! Run configuration parameters character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters character(len=:), allocatable :: display_style !! Style of the output display {"STANDARD", "COMPACT", "PROGRESS"}). Default is "STANDARD" @@ -34,7 +34,6 @@ program swiftest_driver call swiftest_io_get_args(integrator, param_file_name, display_style) !> Read in the user-defined parameters file and the initial conditions of the nbody_system - allocate(swiftest_parameters :: param) param%integrator = trim(adjustl(integrator)) param%display_style = trim(adjustl(display_style)) call param%read_in(param_file_name) From 066e085c7a290a9a79243171b9b5113de8bc4562 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 28 Mar 2023 11:47:59 -0400 Subject: [PATCH 014/149] Testing making the nbody_system object a coarray variable --- src/swiftest/swiftest_driver.f90 | 13 +++++++++++++ src/swiftest/swiftest_module.f90 | 8 ++++++-- src/swiftest/swiftest_util.f90 | 29 +++++++++++++++++++++++------ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 321ed2487..cab050b80 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -19,7 +19,11 @@ program swiftest_driver use swiftest implicit none +#ifdef COARRAY + class(swiftest_nbody_system), allocatable :: nbody_system[:] !! Polymorphic object containing the nbody system to be integrated +#else class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated +#endif type(swiftest_parameters) :: param !! Run configuration parameters character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters @@ -77,6 +81,12 @@ program swiftest_driver !$ write(param%display_unit,'(a)') ' ------------------' !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads +#ifdef COARRAY + if (this_image() == 1) then + write(param%display_unit,'(a)') ' Coarray parameters:' + write(param%display_unit,'(a)') ' -------------------' + write(param%display_unit,'(a)') ' Number of images = ', num_images() +#endif call nbody_system%initialize(param) @@ -138,6 +148,9 @@ program swiftest_driver call nbody_system%display_run_information(param, integration_timer, phase="last") end associate +#ifdef COARRAY + end if ! this_image() == 1 +#endif end associate call base_util_exit(SUCCESS) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index b4b43b526..c4a105644 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -1060,8 +1060,12 @@ end subroutine swiftest_util_setup_body module subroutine swiftest_util_setup_construct_system(nbody_system, param) implicit none - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters +#ifdef COARRAY + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system[:] !! Swiftest nbody_system object +#else + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object +#endif + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_construct_system module subroutine swiftest_util_setup_initialize_particle_info_system(self, param) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 18b6d22df..0b8f3e0e2 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2694,7 +2694,6 @@ module subroutine swiftest_util_set_rhill_approximate(self,cb) return end subroutine swiftest_util_set_rhill_approximate - module subroutine swiftest_util_setup_construct_system(nbody_system, param) !! author: David A. Minton !! @@ -2702,14 +2701,22 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) !! implicit none ! Arguments - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters +#ifdef COARRAY + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system[:] !! Swiftest nbody_system object +#else + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object +#endif + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters select case(param%integrator) case (INT_BS) write(*,*) 'Bulirsch-Stoer integrator not yet enabled' - case (INT_HELIO) + case (INT_HELIO) +#ifdef COARRAY + allocate(helio_nbody_system :: nbody_system[*]) +#else allocate(helio_nbody_system :: nbody_system) +#endif select type(nbody_system) class is (helio_nbody_system) allocate(helio_cb :: nbody_system%cb) @@ -2723,7 +2730,11 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) case (INT_TU4) write(*,*) 'INT_TU4 integrator not yet enabled' case (INT_WHM) +#ifdef COARRAY + allocate(whm_nbody_system :: nbody_system[*]) +#else allocate(whm_nbody_system :: nbody_system) +#endif select type(nbody_system) class is (whm_nbody_system) allocate(whm_cb :: nbody_system%cb) @@ -2733,7 +2744,11 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) end select param%collision_model = "MERGE" case (INT_RMVS) +#ifdef COARRAY + allocate(rmvs_nbody_system :: nbody_system[*]) +#else allocate(rmvs_nbody_system :: nbody_system) +#endif select type(nbody_system) class is (rmvs_nbody_system) allocate(rmvs_cb :: nbody_system%cb) @@ -2743,7 +2758,11 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) end select param%collision_model = "MERGE" case (INT_SYMBA) +#ifdef COARRAY + allocate(symba_nbody_system :: nbody_system[*]) +#else allocate(symba_nbody_system :: nbody_system) +#endif select type(nbody_system) class is (symba_nbody_system) allocate(symba_cb :: nbody_system%cb) @@ -2768,10 +2787,8 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) allocate(swiftest_particle_info :: nbody_system%cb%info) - nbody_system%t = param%tstart - return end subroutine swiftest_util_setup_construct_system From 7357bdfe4f00acad045e0c8afae9386dcaaa8107 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 28 Mar 2023 12:14:14 -0400 Subject: [PATCH 015/149] Fixed syntax in coarray image reporting --- src/swiftest/swiftest_driver.f90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index cab050b80..89111e0c3 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -85,7 +85,8 @@ program swiftest_driver if (this_image() == 1) then write(param%display_unit,'(a)') ' Coarray parameters:' write(param%display_unit,'(a)') ' -------------------' - write(param%display_unit,'(a)') ' Number of images = ', num_images() + write(param%display_unit,'(a,i3)') ' Number of images = ', num_images() + if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() #endif call nbody_system%initialize(param) From 2b62ee1ce2af11862bebd6d03e0a1a44bfb75dad Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 28 Mar 2023 12:21:31 -0400 Subject: [PATCH 016/149] Updated snapshot system to use coarray version of nbody_system --- src/swiftest/swiftest_module.f90 | 12 ++++++++---- src/swiftest/swiftest_util.f90 | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index c4a105644..bb43c40b9 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -1659,11 +1659,15 @@ end subroutine swiftest_util_snapshot_save module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, arg) implicit none - class(swiftest_storage), intent(inout) :: self !! Swiftest storage object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_storage), intent(inout) :: self !! Swiftest storage object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters +#ifdef COARRAY + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store +#else class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store - real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time - character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in encounter snapshots) +#endif + real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time + character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in encounter snapshots) end subroutine swiftest_util_snapshot_system end interface diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 0b8f3e0e2..24e98a4ae 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -3067,7 +3067,11 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar ! Arguments class(swiftest_storage), intent(inout) :: self !! Swiftest storage object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters +#ifdef COARRAY + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store +#else class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store +#endif real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) ! Internals From 7a06aff7bb9b040fe560be4b17780a7f327a8dde Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 28 Mar 2023 14:15:49 -0400 Subject: [PATCH 017/149] Added collect and distribute methods for coarrays --- src/swiftest/swiftest_driver.f90 | 93 ++++++++++++++++++-------------- src/swiftest/swiftest_module.f90 | 29 +++++++--- src/swiftest/swiftest_util.f90 | 71 +++++++++++++++++++++--- 3 files changed, 141 insertions(+), 52 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 89111e0c3..bcc4d3fa1 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -81,17 +81,17 @@ program swiftest_driver !$ write(param%display_unit,'(a)') ' ------------------' !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads -#ifdef COARRAY - if (this_image() == 1) then - write(param%display_unit,'(a)') ' Coarray parameters:' - write(param%display_unit,'(a)') ' -------------------' - write(param%display_unit,'(a,i3)') ' Number of images = ', num_images() - if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() +#ifdef COARRAY + ! Only execute file file I/O and reporting on image 1 + if (this_image() == 1) then + write(param%display_unit,'(a)') ' Coarray parameters:' + write(param%display_unit,'(a)') ' -------------------' + write(param%display_unit,'(a,i3)') ' Number of images = ', num_images() + if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() #endif - call nbody_system%initialize(param) + call nbody_system%initialize(param) - associate (system_history => nbody_system%system_history) ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. call nbody_system%display_run_information(param, integration_timer, phase="first") if (param%lenergy) then @@ -102,32 +102,40 @@ program swiftest_driver end if call nbody_system%conservation_report(param, lterminal=.true.) end if - call system_history%take_snapshot(param,nbody_system) + call nbody_system%system_history%take_snapshot(param,nbody_system) call nbody_system%dump(param) - do iloop = istart, nloops - !> Step the nbody_system forward in time - call integration_timer%start() - call nbody_system%step(param, nbody_system%t, dt) - call integration_timer%stop() - - nbody_system%t = t0 + iloop * dt - - !> Evaluate any discards or collisional outcomes - call nbody_system%discard(param) - - !> If the loop counter is at the output cadence value, append the data file with a single frame - if (istep_out > 0) then - iout = iout + 1 - if ((iout == istep) .or. (iloop == nloops)) then - iout = 0 - idump = idump + 1 - if (ltstretch) then - nout = nout + 1 - istep = floor(istep_out * fstep_out**nout, kind=I4B) - end if - - call system_history%take_snapshot(param,nbody_system) +#ifdef COARRAY + ! Distribute test particles to the various images + call nbody_system%coarray_distribute() + end if ! this_image() == 1 +#endif + do iloop = istart, nloops + !> Step the nbody_system forward in time + call integration_timer%start() + call nbody_system%step(param, nbody_system%t, dt) + call integration_timer%stop() + + nbody_system%t = t0 + iloop * dt + + !> Evaluate any discards or collisional outcomes + call nbody_system%discard(param) + + !> If the loop counter is at the output cadence value, append the data file with a single frame + if (istep_out > 0) then + iout = iout + 1 + if ((iout == istep) .or. (iloop == nloops)) then + iout = 0 + idump = idump + 1 + if (ltstretch) then + nout = nout + 1 + istep = floor(istep_out * fstep_out**nout, kind=I4B) + end if +#ifdef COARRAY + if (this_image() == 1) then + call nbody_system%coarray_collect() +#endif + call nbody_system%system_history%take_snapshot(param,nbody_system) if (idump == dump_cadence) then idump = 0 @@ -138,20 +146,27 @@ program swiftest_driver call nbody_system%display_run_information(param, integration_timer) call integration_timer%reset() if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) - - end if +#ifdef COARRAY + call nbody_system%coarray_distribute() + end if + sync all +#endif end if + end if - end do - ! Dump any remaining history if it exists + end do + ! Dump any remaining history if it exists +#ifdef COARRAY + if (this_image() == 1) then +#endif + call nbody_system%coarray_collect() call nbody_system%dump(param) - call system_history%dump(param) + call nbody_system%system_history%dump(param) call nbody_system%display_run_information(param, integration_timer, phase="last") - - end associate #ifdef COARRAY end if ! this_image() == 1 #endif + end associate call base_util_exit(SUCCESS) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index bb43c40b9..a569e0d98 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -393,7 +393,11 @@ module swiftest procedure :: read_in => swiftest_io_read_in_system !! Reads the initial conditions for an nbody system procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body - procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer +#ifdef COARRAY + procedure :: coarray_collect => swiftest_util_coarray_collect_system !! Collects test particles from distributed images into image #1 + procedure :: coarray_distribute => swiftest_util_coarray_distribute_system !! Distributes test particles from image #1 out to all images +#endif + procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum procedure :: get_idvals => swiftest_util_get_idvalues_system !! Returns an array of all id values in use in the nbody_system procedure :: rescale => swiftest_util_rescale_system !! Rescales the nbody_system into a new set of units @@ -1184,6 +1188,18 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_tp +#ifdef COARRAY + module subroutine swiftest_util_coarray_collect_system(self) + implicit none + class(swiftest_nbody_system), codimension[*], intent(inout) :: self + end subroutine swiftest_util_coarray_collect_system + + module subroutine swiftest_util_coarray_distribute_system(self) + implicit none + class(swiftest_nbody_system), codimension[*], intent(inout) :: self + end subroutine swiftest_util_coarray_distribute_system +#endif + module subroutine swiftest_util_coord_b2h_pl(self, cb) implicit none class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object @@ -1659,19 +1675,18 @@ end subroutine swiftest_util_snapshot_save module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, arg) implicit none - class(swiftest_storage), intent(inout) :: self !! Swiftest storage object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_storage), intent(inout) :: self !! Swiftest storage object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters #ifdef COARRAY class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store #else - class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store #endif - real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time - character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in encounter snapshots) + real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time + character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) end subroutine swiftest_util_snapshot_system end interface - interface swiftest_util_sort pure module subroutine swiftest_util_sort_i4b(arr) implicit none diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 24e98a4ae..7661115e6 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -354,6 +354,65 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) return end subroutine swiftest_util_append_tp +#ifdef COARRAY + module subroutine swiftest_util_coarray_collect_system(self) + !! author: David A. Minton + !! + !! Distributes test particles from image #1 out to all images. + implicit none + ! Arguments + class(swiftest_nbody_system), codimension[*], intent(inout) :: self + ! Internals + integer(I4B) :: i,j + integer(I4B), dimension(num_images()) :: ntp + class(swiftest_tp), allocatable :: tp_img + + ntp(this_image()) = self%tp%nbody + sync all + if (this_image() == 1) then + do i = 2, num_images() + allocate(tp_img, source=self[i]%tp) + call self%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) + deallocate(tp_img) + end do + end if + + return + end subroutine swiftest_util_coarray_collect_system + + + module subroutine swiftest_util_coarray_distribute_system(self) + !! author: David A. Minton + !! + !! Distributes test particles from image #1 out to all images. + implicit none + ! Arguments + class(swiftest_nbody_system), codimension[*], intent(inout) :: self + ! Internals + integer(I4B) :: i, istart, iend, ntot, num_per_image + class(swiftest_tp), allocatable :: tp_orig + logical, dimension(:), allocatable :: lspill_list + + sync all + ntot = self[1]%tp%nbody + if (ntot == 0) return + allocate(lspill_list(ntot)) + allocate(tp_orig, source=self[1]%tp) + if (allocated(self%tp)) deallocate(self%tp) + num_per_image = ntot / num_images() + istart = (this_image() - 1) * num_per_image + 1 + if (this_image() == num_images()) then + iend = ntot + else + iend = this_image() * num_per_image + end if + lspill_list(:) = .false. + lspill_list(istart:iend) = .true. + call tp_orig%spill(self%tp,lspill_list(:), ldestructive=.false.) + + return + end subroutine swiftest_util_coarray_distribute_system +#endif module subroutine swiftest_util_coord_h2b_pl(self, cb) !! author: David A. Minton @@ -3065,15 +3124,15 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar !! Takes a snapshot of the nbody_system for later file storage implicit none ! Arguments - class(swiftest_storage), intent(inout) :: self !! Swiftest storage object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_storage), intent(inout) :: self !! Swiftest storage object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters #ifdef COARRAY - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store #else - class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store #endif - real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time - character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) + real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time + character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) ! Internals class(swiftest_nbody_system), allocatable :: snapshot From 7b4857841ddbbd587554f3661d1def8678f27850 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 4 Apr 2023 17:11:14 -0400 Subject: [PATCH 018/149] Created coarray infrastructure that is the basis for being able to broadcast particle data between coarray images --- src/CMakeLists.txt | 10 +- src/base/base_module.f90 | 116 ++++++++++- src/coarray/coarray_module.f90 | 336 +++++++++++++++++++++++++++++++ src/rmvs/rmvs_util.f90 | 10 +- src/swiftest/swiftest_driver.f90 | 35 ++-- src/swiftest/swiftest_io.f90 | 27 ++- src/swiftest/swiftest_module.f90 | 19 +- src/swiftest/swiftest_util.f90 | 87 +++++--- 8 files changed, 561 insertions(+), 79 deletions(-) create mode 100644 src/coarray/coarray_module.f90 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0ad615170..0def91f34 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -83,7 +83,15 @@ SET(FAST_MATH_FILES ${SRC}/swiftest/swiftest_driver.f90 ) -set(SWIFTEST_src ${FAST_MATH_FILES} ${STRICT_MATH_FILES}) +SET(COARRAY_FILES + ${SRC}/coarray/coarray_module.f90 +) + +IF(USE_COARRAY) + set(SWIFTEST_src ${COARRAY_FILES} ${FAST_MATH_FILES} ${STRICT_MATH_FILES}) +ELSE() + set(SWIFTEST_src ${FAST_MATH_FILES} ${STRICT_MATH_FILES}) +ENDIF(USE_COARRAY) # Define the executable in terms of the source files ADD_EXECUTABLE(${SWIFTEST_DRIVER} ${SWIFTEST_src}) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 042c53909..99b0cd145 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -13,15 +13,17 @@ module base !! Base type definitions. This allows the collision and encounter modules to be defined before the swiftest module. !! use globals +#ifdef COARRAY + use coarray +#endif implicit none public - !> User defined parameters that are read in from the parameters input file. !> Each paramter is initialized to a default values. type, abstract :: base_parameters - character(len=:), allocatable :: integrator !! Name of the nbody integrator used - character(len=:), allocatable :: param_file_name !! The name of the parameter file + character(STRMAX) :: integrator !! Name of the nbody integrator used + character(STRMAX) :: param_file_name !! The name of the parameter file real(DP) :: t0 = 0.0_DP !! Integration reference time real(DP) :: tstart = -1.0_DP !! Integration start time real(DP) :: tstop = -1.0_DP !! Integration stop time @@ -97,9 +99,9 @@ module base logical :: lfirstkick = .true. !! Initiate the first kick in a symplectic step logical :: lrestart = .false. !! Indicates whether or not this is a restarted run - character(len=:), allocatable :: display_style !! Style of the output display {"STANDARD", "COMPACT"}). Default is "STANDARD" - integer(I4B) :: display_unit !! File unit number for display (either to stdout or to a log file) - logical :: log_output = .false. !! Logs the output to file instead of displaying it on the terminal + character(NAMELEN) :: display_style !! Style of the output display {"STANDARD", "COMPACT"}). Default is "STANDARD" + integer(I4B) :: display_unit !! File unit number for display (either to stdout or to a log file) + logical :: log_output = .false. !! Logs the output to file instead of displaying it on the terminal ! Future features not implemented or in development logical :: lgr = .false. !! Turn on GR @@ -111,6 +113,9 @@ module base procedure(abstract_io_param_reader), deferred :: reader procedure(abstract_io_param_writer), deferred :: writer procedure(abstract_io_read_in_param), deferred :: read_in +#ifdef COARRAY + procedure :: cobroadcast => base_cobroadcast_param +#endif end type base_parameters abstract interface @@ -240,9 +245,6 @@ subroutine base_util_dealloc_param(self) ! Arguments class(base_parameters),intent(inout) :: self !! Collection of parameters - if (allocated(self%integrator)) deallocate(self%integrator) - if (allocated(self%param_file_name)) deallocate(self%param_file_name) - if (allocated(self%display_style)) deallocate(self%display_style) if (allocated(self%seed)) deallocate(self%seed) return @@ -430,5 +432,101 @@ subroutine base_final_storage_frame(self) return end subroutine base_final_storage_frame +#ifdef COARRAY + subroutine base_cobroadcast_param(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 parameter to all other images in a parameter coarray + implicit none + ! Arguments + class(base_parameters),intent(inout),codimension[*] :: self !! Collection of parameters + ! Internals + integer(I4B) :: i + + call cocopy(self%integrator) + call cocopy(self%param_file_name) + call cocopy(self%t0) + call cocopy(self%tstart) + call cocopy(self%tstop) + call cocopy(self%dt) + call cocopy(self%iloop) + call cocopy(self%nloops) + call cocopy(self%incbfile) + call cocopy(self%inplfile) + call cocopy(self%intpfile) + call cocopy(self%nc_in) + call cocopy(self%in_type) + call cocopy(self%in_form) + call cocopy(self%istep_out) + call cocopy(self%nstep_out) + call cocopy(self%fstep_out) + call cocopy(self%ltstretch) + call cocopy(self%outfile) + call cocopy(self%out_type) + call cocopy(self%out_form) + call cocopy(self%out_stat) + call cocopy(self%dump_cadence) + call cocopy(self%rmin) + call cocopy(self%rmax) + call cocopy(self%rmaxu) + call cocopy(self%qmin) + call cocopy(self%qmin_coord) + call cocopy(self%qmin_alo) + call cocopy(self%qmin_ahi) + call cocopy(self%MU2KG) + call cocopy(self%TU2S) + call cocopy(self%DU2M) + call cocopy(self%GU) + call cocopy(self%inv_c2) + call cocopy(self%GMTINY) + call cocopy(self%min_GMfrag) + call cocopy(self%nfrag_reduction) + call cocopy(self%lmtiny_pl) + call cocopy(self%collision_model) + call cocopy(self%encounter_save) + call cocopy(self%lenc_save_trajectory) + call cocopy(self%lenc_save_closest ) + call cocopy(self%interaction_loops ) + call cocopy(self%encounter_check_plpl) + call cocopy(self%encounter_check_pltp) + call cocopy(self%lflatten_interactions) + call cocopy(self%lencounter_sas_plpl) + call cocopy(self%lencounter_sas_pltp ) + call cocopy(self%lrhill_present) + call cocopy(self%lextra_force ) + call cocopy(self%lbig_discard ) + call cocopy(self%lclose ) + call cocopy(self%lenergy ) + call cocopy(self%loblatecb ) + call cocopy(self%lrotation ) + call cocopy(self%ltides ) + call cocopy(self%E_orbit_orig ) + call cocopy(self%GMtot_orig ) + do i = 1, NDIM + call cocopy(self%L_total_orig(i)) + call cocopy(self%L_orbit_orig(i)) + call cocopy(self%L_spin_orig(i)) + call cocopy(self%L_escape(i)) + end do + call cocopy(self%GMescape ) + call cocopy(self%E_collisions ) + call cocopy(self%E_untracked ) + call cocopy(self%lfirstenergy) + call cocopy(self%lfirstkick ) + call cocopy(self%lrestart ) + call cocopy(self%display_style) + call cocopy(self%display_unit ) + call cocopy(self%log_output ) + call cocopy(self%lgr ) + call cocopy(self%lyarkovsky) + call cocopy(self%lyorp ) + call cocopy(self%seed) + + return + end subroutine base_cobroadcast_param + +#endif + + end module base diff --git a/src/coarray/coarray_module.f90 b/src/coarray/coarray_module.f90 new file mode 100644 index 000000000..e9905b9df --- /dev/null +++ b/src/coarray/coarray_module.f90 @@ -0,0 +1,336 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +module coarray + !! author: David A. Minton + !! + !! Utilities that are used for coarray test particles + !! + use globals + implicit none + public + + interface cocopy + module procedure coarray_component_copy_char + module procedure coarray_component_copy_DP + module procedure coarray_component_copy_DP_arr1D + module procedure coarray_component_copy_I4B + module procedure coarray_component_copy_I4B_arr1D + module procedure coarray_component_copy_I8B + module procedure coarray_component_copy_lgt + module procedure coarray_component_copy_QP + end interface + + contains + + subroutine coarray_component_copy_char(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! character scalar version + implicit none + ! Arguments + character(len=*), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + character(len=STRMAX),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_char + + + subroutine coarray_component_copy_DP(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) scalar version + implicit none + ! Arguments + real(DP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_DP + + + subroutine coarray_component_copy_DP_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) allocatable array version + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + n = size(var) + sync all + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var(:) + end do + end if + sync all + if (this_image() /= si) then + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_copy_DP_arr1D + + + subroutine coarray_component_copy_DP_arr2D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) 2D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n1[*], n2[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + n1 = size(var,dim=1) + n2 = size(var,dim=2) + sync all + allocate(tmp(n1[si],n2[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:,:)[img] = var(:,:) + end do + end if + sync all + if (this_image() /= si) then + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_copy_DP_arr2D + + + subroutine coarray_component_copy_I4B(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) scalar version + implicit none + ! Arguments + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I4B),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_I4B + + + subroutine coarray_component_copy_I4B_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I4B), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + n = size(var) + sync all + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + end if + sync all + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + + return + end subroutine coarray_component_copy_I4B_arr1D + + + subroutine coarray_component_copy_I8B(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) scalar version + implicit none + ! Arguments + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I8B),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_I8B + + + subroutine coarray_component_copy_lgt(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! logical scalar version + implicit none + ! Arguments + logical, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + logical,save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_lgt + + + subroutine coarray_component_copy_QP(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) scalar version + implicit none + ! Arguments + real(QP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(QP),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + end subroutine coarray_component_copy_QP + +end module coarray \ No newline at end of file diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index b9462840b..c0f987aa9 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -413,11 +413,13 @@ module subroutine rmvs_util_setup_tp(self, n, param) call self%whm_tp%setup(n, param) if (n <= 0) return - allocate(self%lperi(n)) - allocate(self%plperP(n)) - allocate(self%plencP(n)) + if (allocated(self%lperi)) deallocate(self%lperi); allocate(self%lperi(n)) + if (allocated(self%plperP)) deallocate(self%plperP); allocate(self%plperP(n)) + if (allocated(self%plencP)) deallocate(self%plencP); allocate(self%plencP(n)) - if (self%lplanetocentric) allocate(self%rheliocentric(NDIM, n)) + if (self%lplanetocentric) then + if (allocated(self%rheliocentric)) deallocate(self%rheliocentric); allocate(self%rheliocentric(NDIM, n)) + end if self%lperi(:) = .false. diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index bcc4d3fa1..31eccbdf9 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -20,7 +20,7 @@ program swiftest_driver implicit none #ifdef COARRAY - class(swiftest_nbody_system), allocatable :: nbody_system[:] !! Polymorphic object containing the nbody system to be integrated + class(swiftest_nbody_system), allocatable :: nbody_system[:] !! Polymorphic object containing the nbody system to be integrated #else class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated #endif @@ -77,21 +77,25 @@ program swiftest_driver !> Define the maximum number of threads nthreads = 1 ! In the *serial* case !$ nthreads = omp_get_max_threads() ! In the *parallel* case - !$ write(param%display_unit,'(a)') ' OpenMP parameters:' - !$ write(param%display_unit,'(a)') ' ------------------' - !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads - !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads -#ifdef COARRAY - ! Only execute file file I/O and reporting on image 1 +#ifdef COARRAY if (this_image() == 1) then - write(param%display_unit,'(a)') ' Coarray parameters:' - write(param%display_unit,'(a)') ' -------------------' - write(param%display_unit,'(a,i3)') ' Number of images = ', num_images() +#endif + !$ write(param%display_unit,'(a)') ' OpenMP parameters:' + !$ write(param%display_unit,'(a)') ' ------------------' + !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads + !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads +#ifdef COARRAY + write(param%display_unit,*) ' Coarray parameters:' + write(param%display_unit,*) ' -------------------' + write(param%display_unit,*) ' Number of images = ', num_images() if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() + end if #endif +#ifdef COARRAY + if (this_image() == 1) then +#endif call nbody_system%initialize(param) - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. call nbody_system%display_run_information(param, integration_timer, phase="first") if (param%lenergy) then @@ -107,8 +111,8 @@ program swiftest_driver #ifdef COARRAY ! Distribute test particles to the various images - call nbody_system%coarray_distribute() end if ! this_image() == 1 + call nbody_system%coarray_distribute() #endif do iloop = istart, nloops !> Step the nbody_system forward in time @@ -132,8 +136,8 @@ program swiftest_driver istep = floor(istep_out * fstep_out**nout, kind=I4B) end if #ifdef COARRAY + call nbody_system%coarray_collect() if (this_image() == 1) then - call nbody_system%coarray_collect() #endif call nbody_system%system_history%take_snapshot(param,nbody_system) @@ -147,9 +151,8 @@ program swiftest_driver call integration_timer%reset() if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY - call nbody_system%coarray_distribute() end if - sync all + call nbody_system%coarray_distribute() #endif end if end if @@ -157,9 +160,9 @@ program swiftest_driver end do ! Dump any remaining history if it exists #ifdef COARRAY + call nbody_system%coarray_collect() if (this_image() == 1) then #endif - call nbody_system%coarray_collect() call nbody_system%dump(param) call nbody_system%system_history%dump(param) call nbody_system%display_run_information(param, integration_timer, phase="last") diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 32f722a7b..928f6eccd 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1882,10 +1882,16 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i logical :: seed_set = .false. !! Is the random seed set in the input file? character(len=:), allocatable :: integrator real(DP) :: tratio, y - - - ! Parse the file line by line, extracting tokens then matching them up with known parameters if possible + type(swiftest_parameters), codimension[*], save :: coparam + +#ifdef COARRAY + if (this_image() == 1) then + coparam = self + associate(param => coparam) +#else associate(param => self) +#endif + ! Parse the file line by line, extracting tokens then matching them up with known parameters if possible call random_seed(size = nseeds) if (allocated(param%seed)) deallocate(param%seed) allocate(param%seed(nseeds)) @@ -2309,11 +2315,22 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i iostat = 0 call param%set_display(param%display_style) - ! Print the contents of the parameter file to standard output if (.not.param%lrestart) call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) - end associate +#ifdef COARRAY + end if ! this_image() == 1 + call coparam%cobroadcast() + select type(self) + type is (swiftest_parameters) + self = coparam + end select + + write(*,*) "Image: ", this_image(),"tstop: ",self%tstop + write(*,*) "Image: ", this_image(),"seed: ",self%seed + sync all + stop +#endif return 667 continue diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index a569e0d98..55f63ec6f 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -150,7 +150,6 @@ module swiftest procedure :: fill => swiftest_util_fill_body !! "Fills" bodies from one object into another depending on the results of a mask (uses the UNPACK intrinsic) procedure :: get_peri => swiftest_util_peri_body !! Determine nbody_system pericenter passages for test particles procedure :: resize => swiftest_util_resize_body !! Checks the current size of a Swiftest body against the requested size and resizes it if it is too small. - procedure :: set_ir3 => swiftest_util_set_ir3h !! Sets the inverse heliocentric radius term (1/rh**3) procedure :: sort => swiftest_util_sort_body !! Sorts body arrays by a sortable componen procedure :: rearrange => swiftest_util_sort_rearrange_body !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods @@ -394,8 +393,8 @@ module swiftest procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body #ifdef COARRAY - procedure :: coarray_collect => swiftest_util_coarray_collect_system !! Collects test particles from distributed images into image #1 - procedure :: coarray_distribute => swiftest_util_coarray_distribute_system !! Distributes test particles from image #1 out to all images + procedure :: coarray_collect => swiftest_util_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system + procedure :: coarray_distribute => swiftest_util_coarray_distribute_system !! Distributes test particles from image #1 out to all images. #endif procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum @@ -1081,7 +1080,7 @@ end subroutine swiftest_util_setup_initialize_particle_info_system module subroutine swiftest_util_setup_initialize_system(self, param) implicit none class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_initialize_system module subroutine swiftest_util_setup_pl(self, n, param) @@ -1189,14 +1188,14 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) end subroutine swiftest_util_append_tp #ifdef COARRAY - module subroutine swiftest_util_coarray_collect_system(self) + module subroutine swiftest_util_coarray_collect_system(nbody_system) implicit none - class(swiftest_nbody_system), codimension[*], intent(inout) :: self + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] end subroutine swiftest_util_coarray_collect_system - module subroutine swiftest_util_coarray_distribute_system(self) + module subroutine swiftest_util_coarray_distribute_system(nbody_system) implicit none - class(swiftest_nbody_system), codimension[*], intent(inout) :: self + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] end subroutine swiftest_util_coarray_distribute_system #endif @@ -1677,11 +1676,7 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar implicit none class(swiftest_storage), intent(inout) :: self !! Swiftest storage object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters -#ifdef COARRAY - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store -#else class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store -#endif real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) end subroutine swiftest_util_snapshot_system diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 7661115e6..44c3bc17f 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -355,60 +355,81 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) end subroutine swiftest_util_append_tp #ifdef COARRAY - module subroutine swiftest_util_coarray_collect_system(self) + module subroutine swiftest_util_coarray_collect_system(nbody_system) !! author: David A. Minton !! - !! Distributes test particles from image #1 out to all images. + !! Collects all the test particles from other images into the image #1 test particle system implicit none ! Arguments - class(swiftest_nbody_system), codimension[*], intent(inout) :: self + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] ! Internals integer(I4B) :: i,j integer(I4B), dimension(num_images()) :: ntp class(swiftest_tp), allocatable :: tp_img - ntp(this_image()) = self%tp%nbody - sync all - if (this_image() == 1) then - do i = 2, num_images() - allocate(tp_img, source=self[i]%tp) - call self%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) - deallocate(tp_img) - end do - end if + ! ntp(this_image()) = nbody_system%tp%nbody + ! sync all + ! if (this_image() == 1) then + ! write(*,*) "Collecting test particles" + ! write(*,*) "Image ",1," ntp: ",ntp(1) + ! do i = 2, num_images() + ! write(*,*) "Image ",i," ntp: ",ntp(i) + ! allocate(tp_img, source=nbody_system[i]%tp) + ! call nbody_system%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) + ! deallocate(tp_img) + ! end do + ! write(*,*) "Total test particles: ",nbody_system%tp%nbody + ! end if return end subroutine swiftest_util_coarray_collect_system - module subroutine swiftest_util_coarray_distribute_system(self) + module subroutine swiftest_util_coarray_distribute_system(nbody_system) !! author: David A. Minton !! !! Distributes test particles from image #1 out to all images. implicit none ! Arguments - class(swiftest_nbody_system), codimension[*], intent(inout) :: self + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] ! Internals - integer(I4B) :: i, istart, iend, ntot, num_per_image + integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy class(swiftest_tp), allocatable :: tp_orig logical, dimension(:), allocatable :: lspill_list - - sync all - ntot = self[1]%tp%nbody - if (ntot == 0) return - allocate(lspill_list(ntot)) - allocate(tp_orig, source=self[1]%tp) - if (allocated(self%tp)) deallocate(self%tp) - num_per_image = ntot / num_images() - istart = (this_image() - 1) * num_per_image + 1 - if (this_image() == num_images()) then - iend = ntot - else - iend = this_image() * num_per_image - end if - lspill_list(:) = .false. - lspill_list(istart:iend) = .true. - call tp_orig%spill(self%tp,lspill_list(:), ldestructive=.false.) + integer(I4B), codimension[*],save :: ntp + class(swiftest_nbody_system), allocatable :: tmp_system + class(swiftest_tp), allocatable :: tp + + ! ntp = nbody_system%tp%nbody + ! sync all + + ! ntot = ntp[1] + ! if (ntot == 0) return + + ! allocate(tp, mold=nbody_system%tp) + + ! write(*,*) "Image ",this_image(), "Distributing ",ntot + ! allocate(lspill_list(ntot)) + ! num_per_image = ntot / num_images() + ! istart = (this_image() - 1) * num_per_image + 1 + ! if (this_image() == num_images()) then + ! iend = ntot + ! else + ! iend = this_image() * num_per_image + ! end if + + ! if (this_image() == 1) then + ! lspill_list(:) = .true. + ! lspill_list(istart:iend) = .false. + ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) + ! else + ! lspill_list(:) = .false. + ! lspill_list(istart:iend) = .true. + ! tp%nbody = ntot + ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) + ! end if + + ! write(*,*) "Image ",this_image(), "ntp: ",nbody_system%tp%nbody return end subroutine swiftest_util_coarray_distribute_system @@ -2895,6 +2916,7 @@ module subroutine swiftest_util_setup_initialize_system(self, param) class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals if (allocated(self%system_history)) then call self%system_history%dealloc() deallocate(self%system_history) @@ -2904,6 +2926,7 @@ module subroutine swiftest_util_setup_initialize_system(self, param) allocate(swiftest_netcdf_parameters :: self%system_history%nc) associate(nbody_system => self, cb => self%cb, pl => self%pl, tp => self%tp, nc => self%system_history%nc) + call nbody_system%read_in(param) call nbody_system%read_in(param) call nbody_system%validate_ids(param) call nbody_system%set_msys() From 8bea0d950c37926f72863d78164dff634e21ae25 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 6 Apr 2023 08:06:29 -0400 Subject: [PATCH 019/149] Refactored cocopy to coclone to reflect what is actually being done by this procedure --- src/base/base_module.f90 | 153 ++++++++++++++++----------------- src/coarray/coarray_module.f90 | 2 +- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 99b0cd145..d9d456958 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -443,84 +443,84 @@ subroutine base_cobroadcast_param(self) ! Internals integer(I4B) :: i - call cocopy(self%integrator) - call cocopy(self%param_file_name) - call cocopy(self%t0) - call cocopy(self%tstart) - call cocopy(self%tstop) - call cocopy(self%dt) - call cocopy(self%iloop) - call cocopy(self%nloops) - call cocopy(self%incbfile) - call cocopy(self%inplfile) - call cocopy(self%intpfile) - call cocopy(self%nc_in) - call cocopy(self%in_type) - call cocopy(self%in_form) - call cocopy(self%istep_out) - call cocopy(self%nstep_out) - call cocopy(self%fstep_out) - call cocopy(self%ltstretch) - call cocopy(self%outfile) - call cocopy(self%out_type) - call cocopy(self%out_form) - call cocopy(self%out_stat) - call cocopy(self%dump_cadence) - call cocopy(self%rmin) - call cocopy(self%rmax) - call cocopy(self%rmaxu) - call cocopy(self%qmin) - call cocopy(self%qmin_coord) - call cocopy(self%qmin_alo) - call cocopy(self%qmin_ahi) - call cocopy(self%MU2KG) - call cocopy(self%TU2S) - call cocopy(self%DU2M) - call cocopy(self%GU) - call cocopy(self%inv_c2) - call cocopy(self%GMTINY) - call cocopy(self%min_GMfrag) - call cocopy(self%nfrag_reduction) - call cocopy(self%lmtiny_pl) - call cocopy(self%collision_model) - call cocopy(self%encounter_save) - call cocopy(self%lenc_save_trajectory) - call cocopy(self%lenc_save_closest ) - call cocopy(self%interaction_loops ) - call cocopy(self%encounter_check_plpl) - call cocopy(self%encounter_check_pltp) - call cocopy(self%lflatten_interactions) - call cocopy(self%lencounter_sas_plpl) - call cocopy(self%lencounter_sas_pltp ) - call cocopy(self%lrhill_present) - call cocopy(self%lextra_force ) - call cocopy(self%lbig_discard ) - call cocopy(self%lclose ) - call cocopy(self%lenergy ) - call cocopy(self%loblatecb ) - call cocopy(self%lrotation ) - call cocopy(self%ltides ) - call cocopy(self%E_orbit_orig ) - call cocopy(self%GMtot_orig ) + call coclone(self%integrator) + call coclone(self%param_file_name) + call coclone(self%t0) + call coclone(self%tstart) + call coclone(self%tstop) + call coclone(self%dt) + call coclone(self%iloop) + call coclone(self%nloops) + call coclone(self%incbfile) + call coclone(self%inplfile) + call coclone(self%intpfile) + call coclone(self%nc_in) + call coclone(self%in_type) + call coclone(self%in_form) + call coclone(self%istep_out) + call coclone(self%nstep_out) + call coclone(self%fstep_out) + call coclone(self%ltstretch) + call coclone(self%outfile) + call coclone(self%out_type) + call coclone(self%out_form) + call coclone(self%out_stat) + call coclone(self%dump_cadence) + call coclone(self%rmin) + call coclone(self%rmax) + call coclone(self%rmaxu) + call coclone(self%qmin) + call coclone(self%qmin_coord) + call coclone(self%qmin_alo) + call coclone(self%qmin_ahi) + call coclone(self%MU2KG) + call coclone(self%TU2S) + call coclone(self%DU2M) + call coclone(self%GU) + call coclone(self%inv_c2) + call coclone(self%GMTINY) + call coclone(self%min_GMfrag) + call coclone(self%nfrag_reduction) + call coclone(self%lmtiny_pl) + call coclone(self%collision_model) + call coclone(self%encounter_save) + call coclone(self%lenc_save_trajectory) + call coclone(self%lenc_save_closest ) + call coclone(self%interaction_loops ) + call coclone(self%encounter_check_plpl) + call coclone(self%encounter_check_pltp) + call coclone(self%lflatten_interactions) + call coclone(self%lencounter_sas_plpl) + call coclone(self%lencounter_sas_pltp ) + call coclone(self%lrhill_present) + call coclone(self%lextra_force ) + call coclone(self%lbig_discard ) + call coclone(self%lclose ) + call coclone(self%lenergy ) + call coclone(self%loblatecb ) + call coclone(self%lrotation ) + call coclone(self%ltides ) + call coclone(self%E_orbit_orig ) + call coclone(self%GMtot_orig ) do i = 1, NDIM - call cocopy(self%L_total_orig(i)) - call cocopy(self%L_orbit_orig(i)) - call cocopy(self%L_spin_orig(i)) - call cocopy(self%L_escape(i)) + call coclone(self%L_total_orig(i)) + call coclone(self%L_orbit_orig(i)) + call coclone(self%L_spin_orig(i)) + call coclone(self%L_escape(i)) end do - call cocopy(self%GMescape ) - call cocopy(self%E_collisions ) - call cocopy(self%E_untracked ) - call cocopy(self%lfirstenergy) - call cocopy(self%lfirstkick ) - call cocopy(self%lrestart ) - call cocopy(self%display_style) - call cocopy(self%display_unit ) - call cocopy(self%log_output ) - call cocopy(self%lgr ) - call cocopy(self%lyarkovsky) - call cocopy(self%lyorp ) - call cocopy(self%seed) + call coclone(self%GMescape ) + call coclone(self%E_collisions ) + call coclone(self%E_untracked ) + call coclone(self%lfirstenergy) + call coclone(self%lfirstkick ) + call coclone(self%lrestart ) + call coclone(self%display_style) + call coclone(self%display_unit ) + call coclone(self%log_output ) + call coclone(self%lgr ) + call coclone(self%lyarkovsky) + call coclone(self%lyorp ) + call coclone(self%seed) return end subroutine base_cobroadcast_param @@ -528,5 +528,4 @@ end subroutine base_cobroadcast_param #endif - end module base diff --git a/src/coarray/coarray_module.f90 b/src/coarray/coarray_module.f90 index e9905b9df..a8b9f02ab 100644 --- a/src/coarray/coarray_module.f90 +++ b/src/coarray/coarray_module.f90 @@ -16,7 +16,7 @@ module coarray implicit none public - interface cocopy + interface coclone module procedure coarray_component_copy_char module procedure coarray_component_copy_DP module procedure coarray_component_copy_DP_arr1D From fe00a04ae9923e22006971793bd11143465c2a19 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 6 Apr 2023 11:22:46 -0400 Subject: [PATCH 020/149] More coarray broadcasting infrastructure --- src/CMakeLists.txt | 1 + src/base/base_module.f90 | 6 +- src/coarray/coarray_module.f90 | 43 ++++++- src/swiftest/swiftest_coarray.f90 | 203 ++++++++++++++++++++++++++++++ src/swiftest/swiftest_io.f90 | 2 +- src/swiftest/swiftest_module.f90 | 66 +++++++--- src/swiftest/swiftest_util.f90 | 80 ------------ 7 files changed, 296 insertions(+), 105 deletions(-) create mode 100644 src/swiftest/swiftest_coarray.f90 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0def91f34..7dade4335 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -85,6 +85,7 @@ SET(FAST_MATH_FILES SET(COARRAY_FILES ${SRC}/coarray/coarray_module.f90 + ${SRC}/swiftest/swiftest_coarray.f90 ) IF(USE_COARRAY) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index d9d456958..f40f3e67e 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -114,7 +114,7 @@ module base procedure(abstract_io_param_writer), deferred :: writer procedure(abstract_io_read_in_param), deferred :: read_in #ifdef COARRAY - procedure :: cobroadcast => base_cobroadcast_param + procedure :: coclone => base_coclone_param #endif end type base_parameters @@ -433,7 +433,7 @@ subroutine base_final_storage_frame(self) end subroutine base_final_storage_frame #ifdef COARRAY - subroutine base_cobroadcast_param(self) + subroutine base_coclone_param(self) !! author: David A. Minton !! !! Broadcasts the image 1 parameter to all other images in a parameter coarray @@ -523,7 +523,7 @@ subroutine base_cobroadcast_param(self) call coclone(self%seed) return - end subroutine base_cobroadcast_param + end subroutine base_coclone_param #endif diff --git a/src/coarray/coarray_module.f90 b/src/coarray/coarray_module.f90 index a8b9f02ab..b174e1b34 100644 --- a/src/coarray/coarray_module.f90 +++ b/src/coarray/coarray_module.f90 @@ -20,10 +20,12 @@ module coarray module procedure coarray_component_copy_char module procedure coarray_component_copy_DP module procedure coarray_component_copy_DP_arr1D + module procedure coarray_component_copy_DP_arr2D module procedure coarray_component_copy_I4B module procedure coarray_component_copy_I4B_arr1D module procedure coarray_component_copy_I8B module procedure coarray_component_copy_lgt + module procedure coarray_component_copy_lgt_arr1D module procedure coarray_component_copy_QP end interface @@ -97,7 +99,7 @@ subroutine coarray_component_copy_DP_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! real(DP) allocatable array version + !! real(DP) 1D allocatable array version implicit none ! Arguments real(DP), dimension(:), allocatable, intent(inout) :: var @@ -206,7 +208,7 @@ subroutine coarray_component_copy_I4B_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! integer(I4B) allocatable array version + !! integer(I4B) 1D allocatable array version implicit none ! Arguments integer(I4B), dimension(:), allocatable, intent(inout) :: var @@ -302,6 +304,43 @@ subroutine coarray_component_copy_lgt(var,src_img) end subroutine coarray_component_copy_lgt + subroutine coarray_component_copy_lgt_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! logical 1D allocatable array version + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + logical, dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + n = size(var) + sync all + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + end if + sync all + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + + return + end subroutine coarray_component_copy_lgt_arr1D + + + subroutine coarray_component_copy_QP(var,src_img) !! author: David A. Minton !! diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 new file mode 100644 index 000000000..c9db7077c --- /dev/null +++ b/src/swiftest/swiftest_coarray.f90 @@ -0,0 +1,203 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +submodule (swiftest) s_swiftest_coarray + use coarray +contains + + module subroutine swiftest_coarray_coclone_body(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_body),intent(inout),codimension[*] :: self !! Swiftest body object + + call coclone(self%lfirst) + call coclone(self%nbody) + call coclone(self%id) + call coclone(self%info) + call coclone(self%lmask) + call coclone(self%status) + call coclone(self%ldiscard) + call coclone(self%lcollision) + call coclone(self%lencounter) + call coclone(self%mu) + call coclone(self%rh) + call coclone(self%vh) + call coclone(self%rb) + call coclone(self%vb) + call coclone(self%ah) + call coclone(self%aobl) + call coclone(self%agr) + call coclone(self%atide) + call coclone(self%ir3h) + call coclone(self%isperi) + call coclone(self%peri) + call coclone(self%atp) + call coclone(self%a) + call coclone(self%e) + call coclone(self%inc) + call coclone(self%capom) + call coclone(self%omega) + call coclone(self%capm) + + return + end subroutine swiftest_coarray_coclone_body + + + module subroutine swiftest_coarray_component_copy_info(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! swiftest_particle_info scalar version + implicit none + ! Arguments + type(swiftest_particle_info), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + type(swiftest_particle_info),save :: tmp[*] + integer(I4B) :: img, si + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + end if + sync all + var = tmp[si] + + return + + end subroutine swiftest_coarray_component_copy_info + + + module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! swiftest_particle_info 1D allocatable array version + implicit none + ! Arguments + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + type(swiftest_particle_info), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + n = size(var) + sync all + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + end if + sync all + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + + return + end subroutine swiftest_coarray_component_copy_info_arr1D + + + module subroutine swiftest_coarray_collect_system(nbody_system) + !! author: David A. Minton + !! + !! Collects all the test particles from other images into the image #1 test particle system + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + ! Internals + integer(I4B) :: i,j + integer(I4B), dimension(num_images()) :: ntp + class(swiftest_tp), allocatable :: tp_img + + ! ntp(this_image()) = nbody_system%tp%nbody + ! sync all + ! if (this_image() == 1) then + ! write(*,*) "Collecting test particles" + ! write(*,*) "Image ",1," ntp: ",ntp(1) + ! do i = 2, num_images() + ! write(*,*) "Image ",i," ntp: ",ntp(i) + ! allocate(tp_img, source=nbody_system[i]%tp) + ! call nbody_system%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) + ! deallocate(tp_img) + ! end do + ! write(*,*) "Total test particles: ",nbody_system%tp%nbody + ! end if + + return + end subroutine swiftest_coarray_collect_system + + + module subroutine swiftest_coarray_distribute_system(nbody_system) + !! author: David A. Minton + !! + !! Distributes test particles from image #1 out to all images. + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + ! Internals + integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy + class(swiftest_tp), allocatable :: tp_orig + logical, dimension(:), allocatable :: lspill_list + integer(I4B), codimension[*],save :: ntp + class(swiftest_nbody_system), allocatable :: tmp_system + class(swiftest_tp), allocatable :: tp + + ! ntp = nbody_system%tp%nbody + ! sync all + + ! ntot = ntp[1] + ! if (ntot == 0) return + + ! allocate(tp, mold=nbody_system%tp) + + ! write(*,*) "Image ",this_image(), "Distributing ",ntot + ! allocate(lspill_list(ntot)) + ! num_per_image = ntot / num_images() + ! istart = (this_image() - 1) * num_per_image + 1 + ! if (this_image() == num_images()) then + ! iend = ntot + ! else + ! iend = this_image() * num_per_image + ! end if + + ! if (this_image() == 1) then + ! lspill_list(:) = .true. + ! lspill_list(istart:iend) = .false. + ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) + ! else + ! lspill_list(:) = .false. + ! lspill_list(istart:iend) = .true. + ! tp%nbody = ntot + ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) + ! end if + + ! write(*,*) "Image ",this_image(), "ntp: ",nbody_system%tp%nbody + + return + end subroutine swiftest_coarray_distribute_system + +end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 928f6eccd..610f623d1 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2320,7 +2320,7 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i end associate #ifdef COARRAY end if ! this_image() == 1 - call coparam%cobroadcast() + call coparam%coclone() select type(self) type is (swiftest_parameters) self = coparam diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 55f63ec6f..fc5819ac1 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -43,6 +43,9 @@ module swiftest use io_progress_bar use netcdf_io use solver +#ifdef COARRAY + use coarray +#endif !use advisor_annotate !$ use omp_lib implicit none @@ -155,6 +158,9 @@ module swiftest procedure :: rearrange => swiftest_util_sort_rearrange_body !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods procedure :: spill => swiftest_util_spill_body !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) generic :: read_frame => read_frame_bin !! Add the generic read frame for Fortran binary files +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_body !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type swiftest_body @@ -393,8 +399,8 @@ module swiftest procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body #ifdef COARRAY - procedure :: coarray_collect => swiftest_util_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system - procedure :: coarray_distribute => swiftest_util_coarray_distribute_system !! Distributes test particles from image #1 out to all images. + procedure :: coarray_collect => swiftest_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system + procedure :: coarray_distribute => swiftest_coarray_distribute_system !! Distributes test particles from image #1 out to all images. #endif procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum @@ -447,11 +453,11 @@ end subroutine abstract_set_mu subroutine abstract_step_body(self, nbody_system, param, t, dt) import DP, swiftest_body, swiftest_nbody_system, swiftest_parameters implicit none - class(swiftest_body), intent(inout) :: self !! Swiftest body object + class(swiftest_body), intent(inout) :: self !! Swiftest body object class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - real(DP), intent(in) :: t !! Simulation time - real(DP), intent(in) :: dt !! Current stepsize + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + real(DP), intent(in) :: t !! Simulation time + real(DP), intent(in) :: dt !! Current stepsize end subroutine abstract_step_body subroutine abstract_step_system(self, param, t, dt) @@ -464,7 +470,6 @@ subroutine abstract_step_system(self, param, t, dt) end subroutine abstract_step_system end interface - interface module subroutine swiftest_discard_pl(self, nbody_system, param) implicit none @@ -1187,18 +1192,6 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_tp -#ifdef COARRAY - module subroutine swiftest_util_coarray_collect_system(nbody_system) - implicit none - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - end subroutine swiftest_util_coarray_collect_system - - module subroutine swiftest_util_coarray_distribute_system(nbody_system) - implicit none - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - end subroutine swiftest_util_coarray_distribute_system -#endif - module subroutine swiftest_util_coord_b2h_pl(self, cb) implicit none class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object @@ -1957,6 +1950,41 @@ module subroutine swiftest_util_version() end subroutine swiftest_util_version end interface +#ifdef COARRAY + interface + module subroutine swiftest_coarray_collect_system(nbody_system) + implicit none + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + end subroutine swiftest_coarray_collect_system + + module subroutine swiftest_coarray_distribute_system(nbody_system) + implicit none + class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + end subroutine swiftest_coarray_distribute_system + end interface + + interface coclone + module subroutine swiftest_coarray_component_copy_info(var,src_img) + implicit none + type(swiftest_particle_info), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine swiftest_coarray_component_copy_info + + module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) + implicit none + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine swiftest_coarray_component_copy_info_arr1D + end interface + + interface + module subroutine swiftest_coarray_coclone_body(self) + implicit none + class(swiftest_body),intent(inout),codimension[*] :: self !! Swiftest body object + end subroutine swiftest_coarray_coclone_body + end interface +#endif + contains subroutine swiftest_final_kin(self) !! author: David A. Minton diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 44c3bc17f..7e5bd548a 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -354,86 +354,6 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) return end subroutine swiftest_util_append_tp -#ifdef COARRAY - module subroutine swiftest_util_coarray_collect_system(nbody_system) - !! author: David A. Minton - !! - !! Collects all the test particles from other images into the image #1 test particle system - implicit none - ! Arguments - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - ! Internals - integer(I4B) :: i,j - integer(I4B), dimension(num_images()) :: ntp - class(swiftest_tp), allocatable :: tp_img - - ! ntp(this_image()) = nbody_system%tp%nbody - ! sync all - ! if (this_image() == 1) then - ! write(*,*) "Collecting test particles" - ! write(*,*) "Image ",1," ntp: ",ntp(1) - ! do i = 2, num_images() - ! write(*,*) "Image ",i," ntp: ",ntp(i) - ! allocate(tp_img, source=nbody_system[i]%tp) - ! call nbody_system%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) - ! deallocate(tp_img) - ! end do - ! write(*,*) "Total test particles: ",nbody_system%tp%nbody - ! end if - - return - end subroutine swiftest_util_coarray_collect_system - - - module subroutine swiftest_util_coarray_distribute_system(nbody_system) - !! author: David A. Minton - !! - !! Distributes test particles from image #1 out to all images. - implicit none - ! Arguments - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - ! Internals - integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy - class(swiftest_tp), allocatable :: tp_orig - logical, dimension(:), allocatable :: lspill_list - integer(I4B), codimension[*],save :: ntp - class(swiftest_nbody_system), allocatable :: tmp_system - class(swiftest_tp), allocatable :: tp - - ! ntp = nbody_system%tp%nbody - ! sync all - - ! ntot = ntp[1] - ! if (ntot == 0) return - - ! allocate(tp, mold=nbody_system%tp) - - ! write(*,*) "Image ",this_image(), "Distributing ",ntot - ! allocate(lspill_list(ntot)) - ! num_per_image = ntot / num_images() - ! istart = (this_image() - 1) * num_per_image + 1 - ! if (this_image() == num_images()) then - ! iend = ntot - ! else - ! iend = this_image() * num_per_image - ! end if - - ! if (this_image() == 1) then - ! lspill_list(:) = .true. - ! lspill_list(istart:iend) = .false. - ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) - ! else - ! lspill_list(:) = .false. - ! lspill_list(istart:iend) = .true. - ! tp%nbody = ntot - ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) - ! end if - - ! write(*,*) "Image ",this_image(), "ntp: ",nbody_system%tp%nbody - - return - end subroutine swiftest_util_coarray_distribute_system -#endif module subroutine swiftest_util_coord_h2b_pl(self, cb) !! author: David A. Minton From be4985cb259c4cb27dcbbc126215a3bbbb1e87e4 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 14 Apr 2023 10:03:29 -0400 Subject: [PATCH 021/149] Most coarray methods implemented, but not fully tested --- src/CMakeLists.txt | 2 + src/base/base_module.f90 | 2 +- src/coarray/coarray_module.f90 | 637 ++++++++++++++++++++++++++---- src/helio/helio_module.f90 | 7 +- src/helio/helio_util.f90 | 9 +- src/rmvs/rmvs_coarray.f90 | 139 +++++++ src/rmvs/rmvs_module.f90 | 55 ++- src/rmvs/rmvs_util.f90 | 9 +- src/swiftest/swiftest_coarray.f90 | 561 ++++++++++++++++++++++---- src/swiftest/swiftest_discard.f90 | 2 +- src/swiftest/swiftest_driver.f90 | 35 +- src/swiftest/swiftest_io.f90 | 60 ++- src/swiftest/swiftest_module.f90 | 138 +++++-- src/swiftest/swiftest_util.f90 | 61 +-- src/symba/symba_module.f90 | 8 +- src/symba/symba_util.f90 | 9 +- src/whm/whm_coarray.f90 | 34 ++ src/whm/whm_module.f90 | 25 +- src/whm/whm_util.f90 | 9 +- 19 files changed, 1484 insertions(+), 318 deletions(-) create mode 100644 src/rmvs/rmvs_coarray.f90 create mode 100644 src/whm/whm_coarray.f90 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7dade4335..4e1caebba 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -86,6 +86,8 @@ SET(FAST_MATH_FILES SET(COARRAY_FILES ${SRC}/coarray/coarray_module.f90 ${SRC}/swiftest/swiftest_coarray.f90 + ${SRC}/whm/whm_coarray.f90 + ${SRC}/rmvs/rmvs_coarray.f90 ) IF(USE_COARRAY) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index f40f3e67e..3c33a47c1 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -387,7 +387,7 @@ subroutine base_util_snapshot_save(self, snapshot) !! Memory usage grows by a factor of 2 each time it fills up, but no more. implicit none ! Arguments - class(base_storage), intent(inout) :: self !! Storage ncounter storage object + class(base_storage), intent(inout) :: self !! Storage encounter storage object class(*), intent(in) :: snapshot !! Object to snapshot ! Internals integer(I4B) :: nnew, nold diff --git a/src/coarray/coarray_module.f90 b/src/coarray/coarray_module.f90 index b174e1b34..597945761 100644 --- a/src/coarray/coarray_module.f90 +++ b/src/coarray/coarray_module.f90 @@ -17,21 +17,32 @@ module coarray public interface coclone - module procedure coarray_component_copy_char - module procedure coarray_component_copy_DP - module procedure coarray_component_copy_DP_arr1D - module procedure coarray_component_copy_DP_arr2D - module procedure coarray_component_copy_I4B - module procedure coarray_component_copy_I4B_arr1D - module procedure coarray_component_copy_I8B - module procedure coarray_component_copy_lgt - module procedure coarray_component_copy_lgt_arr1D - module procedure coarray_component_copy_QP + module procedure coarray_component_clone_char + module procedure coarray_component_clone_DP + module procedure coarray_component_clone_DP_arr1D + module procedure coarray_component_clone_DP_arr2D + module procedure coarray_component_clone_I4B + module procedure coarray_component_clone_I4B_arr1D + module procedure coarray_component_clone_I4B_arr2D + module procedure coarray_component_clone_I8B + module procedure coarray_component_clone_lgt + module procedure coarray_component_clone_lgt_arr1D + module procedure coarray_component_clone_QP + end interface + + interface cocollect + module procedure coarray_component_collect_DP_arr1D + module procedure coarray_component_collect_DP_arr2D + module procedure coarray_component_collect_I4B + module procedure coarray_component_collect_I4B_arr1D + module procedure coarray_component_collect_I4B_arr2D + module procedure coarray_component_collect_I8B + module procedure coarray_component_collect_lgt_arr1D end interface contains - subroutine coarray_component_copy_char(var,src_img) + subroutine coarray_component_clone_char(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -55,15 +66,17 @@ subroutine coarray_component_copy_char(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] return - end subroutine coarray_component_copy_char + end subroutine coarray_component_clone_char - subroutine coarray_component_copy_DP(var,src_img) + subroutine coarray_component_clone_DP(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -87,15 +100,18 @@ subroutine coarray_component_copy_DP(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] + return - end subroutine coarray_component_copy_DP + end subroutine coarray_component_clone_DP - subroutine coarray_component_copy_DP_arr1D(var,src_img) + subroutine coarray_component_clone_DP_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -108,32 +124,37 @@ subroutine coarray_component_copy_DP_arr1D(var,src_img) real(DP), dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si integer(I4B), save :: n[*] - + logical, save :: isalloc[*] + if (present(src_img)) then si = src_img else si = 1 end if - n = size(var) sync all + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var(:) + tmp(:)[img] = var end do - end if - sync all - if (this_image() /= si) then + sync images(*) + else + sync images(si) if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if return - end subroutine coarray_component_copy_DP_arr1D + end subroutine coarray_component_clone_DP_arr1D - subroutine coarray_component_copy_DP_arr2D(var,src_img) + subroutine coarray_component_clone_DP_arr2D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -146,33 +167,40 @@ subroutine coarray_component_copy_DP_arr2D(var,src_img) real(DP), dimension(:,:), codimension[:], allocatable :: tmp integer(I4B) :: img, si integer(I4B), save :: n1[*], n2[*] - + logical, save :: isalloc[*] + if (present(src_img)) then si = src_img else si = 1 end if - n1 = size(var,dim=1) - n2 = size(var,dim=2) sync all + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + end if + sync all + if (.not. isalloc[si]) return + allocate(tmp(n1[si],n2[si])[*]) if (this_image() == si) then do img = 1, num_images() tmp(:,:)[img] = var(:,:) end do - end if - sync all - if (this_image() /= si) then + sync images(*) + else + sync images(si) if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if - + return - end subroutine coarray_component_copy_DP_arr2D + end subroutine coarray_component_clone_DP_arr2D - subroutine coarray_component_copy_I4B(var,src_img) + subroutine coarray_component_clone_I4B(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -196,15 +224,17 @@ subroutine coarray_component_copy_I4B(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] return - end subroutine coarray_component_copy_I4B - + end subroutine coarray_component_clone_I4B + - subroutine coarray_component_copy_I4B_arr1D(var,src_img) + subroutine coarray_component_clone_I4B_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -217,30 +247,83 @@ subroutine coarray_component_copy_I4B_arr1D(var,src_img) integer(I4B), dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si integer(I4B), save :: n[*] - + logical, save :: isalloc[*] + if (present(src_img)) then si = src_img else si = 1 end if - n = size(var) sync all + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() tmp(:)[img] = var end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) end if + + return + end subroutine coarray_component_clone_I4B_arr1D + + + subroutine coarray_component_clone_I4B_arr2D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) 2D allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n1[*], n2[*] + logical, save :: isalloc[*] + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + sync all - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + end if + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n1[si],n2[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:,:)[img] = var(:,:) + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + return - end subroutine coarray_component_copy_I4B_arr1D + end subroutine coarray_component_clone_I4B_arr2D - subroutine coarray_component_copy_I8B(var,src_img) + subroutine coarray_component_clone_I8B(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -264,15 +347,17 @@ subroutine coarray_component_copy_I8B(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] return - end subroutine coarray_component_copy_I8B + end subroutine coarray_component_clone_I8B - subroutine coarray_component_copy_lgt(var,src_img) + subroutine coarray_component_clone_lgt(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -296,15 +381,18 @@ subroutine coarray_component_copy_lgt(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] + return - end subroutine coarray_component_copy_lgt + end subroutine coarray_component_clone_lgt - subroutine coarray_component_copy_lgt_arr1D(var,src_img) + subroutine coarray_component_clone_lgt_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -317,31 +405,37 @@ subroutine coarray_component_copy_lgt_arr1D(var,src_img) logical, dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si integer(I4B), save :: n[*] - + logical, save :: isalloc[*] + if (present(src_img)) then si = src_img else si = 1 end if - n = size(var) sync all + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() tmp(:)[img] = var end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) end if - sync all - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) return - end subroutine coarray_component_copy_lgt_arr1D + end subroutine coarray_component_clone_lgt_arr1D - - subroutine coarray_component_copy_QP(var,src_img) + subroutine coarray_component_clone_QP(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -365,11 +459,420 @@ subroutine coarray_component_copy_QP(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] return - end subroutine coarray_component_copy_QP + end subroutine coarray_component_clone_QP + + + subroutine coarray_component_collect_DP_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! real(DP) 1D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + real(DP), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n[img] + end do + + allocate(tmp(ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n + do img = 1, this_image() - 1 + istart = istart + n[img] + iend = iend + n[img] + end do + + if (isalloc) then + tmp(istart:iend) = var(:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(ntot)) + istart = 1 + iend = n + do img = 1, num_images() + var(istart:iend) = tmp[img](istart:iend) + istart = istart + n[img] + iend = iend + n[img] + end do + end if + + return + end subroutine coarray_component_collect_DP_arr1D + + + subroutine coarray_component_collect_DP_arr2D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! real(DP) 2D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n1[*], n2[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + else + n1 = 0 + n2 = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n2[img] + end do + + allocate(tmp(n1,ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n2 + do img = 1, this_image() - 1 + istart = istart + n2[img] + iend = iend + n2[img] + end do + + if (isalloc) then + tmp(:,istart:iend) = var(:,:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(n1,ntot)) + + istart = 1 + iend = n2 + do img = 1, num_images() + var(:,istart:iend) = tmp[img](:,istart:iend) + istart = istart + n2[img] + iend = iend + n2[img] + end do + end if + + return + end subroutine coarray_component_collect_DP_arr2D + + + subroutine coarray_component_collect_I4B(var,dest_img) + !! author: David A. Minton + !! + !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 + !! integer(I4B) version + implicit none + ! Arguments + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), allocatable :: tmp[:] + integer(I4B) :: img, di + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + allocate(tmp[*], source=var) + sync all + + if (this_image() == di) then + var = 0 + do img = 1, num_images() + var = var + tmp[img] + end do + else + var = 0 + end if + + return + end subroutine coarray_component_collect_I4B + + + subroutine coarray_component_collect_I8B(var,dest_img) + !! author: David A. Minton + !! + !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 + !! integer(I8B) version + implicit none + ! Arguments + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I8B), allocatable :: tmp[:] + integer(I4B) :: img, di + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + allocate(tmp[*], source=var) + sync all + + if (this_image() == di) then + var = 0 + do img = 1, num_images() + var = var + tmp[img] + end do + else + var = 0 + end if + + return + end subroutine coarray_component_collect_I8B + + + subroutine coarray_component_collect_I4B_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! integer(I4B) 1D allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n[img] + end do + + allocate(tmp(ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n + do img = 1, this_image() - 1 + istart = istart + n[img] + iend = iend + n[img] + end do + + if (isalloc) then + tmp(istart:iend) = var(:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(ntot)) + istart = 1 + iend = n + do img = 1, num_images() + var(istart:iend) = tmp[img](istart:iend) + istart = istart + n[img] + iend = iend + n[img] + end do + end if + + return + end subroutine coarray_component_collect_I4B_arr1D + + + subroutine coarray_component_collect_I4B_arr2D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! integer(I4B) 2D allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n1[*], n2[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + else + n1 = 0 + n2 = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n2[img] + end do + + allocate(tmp(n1,ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n2 + do img = 1, this_image() - 1 + istart = istart + n2[img] + iend = iend + n2[img] + end do + + if (isalloc) then + tmp(:,istart:iend) = var(:,:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(n1,ntot)) + + istart = 1 + iend = n2 + do img = 1, num_images() + var(:,istart:iend) = tmp[img](:,istart:iend) + istart = istart + n2[img] + iend = iend + n2[img] + end do + end if + + return + end subroutine coarray_component_collect_I4B_arr2D + + + subroutine coarray_component_collect_lgt_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! logical 1D allocatable array version + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + logical, dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n[img] + end do + + allocate(tmp(ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n + do img = 1, this_image() - 1 + istart = istart + n[img] + iend = iend + n[img] + end do + + if (isalloc) then + tmp(istart:iend) = var(:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(ntot)) + istart = 1 + iend = n + do img = 1, num_images() + var(istart:iend) = tmp[img](istart:iend) + istart = istart + n[img] + iend = iend + n[img] + end do + end if + + return + end subroutine coarray_component_collect_lgt_arr1D + end module coarray \ No newline at end of file diff --git a/src/helio/helio_module.f90 b/src/helio/helio_module.f90 index cd5581837..182512e53 100644 --- a/src/helio/helio_module.f90 +++ b/src/helio/helio_module.f90 @@ -165,10 +165,11 @@ module subroutine helio_kick_vb_tp(self, nbody_system, param, t, dt, lbeg) logical, intent(in) :: lbeg !! Logical flag indicating whether this is the beginning of the half step or not. end subroutine helio_kick_vb_tp - module subroutine helio_util_setup_initialize_system(self, param) + module subroutine helio_util_setup_initialize_system(self, system_history, param) implicit none - class(helio_nbody_system), intent(inout) :: self !! Helio nbody system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(helio_nbody_system), intent(inout) :: self !! Helio nbody system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine helio_util_setup_initialize_system module subroutine helio_step_pl(self, nbody_system, param, t, dt) diff --git a/src/helio/helio_util.f90 b/src/helio/helio_util.f90 index 94b464502..bb032defb 100644 --- a/src/helio/helio_util.f90 +++ b/src/helio/helio_util.f90 @@ -11,17 +11,18 @@ use swiftest contains - module subroutine helio_util_setup_initialize_system(self, param) + module subroutine helio_util_setup_initialize_system(self, system_history, param) !! author: David A. Minton !! !! Initialize a Helio nbody system from files, converting all heliocentric quantities to barycentric. !! implicit none ! Arguments - class(helio_nbody_system), intent(inout) :: self !! Helio nbody system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(helio_nbody_system), intent(inout) :: self !! Helio nbody system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - call swiftest_util_setup_initialize_system(self, param) + call swiftest_util_setup_initialize_system(self, system_history, param) call self%pl%sort("mass", ascending=.false.) call self%pl%vh2vb(self%cb) call self%tp%h2b(self%cb) diff --git a/src/rmvs/rmvs_coarray.f90 b/src/rmvs/rmvs_coarray.f90 new file mode 100644 index 000000000..61c638c60 --- /dev/null +++ b/src/rmvs/rmvs_coarray.f90 @@ -0,0 +1,139 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +submodule (rmvs) s_rmvs_coarray +use coarray +use swiftest +use whm +contains + + module subroutine rmvs_coarray_coclone_cb(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_cb),intent(inout),codimension[*] :: self !! RMVS pl object + + call coclone(self%outer) + call coclone(self%inner) + call coclone(self%lplanetocentric) + + call swiftest_coarray_coclone_cb(self) + + return + end subroutine rmvs_coarray_coclone_cb + + + module subroutine rmvs_coarray_coclone_pl(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_pl),intent(inout),codimension[*] :: self !! RMVS pl object + + call coclone(self%nenc) + call coclone(self%tpenc1P) + call coclone(self%plind) + call coclone(self%outer) + call coclone(self%lplanetocentric) + + call whm_coarray_coclone_pl(self) + + return + end subroutine rmvs_coarray_coclone_pl + + + + module subroutine rmvs_coarray_coclone_tp(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_tp),intent(inout),codimension[*] :: self !! RMVS pl object + + call coclone(self%lperi) + call coclone(self%plperP) + call coclone(self%plencP) + call coclone(self%index) + call coclone(self%ipleP) + call coclone(self%lplanetocentric) + + call swiftest_coarray_coclone_tp(self) + + return + end subroutine rmvs_coarray_coclone_tp + + + module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) + implicit none + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! swiftest_particle_info scalar version + ! Arguments + type(rmvs_interp), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + type(rmvs_interp), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine rmvs_coarray_component_clone_interp_arr1D + + + module subroutine rmvs_coarray_cocollect_tp(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_tp),intent(inout),codimension[*] :: self !! RMVS pl object + + call cocollect(self%lperi) + call cocollect(self%plperP) + call cocollect(self%plencP) + call cocollect(self%index) + call cocollect(self%ipleP) + + call swiftest_coarray_cocollect_tp(self) + + return + end subroutine rmvs_coarray_cocollect_tp + +end submodule s_rmvs_coarray \ No newline at end of file diff --git a/src/rmvs/rmvs_module.f90 b/src/rmvs/rmvs_module.f90 index 733a81e60..0f4c0f4b1 100644 --- a/src/rmvs/rmvs_module.f90 +++ b/src/rmvs/rmvs_module.f90 @@ -56,6 +56,9 @@ module rmvs contains procedure :: dealloc => rmvs_util_dealloc_cb !! Deallocates all allocatable arrays final :: rmvs_final_cb !! Finalizes the RMVS central body object - deallocates all allocatables +#ifdef COARRAY + procedure :: coclone => rmvs_coarray_coclone_cb !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type rmvs_cb @@ -88,6 +91,9 @@ module rmvs procedure :: rearrange => rmvs_util_sort_rearrange_tp !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods procedure :: spill => rmvs_util_spill_tp !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) final :: rmvs_final_tp !! Finalizes the RMVS test particle object - deallocates all allocatables +#ifdef COARRAY + procedure :: coclone => rmvs_coarray_coclone_tp !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type rmvs_tp @@ -101,7 +107,7 @@ module rmvs class(rmvs_nbody_system), dimension(:), allocatable :: planetocentric !! Planetocentric version of the massive body objects (one for each massive body) logical :: lplanetocentric = .false. !! Flag that indicates that the object is a planetocentric set of masive bodies used for close encounter calculations contains - procedure :: setup => rmvs_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess + procedure :: setup => rmvs_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess procedure :: append => rmvs_util_append_pl !! Appends elements from one structure to another procedure :: dealloc => rmvs_util_dealloc_pl !! Deallocates all allocatable arrays procedure :: fill => rmvs_util_fill_pl !! "Fills" bodies from one object into another depending on the results of a mask (uses the UNPACK intrinsic) @@ -109,7 +115,10 @@ module rmvs procedure :: sort => rmvs_util_sort_pl !! Sorts body arrays by a sortable componen procedure :: rearrange => rmvs_util_sort_rearrange_pl !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods procedure :: spill => rmvs_util_spill_pl !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) - final :: rmvs_final_pl !! Finalizes the RMVS massive body object - deallocates all allocatables + final :: rmvs_final_pl !! Finalizes the RMVS massive body object - deallocates all allocatables +#ifdef COARRAY + procedure :: coclone => rmvs_coarray_coclone_pl !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type rmvs_pl interface @@ -145,10 +154,11 @@ module subroutine rmvs_util_setup_pl(self, n, param) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters end subroutine rmvs_util_setup_pl - module subroutine rmvs_util_setup_initialize_system(self, param) + module subroutine rmvs_util_setup_initialize_system(self, system_history, param) implicit none - class(rmvs_nbody_system), intent(inout) :: self !! RMVS system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(rmvs_nbody_system), intent(inout) :: self !! RMVS system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine rmvs_util_setup_initialize_system module subroutine rmvs_util_setup_tp(self, n, param) @@ -275,6 +285,41 @@ end subroutine rmvs_step_system end interface + +#ifdef COARRAY + interface coclone + module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) + implicit none + type(rmvs_interp), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine rmvs_coarray_component_clone_interp_arr1D + end interface + + interface cocollect + module subroutine rmvs_coarray_cocollect_tp(self) + implicit none + class(rmvs_tp),intent(inout),codimension[*] :: self !! RMVS pl object + end subroutine rmvs_coarray_cocollect_tp + end interface + + interface + module subroutine rmvs_coarray_coclone_cb(self) + implicit none + class(rmvs_cb),intent(inout),codimension[*] :: self !! RMVS tp object + end subroutine rmvs_coarray_coclone_cb + + module subroutine rmvs_coarray_coclone_pl(self) + implicit none + class(rmvs_pl),intent(inout),codimension[*] :: self !! RMVS pl object + end subroutine rmvs_coarray_coclone_pl + + module subroutine rmvs_coarray_coclone_tp(self) + implicit none + class(rmvs_tp),intent(inout),codimension[*] :: self !! RMVS tp object + end subroutine rmvs_coarray_coclone_tp + end interface +#endif + contains subroutine rmvs_final_cb(self) diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index c0f987aa9..7100f7019 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -323,7 +323,7 @@ module subroutine rmvs_util_setup_pl(self, n, param) end subroutine rmvs_util_setup_pl - module subroutine rmvs_util_setup_initialize_system(self, param) + module subroutine rmvs_util_setup_initialize_system(self, system_history, param) !! author: David A. Minton !! !! Initialize an RMVS nbody system from files and sets up the planetocentric structures. @@ -335,13 +335,14 @@ module subroutine rmvs_util_setup_initialize_system(self, param) !! to use during close encounters. implicit none ! Arguments - class(rmvs_nbody_system), intent(inout) :: self !! RMVS system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(rmvs_nbody_system), intent(inout) :: self !! RMVS system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i, j ! Call parent method - call whm_util_setup_initialize_system(self, param) + call whm_util_setup_initialize_system(self, system_history, param) ! Set up the pl-tp planetocentric encounter structures for pl and cb. The planetocentric tp structures are ! generated as necessary during close encounter steps. diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index c9db7077c..08590c112 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -52,7 +52,170 @@ module subroutine swiftest_coarray_coclone_body(self) end subroutine swiftest_coarray_coclone_body - module subroutine swiftest_coarray_component_copy_info(var,src_img) + module subroutine swiftest_coarray_coclone_cb(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_cb),intent(inout),codimension[*] :: self !! Swiftest body object + ! Internals + integer(I4B) :: i + + call coclone(self%info) + call coclone(self%id) + call coclone(self%mass) + call coclone(self%Gmass) + call coclone(self%radius) + call coclone(self%density) + call coclone(self%j2rp2) + call coclone(self%j4rp4) + call coclone(self%k2) + call coclone(self%Q) + call coclone(self%tlag) + call coclone(self%GM0) + call coclone(self%dGM) + call coclone(self%R0) + call coclone(self%dR) + + do i = 1, NDIM + call coclone(self%aobl(i)) + call coclone(self%atide(i)) + call coclone(self%aoblbeg(i)) + call coclone(self%aoblend(i)) + call coclone(self%atidebeg(i)) + call coclone(self%atideend(i)) + call coclone(self%rb(i)) + call coclone(self%vb(i)) + call coclone(self%agr(i)) + call coclone(self%Ip(i)) + call coclone(self%rot(i)) + call coclone(self%L0(i)) + call coclone(self%dL(i)) + end do + + return + end subroutine swiftest_coarray_coclone_cb + + + module subroutine swiftest_coarray_coclone_pl(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_pl),intent(inout),codimension[*] :: self !! Swiftest body object + + call coclone(self%mass) + call coclone(self%Gmass) + call coclone(self%rhill) + call coclone(self%renc) + call coclone(self%radius) + call coclone(self%density) + call coclone(self%rbeg) + call coclone(self%rend) + call coclone(self%vbeg) + call coclone(self%Ip) + call coclone(self%rot) + call coclone(self%k2) + call coclone(self%Q ) + call coclone(self%tlag) + call coclone(self%k_plpl) + call coclone(self%nplpl) + call coclone(self%kin) + call coclone(self%lmtiny) + call coclone(self%nplm) + call coclone(self%nplplm) + call coclone(self%nplenc) + call coclone(self%ntpenc) + + call swiftest_coarray_coclone_body(self) + + return + end subroutine swiftest_coarray_coclone_pl + + + module subroutine swiftest_coarray_coclone_tp(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_tp),intent(inout),codimension[*] :: self !! Swiftest body object + + call coclone(self%k_pltp) + call coclone(self%npltp) + call coclone(self%nplenc) + call swiftest_coarray_coclone_body(self) + + return + end subroutine swiftest_coarray_coclone_tp + + + module subroutine swiftest_coarray_coclone_system(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_nbody_system),intent(inout),codimension[*] :: self !! Swiftest body object + ! Internals + integer(I4B) :: i, img + + call self%cb%coclone() + call self%pl%coclone() + call self%tp%coclone() + + call coclone(self%maxid) + call coclone(self%t) + call coclone(self%GMtot) + call coclone(self%ke_orbit) + call coclone(self%ke_spin) + call coclone(self%pe) + call coclone(self%be) + call coclone(self%te) + call coclone(self%oblpot) + do i = 1, NDIM + call coclone(self%L_orbit(i)) + call coclone(self%L_spin(i)) + call coclone(self%L_total(i)) + call coclone(self%L_total_orig(i)) + call coclone(self%L_orbit_orig(i)) + call coclone(self%L_spin_orig(i)) + call coclone(self%L_escape(i)) + end do + call coclone(self%ke_orbit_orig) + call coclone(self%ke_spin_orig) + call coclone(self%pe_orig) + call coclone(self%be_orig) + call coclone(self%te_orig) + call coclone(self%be_cb) + call coclone(self%E_orbit_orig) + call coclone(self%GMtot_orig) + call coclone(self%GMescape) + call coclone(self%E_collisions) + call coclone(self%E_untracked) + call coclone(self%ke_orbit_error) + call coclone(self%ke_spin_error) + call coclone(self%pe_error) + call coclone(self%be_error) + call coclone(self%E_orbit_error) + call coclone(self%Ecoll_error) + call coclone(self%E_untracked_error) + call coclone(self%te_error) + call coclone(self%L_orbit_error) + call coclone(self%L_spin_error) + call coclone(self%L_escape_error) + call coclone(self%L_total_error) + call coclone(self%Mtot_error) + call coclone(self%Mescape_error) + call coclone(self%lbeg) + + return + end subroutine swiftest_coarray_coclone_system + + + module subroutine swiftest_coarray_component_clone_info(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -76,16 +239,17 @@ module subroutine swiftest_coarray_component_copy_info(var,src_img) do img = 1, num_images() tmp[img] = var end do + sync images(*) + else + sync images(si) + var = tmp[si] end if - sync all - var = tmp[si] - + return - - end subroutine swiftest_coarray_component_copy_info + end subroutine swiftest_coarray_component_clone_info - module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) + module subroutine swiftest_coarray_component_clone_info_arr1D(var,src_img) !! author: David A. Minton !! !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 @@ -98,6 +262,7 @@ module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) type(swiftest_particle_info), dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si integer(I4B), save :: n[*] + logical, save :: isalloc[*] if (present(src_img)) then si = src_img @@ -105,99 +270,321 @@ module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) si = 1 end if - n = size(var) sync all + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() tmp(:)[img] = var end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine swiftest_coarray_component_clone_info_arr1D + + + module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) + implicit none + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! swiftest_kinship allocatable array version + ! Arguments + type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + type(swiftest_kinship), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + if (present(src_img)) then + si = src_img + else + si = 1 end if + sync all - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if return - end subroutine swiftest_coarray_component_copy_info_arr1D - - - module subroutine swiftest_coarray_collect_system(nbody_system) - !! author: David A. Minton - !! - !! Collects all the test particles from other images into the image #1 test particle system - implicit none - ! Arguments - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - ! Internals - integer(I4B) :: i,j - integer(I4B), dimension(num_images()) :: ntp - class(swiftest_tp), allocatable :: tp_img - - ! ntp(this_image()) = nbody_system%tp%nbody - ! sync all - ! if (this_image() == 1) then - ! write(*,*) "Collecting test particles" - ! write(*,*) "Image ",1," ntp: ",ntp(1) - ! do i = 2, num_images() - ! write(*,*) "Image ",i," ntp: ",ntp(i) - ! allocate(tp_img, source=nbody_system[i]%tp) - ! call nbody_system%tp%append(tp_img,lsource_mask=[(.true., j = 1, ntp(i))]) - ! deallocate(tp_img) - ! end do - ! write(*,*) "Total test particles: ",nbody_system%tp%nbody - ! end if - - return + end subroutine swiftest_coarray_component_clone_kin_arr1D + + + module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! swiftest_particle_info 1D allocatable array version + implicit none + ! Arguments + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + type(swiftest_particle_info), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, ti, di, ntot, istart, iend + integer(I4B), save :: n[*] + logical, save :: isalloc[*] + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + sync all + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + sync all + ntot = 0 + do img = 1, num_images() + ntot = ntot + n[img] + end do + + allocate(tmp(ntot)[*]) + + ti = this_image() + + istart = 1 + iend = n + do img = 1, this_image() - 1 + istart = istart + n[img] + iend = iend + n[img] + end do + + if (isalloc) then + tmp(istart:iend) = var(:) + deallocate(var) + end if + + sync all + if (this_image() == di) then + allocate(var(ntot)) + istart = 1 + iend = n + do img = 1, num_images() + var(istart:iend) = tmp[img](istart:iend) + istart = istart + n[img] + iend = iend + n[img] + end do + end if + + return + end subroutine swiftest_coarray_component_collect_info_arr1D + + + module subroutine swiftest_coarray_cocollect_body(self) + !! author: David A. Minton + !! + !! Collects all body object array components from all images and combines them into the image 1 body object + implicit none + ! Arguments + class(swiftest_body),intent(inout), codimension[*] :: self !! Swiftest body object + integer(I4B) :: i + + call cocollect(self%nbody) + call cocollect(self%id) + call cocollect(self%info) + call cocollect(self%lmask) + call cocollect(self%status) + call cocollect(self%ldiscard) + call cocollect(self%lcollision) + call cocollect(self%lencounter) + call cocollect(self%mu) + call cocollect(self%rh) + call cocollect(self%vh) + call cocollect(self%rb) + call cocollect(self%vb) + call cocollect(self%ah) + call cocollect(self%aobl) + call cocollect(self%agr) + call cocollect(self%atide) + call cocollect(self%ir3h) + call cocollect(self%isperi) + call cocollect(self%peri) + call cocollect(self%atp) + call cocollect(self%a) + call cocollect(self%e) + call cocollect(self%inc) + call cocollect(self%capom) + call cocollect(self%omega) + call cocollect(self%capm) + + return + end subroutine swiftest_coarray_cocollect_body + + + module subroutine swiftest_coarray_cocollect_tp(self) + !! author: David A. Minton + !! + !! Collects all object array components from all images and combines them into the image 1 object + implicit none + ! Arguments + class(swiftest_tp),intent(inout),codimension[*] :: self !! Swiftest body object + + call cocollect(self%k_pltp) + call cocollect(self%npltp) + call cocollect(self%nplenc) + call swiftest_coarray_cocollect_body(self) + + return + end subroutine swiftest_coarray_cocollect_tp + + + module subroutine swiftest_coarray_collect_system(nbody_system, param) + !! author: David A. Minton + !! + !! Collects all the test particles from other images into the image #1 test particle system + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + integer(I4B) :: i,j + integer(I4B), codimension[*], save :: ntp + class(swiftest_tp), allocatable, codimension[:] :: cotp + character(len=NAMELEN) :: image_num_char + + sync all + if (this_image() == 1) then + write(image_num_char,*) num_images() + write(param%display_unit,*) " Collecting test particles from " // trim(adjustl(image_num_char)) // " images." + end if + + allocate(cotp[*], source=nbody_system%tp) + call cotp%cocollect() + deallocate(nbody_system%tp) + allocate(nbody_system%tp, source=cotp) + + deallocate(cotp) + + if (this_image() == 1) then + write(param%display_unit,*) " Done collecting" + do i = 1, nbody_system%tp%nbody + write(*,*) i,"mu ",nbody_system%tp%mu(i) + end do + end if + + return end subroutine swiftest_coarray_collect_system - module subroutine swiftest_coarray_distribute_system(nbody_system) - !! author: David A. Minton - !! - !! Distributes test particles from image #1 out to all images. - implicit none - ! Arguments - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] - ! Internals - integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy - class(swiftest_tp), allocatable :: tp_orig - logical, dimension(:), allocatable :: lspill_list - integer(I4B), codimension[*],save :: ntp - class(swiftest_nbody_system), allocatable :: tmp_system - class(swiftest_tp), allocatable :: tp - - ! ntp = nbody_system%tp%nbody - ! sync all - - ! ntot = ntp[1] - ! if (ntot == 0) return - - ! allocate(tp, mold=nbody_system%tp) - - ! write(*,*) "Image ",this_image(), "Distributing ",ntot - ! allocate(lspill_list(ntot)) - ! num_per_image = ntot / num_images() - ! istart = (this_image() - 1) * num_per_image + 1 - ! if (this_image() == num_images()) then - ! iend = ntot - ! else - ! iend = this_image() * num_per_image - ! end if - - ! if (this_image() == 1) then - ! lspill_list(:) = .true. - ! lspill_list(istart:iend) = .false. - ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) - ! else - ! lspill_list(:) = .false. - ! lspill_list(istart:iend) = .true. - ! tp%nbody = ntot - ! call nbody_system%tp%spill(tp,lspill_list(:), ldestructive=.true.) - ! end if - - ! write(*,*) "Image ",this_image(), "ntp: ",nbody_system%tp%nbody - - return + module subroutine swiftest_coarray_distribute_system(nbody_system, param) + !! author: David A. Minton + !! + !! Distributes test particles from image #1 out to all images. + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy + class(swiftest_tp), allocatable :: tp + logical, dimension(:), allocatable :: lspill_list + integer(I4B), codimension[*], save :: ntp + character(len=NAMELEN) :: image_num_char + class(swiftest_tp), allocatable, codimension[:] :: cotp + class(swiftest_tp), allocatable :: tmp + + sync all + if (this_image() == 1) then + write(image_num_char,*) num_images() + write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." + end if + + ntp = nbody_system%tp%nbody + sync all + ntot = ntp[1] + if (ntot == 0) return + + allocate(lspill_list(ntot)) + num_per_image = ntot / num_images() + istart = (this_image() - 1) * num_per_image + 1 + if (this_image() == num_images()) then + iend = ntot + else + iend = this_image() * num_per_image + end if + + lspill_list(:) = .true. + lspill_list(istart:iend) = .false. + + allocate(cotp[*], source=nbody_system%tp) + call cotp%coclone() + if (this_image() /= 1) then + deallocate(nbody_system%tp) + allocate(nbody_system%tp, source=cotp) + end if + allocate(tmp, mold=nbody_system%tp) + call nbody_system%tp%spill(tmp, lspill_list(:), ldestructive=.true.) + + deallocate(tmp, cotp) + + if (this_image() == 1) then + write(param%display_unit,*) " Done distributing" + do i = 1, nbody_system%tp%nbody + write(*,*) i,"mu ",nbody_system%tp%mu(i) + end do + end if + + return end subroutine swiftest_coarray_distribute_system + + module subroutine swiftest_coarray_initialize_system(nbody_system, param) + !! author: David A. Minton + !! + !! Distributes test particles from image #1 out to all images. + implicit none + ! Arguments + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + class(swiftest_nbody_system), allocatable, codimension[:] :: tmp_system + character(len=NAMELEN) :: image_num_char + + sync all + if (this_image() == 1) then + write(image_num_char,*) num_images() + write(param%display_unit,*) " Cloning nbody system to " // trim(adjustl(image_num_char)) // " images." + end if + allocate(tmp_system[*], source=nbody_system) + call tmp_system%coclone() + if (this_image() /= 1) then + if (allocated(nbody_system)) deallocate(nbody_system) + allocate(nbody_system, source=tmp_system) + end if + + return + end subroutine swiftest_coarray_initialize_system + + end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_discard.f90 b/src/swiftest/swiftest_discard.f90 index 6af0c9a8f..da9208f6a 100644 --- a/src/swiftest/swiftest_discard.f90 +++ b/src/swiftest/swiftest_discard.f90 @@ -25,7 +25,7 @@ module subroutine swiftest_discard_system(self, param) lpl_check = allocated(self%pl_discards) ltp_check = allocated(self%tp_discards) - associate(nbody_system => self, tp => self%tp, pl => self%pl, tp_discards => self%tp_discards, pl_discards => self%pl_discards, nc => self%system_history%nc) + associate(nbody_system => self, tp => self%tp, pl => self%pl, tp_discards => self%tp_discards, pl_discards => self%pl_discards) lpl_discards = .false. ltp_discards = .false. if (lpl_check .and. pl%nbody > 0) then diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 31eccbdf9..9e139c986 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -19,12 +19,9 @@ program swiftest_driver use swiftest implicit none -#ifdef COARRAY - class(swiftest_nbody_system), allocatable :: nbody_system[:] !! Polymorphic object containing the nbody system to be integrated -#else class(swiftest_nbody_system), allocatable :: nbody_system !! Polymorphic object containing the nbody system to be integrated -#endif type(swiftest_parameters) :: param !! Run configuration parameters + class(swiftest_storage), allocatable :: system_history !! Stores the system history between output dumps character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters character(len=:), allocatable :: display_style !! Style of the output display {"STANDARD", "COMPACT", "PROGRESS"}). Default is "STANDARD" @@ -95,24 +92,29 @@ program swiftest_driver #ifdef COARRAY if (this_image() == 1) then #endif - call nbody_system%initialize(param) + call nbody_system%initialize(system_history, param) +#ifdef COARRAY + end if ! this_image() == 1 + call swiftest_coarray_initialize_system(nbody_system, param) + if (this_image() == 1) then +#endif ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. call nbody_system%display_run_information(param, integration_timer, phase="first") if (param%lenergy) then if (param%lrestart) then - call nbody_system%get_t0_values(param) + call nbody_system%get_t0_values(system_history%nc, param) else call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum end if call nbody_system%conservation_report(param, lterminal=.true.) end if - call nbody_system%system_history%take_snapshot(param,nbody_system) - call nbody_system%dump(param) + call system_history%take_snapshot(param,nbody_system) + call nbody_system%dump(param, system_history) #ifdef COARRAY - ! Distribute test particles to the various images end if ! this_image() == 1 - call nbody_system%coarray_distribute() + ! Distribute test particles to the various images + call nbody_system%coarray_distribute(param) #endif do iloop = istart, nloops !> Step the nbody_system forward in time @@ -136,14 +138,14 @@ program swiftest_driver istep = floor(istep_out * fstep_out**nout, kind=I4B) end if #ifdef COARRAY - call nbody_system%coarray_collect() + call nbody_system%coarray_collect(param) if (this_image() == 1) then #endif - call nbody_system%system_history%take_snapshot(param,nbody_system) + call system_history%take_snapshot(param,nbody_system) if (idump == dump_cadence) then idump = 0 - call nbody_system%dump(param) + call nbody_system%dump(param, system_history) end if call integration_timer%report(message="Integration steps:", unit=display_unit) @@ -152,7 +154,7 @@ program swiftest_driver if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY end if - call nbody_system%coarray_distribute() + call nbody_system%coarray_distribute(param) #endif end if end if @@ -160,11 +162,10 @@ program swiftest_driver end do ! Dump any remaining history if it exists #ifdef COARRAY - call nbody_system%coarray_collect() + call nbody_system%coarray_collect(param) if (this_image() == 1) then #endif - call nbody_system%dump(param) - call nbody_system%system_history%dump(param) + call nbody_system%dump(param, system_history) call nbody_system%display_run_information(param, integration_timer, phase="last") #ifdef COARRAY end if ! this_image() == 1 diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 610f623d1..1cf6f93e7 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -127,7 +127,7 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) integer(I4B), parameter :: EGYIU = 72 character(len=*), parameter :: EGYTERMFMT = '(" DL/L0 = ", ES12.5, "; DE_orbit/|E0| = ", ES12.5, "; DE_total/|E0| = ", ES12.5, "; DM/M0 = ", ES12.5)' - associate(nbody_system => self, pl => self%pl, cb => self%cb, npl => self%pl%nbody, display_unit => param%display_unit, nc => self%system_history%nc) + associate(nbody_system => self, pl => self%pl, cb => self%cb, npl => self%pl%nbody, display_unit => param%display_unit) select type(self) class is (helio_nbody_system) ! Don't convert vh to vb for Helio-based integrators, because they are already have that calculated @@ -189,8 +189,6 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) if (abs(nbody_system%Mtot_error) > 100 * epsilon(nbody_system%Mtot_error)) then write(*,*) "Severe error! Mass not conserved! Halting!" ! Save the frame of data to the bin file in the slot just after the present one for diagnostics - call self%write_frame(nc, param) - call nc%close() call base_util_exit(FAILURE) end if end if @@ -239,14 +237,14 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t if (phase_val == 0) then if (param%lrestart) then - write(param%display_unit, *) " *************** Swiftest restart " // param%integrator // " *************** " + write(param%display_unit, *) " *************** Swiftest restart " // trim(adjustl(param%integrator)) // " *************** " else - write(param%display_unit, *) " *************** Swiftest start " // param%integrator // " *************** " + write(param%display_unit, *) " *************** Swiftest start " // trim(adjustl(param%integrator)) // " *************** " end if if (param%display_style == "PROGRESS") then call pbar%reset(param%nloops) else if (param%display_style == "COMPACT") then - write(*,*) "SWIFTEST START " // param%integrator + write(*,*) "SWIFTEST START " // trim(adjustl(param%integrator)) end if end if @@ -264,8 +262,8 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t end if if (phase_val == -1) then - write(param%display_unit, *)" *************** Swiftest stop " // param%integrator // " *************** " - if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // param%integrator + write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " + if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) end if return @@ -303,7 +301,7 @@ module subroutine swiftest_io_dump_param(self, param_file_name) end subroutine swiftest_io_dump_param - module subroutine swiftest_io_dump_system(self, param) + module subroutine swiftest_io_dump_system(self, param, system_history) !! author: David A. Minton !! !! Dumps the state of the nbody_system to files in case the simulation is interrupted. @@ -313,6 +311,7 @@ module subroutine swiftest_io_dump_system(self, param) ! Arguments class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_storage), intent(inout) :: system_history !! Stores the system history between output dumps ! Internals class(swiftest_parameters), allocatable :: param_restart !! Local parameters variable used to parameters change input file names !! to dump file-specific values without changing the user-defined values @@ -324,7 +323,7 @@ module subroutine swiftest_io_dump_system(self, param) if (allocated(self%collision_history)) call self%collision_history%dump(param) ! Dump the nbody_system history to file - call self%system_history%dump(param) + call system_history%dump(param) allocate(param_restart, source=param) param_restart%in_form = "XV" @@ -353,7 +352,7 @@ module subroutine swiftest_io_dump_storage(self, param) implicit none ! Arguments class(swiftest_storage), intent(inout) :: self !! Swiftest simulation history storage object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i @@ -366,7 +365,7 @@ module subroutine swiftest_io_dump_storage(self, param) if (allocated(self%frame(i)%item)) then select type(nbody_system => self%frame(i)%item) class is (swiftest_nbody_system) - call nbody_system%write_frame(param) + call nbody_system%write_frame(nc, param) end select deallocate(self%frame(i)%item) end if @@ -566,15 +565,16 @@ module subroutine swiftest_io_netcdf_flush(self, param) end subroutine swiftest_io_netcdf_flush - module subroutine swiftest_io_netcdf_get_t0_values_system(self, param) + module subroutine swiftest_io_netcdf_get_t0_values_system(self, nc, param) !! author: David A. Minton !! !! Gets the t0 values of various parameters such as energy and momentum !! implicit none ! Arguments - class(swiftest_nbody_system), intent(inout) :: self - class(swiftest_parameters), intent(inout) :: param + class(swiftest_nbody_system), intent(inout) :: self + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset + class(swiftest_parameters), intent(inout) :: param ! Internals integer(I4B) :: itmax, idmax, tslot real(DP), dimension(:), allocatable :: vals @@ -582,7 +582,7 @@ module subroutine swiftest_io_netcdf_get_t0_values_system(self, param) real(DP), dimension(NDIM) :: rot0, Ip0, L real(DP) :: mass0 - associate (nc => self%system_history%nc, cb => self%cb) + associate (cb => self%cb) call nc%open(param, readonly=.true.) call nc%find_tslot(param%t0, tslot) call netcdf_io_check( nf90_inquire_dimension(nc%id, nc%time_dimid, len=itmax), "netcdf_io_get_t0_values_system time_dimid" ) @@ -2318,6 +2318,7 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i ! Print the contents of the parameter file to standard output if (.not.param%lrestart) call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) end associate + #ifdef COARRAY end if ! this_image() == 1 call coparam%coclone() @@ -2325,11 +2326,6 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i type is (swiftest_parameters) self = coparam end select - - write(*,*) "Image: ", this_image(),"tstop: ",self%tstop - write(*,*) "Image: ", this_image(),"seed: ",self%seed - sync all - stop #endif return @@ -2751,14 +2747,15 @@ module subroutine swiftest_io_read_in_cb(self, param) end subroutine swiftest_io_read_in_cb - module subroutine swiftest_io_read_in_system(self, param) + module subroutine swiftest_io_read_in_system(self, nc, param) !! author: David A. Minton and Carlisle A. Wishard !! !! Reads in the nbody_system from input files implicit none ! Arguments - class(swiftest_nbody_system), intent(inout) :: self - class(swiftest_parameters), intent(inout) :: param + class(swiftest_nbody_system), intent(inout) :: self + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset + class(swiftest_parameters), intent(inout) :: param ! Internals integer(I4B) :: ierr, i class(swiftest_parameters), allocatable :: tmp_param @@ -2781,13 +2778,13 @@ module subroutine swiftest_io_read_in_system(self, param) self%E_untracked = param%E_untracked else allocate(tmp_param, source=param) - self%system_history%nc%file_name = param%nc_in + nc%file_name = param%nc_in tmp_param%out_form = param%in_form if (.not. param%lrestart) then ! Turn off energy computation so we don't have to feed it into the initial conditions tmp_param%lenergy = .false. end if - ierr = self%read_frame(self%system_history%nc, tmp_param) + ierr = self%read_frame(nc, tmp_param) deallocate(tmp_param) if (ierr /=0) call base_util_exit(FAILURE) end if @@ -3003,7 +3000,7 @@ module subroutine swiftest_io_toupper(string) end subroutine swiftest_io_toupper - module subroutine swiftest_io_write_frame_system(self, param) + module subroutine swiftest_io_write_frame_system(self, nc, param) !! author: The Purdue Swiftest Team - David A. Minton, Carlisle A. Wishard, Jennifer L.L. Pouplin, and Jacob R. Elliott !! !! Write a frame (header plus records for each massive body and active test particle) to output binary file @@ -3013,14 +3010,15 @@ module subroutine swiftest_io_write_frame_system(self, param) !! Adapted from Hal Levison's Swift routine io_write_frame.f implicit none ! Arguments - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals logical, save :: lfirst = .true. !! Flag to determine if this is the first call of this method character(len=STRMAX) :: errmsg logical :: fileExists - associate (nc => self%system_history%nc, pl => self%pl, tp => self%tp, npl => self%pl%nbody, ntp => self%tp%nbody) + associate (pl => self%pl, tp => self%tp, npl => self%pl%nbody, ntp => self%tp%nbody) nc%file_name = param%outfile if (lfirst) then inquire(file=param%outfile, exist=fileExists) @@ -3045,7 +3043,7 @@ module subroutine swiftest_io_write_frame_system(self, param) lfirst = .false. end if - call self%write_frame(nc, param) + call swiftest_io_netcdf_write_frame_system(self, nc, param) end associate return diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index fc5819ac1..6388929ac 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -159,7 +159,8 @@ module swiftest procedure :: spill => swiftest_util_spill_body !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) generic :: read_frame => read_frame_bin !! Add the generic read frame for Fortran binary files #ifdef COARRAY - procedure :: coclone => swiftest_coarray_coclone_body !! Clones the image 1 body object to all other images in the coarray structure. + procedure :: coclone => swiftest_coarray_coclone_body !! Clones the image 1 body object to all other images in the coarray structure. + procedure :: cocollect => swiftest_coarray_cocollect_body !! Collects all body object array components from all images and combines them into the image 1 body object #endif end type swiftest_body @@ -218,6 +219,10 @@ module swiftest procedure :: read_in => swiftest_io_read_in_cb !! Read in central body initial conditions from an ASCII file procedure :: write_frame => swiftest_io_netcdf_write_frame_cb !! I/O routine for writing out a single frame of time-series data for all bodies in the system in NetCDF format procedure :: write_info => swiftest_io_netcdf_write_info_cb !! Dump contents of particle information metadata to file + +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_cb !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type swiftest_cb @@ -277,6 +282,9 @@ module swiftest procedure :: rearrange => swiftest_util_sort_rearrange_pl !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods procedure :: spill => swiftest_util_spill_pl !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) generic :: set_renc => set_renc_I4B, set_renc_DP +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_pl !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type swiftest_pl @@ -307,6 +315,10 @@ module swiftest procedure :: sort => swiftest_util_sort_tp !! Sorts body arrays by a sortable component procedure :: rearrange => swiftest_util_sort_rearrange_tp !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods procedure :: spill => swiftest_util_spill_tp !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_tp !! Clones the image 1 object to all other images in the coarray structure. + procedure :: cocollect => swiftest_coarray_cocollect_tp !! Collects all object array components from all images and combines them into the image 1 object +#endif end type swiftest_tp @@ -330,7 +342,6 @@ module swiftest class(collision_basic), allocatable :: collider !! Collision system object class(encounter_storage), allocatable :: encounter_history !! Stores encounter history for later retrieval and saving to file class(collision_storage), allocatable :: collision_history !! Stores encounter history for later retrieval and saving to file - class(swiftest_storage), allocatable :: system_history !! Stores the system history between output dumps integer(I4B) :: maxid = -1 !! The current maximum particle id number real(DP) :: t = -1.0_DP !! Integration current time @@ -391,17 +402,12 @@ module swiftest procedure :: dump => swiftest_io_dump_system !! Dump the state of the nbody_system to a file procedure :: get_t0_values => swiftest_io_netcdf_get_t0_values_system !! Validates the dump file to check whether the dump file initial conditions duplicate the last frame of the netcdf output. procedure :: read_frame => swiftest_io_netcdf_read_frame_system !! Read in a frame of input data from file - procedure :: write_frame_netcdf => swiftest_io_netcdf_write_frame_system !! Write a frame of input data from file procedure :: read_hdr => swiftest_io_netcdf_read_hdr_system !! Read a header for an output frame in NetCDF format procedure :: write_hdr => swiftest_io_netcdf_write_hdr_system !! Write a header for an output frame in NetCDF format procedure :: read_particle_info => swiftest_io_netcdf_read_particle_info_system !! Read in particle metadata from file procedure :: read_in => swiftest_io_read_in_system !! Reads the initial conditions for an nbody system - procedure :: write_frame_system => swiftest_io_write_frame_system !! Write a frame of input data from file + procedure :: write_frame => swiftest_io_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body -#ifdef COARRAY - procedure :: coarray_collect => swiftest_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system - procedure :: coarray_distribute => swiftest_coarray_distribute_system !! Distributes test particles from image #1 out to all images. -#endif procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum procedure :: get_idvals => swiftest_util_get_idvalues_system !! Returns an array of all id values in use in the nbody_system @@ -411,7 +417,11 @@ module swiftest ! procedure :: step_spin => tides_step_spin_system !! Steps the spins of the massive & central bodies due to tides. procedure :: set_msys => swiftest_util_set_msys !! Sets the value of msys from the masses of nbody_system bodies. procedure :: validate_ids => swiftest_util_valid_id_system !! Validate the numerical ids passed to the nbody_system and save the maximum value - generic :: write_frame => write_frame_system, write_frame_netcdf !! Generic method call for reading a frame of output data +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_system !! Clones the image 1 body object to all other images in the coarray structure. + procedure :: coarray_collect => swiftest_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system + procedure :: coarray_distribute => swiftest_coarray_distribute_system !! Distributes test particles from image #1 out to all images. +#endif end type swiftest_nbody_system @@ -602,10 +612,11 @@ module subroutine swiftest_io_dump_param(self, param_file_name) character(len=*), intent(in) :: param_file_name !! Parameter input file name (i.e. param.in) end subroutine swiftest_io_dump_param - module subroutine swiftest_io_dump_system(self, param) + module subroutine swiftest_io_dump_system(self, param, system_history) implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_storage), intent(inout) :: system_history !! Stores the system history between output dumps end subroutine swiftest_io_dump_system module subroutine swiftest_io_dump_storage(self, param) @@ -649,10 +660,11 @@ module subroutine swiftest_io_netcdf_flush(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_io_netcdf_flush - module subroutine swiftest_io_netcdf_get_t0_values_system(self, param) + module subroutine swiftest_io_netcdf_get_t0_values_system(self, nc, param) implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody system object + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_io_netcdf_get_t0_values_system module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) @@ -847,9 +859,10 @@ module subroutine swiftest_io_read_in_param(self, param_file_name) character(len=*), intent(in) :: param_file_name !! Parameter input file name (i.e. param.in) end subroutine swiftest_io_read_in_param - module subroutine swiftest_io_read_in_system(self, param) + module subroutine swiftest_io_read_in_system(self, nc, param) implicit none - class(swiftest_nbody_system), intent(inout) :: self + class(swiftest_nbody_system), intent(inout) :: self + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset class(swiftest_parameters), intent(inout) :: param end subroutine swiftest_io_read_in_system @@ -880,9 +893,10 @@ module subroutine swiftest_io_toupper(string) character(*), intent(inout) :: string !! String to make upper case end subroutine swiftest_io_toupper - module subroutine swiftest_io_write_frame_system(self, param) + module subroutine swiftest_io_write_frame_system(self, nc, param) implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_io_write_frame_system @@ -1068,12 +1082,8 @@ end subroutine swiftest_util_setup_body module subroutine swiftest_util_setup_construct_system(nbody_system, param) implicit none -#ifdef COARRAY - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system[:] !! Swiftest nbody_system object -#else - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object -#endif - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_construct_system module subroutine swiftest_util_setup_initialize_particle_info_system(self, param) @@ -1082,10 +1092,11 @@ module subroutine swiftest_util_setup_initialize_particle_info_system(self, para class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_initialize_particle_info_system - module subroutine swiftest_util_setup_initialize_system(self, param) + module subroutine swiftest_util_setup_initialize_system(self, system_history, param) implicit none - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_util_setup_initialize_system module subroutine swiftest_util_setup_pl(self, n, param) @@ -1952,29 +1963,51 @@ end subroutine swiftest_util_version #ifdef COARRAY interface - module subroutine swiftest_coarray_collect_system(nbody_system) + module subroutine swiftest_coarray_collect_system(nbody_system, param) implicit none - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_coarray_collect_system - module subroutine swiftest_coarray_distribute_system(nbody_system) + module subroutine swiftest_coarray_distribute_system(nbody_system, param) implicit none - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_coarray_distribute_system + + module subroutine swiftest_coarray_initialize_system(nbody_system, param) + implicit none + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + end subroutine swiftest_coarray_initialize_system end interface interface coclone - module subroutine swiftest_coarray_component_copy_info(var,src_img) + module subroutine swiftest_coarray_component_clone_info(var,src_img) implicit none type(swiftest_particle_info), intent(inout) :: var integer(I4B), intent(in),optional :: src_img - end subroutine swiftest_coarray_component_copy_info + end subroutine swiftest_coarray_component_clone_info - module subroutine swiftest_coarray_component_copy_info_arr1D(var,src_img) + module subroutine swiftest_coarray_component_clone_info_arr1D(var,src_img) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: var integer(I4B), intent(in),optional :: src_img - end subroutine swiftest_coarray_component_copy_info_arr1D + end subroutine swiftest_coarray_component_clone_info_arr1D + + module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) + implicit none + type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine swiftest_coarray_component_clone_kin_arr1D + end interface + + interface cocollect + module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) + implicit none + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine end interface interface @@ -1982,7 +2015,40 @@ module subroutine swiftest_coarray_coclone_body(self) implicit none class(swiftest_body),intent(inout),codimension[*] :: self !! Swiftest body object end subroutine swiftest_coarray_coclone_body + + module subroutine swiftest_coarray_coclone_cb(self) + implicit none + class(swiftest_cb),intent(inout),codimension[*] :: self !! Swiftest cb object + end subroutine swiftest_coarray_coclone_cb + + module subroutine swiftest_coarray_coclone_pl(self) + implicit none + class(swiftest_pl),intent(inout),codimension[*] :: self !! Swiftest pl object + end subroutine swiftest_coarray_coclone_pl + + module subroutine swiftest_coarray_coclone_tp(self) + implicit none + class(swiftest_tp),intent(inout),codimension[*] :: self !! Swiftest tp object + end subroutine swiftest_coarray_coclone_tp + + module subroutine swiftest_coarray_coclone_system(self) + implicit none + class(swiftest_nbody_system),intent(inout),codimension[*] :: self !! Swiftest nbody system object + end subroutine swiftest_coarray_coclone_system + + module subroutine swiftest_coarray_cocollect_body(self) + !! Collects all body object array components from all images and combines them into the image 1 body object + implicit none + class(swiftest_body),intent(inout), codimension[*] :: self !! Swiftest body object + end subroutine swiftest_coarray_cocollect_body + + module subroutine swiftest_coarray_cocollect_tp(self) + !! Collects all body object array components from all images and combines them into the image 1 body object + implicit none + class(swiftest_tp),intent(inout), codimension[*] :: self !! Swiftest tp object + end subroutine swiftest_coarray_cocollect_tp end interface + #endif contains diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 7e5bd548a..77747170b 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -896,7 +896,6 @@ module subroutine swiftest_util_dealloc_system(self) if (allocated(self%collider)) deallocate(self%collider) if (allocated(self%encounter_history)) deallocate(self%encounter_history) if (allocated(self%collision_history)) deallocate(self%collision_history) - if (allocated(self%system_history)) deallocate(self%system_history) self%t = -1.0_DP self%GMtot = 0.0_DP @@ -2695,28 +2694,19 @@ module subroutine swiftest_util_set_rhill_approximate(self,cb) end subroutine swiftest_util_set_rhill_approximate module subroutine swiftest_util_setup_construct_system(nbody_system, param) + !! author: David A. Minton !! !! Constructor for a Swiftest nbody system. Creates the nbody system object based on the user-input integrator !! implicit none ! Arguments -#ifdef COARRAY - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system[:] !! Swiftest nbody_system object -#else - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object -#endif - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - + class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody_system object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters select case(param%integrator) case (INT_BS) write(*,*) 'Bulirsch-Stoer integrator not yet enabled' case (INT_HELIO) -#ifdef COARRAY - allocate(helio_nbody_system :: nbody_system[*]) -#else - allocate(helio_nbody_system :: nbody_system) -#endif select type(nbody_system) class is (helio_nbody_system) allocate(helio_cb :: nbody_system%cb) @@ -2730,11 +2720,7 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) case (INT_TU4) write(*,*) 'INT_TU4 integrator not yet enabled' case (INT_WHM) -#ifdef COARRAY - allocate(whm_nbody_system :: nbody_system[*]) -#else allocate(whm_nbody_system :: nbody_system) -#endif select type(nbody_system) class is (whm_nbody_system) allocate(whm_cb :: nbody_system%cb) @@ -2744,11 +2730,7 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) end select param%collision_model = "MERGE" case (INT_RMVS) -#ifdef COARRAY - allocate(rmvs_nbody_system :: nbody_system[*]) -#else allocate(rmvs_nbody_system :: nbody_system) -#endif select type(nbody_system) class is (rmvs_nbody_system) allocate(rmvs_cb :: nbody_system%cb) @@ -2758,11 +2740,7 @@ module subroutine swiftest_util_setup_construct_system(nbody_system, param) end select param%collision_model = "MERGE" case (INT_SYMBA) -#ifdef COARRAY - allocate(symba_nbody_system :: nbody_system[*]) -#else allocate(symba_nbody_system :: nbody_system) -#endif select type(nbody_system) class is (symba_nbody_system) allocate(symba_cb :: nbody_system%cb) @@ -2826,28 +2804,28 @@ module subroutine swiftest_util_setup_initialize_particle_info_system(self, para end subroutine swiftest_util_setup_initialize_particle_info_system - module subroutine swiftest_util_setup_initialize_system(self, param) + module subroutine swiftest_util_setup_initialize_system(self, system_history, param) !! author: David A. Minton !! !! Wrapper method to initialize a basic Swiftest nbody system from files !! implicit none ! Arguments - class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - if (allocated(self%system_history)) then - call self%system_history%dealloc() - deallocate(self%system_history) + if (allocated(system_history)) then + call system_history%dealloc() + deallocate(system_history) end if - allocate(swiftest_storage :: self%system_history) - call self%system_history%setup(param%dump_cadence) - allocate(swiftest_netcdf_parameters :: self%system_history%nc) + allocate(swiftest_storage :: system_history) + call system_history%setup(param%dump_cadence) + allocate(swiftest_netcdf_parameters :: system_history%nc) - associate(nbody_system => self, cb => self%cb, pl => self%pl, tp => self%tp, nc => self%system_history%nc) - call nbody_system%read_in(param) - call nbody_system%read_in(param) + associate(nbody_system => self, cb => self%cb, pl => self%pl, tp => self%tp, nc => system_history%nc) + call nbody_system%read_in(nc, param) call nbody_system%validate_ids(param) call nbody_system%set_msys() call pl%set_mu(cb) @@ -2867,7 +2845,7 @@ module subroutine swiftest_util_setup_initialize_system(self, param) ! Write initial conditions to file nc%file_name = param%outfile - call nbody_system%write_frame(param) + call nbody_system%write_frame(nc, param) call nc%close() end associate @@ -3069,11 +3047,7 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar ! Arguments class(swiftest_storage), intent(inout) :: self !! Swiftest storage object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters -#ifdef COARRAY - class(swiftest_nbody_system), intent(inout) :: nbody_system[*] !! Swiftest nbody system object to store -#else class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store -#endif real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time character(*), intent(in), optional :: arg !! Optional argument (needed for extended storage type used in collision snapshots) ! Internals @@ -3102,9 +3076,6 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar allocate(snapshot%cb, source=nbody_system%cb ) allocate(snapshot%pl, source=nbody_system%pl ) allocate(snapshot%tp, source=nbody_system%tp ) - allocate(snapshot%system_history) - allocate(snapshot%system_history%nc, source=nbody_system%system_history%nc) - snapshot%system_history%nc%lfile_is_open = .true. snapshot%t = nbody_system%t snapshot%GMtot = nbody_system%GMtot diff --git a/src/symba/symba_module.f90 b/src/symba/symba_module.f90 index efa8dda8f..85e5ca3bc 100644 --- a/src/symba/symba_module.f90 +++ b/src/symba/symba_module.f90 @@ -243,10 +243,12 @@ module subroutine symba_util_dealloc_system(self) class(symba_nbody_system), intent(inout) :: self end subroutine symba_util_dealloc_system - module subroutine symba_util_setup_initialize_system(self, param) + + module subroutine symba_util_setup_initialize_system(self, system_history, param) implicit none - class(symba_nbody_system), intent(inout) :: self !! SyMBA nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(symba_nbody_system), intent(inout) :: self !! SyMBA nbody_system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine symba_util_setup_initialize_system module subroutine symba_util_setup_pl(self, n, param) diff --git a/src/symba/symba_util.f90 b/src/symba/symba_util.f90 index a30c08bac..ad1d50383 100644 --- a/src/symba/symba_util.f90 +++ b/src/symba/symba_util.f90 @@ -273,7 +273,7 @@ module subroutine symba_util_set_renc(self, scale) end subroutine symba_util_set_renc - module subroutine symba_util_setup_initialize_system(self, param) + module subroutine symba_util_setup_initialize_system(self, system_history, param) !! author: David A. Minton !! !! Initialize an SyMBA nbody system from files and sets up the planetocentric structures. @@ -281,8 +281,9 @@ module subroutine symba_util_setup_initialize_system(self, param) !! implicit none ! Arguments - class(symba_nbody_system), intent(inout) :: self !! SyMBA nbody_system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(symba_nbody_system), intent(inout) :: self !! SyMBA nbody_system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals type(encounter_storage) :: encounter_history type(collision_storage) :: collision_history @@ -291,7 +292,7 @@ module subroutine symba_util_setup_initialize_system(self, param) call collision_history%setup(4096) ! Call parent method associate(nbody_system => self) - call helio_util_setup_initialize_system(nbody_system, param) + call helio_util_setup_initialize_system(nbody_system, system_history, param) call nbody_system%pltp_encounter%setup(0_I8B) call nbody_system%plpl_encounter%setup(0_I8B) call nbody_system%plpl_collision%setup(0_I8B) diff --git a/src/whm/whm_coarray.f90 b/src/whm/whm_coarray.f90 new file mode 100644 index 000000000..5a8de4c32 --- /dev/null +++ b/src/whm/whm_coarray.f90 @@ -0,0 +1,34 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +submodule (whm) s_whm_coarray +use coarray +use swiftest +contains + + module subroutine whm_coarray_coclone_pl(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(whm_pl),intent(inout),codimension[*] :: self !! WHM pl object + + call coclone(self%eta) + call coclone(self%xj) + call coclone(self%vj) + call coclone(self%muj) + call coclone(self%ir3j) + + call swiftest_coarray_coclone_pl(self) + + return + end subroutine whm_coarray_coclone_pl + +end submodule s_whm_coarray \ No newline at end of file diff --git a/src/whm/whm_module.f90 b/src/whm/whm_module.f90 index d2d10f971..38e73f190 100644 --- a/src/whm/whm_module.f90 +++ b/src/whm/whm_module.f90 @@ -48,10 +48,13 @@ module whm procedure :: set_mu => whm_util_set_mu_eta_pl !! Sets the Jacobi mass value for all massive bodies. procedure :: sort => whm_util_sort_pl !! Sort a WHM massive body object in-place. procedure :: rearrange => whm_util_sort_rearrange_pl !! Rearranges the order of array elements of body based on an input index array. Used in sorting methods - procedure :: spill => whm_util_spill_pl !!"Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) - procedure :: setup => whm_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess + procedure :: spill => whm_util_spill_pl !! "Spills" bodies from one object to another depending on the results of a mask (uses the PACK intrinsic) + procedure :: setup => whm_util_setup_pl !! Constructor method - Allocates space for the input number of bodiess procedure :: step => whm_step_pl !! Steps the body forward one stepsize - final :: whm_final_pl !! Finalizes the WHM massive body object - deallocates all allocatables + final :: whm_final_pl !! Finalizes the WHM massive body object - deallocates all allocatables +#ifdef COARRAY + procedure :: coclone => whm_coarray_coclone_pl !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type whm_pl @@ -180,10 +183,11 @@ module subroutine whm_util_setup_pl(self, n, param) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters end subroutine whm_util_setup_pl - module subroutine whm_util_setup_initialize_system(self, param) + module subroutine whm_util_setup_initialize_system(self, system_history, param) implicit none - class(whm_nbody_system), intent(inout) :: self !! WHM nbody system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(whm_nbody_system), intent(inout) :: self !! WHM nbody system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine whm_util_setup_initialize_system module subroutine whm_step_pl(self, nbody_system, param, t, dt) @@ -270,6 +274,15 @@ module subroutine whm_util_sort_rearrange_pl(self, ind) end subroutine whm_util_sort_rearrange_pl end interface +#ifdef COARRAY + interface + module subroutine whm_coarray_coclone_pl(self) + implicit none + class(whm_pl),intent(inout),codimension[*] :: self !! WHM pl object + end subroutine whm_coarray_coclone_pl + end interface +#endif + contains subroutine whm_final_pl(self) diff --git a/src/whm/whm_util.f90 b/src/whm/whm_util.f90 index 751e97d2a..13105047e 100644 --- a/src/whm/whm_util.f90 +++ b/src/whm/whm_util.f90 @@ -200,17 +200,18 @@ module subroutine whm_util_set_mu_eta_pl(self, cb) end subroutine whm_util_set_mu_eta_pl - module subroutine whm_util_setup_initialize_system(self, param) + module subroutine whm_util_setup_initialize_system(self, system_history, param) !! author: David A. Minton !! !! Initialize a WHM nbody system from files !! implicit none ! Arguments - class(whm_nbody_system), intent(inout) :: self !! WHM nbody system object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(whm_nbody_system), intent(inout) :: self !! WHM nbody system object + class(swiftest_storage), allocatable, intent(inout) :: system_history !! Stores the system history between output dumps + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - call swiftest_util_setup_initialize_system(self, param) + call swiftest_util_setup_initialize_system(self, system_history, param) ! First we need to make sure that the massive bodies are sorted by heliocentric distance before computing jacobies call swiftest_util_set_ir3h(self%pl) call self%pl%sort("ir3h", ascending=.false.) From 042a882ac17e3349b1ad3f5f668632932f2a6980 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 14 Apr 2023 10:17:08 -0400 Subject: [PATCH 022/149] Moved coarray conditional preprocessor statement to above a coarray declaration --- src/swiftest/swiftest_io.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 1cf6f93e7..71ba57b7f 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1882,9 +1882,9 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i logical :: seed_set = .false. !! Is the random seed set in the input file? character(len=:), allocatable :: integrator real(DP) :: tratio, y +#ifdef COARRAY type(swiftest_parameters), codimension[*], save :: coparam -#ifdef COARRAY if (this_image() == 1) then coparam = self associate(param => coparam) From 2e2e97cf30ff60d5ae941cc9229247c8683744ee Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 20 Apr 2023 13:24:55 -0400 Subject: [PATCH 023/149] Major restructing to move a bunch of basic utillity functions that operate on standard data types (DP, I4B, etc) to the base_module. This will make it easier to use them inside the coarray module. Also built coarray submodules to put implementations in. Restructuring still needs more thorough testing --- src/CMakeLists.txt | 2 + src/base/base_module.f90 | 1729 +++++++++++++++++++- src/coarray/coarray_clone.f90 | 494 ++++++ src/coarray/coarray_collect.f90 | 296 ++++ src/coarray/coarray_module.f90 | 992 ++---------- src/collision/collision_util.f90 | 2 +- src/encounter/encounter_check.f90 | 20 +- src/encounter/encounter_module.f90 | 6 +- src/encounter/encounter_util.f90 | 65 +- src/fraggle/fraggle_util.f90 | 4 +- src/rmvs/rmvs_coarray.f90 | 48 +- src/rmvs/rmvs_module.f90 | 17 + src/rmvs/rmvs_util.f90 | 104 +- src/swiftest/swiftest_coarray.f90 | 124 +- src/swiftest/swiftest_driver.f90 | 46 +- src/swiftest/swiftest_io.f90 | 6 +- src/swiftest/swiftest_module.f90 | 297 +--- src/swiftest/swiftest_util.f90 | 2370 +++++----------------------- src/symba/symba_util.f90 | 64 +- src/whm/whm_util.f90 | 62 +- 20 files changed, 3389 insertions(+), 3359 deletions(-) create mode 100644 src/coarray/coarray_clone.f90 create mode 100644 src/coarray/coarray_collect.f90 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e1caebba..d2a04991c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -85,6 +85,8 @@ SET(FAST_MATH_FILES SET(COARRAY_FILES ${SRC}/coarray/coarray_module.f90 + ${SRC}/coarray/coarray_clone.f90 + ${SRC}/coarray/coarray_collect.f90 ${SRC}/swiftest/swiftest_coarray.f90 ${SRC}/whm/whm_coarray.f90 ${SRC}/rmvs/rmvs_coarray.f90 diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 3c33a47c1..84e5a1f2e 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -220,8 +220,285 @@ end subroutine abstract_util_dealloc_object type, abstract :: base_nbody_system end type base_nbody_system + + interface util_append + module procedure base_util_append_arr_char_string + module procedure base_util_append_arr_DP + module procedure base_util_append_arr_DPvec + module procedure base_util_append_arr_I4B + module procedure base_util_append_arr_logical + end interface + + interface util_fill + module procedure base_util_fill_arr_char_string + module procedure base_util_fill_arr_DP + module procedure base_util_fill_arr_DPvec + module procedure base_util_fill_arr_I4B + module procedure base_util_fill_arr_logical + end interface + + interface util_resize + module procedure base_util_resize_arr_char_string + module procedure base_util_resize_arr_DP + module procedure base_util_resize_arr_DPvec + module procedure base_util_resize_arr_I4B + module procedure base_util_resize_arr_logical + end interface + + interface util_sort + module procedure base_util_sort_i4b + module procedure base_util_sort_index_i4b + module procedure base_util_sort_index_I4B_I8Bind + module procedure base_util_sort_index_I8B_I8Bind + module procedure base_util_sort_sp + module procedure base_util_sort_index_sp + module procedure base_util_sort_dp + module procedure base_util_sort_index_dp + end interface + + interface util_sort_rearrange + module procedure base_util_sort_rearrange_arr_char_string + module procedure base_util_sort_rearrange_arr_DP + module procedure base_util_sort_rearrange_arr_DPvec + module procedure base_util_sort_rearrange_arr_I4B + module procedure base_util_sort_rearrange_arr_I4B_I8Bind + module procedure base_util_sort_rearrange_arr_logical + module procedure base_util_sort_rearrange_arr_logical_I8Bind + end interface + + interface util_spill + module procedure base_util_spill_arr_char_string + module procedure base_util_spill_arr_DP + module procedure base_util_spill_arr_DPvec + module procedure base_util_spill_arr_I4B + module procedure base_util_spill_arr_I8B + module procedure base_util_spill_arr_logical + end interface + + interface util_unique + module procedure base_util_unique_DP + module procedure base_util_unique_I4B + end interface + contains + subroutine base_util_append_arr_char_string(arr, source, nold, lsource_mask) + !! author: David A. Minton + !! + !! Append a single array of character string type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + implicit none + ! Arguments + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array + character(len=STRMAX), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + ! Internals + integer(I4B) :: nnew, nsrc, nend_orig + + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) + else + nsrc = size(source) + end if + if (nsrc == 0) return + + if (.not.allocated(arr)) then + nend_orig = 0 + allocate(arr(nsrc)) + else + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) + end if + nnew = nend_orig + nsrc + + if (present(lsource_mask)) then + arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + else + arr(nold + 1:nnew) = source(1:nsrc) + end if + + return + end subroutine base_util_append_arr_char_string + + + subroutine base_util_append_arr_DP(arr, source, nold, lsource_mask) + !! author: David A. Minton + !! + !! Append a single array of double precision type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array + real(DP), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + ! Internals + integer(I4B) :: nnew, nsrc, nend_orig + + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) + else + nsrc = size(source) + end if + if (nsrc == 0) return + + if (.not.allocated(arr)) then + nend_orig = 0 + allocate(arr(nsrc)) + else + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) + end if + nnew = nend_orig + nsrc + + if (present(lsource_mask)) then + arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + else + arr(nold + 1:nnew) = source(1:nsrc) + end if + + return + end subroutine base_util_append_arr_DP + + + subroutine base_util_append_arr_DPvec(arr, source, nold, lsource_mask) + !! author: David A. Minton + !! + !! Append a single array of double precision vector type of size (NDIM, n) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array + real(DP), dimension(:,:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + ! Internals + integer(I4B) :: nnew, nsrc, nend_orig + + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) + else + nsrc = size(source,dim=2) + end if + if (nsrc == 0) return + + if (.not.allocated(arr)) then + nend_orig = 0 + allocate(arr(NDIM,nsrc)) + else + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr,dim=2) + end if + call util_resize(arr, nend_orig + nsrc) + end if + nnew = nend_orig + nsrc + + if (present(lsource_mask)) then + arr(1, nold + 1:nnew) = pack(source(1,1:nsrc), lsource_mask(1:nsrc)) + arr(2, nold + 1:nnew) = pack(source(2,1:nsrc), lsource_mask(1:nsrc)) + arr(3, nold + 1:nnew) = pack(source(3,1:nsrc), lsource_mask(1:nsrc)) + else + arr(:,nold + 1:nnew) = source(:,1:nsrc) + end if + + return + end subroutine base_util_append_arr_DPvec + + + subroutine base_util_append_arr_I4B(arr, source, nold, lsource_mask) + !! author: David A. Minton + !! + !! Append a single array of integer(I4B) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + ! Internals + integer(I4B) :: nnew, nsrc, nend_orig + + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) + else + nsrc = size(source) + end if + if (nsrc == 0) return + + if (.not.allocated(arr)) then + nend_orig = 0 + allocate(arr(nsrc)) + else + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) + end if + nnew = nend_orig + nsrc + + if (present(lsource_mask)) then + arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + else + arr(nold + 1:nnew) = source(1:nsrc) + end if + + return + end subroutine base_util_append_arr_I4B + + + subroutine base_util_append_arr_logical(arr, source, nold, lsource_mask) + !! author: David A. Minton + !! + !! Append a single array of logical type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array + logical, dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + ! Internals + integer(I4B) :: nnew, nsrc, nend_orig + + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) + else + nsrc = size(source) + end if + if (nsrc == 0) return + + if (.not.allocated(arr)) then + nend_orig = 0 + allocate(arr(nsrc)) + else + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) + end if + nnew = nend_orig + nsrc + + if (present(lsource_mask)) then + arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + else + arr(nold + 1:nnew) = source(1:nsrc) + end if + + return + end subroutine base_util_append_arr_logical + + subroutine base_util_copy_store(self, source) !! author: David A. Minton !! @@ -303,6 +580,109 @@ subroutine base_util_exit(code) end subroutine base_util_exit + subroutine base_util_fill_arr_char_string(keeps, inserts, lfill_list) + !! author: David A. Minton + !! + !! Performs a fill operation on a single array of type character strings + !! This is the inverse of a spill operation + implicit none + ! Arguments + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + character(len=STRMAX), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + + if (.not.allocated(keeps) .or. .not.allocated(inserts)) return + + keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) + keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) + + return + end subroutine base_util_fill_arr_char_string + + + subroutine base_util_fill_arr_DP(keeps, inserts, lfill_list) + !! author: David A. Minton + !! + !! Performs a fill operation on a single array of type DP + !! This is the inverse of a spill operation + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + real(DP), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + + if (.not.allocated(keeps) .or. .not.allocated(inserts)) return + + keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) + keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) + + return + end subroutine base_util_fill_arr_DP + + + subroutine base_util_fill_arr_DPvec(keeps, inserts, lfill_list) + !! author: David A. Minton + !! + !! Performs a fill operation on a single array of DP vectors with shape (NDIM, n) + !! This is the inverse of a spill operation + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep + real(DP), dimension(:,:), allocatable, intent(in) :: inserts !! Array of values to insert into keep + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + ! Internals + integer(I4B) :: i + + if (.not.allocated(keeps) .or. .not.allocated(inserts)) return + + do i = 1, NDIM + keeps(i,:) = unpack(keeps(i,:), .not.lfill_list(:), keeps(i,:)) + keeps(i,:) = unpack(inserts(i,:), lfill_list(:), keeps(i,:)) + end do + + return + end subroutine base_util_fill_arr_DPvec + + + subroutine base_util_fill_arr_I4B(keeps, inserts, lfill_list) + !! author: David A. Minton + !! + !! Performs a fill operation on a single array of type I4B + !! This is the inverse of a spill operation + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + integer(I4B), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + + if (.not.allocated(keeps) .or. .not.allocated(inserts)) return + + keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) + keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) + + return + end subroutine base_util_fill_arr_I4B + + + subroutine base_util_fill_arr_logical(keeps, inserts, lfill_list) + !! author: David A. Minton + !! + !! Performs a fill operation on a single array of logicals + !! This is the inverse of a spill operation + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + logical, dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + + if (.not.allocated(keeps) .or. .not.allocated(inserts)) return + + keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) + keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) + + return + end subroutine base_util_fill_arr_logical + subroutine base_util_reset_storage(self) !! author: David A. Minton !! @@ -330,6 +710,237 @@ subroutine base_util_reset_storage(self) return end subroutine base_util_reset_storage + + subroutine base_util_resize_arr_char_string(arr, nnew) + !! author: David A. Minton + !! + !! Resizes an array component of type character string. nnew = 0 will deallocate. + implicit none + ! Arguments + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size + ! Internals + character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + integer(I4B) :: nold !! Old size + + if (nnew < 0) return + + if (nnew == 0) then + if (allocated(arr)) deallocate(arr) + return + end if + + if (allocated(arr)) then + nold = size(arr) + else + nold = 0 + end if + + if (nnew == nold) return + + allocate(tmp(nnew)) + if (nold > 0) then + if (nnew > nold) then + tmp(1:nold) = arr(1:nold) + tmp(nold+1:nnew) = "" + else + tmp(1:nnew) = arr(1:nnew) + end if + else + tmp(1:nnew) = "" + end if + call move_alloc(tmp, arr) + + return + end subroutine base_util_resize_arr_char_string + + + subroutine base_util_resize_arr_DP(arr, nnew) + !! author: David A. Minton + !! + !! Resizes an array component of double precision type. Passing nnew = 0 will deallocate. + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size + ! Internals + real(DP), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + integer(I4B) :: nold !! Old size + real(DP), parameter :: init_val = 0.0_DP + + if (nnew < 0) return + + if (nnew == 0) then + if (allocated(arr)) deallocate(arr) + return + end if + + if (allocated(arr)) then + nold = size(arr) + else + nold = 0 + end if + + if (nnew == nold) return + + allocate(tmp(nnew)) + if (nold > 0) then + if (nnew > nold) then + tmp(1:nold) = arr(1:nold) + tmp(nold+1:nnew) = init_val + else + tmp(1:nnew) = arr(1:nnew) + end if + else + tmp(1:nnew) = init_val + end if + call move_alloc(tmp, arr) + + return + end subroutine base_util_resize_arr_DP + + + subroutine base_util_resize_arr_DPvec(arr, nnew) + !! author: David A. Minton + !! + !! Resizes an array component of double precision vectors of size (NDIM, n). Passing nnew = 0 will deallocate. + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size + ! Internals + real(DP), dimension(:,:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + integer(I4B) :: nold !! Old size + real(DP), dimension(NDIM), parameter :: init_val = 0.0_DP + integer(I4B) :: i + + if (nnew < 0) return + + if (nnew == 0) then + if (allocated(arr)) deallocate(arr) + return + end if + + if (allocated(arr)) then + nold = size(arr, dim=2) + else + nold = 0 + end if + + if (nnew == nold) return + + allocate(tmp(NDIM, nnew)) + if (nold > 0) then + if (nnew > nold) then + tmp(:,1:nold) = arr(:,1:nold) + do i = nold+1, nnew + tmp(:,i) = init_val(:) + end do + else + tmp(:,1:nnew) = arr(:,1:nnew) + end if + else + do i = 1, nnew + tmp(:, i) = init_val(:) + end do + end if + call move_alloc(tmp, arr) + + return + + return + end subroutine base_util_resize_arr_DPvec + + + subroutine base_util_resize_arr_I4B(arr, nnew) + !! author: David A. Minton + !! + !! Resizes an array component of integer type. Passing nnew = 0 will deallocate. + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size + ! Internals + integer(I4B), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + integer(I4B) :: nold !! Old size + integer(I4B), parameter :: init_val = -1 + + if (nnew < 0) return + + if (nnew == 0) then + if (allocated(arr)) deallocate(arr) + return + end if + + if (allocated(arr)) then + nold = size(arr) + else + nold = 0 + end if + + if (nnew == nold) return + + allocate(tmp(nnew)) + if (nold > 0) then + if (nnew > nold) then + tmp(1:nold) = arr(1:nold) + tmp(nold+1:nnew) = init_val + else + tmp(1:nnew) = arr(1:nnew) + end if + else + tmp(1:nnew) = init_val + end if + call move_alloc(tmp, arr) + + return + end subroutine base_util_resize_arr_I4B + + + subroutine base_util_resize_arr_logical(arr, nnew) + !! author: David A. Minton + !! + !! Resizes an array component of logical type. Passing nnew = 0 will deallocate. + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size + ! Internals + logical, dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + integer(I4B) :: nold !! Old size + logical, parameter :: init_val = .false. + + if (nnew < 0) return + + if (nnew == 0) then + if (allocated(arr)) deallocate(arr) + return + end if + + if (allocated(arr)) then + nold = size(arr) + else + nold = 0 + end if + + if (nnew == nold) return + + allocate(tmp(nnew)) + if (nold > 0) then + if (nnew > nold) then + tmp(1:nold) = arr(1:nold) + tmp(nold+1:nnew) = init_val + else + tmp(1:nnew) = arr(1:nnew) + end if + else + tmp(1:nnew) = init_val + end if + call move_alloc(tmp, arr) + + return + end subroutine base_util_resize_arr_logical + subroutine base_util_resize_storage(self, nnew) !! author: David A. Minton @@ -405,35 +1016,1114 @@ subroutine base_util_snapshot_save(self, snapshot) return end subroutine base_util_snapshot_save - - subroutine base_final_storage(self) + + subroutine base_util_spill_arr_char_string(keeps, discards, lspill_list, ldestructive) !! author: David A. Minton !! - !! Finalizer for the storage object + !! Performs a spill operation on a single array of type character strings + !! This is the inverse of a spill operation implicit none ! Arguments - class(base_storage), intent(inout) :: self + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + ! Internals + integer(I4B) :: nspill, nkeep, nlist + character(len=STRMAX), dimension(:), allocatable :: tmp !! Array of values to keep - call self%dealloc() - return - end subroutine base_final_storage + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(nspill)) + else if (size(discards) /= nspill) then + deallocate(discards) + allocate(discards(nspill)) + end if + + discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(nkeep)) + tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if - subroutine base_final_storage_frame(self) - !! author: David A. Minton - !! - !! Finalizer for the storage frame data type - implicit none - type(base_storage_frame) :: self - - if (allocated(self%item)) deallocate(self%item) - return - end subroutine base_final_storage_frame + end subroutine base_util_spill_arr_char_string + -#ifdef COARRAY - subroutine base_coclone_param(self) + subroutine base_util_spill_arr_DP(keeps, discards, lspill_list, ldestructive) + !! author: David A. Minton + !! + !! Performs a spill operation on a single array of type DP + !! This is the inverse of a spill operation + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + real(DP), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + ! Internals + integer(I4B) :: nspill, nkeep, nlist + real(DP), dimension(:), allocatable :: tmp !! Array of values to keep + + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(nspill)) + else if (size(discards) /= nspill) then + deallocate(discards) + allocate(discards(nspill)) + end if + + discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(nkeep)) + tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if + + return + end subroutine base_util_spill_arr_DP + + + subroutine base_util_spill_arr_DPvec(keeps, discards, lspill_list, ldestructive) + !! author: David A. Minton + !! + !! Performs a spill operation on a single array of DP vectors with shape (NDIM, n) + !! This is the inverse of a spill operation + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep + real(DP), dimension(:,:), allocatable, intent(inout) :: discards !! Array discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + ! Internals + integer(I4B) :: i, nspill, nkeep, nlist + real(DP), dimension(:,:), allocatable :: tmp !! Array of values to keep + + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(NDIM, nspill)) + else if (size(discards, dim=2) /= nspill) then + deallocate(discards) + allocate(discards(NDIM, nspill)) + end if + + do i = 1, NDIM + discards(i,:) = pack(keeps(i,1:nlist), lspill_list(1:nlist)) + end do + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(NDIM, nkeep)) + do i = 1, NDIM + tmp(i, :) = pack(keeps(i, 1:nlist), .not. lspill_list(1:nlist)) + end do + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if + + return + end subroutine base_util_spill_arr_DPvec + + + subroutine base_util_spill_arr_I4B(keeps, discards, lspill_list, ldestructive) + !! author: David A. Minton + !! + !! Performs a spill operation on a single array of type I4B + !! This is the inverse of a spill operation + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + integer(I4B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + ! Internals + integer(I4B) :: nspill, nkeep, nlist + integer(I4B), dimension(:), allocatable :: tmp !! Array of values to keep + + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(nspill)) + else if (size(discards) /= nspill) then + deallocate(discards) + allocate(discards(nspill)) + end if + + discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(nkeep)) + tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if + + return + end subroutine base_util_spill_arr_I4B + + + subroutine base_util_spill_arr_I8B(keeps, discards, lspill_list, ldestructive) + !! author: David A. Minton + !! + !! Performs a spill operation on a single array of type I4B + !! This is the inverse of a spill operation + implicit none + ! Arguments + integer(I8B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + integer(I8B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + ! Internals + integer(I4B) :: nspill, nkeep, nlist + integer(I8B), dimension(:), allocatable :: tmp !! Array of values to keep + + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(nspill)) + else if (size(discards) /= nspill) then + deallocate(discards) + allocate(discards(nspill)) + end if + + discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(nkeep)) + tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if + + return + end subroutine base_util_spill_arr_I8B + + + subroutine base_util_spill_arr_logical(keeps, discards, lspill_list, ldestructive) + !! author: David A. Minton + !! + !! Performs a spill operation on a single array of logicals + !! This is the inverse of a spill operation + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + logical, dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or no + ! Internals + integer(I4B) :: nspill, nkeep, nlist + logical, dimension(:), allocatable :: tmp !! Array of values to keep + + nkeep = count(.not.lspill_list(:)) + nspill = count(lspill_list(:)) + nlist = size(lspill_list(:)) + + if (.not.allocated(keeps) .or. nspill == 0) return + if (.not.allocated(discards)) then + allocate(discards(nspill)) + else if (size(discards) /= nspill) then + deallocate(discards) + allocate(discards(nspill)) + end if + + discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) + if (ldestructive) then + if (nkeep > 0) then + allocate(tmp(nkeep)) + tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) + call move_alloc(tmp, keeps) + else + deallocate(keeps) + end if + end if + + return + end subroutine base_util_spill_arr_logical + + + pure subroutine base_util_sort_dp(arr) + !! author: David A. Minton + !! + !! Sort input DP precision array in place into ascending numerical order using quicksort. + !! + implicit none + ! Arguments + real(DP), dimension(:), intent(inout) :: arr + + call base_util_sort_qsort_DP(arr) + + return + end subroutine base_util_sort_dp + + + pure subroutine base_util_sort_index_dp(arr, ind) + !! author: David A. Minton + !! + !! Sort input DP precision array by index in ascending numerical order using quick sort. + !! This algorithm works well for partially sorted arrays (which is usually the case here). + !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously + !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. + !! + implicit none + ! Arguments + real(DP), dimension(:), intent(in) :: arr + integer(I4B), dimension(:), allocatable, intent(inout) :: ind + ! Internals + integer(I4B) :: n, i + real(DP), dimension(:), allocatable :: tmparr + + n = size(arr) + if (.not.allocated(ind)) then + allocate(ind(n)) + ind = [(i, i=1, n)] + end if + allocate(tmparr, mold=arr) + tmparr(:) = arr(ind(:)) + call base_util_sort_qsort_DP(tmparr, ind) + + return + end subroutine base_util_sort_index_dp + + + recursive pure subroutine base_util_sort_qsort_DP(arr, ind) + !! author: David A. Minton + !! + !! Sort input DP precision array by index in ascending numerical order using quicksort sort. + !! + implicit none + ! Arguments + real(DP), dimension(:), intent(inout) :: arr + integer(I4B),dimension(:),intent(out), optional :: ind + !! Internals + integer :: iq + + if (size(arr) > 1) then + if (present(ind)) then + call base_util_sort_partition_DP(arr, iq, ind) + call base_util_sort_qsort_DP(arr(:iq-1),ind(:iq-1)) + call base_util_sort_qsort_DP(arr(iq:), ind(iq:)) + else + call base_util_sort_partition_DP(arr, iq) + call base_util_sort_qsort_DP(arr(:iq-1)) + call base_util_sort_qsort_DP(arr(iq:)) + end if + end if + + return + end subroutine base_util_sort_qsort_DP + + + pure subroutine base_util_sort_partition_DP(arr, marker, ind) + !! author: David A. Minton + !! + !! Partition function for quicksort on DP type + !! + implicit none + ! Arguments + real(DP), intent(inout), dimension(:) :: arr + integer(I4B), intent(inout), dimension(:), optional :: ind + integer(I4B), intent(out) :: marker + ! Internals + integer(I4B) :: i, j, itmp, narr, ipiv + real(DP) :: temp + real(DP) :: x ! pivot point + + narr = size(arr) + + ! Get center as pivot, as this is likely partially sorted + ipiv = narr / 2 + x = arr(ipiv) + i = 0 + j = narr + 1 + + do + j = j - 1 + do + if (arr(j) <= x) exit + j = j - 1 + end do + i = i + 1 + do + if (arr(i) >= x) exit + i = i + 1 + end do + if (i < j) then + ! exchange A(i) and A(j) + temp = arr(i) + arr(i) = arr(j) + arr(j) = temp + if (present(ind)) then + itmp = ind(i) + ind(i) = ind(j) + ind(j) = itmp + end if + else if (i == j) then + marker = i + 1 + return + else + marker = i + return + endif + end do + + return + end subroutine base_util_sort_partition_DP + + + pure subroutine base_util_sort_i4b(arr) + !! author: David A. Minton + !! + !! Sort input integer array in place into ascending numerical order using quick sort. + !! This algorithm works well for partially sorted arrays (which is usually the case here) + !! + implicit none + ! Arguments + integer(I4B), dimension(:), intent(inout) :: arr + + call base_util_sort_qsort_I4B(arr) + + return + end subroutine base_util_sort_i4b + + + pure subroutine base_util_sort_index_I4B(arr, ind) + !! author: David A. Minton + !! + !! Sort input integer array by index in ascending numerical order using quicksort. + !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously + !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. + !! + implicit none + ! Arguments + integer(I4B), dimension(:), intent(in) :: arr + integer(I4B), dimension(:), allocatable, intent(inout) :: ind + ! Internals + integer(I4B) :: n, i + integer(I4B), dimension(:), allocatable :: tmparr + + n = size(arr) + if (.not.allocated(ind)) then + allocate(ind(n)) + ind = [(i, i=1, n)] + end if + allocate(tmparr, mold=arr) + tmparr(:) = arr(ind(:)) + call base_util_sort_qsort_I4B(tmparr, ind) + + return + end subroutine base_util_sort_index_I4B + + + pure subroutine base_util_sort_index_I4B_I8Bind(arr, ind) + !! author: David A. Minton + !! + !! Sort input integer array by index in ascending numerical order using quicksort. + !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously + !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. + !! + implicit none + ! Arguments + integer(I4B), dimension(:), intent(in) :: arr + integer(I8B), dimension(:), allocatable, intent(inout) :: ind + ! Internals + integer(I8B) :: n, i + integer(I4B), dimension(:), allocatable :: tmparr + + n = size(arr) + if (.not.allocated(ind)) then + allocate(ind(n)) + ind = [(i, i=1_I8B, n)] + end if + allocate(tmparr, mold=arr) + tmparr(:) = arr(ind(:)) + call base_util_sort_qsort_I4B_I8Bind(tmparr, ind) + + return + end subroutine base_util_sort_index_I4B_I8Bind + + + pure subroutine base_util_sort_index_I8B_I8Bind(arr, ind) + !! author: David A. Minton + !! + !! Sort input integer array by index in ascending numerical order using quicksort. + !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously + !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. + !! + implicit none + ! Arguments + integer(I8B), dimension(:), intent(in) :: arr + integer(I8B), dimension(:), allocatable, intent(inout) :: ind + ! Internals + integer(I8B) :: n, i + integer(I8B), dimension(:), allocatable :: tmparr + + n = size(arr) + if (.not.allocated(ind)) then + allocate(ind(n)) + ind = [(i, i=1_I8B, n)] + end if + allocate(tmparr, mold=arr) + tmparr(:) = arr(ind(:)) + call base_util_sort_qsort_I8B_I8Bind(tmparr, ind) + + return + end subroutine base_util_sort_index_I8B_I8Bind + + + recursive pure subroutine base_util_sort_qsort_I4B(arr, ind) + !! author: David A. Minton + !! + !! Sort input I4B array by index in ascending numerical order using quicksort. + !! + implicit none + ! Arguments + integer(I4B), dimension(:), intent(inout) :: arr + integer(I4B), dimension(:), intent(out), optional :: ind + ! Internals + integer(I4B) :: iq + + if (size(arr) > 1) then + if (present(ind)) then + call base_util_sort_partition_I4B(arr, iq, ind) + call base_util_sort_qsort_I4B(arr(:iq-1),ind(:iq-1)) + call base_util_sort_qsort_I4B(arr(iq:), ind(iq:)) + else + call base_util_sort_partition_I4B(arr, iq) + call base_util_sort_qsort_I4B(arr(:iq-1)) + call base_util_sort_qsort_I4B(arr(iq:)) + end if + end if + + return + end subroutine base_util_sort_qsort_I4B + + + recursive pure subroutine base_util_sort_qsort_I4B_I8Bind(arr, ind) + !! author: David A. Minton + !! + !! Sort input I4B array by index in ascending numerical order using quicksort. + !! + implicit none + ! Arguments + integer(I4B), dimension(:), intent(inout) :: arr + integer(I8B), dimension(:), intent(out), optional :: ind + ! Internals + integer(I8B) :: iq + + if (size(arr) > 1_I8B) then + if (present(ind)) then + call base_util_sort_partition_I4B_I8Bind(arr, iq, ind) + call base_util_sort_qsort_I4B_I8Bind(arr(:iq-1_I8B),ind(:iq-1_I8B)) + call base_util_sort_qsort_I4B_I8Bind(arr(iq:), ind(iq:)) + else + call base_util_sort_partition_I4B_I8Bind(arr, iq) + call base_util_sort_qsort_I4B_I8Bind(arr(:iq-1_I8B)) + call base_util_sort_qsort_I4B_I8Bind(arr(iq:)) + end if + end if + + return + end subroutine base_util_sort_qsort_I4B_I8Bind + + + recursive pure subroutine base_util_sort_qsort_I8B_I8Bind(arr, ind) + !! author: David A. Minton + !! + !! Sort input I8B array by index in ascending numerical order using quicksort. + !! + implicit none + ! Arguments + integer(I8B), dimension(:), intent(inout) :: arr + integer(I8B), dimension(:), intent(out), optional :: ind + ! Internals + integer(I8B) :: iq + + if (size(arr) > 1_I8B) then + if (present(ind)) then + call base_util_sort_partition_I8B_I8Bind(arr, iq, ind) + call base_util_sort_qsort_I8B_I8Bind(arr(:iq-1_I8B),ind(:iq-1_I8B)) + call base_util_sort_qsort_I8B_I8Bind(arr(iq:), ind(iq:)) + else + call base_util_sort_partition_I8B_I8Bind(arr, iq) + call base_util_sort_qsort_I8B_I8Bind(arr(:iq-1_I8B)) + call base_util_sort_qsort_I8B_I8Bind(arr(iq:)) + end if + end if + + return + end subroutine base_util_sort_qsort_I8B_I8Bind + + + pure subroutine base_util_sort_partition_I4B(arr, marker, ind) + !! author: David A. Minton + !! + !! Partition function for quicksort on I4B type + !! + implicit none + ! Arguments + integer(I4B), intent(inout), dimension(:) :: arr + integer(I4B), intent(inout), dimension(:), optional :: ind + integer(I4B), intent(out) :: marker + ! Internals + integer(I4B) :: i, j, itmp, narr, ipiv + integer(I4B) :: temp + integer(I4B) :: x ! pivot point + + narr = size(arr) + + ! Get center as pivot, as this is likely partially sorted + ipiv = narr / 2 + x = arr(ipiv) + i = 0 + j = narr + 1 + + do + j = j - 1 + do + if (arr(j) <= x) exit + j = j - 1 + end do + i = i + 1 + do + if (arr(i) >= x) exit + i = i + 1 + end do + if (i < j) then + ! exchange A(i) and A(j) + temp = arr(i) + arr(i) = arr(j) + arr(j) = temp + if (present(ind)) then + itmp = ind(i) + ind(i) = ind(j) + ind(j) = itmp + end if + else if (i == j) then + marker = i + 1 + return + else + marker = i + return + endif + end do + + return + end subroutine base_util_sort_partition_I4B + + + pure subroutine base_util_sort_partition_I4B_I8Bind(arr, marker, ind) + !! author: David A. Minton + !! + !! Partition function for quicksort on I4B type + !! + implicit none + ! Arguments + integer(I4B), intent(inout), dimension(:) :: arr + integer(I8B), intent(inout), dimension(:), optional :: ind + integer(I8B), intent(out) :: marker + ! Internals + integer(I8B) :: i, j, itmp, narr, ipiv + integer(I4B) :: temp + integer(I8B) :: x ! pivot point + + narr = size(arr) + + ! Get center as pivot, as this is likely partially sorted + ipiv = narr / 2_I8B + x = arr(ipiv) + i = 0_I8B + j = narr + 1_I8B + + do + j = j - 1_I8B + do + if (arr(j) <= x) exit + j = j - 1_I8B + end do + i = i + 1_I8B + do + if (arr(i) >= x) exit + i = i + 1_I8B + end do + if (i < j) then + ! exchange A(i) and A(j) + temp = arr(i) + arr(i) = arr(j) + arr(j) = temp + if (present(ind)) then + itmp = ind(i) + ind(i) = ind(j) + ind(j) = itmp + end if + else if (i == j) then + marker = i + 1_I8B + return + else + marker = i + return + endif + end do + + return + end subroutine base_util_sort_partition_I4B_I8Bind + + + pure subroutine base_util_sort_partition_I8B_I8Bind(arr, marker, ind) + !! author: David A. Minton + !! + !! Partition function for quicksort on I8B type with I8B index + !! + implicit none + ! Arguments + integer(I8B), intent(inout), dimension(:) :: arr + integer(I8B), intent(inout), dimension(:), optional :: ind + integer(I8B), intent(out) :: marker + ! Internals + integer(I8B) :: i, j, itmp, narr, ipiv + integer(I8B) :: temp + integer(I8B) :: x ! pivot point + + narr = size(arr) + + ! Get center as pivot, as this is likely partially sorted + ipiv = narr / 2_I8B + x = arr(ipiv) + i = 0_I8B + j = narr + 1_I8B + + do + j = j - 1_I8B + do + if (arr(j) <= x) exit + j = j - 1_I8B + end do + i = i + 1_I8B + do + if (arr(i) >= x) exit + i = i + 1_I8B + end do + if (i < j) then + ! exchange A(i) and A(j) + temp = arr(i) + arr(i) = arr(j) + arr(j) = temp + if (present(ind)) then + itmp = ind(i) + ind(i) = ind(j) + ind(j) = itmp + end if + else if (i == j) then + marker = i + 1_I8B + return + else + marker = i + return + endif + end do + + return + end subroutine base_util_sort_partition_I8B_I8Bind + + + pure subroutine base_util_sort_sp(arr) + !! author: David A. Minton + !! + !! Sort input DP precision array in place into ascending numerical order using quicksort. + !! + implicit none + ! Arguments + real(SP), dimension(:), intent(inout) :: arr + + call base_util_sort_qsort_SP(arr) + + return + end subroutine base_util_sort_sp + + + pure subroutine base_util_sort_index_sp(arr, ind) + !! author: David A. Minton + !! + !! Sort input DP precision array by index in ascending numerical order using quicksort. + !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously + !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. + !! + implicit none + ! Arguments + real(SP), dimension(:), intent(in) :: arr + integer(I4B), dimension(:), allocatable, intent(inout) :: ind + ! Internals + integer(I4B) :: n, i + real(SP), dimension(:), allocatable :: tmparr + + n = size(arr) + if (.not.allocated(ind)) then + allocate(ind(n)) + ind = [(i, i=1, n)] + end if + allocate(tmparr, mold=arr) + tmparr(:) = arr(ind(:)) + call base_util_sort_qsort_SP(tmparr, ind) + + return + end subroutine base_util_sort_index_sp + + + recursive pure subroutine base_util_sort_qsort_SP(arr, ind) + !! author: David A. Minton + !! + !! Sort input DP precision array by index in ascending numerical order using quicksort. + !! + implicit none + ! Arguments + real(SP), dimension(:), intent(inout) :: arr + integer(I4B),dimension(:),intent(out), optional :: ind + !! Internals + integer :: iq + + if (size(arr) > 1) then + if (present(ind)) then + call base_util_sort_partition_SP(arr, iq, ind) + call base_util_sort_qsort_SP(arr(:iq-1),ind(:iq-1)) + call base_util_sort_qsort_SP(arr(iq:), ind(iq:)) + else + call base_util_sort_partition_SP(arr, iq) + call base_util_sort_qsort_SP(arr(:iq-1)) + call base_util_sort_qsort_SP(arr(iq:)) + end if + end if + + return + end subroutine base_util_sort_qsort_SP + + + pure subroutine base_util_sort_partition_SP(arr, marker, ind) + !! author: David A. Minton + !! + !! Partition function for quicksort on SP type + !! + implicit none + ! Arguments + real(SP), intent(inout), dimension(:) :: arr + integer(I4B), intent(inout), dimension(:), optional :: ind + integer(I4B), intent(out) :: marker + ! Internals + integer(I4B) :: i, j, itmp, narr, ipiv + real(SP) :: temp + real(SP) :: x ! pivot point + + narr = size(arr) + + ! Get center as pivot, as this is likely partially sorted + ipiv = narr / 2 + x = arr(ipiv) + i = 0 + j = narr + 1 + + do + j = j - 1 + do + if (arr(j) <= x) exit + j = j - 1 + end do + i = i + 1 + do + if (arr(i) >= x) exit + i = i + 1 + end do + if (i < j) then + ! exchange A(i) and A(j) + temp = arr(i) + arr(i) = arr(j) + arr(j) = temp + if (present(ind)) then + itmp = ind(i) + ind(i) = ind(j) + ind(j) = itmp + end if + else if (i == j) then + marker = i + 1 + return + else + marker = i + return + endif + end do + + return + end subroutine base_util_sort_partition_SP + + + pure subroutine base_util_sort_rearrange_arr_char_string(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of character string in-place from an index list. + implicit none + ! Arguments + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary copy of arry used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_char_string + + + pure subroutine base_util_sort_rearrange_arr_DP(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of DP type in-place from an index list. + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + real(DP), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_DP + + + pure subroutine base_util_sort_rearrange_arr_DPvec(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of (NDIM,n) DP-type vectors in-place from an index list. + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + real(DP), dimension(:,:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(:,1:n) = arr(:, ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_DPvec + + + pure subroutine base_util_sort_rearrange_arr_I4B(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of integers in-place from an index list. + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + integer(I4B), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_I4B + + pure subroutine base_util_sort_rearrange_arr_I4B_I8Bind(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of integers in-place from an index list. + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + integer(I4B), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0_I8B) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_I4B_I8Bind + + + pure subroutine base_util_sort_rearrange_arr_logical_I8Bind(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of logicals in-place from an index list. + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + logical, dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_logical_I8Bind + + + pure subroutine base_util_sort_rearrange_arr_logical(arr, ind, n) + !! author: David A. Minton + !! + !! Rearrange a single array of logicals in-place from an index list. + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange + ! Internals + logical, dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation + + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) + tmp(1:n) = arr(ind) + call move_alloc(tmp, arr) + + return + end subroutine base_util_sort_rearrange_arr_logical + + + subroutine base_util_unique_DP(input_array, output_array, index_map) + !! author: David A. Minton + !! + !! Takes an input unsorted integer array and returns a new array of sorted, unique values (DP version) + implicit none + ! Arguments + real(DP), dimension(:), intent(in) :: input_array !! Unsorted input array + real(DP), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values + integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) + ! Internals + real(DP), dimension(:), allocatable :: unique_array + integer(I4B) :: n + real(DP) :: lo, hi + + allocate(unique_array, mold=input_array) + allocate(index_map(size(input_array))) + lo = minval(input_array) - 1 + hi = maxval(input_array) + + n = 0 + do + n = n + 1 + lo = minval(input_array(:), mask=input_array(:) > lo) + unique_array(n) = lo + where(input_array(:) == lo) index_map(:) = n + if (lo >= hi) exit + enddo + allocate(output_array(n), source=unique_array(1:n)) + + return + end subroutine base_util_unique_DP + + + subroutine base_util_unique_I4B(input_array, output_array, index_map) + !! author: David A. Minton + !! + !! Takes an input unsorted integer array and returns a new array of sorted, unique values (I4B version) + implicit none + ! Arguments + integer(I4B), dimension(:), intent(in) :: input_array !! Unsorted input array + integer(I4B), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values + integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) + ! Internals + integer(I4B), dimension(:), allocatable :: unique_array + integer(I4B) :: n, lo, hi + + allocate(unique_array, mold=input_array) + allocate(index_map, mold=input_array) + lo = minval(input_array) - 1 + hi = maxval(input_array) + + n = 0 + do + n = n + 1 + lo = minval(input_array(:), mask=input_array(:) > lo) + unique_array(n) = lo + where(input_array(:) == lo) index_map(:) = n + if (lo >= hi) exit + enddo + allocate(output_array(n), source=unique_array(1:n)) + + return + end subroutine base_util_unique_I4B + + + subroutine base_final_storage(self) + !! author: David A. Minton + !! + !! Finalizer for the storage object + implicit none + ! Arguments + class(base_storage), intent(inout) :: self + + call self%dealloc() + return + end subroutine base_final_storage + + + subroutine base_final_storage_frame(self) + !! author: David A. Minton + !! + !! Finalizer for the storage frame data type + implicit none + type(base_storage_frame) :: self + + if (allocated(self%item)) deallocate(self%item) + + return + end subroutine base_final_storage_frame + +#ifdef COARRAY + subroutine base_coclone_param(self) !! author: David A. Minton !! !! Broadcasts the image 1 parameter to all other images in a parameter coarray @@ -524,7 +2214,6 @@ subroutine base_coclone_param(self) return end subroutine base_coclone_param - #endif diff --git a/src/coarray/coarray_clone.f90 b/src/coarray/coarray_clone.f90 new file mode 100644 index 000000000..4e9a34591 --- /dev/null +++ b/src/coarray/coarray_clone.f90 @@ -0,0 +1,494 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +submodule (coarray) s_coarray_clone + use swiftest +contains + + module subroutine coarray_component_clone_char(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! character scalar version + implicit none + ! Arguments + character(len=*), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + character(len=STRMAX),allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + return + end subroutine coarray_component_clone_char + + + module subroutine coarray_component_clone_DP(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) scalar version + implicit none + ! Arguments + real(DP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP),allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + + return + end subroutine coarray_component_clone_DP + + + module subroutine coarray_component_clone_DP_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) 1D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_clone_DP_arr1D + + + module subroutine coarray_component_clone_DP_arr2D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) 2D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n1[:], n2[:] + logical, allocatable :: isalloc[:] + + allocate(n1[*]) + allocate(n2[*]) + allocate(isalloc[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + end if + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n1[si],n2[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:,:)[img] = var(:,:) + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_clone_DP_arr2D + + + module subroutine coarray_component_clone_DP_vec1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) 1D (NDIM) array version + implicit none + ! Arguments + real(DP), dimension(:), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n[:] + + allocate(n[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + allocate(tmp(NDIM)[*]) + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var(:) + end do + sync images(*) + else + sync images(si) + var(:) = tmp(:)[si] + end if + + deallocate(tmp) + + return + end subroutine coarray_component_clone_DP_vec1D + + + module subroutine coarray_component_clone_DP_vec2D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) 1D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(DP), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + isalloc = allocated(var) + if (isalloc) n = size(var,dim=2) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(NDIM,n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:,:)[img] = var(:,:) + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_clone_DP_vec2D + + + module subroutine coarray_component_clone_I4B(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) scalar version + implicit none + ! Arguments + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I4B),allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + return + end subroutine coarray_component_clone_I4B + + + module subroutine coarray_component_clone_I4B_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) 1D allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I4B), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_clone_I4B_arr1D + + + module subroutine coarray_component_clone_I8B(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! integer(I4B) scalar version + implicit none + ! Arguments + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + integer(I8B),allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + return + end subroutine coarray_component_clone_I8B + + + module subroutine coarray_component_clone_lgt(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! logical scalar version + implicit none + ! Arguments + logical, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + logical,allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + + return + end subroutine coarray_component_clone_lgt + + + module subroutine coarray_component_clone_lgt_arr1D(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! logical 1D allocatable array version + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + logical, dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: img, si + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + isalloc = allocated(var) + if (isalloc) n = size(var) + sync all + if (.not. isalloc[si]) return + + allocate(tmp(n[si])[*]) + if (this_image() == si) then + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) + else + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) + end if + + return + end subroutine coarray_component_clone_lgt_arr1D + + + module subroutine coarray_component_clone_QP(var,src_img) + !! author: David A. Minton + !! + !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 + !! real(DP) scalar version + implicit none + ! Arguments + real(QP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + ! Internals + real(QP),allocatable :: tmp[:] + integer(I4B) :: img, si + + allocate(tmp[*]) + if (present(src_img)) then + si = src_img + else + si = 1 + end if + + !sync all + if (this_image() == si) then + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) + else + sync images(si) + var = tmp[si] + end if + + return + end subroutine coarray_component_clone_QP + +end submodule s_coarray_clone \ No newline at end of file diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 new file mode 100644 index 000000000..f9a65dad1 --- /dev/null +++ b/src/coarray/coarray_collect.f90 @@ -0,0 +1,296 @@ +!! Copyright 2023 - David Minton +!! This file is part of Swiftest. +!! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +!! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +!! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +!! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +!! You should have received a copy of the GNU General Public License along with Swiftest. +!! If not, see: https://www.gnu.org/licenses. + +submodule (coarray) s_coarray_collect + use swiftest +contains + + module subroutine coarray_component_collect_DP_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! real(DP) 1D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + real(DP), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: i,img, ti, di, istart, iend, nmax + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + + sync all + + nmax = 0 + do img = 1, num_images() + if (n[img] > nmax) nmax = n[img] + end do + + allocate(tmp(nmax)[*]) + tmp(1:n) = var(1:n) + + !sync all + if (this_image() == di) then + do img = 1, num_images() + if (img /= di) then + call util_append(var, tmp(:)[img]) + n = n + n[img] + end if + end do + end if + + return + end subroutine coarray_component_collect_DP_arr1D + + + module subroutine coarray_component_collect_DP_arr2D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! real(DP) 2D allocatable array version + implicit none + ! Arguments + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + real(DP), dimension(:,:), codimension[:], allocatable :: tmp + integer(I4B) :: i, img, ti, di, ntot, istart, iend, nmax + integer(I4B), allocatable :: n1[:], n2[:] + logical, allocatable :: isalloc[:] + + allocate(n1[*]) + allocate(n2[*]) + allocate(isalloc[*]) + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + isalloc = allocated(var) + if (isalloc) then + n1 = size(var,dim=1) + n2 = size(var,dim=2) + else + n1 = 0 + n2 = 0 + end if + sync all + + nmax = 0 + do img = 1, num_images() + if (n2[img] > nmax) nmax = n2[img] + end do + + allocate(tmp(NDIM,nmax)[*]) + tmp(:,1:n2) = var(:,1:n2) + + !sync all + if (this_image() == di) then + do img = 1, num_images() + if (img /= di) then + call util_append(var, tmp(:,:)[img]) + n2 = n2 + n2[img] + end if + end do + end if + + return + end subroutine coarray_component_collect_DP_arr2D + + + module subroutine coarray_component_collect_I4B(var,dest_img) + !! author: David A. Minton + !! + !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 + !! integer(I4B) version + implicit none + ! Arguments + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), allocatable :: tmp[:] + integer(I4B) :: img, di + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + allocate(tmp[*], source=var) + + !sync all + if (this_image() == di) then + var = 0 + do img = 1, num_images() + var = var + tmp[img] + end do + else + var = 0 + end if + + return + end subroutine coarray_component_collect_I4B + + + module subroutine coarray_component_collect_I8B(var,dest_img) + !! author: David A. Minton + !! + !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 + !! integer(I8B) version + implicit none + ! Arguments + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I8B), allocatable :: tmp[:] + integer(I4B) :: img, di + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + allocate(tmp[*], source=var) + + !sync all + if (this_image() == di) then + var = 0 + do img = 1, num_images() + var = var + tmp[img] + end do + else + var = 0 + end if + + return + end subroutine coarray_component_collect_I8B + + + module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! integer(I4B) 1D allocatable array version + implicit none + ! Arguments + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + integer(I4B), dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: i,img, ti, di, istart, iend, nmax + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + sync all + nmax = 0 + do img = 1, num_images() + if (n[img] > nmax) nmax = n[img] + end do + + allocate(tmp(nmax)[*]) + tmp(1:n) = var(1:n) + + !sync all + if (this_image() == di) then + do img = 1, num_images() + if (img /= di) then + call util_append(var, tmp(:)[img]) + n = n + n[img] + end if + end do + end if + + return + end subroutine coarray_component_collect_I4B_arr1D + + + module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) + !! author: David A. Minton + !! + !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! logical 1D allocatable array version + implicit none + ! Arguments + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + ! Internals + logical, dimension(:), codimension[:], allocatable :: tmp + integer(I4B) :: i,img, ti, di, ntot, istart, iend + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + + if (present(dest_img)) then + di = dest_img + else + di = 1 + end if + + isalloc = allocated(var) + if (isalloc) then + n = size(var) + else + n = 0 + end if + allocate(tmp[*],source=var) + sync all + + if (this_image() == di) then + do img = 1, num_images() + if (img /= di) then + call util_append(var, tmp(:)[img]) + n = n + n[img] + end if + end do + end if + + return + end subroutine coarray_component_collect_lgt_arr1D + + + +end submodule s_coarray_collect \ No newline at end of file diff --git a/src/coarray/coarray_module.f90 b/src/coarray/coarray_module.f90 index 597945761..b034f8786 100644 --- a/src/coarray/coarray_module.f90 +++ b/src/coarray/coarray_module.f90 @@ -8,871 +8,135 @@ !! If not, see: https://www.gnu.org/licenses. module coarray - !! author: David A. Minton - !! - !! Utilities that are used for coarray test particles - !! - use globals - implicit none - public + !! author: David A. Minton + !! + !! Utilities that are used for coarray test particles + !! + use globals + implicit none + public + + interface coclone + module subroutine coarray_component_clone_char(var,src_img) + implicit none + character(len=*), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_char + + module subroutine coarray_component_clone_DP(var,src_img) + implicit none + real(DP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_DP + + module subroutine coarray_component_clone_DP_arr1D(var,src_img) + implicit none + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_DP_arr1D + + module subroutine coarray_component_clone_DP_arr2D(var,src_img) + implicit none + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_DP_arr2D + + module subroutine coarray_component_clone_I4B(var,src_img) + implicit none + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_I4B + + module subroutine coarray_component_clone_I4B_arr1D(var,src_img) + implicit none + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_I4B_arr1D + + module subroutine coarray_component_clone_I8B(var,src_img) + implicit none + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_I8B + + module subroutine coarray_component_clone_lgt(var,src_img) + implicit none + logical, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_lgt + + module subroutine coarray_component_clone_lgt_arr1D(var,src_img) + implicit none + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_lgt_arr1D + + module subroutine coarray_component_clone_QP(var,src_img) + implicit none + real(QP), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_QP + end interface + + + interface coclonevec + module subroutine coarray_component_clone_DP_vec1D(var,src_img) + implicit none + real(DP), dimension(:), intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_DP_vec1D + + module subroutine coarray_component_clone_DP_vec2D(var,src_img) + implicit none + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: src_img + end subroutine coarray_component_clone_DP_vec2D + end interface coclonevec + + + interface cocollect + module subroutine coarray_component_collect_DP_arr1D(var,dest_img) + implicit none + real(DP), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_DP_arr1D + + + module subroutine coarray_component_collect_DP_arr2D(var,dest_img) + implicit none + real(DP), dimension(:,:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_DP_arr2D + + + module subroutine coarray_component_collect_I4B(var,dest_img) + implicit none + integer(I4B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_I4B + + + module subroutine coarray_component_collect_I8B(var,dest_img) + implicit none + integer(I8B), intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_I8B + + + module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) + implicit none + integer(I4B), dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_I4B_arr1D + + + module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) + implicit none + logical, dimension(:), allocatable, intent(inout) :: var + integer(I4B), intent(in),optional :: dest_img + end subroutine coarray_component_collect_lgt_arr1D + end interface - interface coclone - module procedure coarray_component_clone_char - module procedure coarray_component_clone_DP - module procedure coarray_component_clone_DP_arr1D - module procedure coarray_component_clone_DP_arr2D - module procedure coarray_component_clone_I4B - module procedure coarray_component_clone_I4B_arr1D - module procedure coarray_component_clone_I4B_arr2D - module procedure coarray_component_clone_I8B - module procedure coarray_component_clone_lgt - module procedure coarray_component_clone_lgt_arr1D - module procedure coarray_component_clone_QP - end interface - - interface cocollect - module procedure coarray_component_collect_DP_arr1D - module procedure coarray_component_collect_DP_arr2D - module procedure coarray_component_collect_I4B - module procedure coarray_component_collect_I4B_arr1D - module procedure coarray_component_collect_I4B_arr2D - module procedure coarray_component_collect_I8B - module procedure coarray_component_collect_lgt_arr1D - end interface - - contains - - subroutine coarray_component_clone_char(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! character scalar version - implicit none - ! Arguments - character(len=*), intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - character(len=STRMAX),save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - return - end subroutine coarray_component_clone_char - - - subroutine coarray_component_clone_DP(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! real(DP) scalar version - implicit none - ! Arguments - real(DP), intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - real(DP),save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - - return - end subroutine coarray_component_clone_DP - - - subroutine coarray_component_clone_DP_arr1D(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! real(DP) 1D allocatable array version - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - real(DP), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) n = size(var) - sync all - if (.not. isalloc[si]) return - - allocate(tmp(n[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) - else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - end if - - return - end subroutine coarray_component_clone_DP_arr1D - - - subroutine coarray_component_clone_DP_arr2D(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! real(DP) 2D allocatable array version - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - real(DP), dimension(:,:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si - integer(I4B), save :: n1[*], n2[*] - logical, save :: isalloc[*] - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n1 = size(var,dim=1) - n2 = size(var,dim=2) - end if - sync all - if (.not. isalloc[si]) return - - allocate(tmp(n1[si],n2[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:,:)[img] = var(:,:) - end do - sync images(*) - else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - end if - - return - end subroutine coarray_component_clone_DP_arr2D - - - subroutine coarray_component_clone_I4B(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! integer(I4B) scalar version - implicit none - ! Arguments - integer(I4B), intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - integer(I4B),save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - return - end subroutine coarray_component_clone_I4B - - - subroutine coarray_component_clone_I4B_arr1D(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! integer(I4B) 1D allocatable array version - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - integer(I4B), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) n = size(var) - sync all - if (.not. isalloc[si]) return - - allocate(tmp(n[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) - else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - end if - - return - end subroutine coarray_component_clone_I4B_arr1D - - - subroutine coarray_component_clone_I4B_arr2D(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! integer(I4B) 2D allocatable array version - implicit none - ! Arguments - integer(I4B), dimension(:,:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si - integer(I4B), save :: n1[*], n2[*] - logical, save :: isalloc[*] - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n1 = size(var,dim=1) - n2 = size(var,dim=2) - end if - sync all - if (.not. isalloc[si]) return - - allocate(tmp(n1[si],n2[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:,:)[img] = var(:,:) - end do - sync images(*) - else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - end if - - return - end subroutine coarray_component_clone_I4B_arr2D - - - subroutine coarray_component_clone_I8B(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! integer(I4B) scalar version - implicit none - ! Arguments - integer(I8B), intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - integer(I8B),save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - return - end subroutine coarray_component_clone_I8B - - - subroutine coarray_component_clone_lgt(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! logical scalar version - implicit none - ! Arguments - logical, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - logical,save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - - return - end subroutine coarray_component_clone_lgt - - - subroutine coarray_component_clone_lgt_arr1D(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! logical 1D allocatable array version - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - logical, dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) n = size(var) - sync all - if (.not. isalloc[si]) return - - allocate(tmp(n[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) - else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) - end if - - return - end subroutine coarray_component_clone_lgt_arr1D - - - subroutine coarray_component_clone_QP(var,src_img) - !! author: David A. Minton - !! - !! Copies a component of a coarray derived type from the specified source image to the current local one. The default source image is 1 - !! real(DP) scalar version - implicit none - ! Arguments - real(QP), intent(inout) :: var - integer(I4B), intent(in),optional :: src_img - ! Internals - real(QP),save :: tmp[*] - integer(I4B) :: img, si - - if (present(src_img)) then - si = src_img - else - si = 1 - end if - - sync all - if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) - else - sync images(si) - var = tmp[si] - end if - - return - end subroutine coarray_component_clone_QP - - - subroutine coarray_component_collect_DP_arr1D(var,dest_img) - !! author: David A. Minton - !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 - !! real(DP) 1D allocatable array version - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - real(DP), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n = size(var) - else - n = 0 - end if - sync all - ntot = 0 - do img = 1, num_images() - ntot = ntot + n[img] - end do - - allocate(tmp(ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n - do img = 1, this_image() - 1 - istart = istart + n[img] - iend = iend + n[img] - end do - - if (isalloc) then - tmp(istart:iend) = var(:) - deallocate(var) - end if - - sync all - if (this_image() == di) then - allocate(var(ntot)) - istart = 1 - iend = n - do img = 1, num_images() - var(istart:iend) = tmp[img](istart:iend) - istart = istart + n[img] - iend = iend + n[img] - end do - end if - - return - end subroutine coarray_component_collect_DP_arr1D - - - subroutine coarray_component_collect_DP_arr2D(var,dest_img) - !! author: David A. Minton - !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 - !! real(DP) 2D allocatable array version - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n1[*], n2[*] - logical, save :: isalloc[*] - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n1 = size(var,dim=1) - n2 = size(var,dim=2) - else - n1 = 0 - n2 = 0 - end if - sync all - ntot = 0 - do img = 1, num_images() - ntot = ntot + n2[img] - end do - - allocate(tmp(n1,ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n2 - do img = 1, this_image() - 1 - istart = istart + n2[img] - iend = iend + n2[img] - end do - - if (isalloc) then - tmp(:,istart:iend) = var(:,:) - deallocate(var) - end if - - sync all - if (this_image() == di) then - allocate(var(n1,ntot)) - - istart = 1 - iend = n2 - do img = 1, num_images() - var(:,istart:iend) = tmp[img](:,istart:iend) - istart = istart + n2[img] - iend = iend + n2[img] - end do - end if - - return - end subroutine coarray_component_collect_DP_arr2D - - - subroutine coarray_component_collect_I4B(var,dest_img) - !! author: David A. Minton - !! - !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 - !! integer(I4B) version - implicit none - ! Arguments - integer(I4B), intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - integer(I4B), allocatable :: tmp[:] - integer(I4B) :: img, di - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - allocate(tmp[*], source=var) - sync all - - if (this_image() == di) then - var = 0 - do img = 1, num_images() - var = var + tmp[img] - end do - else - var = 0 - end if - - return - end subroutine coarray_component_collect_I4B - - - subroutine coarray_component_collect_I8B(var,dest_img) - !! author: David A. Minton - !! - !! Sums this component of a coarray derived type from all images and places the value in the destination image component. The default destination image is 1 - !! integer(I8B) version - implicit none - ! Arguments - integer(I8B), intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - integer(I8B), allocatable :: tmp[:] - integer(I4B) :: img, di - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - allocate(tmp[*], source=var) - sync all - - if (this_image() == di) then - var = 0 - do img = 1, num_images() - var = var + tmp[img] - end do - else - var = 0 - end if - - return - end subroutine coarray_component_collect_I8B - - - subroutine coarray_component_collect_I4B_arr1D(var,dest_img) - !! author: David A. Minton - !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 - !! integer(I4B) 1D allocatable array version - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - integer(I4B), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n = size(var) - else - n = 0 - end if - sync all - ntot = 0 - do img = 1, num_images() - ntot = ntot + n[img] - end do - - allocate(tmp(ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n - do img = 1, this_image() - 1 - istart = istart + n[img] - iend = iend + n[img] - end do - - if (isalloc) then - tmp(istart:iend) = var(:) - deallocate(var) - end if - - sync all - if (this_image() == di) then - allocate(var(ntot)) - istart = 1 - iend = n - do img = 1, num_images() - var(istart:iend) = tmp[img](istart:iend) - istart = istart + n[img] - iend = iend + n[img] - end do - end if - - return - end subroutine coarray_component_collect_I4B_arr1D - - - subroutine coarray_component_collect_I4B_arr2D(var,dest_img) - !! author: David A. Minton - !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 - !! integer(I4B) 2D allocatable array version - implicit none - ! Arguments - integer(I4B), dimension(:,:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - integer(I4B), dimension(:,:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n1[*], n2[*] - logical, save :: isalloc[*] - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n1 = size(var,dim=1) - n2 = size(var,dim=2) - else - n1 = 0 - n2 = 0 - end if - sync all - ntot = 0 - do img = 1, num_images() - ntot = ntot + n2[img] - end do - - allocate(tmp(n1,ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n2 - do img = 1, this_image() - 1 - istart = istart + n2[img] - iend = iend + n2[img] - end do - - if (isalloc) then - tmp(:,istart:iend) = var(:,:) - deallocate(var) - end if - - sync all - if (this_image() == di) then - allocate(var(n1,ntot)) - - istart = 1 - iend = n2 - do img = 1, num_images() - var(:,istart:iend) = tmp[img](:,istart:iend) - istart = istart + n2[img] - iend = iend + n2[img] - end do - end if - - return - end subroutine coarray_component_collect_I4B_arr2D - - - subroutine coarray_component_collect_lgt_arr1D(var,dest_img) - !! author: David A. Minton - !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 - !! logical 1D allocatable array version - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: var - integer(I4B), intent(in),optional :: dest_img - ! Internals - logical, dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - - if (present(dest_img)) then - di = dest_img - else - di = 1 - end if - - sync all - isalloc = allocated(var) - if (isalloc) then - n = size(var) - else - n = 0 - end if - sync all - ntot = 0 - do img = 1, num_images() - ntot = ntot + n[img] - end do - - allocate(tmp(ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n - do img = 1, this_image() - 1 - istart = istart + n[img] - iend = iend + n[img] - end do - - if (isalloc) then - tmp(istart:iend) = var(:) - deallocate(var) - end if - - sync all - if (this_image() == di) then - allocate(var(ntot)) - istart = 1 - iend = n - do img = 1, num_images() - var(istart:iend) = tmp[img](istart:iend) - istart = istart + n[img] - iend = iend + n[img] - end do - end if - - return - end subroutine coarray_component_collect_lgt_arr1D end module coarray \ No newline at end of file diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 9eeb1746f..95152c0aa 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -227,7 +227,7 @@ module subroutine collision_util_index_map(self) call self%get_index_values(idvals, tvals) ! Consolidate ids to only unique values - call swiftest_util_unique(idvals,self%idvals,self%idmap) + call util_unique(idvals,self%idvals,self%idmap) self%nid = size(self%idvals) ! Don't consolidate time values (multiple collisions can happen in a single time step) diff --git a/src/encounter/encounter_check.f90 b/src/encounter/encounter_check.f90 index c2013ce6b..06d0a76d4 100644 --- a/src/encounter/encounter_check.f90 +++ b/src/encounter/encounter_check.f90 @@ -103,10 +103,10 @@ module subroutine encounter_check_all_plplm(param, nplm, nplt, rplm, vplm, rplt, call move_alloc(ltmp, lvdotr) nenc = nenc + plmplt_nenc - call swiftest_util_sort(index1, ind) - call swiftest_util_sort_rearrange(index1, ind, nenc) - call swiftest_util_sort_rearrange(index2, ind, nenc) - call swiftest_util_sort_rearrange(lvdotr, ind, nenc) + call util_sort(index1, ind) + call util_sort_rearrange(index1, ind, nenc) + call util_sort_rearrange(index2, ind, nenc) + call util_sort_rearrange(lvdotr, ind, nenc) end if @@ -677,10 +677,10 @@ subroutine encounter_check_remove_duplicates(n, nenc, index1, index2, lvdotr) return end if - call swiftest_util_sort(index1, ind) - call swiftest_util_sort_rearrange(index1, ind, nenc) - call swiftest_util_sort_rearrange(index2, ind, nenc) - call swiftest_util_sort_rearrange(lvdotr, ind, nenc) + call util_sort(index1, ind) + call util_sort_rearrange(index1, ind, nenc) + call util_sort_rearrange(index2, ind, nenc) + call util_sort_rearrange(lvdotr, ind, nenc) ! Get the bounds on the bodies in the first index ibeg(:) = n @@ -706,7 +706,7 @@ subroutine encounter_check_remove_duplicates(n, nenc, index1, index2, lvdotr) khi = iend(i) nenci = khi - klo + 1_I8B if (allocated(ind)) deallocate(ind) - call swiftest_util_sort(index2(klo:khi), ind) + call util_sort(index2(klo:khi), ind) index2(klo:khi) = itmp(klo - 1_I8B + ind(:)) do j = klo + 1_I8B, khi if (index2(j) == index2(j - 1_I8B)) lencounter(j) = .false. @@ -746,7 +746,7 @@ pure module subroutine encounter_check_sort_aabb_1D(self, n, extent_arr) ! Internals integer(I8B) :: i, k - call swiftest_util_sort(extent_arr, self%ind) + call util_sort(extent_arr, self%ind) do concurrent(k = 1_I8B:2_I8B * n) i = self%ind(k) diff --git a/src/encounter/encounter_module.f90 b/src/encounter/encounter_module.f90 index f3c24c763..bcae2bd15 100644 --- a/src/encounter/encounter_module.f90 +++ b/src/encounter/encounter_module.f90 @@ -259,9 +259,9 @@ end subroutine encounter_util_setup_list module subroutine encounter_util_append_list(self, source, lsource_mask) implicit none - class(encounter_list), intent(inout) :: self !! Swiftest encounter list object - class(encounter_list), intent(in) :: source !! Source object to append - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + class(encounter_list), intent(inout) :: self !! Swiftest encounter list object + class(encounter_list), intent(in) :: source !! Source object to append + logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to end subroutine encounter_util_append_list module subroutine encounter_util_copy_list(self, source) diff --git a/src/encounter/encounter_util.f90 b/src/encounter/encounter_util.f90 index 01848d571..f935afe5a 100644 --- a/src/encounter/encounter_util.f90 +++ b/src/encounter/encounter_util.f90 @@ -18,28 +18,27 @@ module subroutine encounter_util_append_list(self, source, lsource_mask) !! This method will automatically resize the destination body if it is too small implicit none ! Arguments - class(encounter_list), intent(inout) :: self !! Swiftest encounter list object - class(encounter_list), intent(in) :: source !! Source object to append - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + class(encounter_list), intent(inout) :: self !! Swiftest encounter list object + class(encounter_list), intent(in) :: source !! Source object to append + logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nold, nsrc nold = int(self%nenc, kind=I4B) - nsrc = int(source%nenc, kind=I4B) - call swiftest_util_append(self%tcollision, source%tcollision, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lclosest, source%lclosest, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lvdotr, source%lvdotr, nold, nsrc, lsource_mask) - call swiftest_util_append(self%status, source%status, nold, nsrc, lsource_mask) - call swiftest_util_append(self%index1, source%index1, nold, nsrc, lsource_mask) - call swiftest_util_append(self%index2, source%index2, nold, nsrc, lsource_mask) - call swiftest_util_append(self%id1, source%id1, nold, nsrc, lsource_mask) - call swiftest_util_append(self%id2, source%id2, nold, nsrc, lsource_mask) - call swiftest_util_append(self%r1, source%r1, nold, nsrc, lsource_mask) - call swiftest_util_append(self%r2, source%r2, nold, nsrc, lsource_mask) - call swiftest_util_append(self%v1, source%v1, nold, nsrc, lsource_mask) - call swiftest_util_append(self%v2, source%v2, nold, nsrc, lsource_mask) - call swiftest_util_append(self%level, source%level, nold, nsrc, lsource_mask) - self%nenc = nold + count(lsource_mask(1:nsrc)) + call util_append(self%tcollision, source%tcollision, nold, lsource_mask) + call util_append(self%lclosest, source%lclosest, nold, lsource_mask) + call util_append(self%lvdotr, source%lvdotr, nold, lsource_mask) + call util_append(self%status, source%status, nold, lsource_mask) + call util_append(self%index1, source%index1, nold, lsource_mask) + call util_append(self%index2, source%index2, nold, lsource_mask) + call util_append(self%id1, source%id1, nold, lsource_mask) + call util_append(self%id2, source%id2, nold, lsource_mask) + call util_append(self%r1, source%r1, nold, lsource_mask) + call util_append(self%r2, source%r2, nold, lsource_mask) + call util_append(self%v1, source%v1, nold, lsource_mask) + call util_append(self%v2, source%v2, nold, lsource_mask) + call util_append(self%level, source%level, nold, lsource_mask) + self%nenc = nold + count(lsource_mask(:)) return end subroutine encounter_util_append_list @@ -283,11 +282,11 @@ module subroutine encounter_util_index_map(self) call encounter_util_get_vals_storage(self, idvals, tvals) ! Consolidate ids to only unique values - call swiftest_util_unique(idvals,self%idvals,self%idmap) + call util_unique(idvals,self%idvals,self%idmap) self%nid = size(self%idvals) ! Consolidate time values to only unique values - call swiftest_util_unique(tvals,self%tvals,self%tmap) + call util_unique(tvals,self%tvals,self%tmap) self%nt = size(self%tvals) return @@ -431,19 +430,19 @@ module subroutine encounter_util_spill_list(self, discards, lspill_list, ldestru integer(I8B) :: nenc_old associate(keeps => self) - call swiftest_util_spill(keeps%tcollision, discards%tcollision, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lvdotr, discards%lvdotr, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lclosest, discards%lclosest, lspill_list, ldestructive) - call swiftest_util_spill(keeps%status, discards%status, lspill_list, ldestructive) - call swiftest_util_spill(keeps%index1, discards%index1, lspill_list, ldestructive) - call swiftest_util_spill(keeps%index2, discards%index2, lspill_list, ldestructive) - call swiftest_util_spill(keeps%id1, discards%id1, lspill_list, ldestructive) - call swiftest_util_spill(keeps%id2, discards%id2, lspill_list, ldestructive) - call swiftest_util_spill(keeps%r1, discards%r1, lspill_list, ldestructive) - call swiftest_util_spill(keeps%r2, discards%r2, lspill_list, ldestructive) - call swiftest_util_spill(keeps%v1, discards%v1, lspill_list, ldestructive) - call swiftest_util_spill(keeps%v2, discards%v2, lspill_list, ldestructive) - call swiftest_util_spill(keeps%level, discards%level, lspill_list, ldestructive) + call util_spill(keeps%tcollision, discards%tcollision, lspill_list, ldestructive) + call util_spill(keeps%lvdotr, discards%lvdotr, lspill_list, ldestructive) + call util_spill(keeps%lclosest, discards%lclosest, lspill_list, ldestructive) + call util_spill(keeps%status, discards%status, lspill_list, ldestructive) + call util_spill(keeps%index1, discards%index1, lspill_list, ldestructive) + call util_spill(keeps%index2, discards%index2, lspill_list, ldestructive) + call util_spill(keeps%id1, discards%id1, lspill_list, ldestructive) + call util_spill(keeps%id2, discards%id2, lspill_list, ldestructive) + call util_spill(keeps%r1, discards%r1, lspill_list, ldestructive) + call util_spill(keeps%r2, discards%r2, lspill_list, ldestructive) + call util_spill(keeps%v1, discards%v1, lspill_list, ldestructive) + call util_spill(keeps%v2, discards%v2, lspill_list, ldestructive) + call util_spill(keeps%level, discards%level, lspill_list, ldestructive) nenc_old = keeps%nenc diff --git a/src/fraggle/fraggle_util.f90 b/src/fraggle/fraggle_util.f90 index 14a3b69df..3b0c42d6a 100644 --- a/src/fraggle/fraggle_util.f90 +++ b/src/fraggle/fraggle_util.f90 @@ -188,8 +188,8 @@ module subroutine fraggle_util_set_mass_dist(self, param) end if ! Sort the distribution in descending order by mass so that the largest fragment is always the first - call swiftest_util_sort(-mass, ind) - call swiftest_util_sort_rearrange(mass, ind, nfrag) + call util_sort(-mass, ind) + call util_sort_rearrange(mass, ind, nfrag) call move_alloc(mass, fragments%mass) fragments%Gmass(:) = G * fragments%mass(:) diff --git a/src/rmvs/rmvs_coarray.f90 b/src/rmvs/rmvs_coarray.f90 index 61c638c60..6f7467df8 100644 --- a/src/rmvs/rmvs_coarray.f90 +++ b/src/rmvs/rmvs_coarray.f90 @@ -31,6 +31,23 @@ module subroutine rmvs_coarray_coclone_cb(self) end subroutine rmvs_coarray_coclone_cb + module subroutine rmvs_coarray_coclone_interp(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_interp),intent(inout),codimension[*] :: self !! RMVS pl object + + call coclone(self%x) + call coclone(self%v) + call coclone(self%aobl) + call coclone(self%atide) + + return + end subroutine rmvs_coarray_coclone_interp + + module subroutine rmvs_coarray_coclone_pl(self) !! author: David A. Minton !! @@ -52,6 +69,26 @@ end subroutine rmvs_coarray_coclone_pl + module subroutine rmvs_coarray_coclone_system(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(rmvs_nbody_system),intent(inout),codimension[*] :: self !! Swiftest body object + ! Internals + integer(I4B) :: i, img + + call coclone(self%lplanetocentric) + call coclone(self%rts) + call coclone(self%vbeg) + + call swiftest_coarray_coclone_system(self) + + return + end subroutine rmvs_coarray_coclone_system + + module subroutine rmvs_coarray_coclone_tp(self) !! author: David A. Minton !! @@ -84,10 +121,9 @@ module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) integer(I4B), intent(in),optional :: src_img ! Internals type(rmvs_interp), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si + integer(I4B) :: i,img, si integer(I4B), save :: n[*] logical, save :: isalloc[*] - if (present(src_img)) then si = src_img @@ -102,10 +138,14 @@ module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) if (.not. isalloc[si]) return allocate(tmp(n[si])[*]) + do i = 1, n[si] + call tmp(i)%coclone() + end do if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var + tmp(:)[img] = var(:) end do + sync images(*) else sync images(si) @@ -128,8 +168,6 @@ module subroutine rmvs_coarray_cocollect_tp(self) call cocollect(self%lperi) call cocollect(self%plperP) call cocollect(self%plencP) - call cocollect(self%index) - call cocollect(self%ipleP) call swiftest_coarray_cocollect_tp(self) diff --git a/src/rmvs/rmvs_module.f90 b/src/rmvs/rmvs_module.f90 index 0f4c0f4b1..b219fd01b 100644 --- a/src/rmvs/rmvs_module.f90 +++ b/src/rmvs/rmvs_module.f90 @@ -35,6 +35,9 @@ module rmvs procedure :: dealloc => rmvs_util_dealloc_system !! Performs RMVS-specific deallocation procedure :: initialize => rmvs_util_setup_initialize_system !! Performs RMVS-specific initilization steps, including generating the close encounter planetocentric structures procedure :: step => rmvs_step_system !! Advance the RMVS nbody system forward in time by one step +#ifdef COARRAY + procedure :: coclone => rmvs_coarray_coclone_system !! Clones the image 1 body object to all other images in the coarray structure. +#endif end type rmvs_nbody_system type, private :: rmvs_interp @@ -45,6 +48,9 @@ module rmvs contains procedure :: dealloc => rmvs_util_dealloc_interp !! Deallocates all allocatable arrays final :: rmvs_final_interp !! Finalizes the RMVS interpolated nbody_system variables object - deallocates all allocatables +#ifdef COARRAY + procedure :: coclone => rmvs_coarray_coclone_interp +#endif end type rmvs_interp @@ -93,6 +99,7 @@ module rmvs final :: rmvs_final_tp !! Finalizes the RMVS test particle object - deallocates all allocatables #ifdef COARRAY procedure :: coclone => rmvs_coarray_coclone_tp !! Clones the image 1 body object to all other images in the coarray structure. + procedure :: cocollect => rmvs_coarray_cocollect_tp !! Clones the image 1 body object to all other images in the coarray structure. #endif end type rmvs_tp @@ -308,11 +315,21 @@ module subroutine rmvs_coarray_coclone_cb(self) class(rmvs_cb),intent(inout),codimension[*] :: self !! RMVS tp object end subroutine rmvs_coarray_coclone_cb + module subroutine rmvs_coarray_coclone_interp(self) + implicit none + class(rmvs_interp),intent(inout),codimension[*] :: self !! RMVS tp object + end subroutine rmvs_coarray_coclone_interp + module subroutine rmvs_coarray_coclone_pl(self) implicit none class(rmvs_pl),intent(inout),codimension[*] :: self !! RMVS pl object end subroutine rmvs_coarray_coclone_pl + module subroutine rmvs_coarray_coclone_system(self) + implicit none + class(rmvs_nbody_system),intent(inout),codimension[*] :: self !! RMVS nbody system object + end subroutine rmvs_coarray_coclone_system + module subroutine rmvs_coarray_coclone_tp(self) implicit none class(rmvs_tp),intent(inout),codimension[*] :: self !! RMVS tp object diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index 7100f7019..224ba2cd1 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -24,19 +24,17 @@ module subroutine rmvs_util_append_pl(self, source, lsource_mask) select type(source) class is (rmvs_pl) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%nenc, source%nenc, nold, nsrc, lsource_mask) - call swiftest_util_append(self%tpenc1P, source%tpenc1P, nold, nsrc, lsource_mask) - call swiftest_util_append(self%plind, source%plind, nold, nsrc, lsource_mask) + call util_append(self%nenc, source%nenc, lsource_mask=lsource_mask) + call util_append(self%tpenc1P, source%tpenc1P, lsource_mask=lsource_mask) + call util_append(self%plind, source%plind, lsource_mask=lsource_mask) - ! The following are not implemented as RMVS doesn't make use of fill operations on pl type - ! So they are here as a placeholder in case someone wants to extend the RMVS class for some reason - !call swiftest_util_append(self%outer, source%outer, nold, nsrc, lsource_mask) - !call swiftest_util_append(self%inner, source%inner, nold, nsrc, lsource_mask) - !call swiftest_util_append(self%planetocentric, source%planetocentric, nold, nsrc, lsource_mask) + ! The following are not implemented as RMVS doesn't make use of fill operations on pl type + ! So they are here as a placeholder in case someone wants to extend the RMVS class for some reason + !call util_append(self%outer, source%outer, lsource_mask=lsource_mask) + !call util_append(self%inner, source%inner, lsource_mask=lsource_mask) + !call util_append(self%planetocentric, source%planetocentric, lsource_mask) - call whm_util_append_pl(self, source, lsource_mask) - end associate + call whm_util_append_pl(self, source, lsource_mask) class default write(*,*) "Invalid object passed to the append method. Source must be of class rmvs_pl or its descendents!" call base_util_exit(FAILURE) @@ -59,13 +57,11 @@ module subroutine rmvs_util_append_tp(self, source, lsource_mask) select type(source) class is (rmvs_tp) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%lperi, source%lperi, nold, nsrc, lsource_mask) - call swiftest_util_append(self%plperP, source%plperP, nold, nsrc, lsource_mask) - call swiftest_util_append(self%plencP, source%plencP, nold, nsrc, lsource_mask) + call util_append(self%lperi, source%lperi, lsource_mask=lsource_mask) + call util_append(self%plperP, source%plperP, lsource_mask=lsource_mask) + call util_append(self%plencP, source%plencP, lsource_mask=lsource_mask) - call swiftest_util_append_tp(self, source, lsource_mask) ! Note: whm_tp does not have its own append method, so we skip back to the base class - end associate + call swiftest_util_append_tp(self, source, lsource_mask) ! Note: whm_tp does not have its own append method, so we skip back to the base class class default write(*,*) "Invalid object passed to the append method. Source must be of class rmvs_tp or its descendents!" call base_util_exit(FAILURE) @@ -177,15 +173,15 @@ module subroutine rmvs_util_fill_pl(self, inserts, lfill_list) associate(keeps => self) select type(inserts) class is (rmvs_pl) - call swiftest_util_fill(keeps%nenc, inserts%nenc, lfill_list) - call swiftest_util_fill(keeps%tpenc1P, inserts%tpenc1P, lfill_list) - call swiftest_util_fill(keeps%plind, inserts%plind, lfill_list) + call util_fill(keeps%nenc, inserts%nenc, lfill_list) + call util_fill(keeps%tpenc1P, inserts%tpenc1P, lfill_list) + call util_fill(keeps%plind, inserts%plind, lfill_list) ! The following are not implemented as RMVS doesn't make use of fill operations on pl type ! So they are here as a placeholder in case someone wants to extend the RMVS class for some reason - !call swiftest_util_fill(keeps%outer, inserts%outer, lfill_list) - !call swiftest_util_fill(keeps%inner, inserts%inner, lfill_list) - !call swiftest_util_fill(keeps%planetocentric, inserts%planetocentric, lfill_list) + !call util_fill(keeps%outer, inserts%outer, lfill_list) + !call util_fill(keeps%inner, inserts%inner, lfill_list) + !call util_fill(keeps%planetocentric, inserts%planetocentric, lfill_list) call whm_util_fill_pl(keeps, inserts, lfill_list) class default @@ -213,9 +209,9 @@ module subroutine rmvs_util_fill_tp(self, inserts, lfill_list) associate(keeps => self) select type(inserts) class is (rmvs_tp) - call swiftest_util_fill(keeps%lperi, inserts%lperi, lfill_list) - call swiftest_util_fill(keeps%plperP, inserts%plperP, lfill_list) - call swiftest_util_fill(keeps%plencP, inserts%plencP, lfill_list) + call util_fill(keeps%lperi, inserts%lperi, lfill_list) + call util_fill(keeps%plperP, inserts%plperP, lfill_list) + call util_fill(keeps%plencP, inserts%plencP, lfill_list) call swiftest_util_fill_tp(keeps, inserts, lfill_list) ! Note: whm_tp does not have its own fill method, so we skip back to the base class class default @@ -237,15 +233,15 @@ module subroutine rmvs_util_resize_pl(self, nnew) class(rmvs_pl), intent(inout) :: self !! RMVS massive body object integer(I4B), intent(in) :: nnew !! New size neded - call swiftest_util_resize(self%nenc, nnew) - call swiftest_util_resize(self%tpenc1P, nnew) - call swiftest_util_resize(self%plind, nnew) + call util_resize(self%nenc, nnew) + call util_resize(self%tpenc1P, nnew) + call util_resize(self%plind, nnew) ! The following are not implemented as RMVS doesn't make use of resize operations on pl type ! So they are here as a placeholder in case someone wants to extend the RMVS class for some reason - !call swiftest_util_resize(self%outer, nnew) - !call swiftest_util_resize(self%inner, nnew) - !call swiftest_util_resize(self%planetocentric, nnew) + !call util_resize(self%outer, nnew) + !call util_resize(self%inner, nnew) + !call util_resize(self%planetocentric, nnew) call whm_util_resize_pl(self, nnew) return @@ -261,10 +257,10 @@ module subroutine rmvs_util_resize_tp(self, nnew) class(rmvs_tp), intent(inout) :: self !! RMVS test particle object integer(I4B), intent(in) :: nnew !! New size neded - call swiftest_util_resize(self%lperi, nnew) - call swiftest_util_resize(self%plperP, nnew) - call swiftest_util_resize(self%plencP, nnew) - call swiftest_util_resize(self%rheliocentric, nnew) + call util_resize(self%lperi, nnew) + call util_resize(self%plperP, nnew) + call util_resize(self%plencP, nnew) + call util_resize(self%rheliocentric, nnew) call swiftest_util_resize_tp(self, nnew) @@ -453,11 +449,11 @@ module subroutine rmvs_util_sort_pl(self, sortby, ascending) associate(pl => self, npl => self%nbody) select case(sortby) case("nenc") - call swiftest_util_sort(direction * pl%nenc(1:npl), ind) + call util_sort(direction * pl%nenc(1:npl), ind) case("tpenc1P") - call swiftest_util_sort(direction * pl%tpenc1P(1:npl), ind) + call util_sort(direction * pl%tpenc1P(1:npl), ind) case("plind") - call swiftest_util_sort(direction * pl%plind(1:npl), ind) + call util_sort(direction * pl%plind(1:npl), ind) case("outer", "inner", "planetocentric", "lplanetocentric") write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' case default ! Look for components in the parent class @@ -497,9 +493,9 @@ module subroutine rmvs_util_sort_tp(self, sortby, ascending) associate(tp => self, ntp => self%nbody) select case(sortby) case("plperP") - call swiftest_util_sort(direction * tp%plperP(1:ntp), ind) + call util_sort(direction * tp%plperP(1:ntp), ind) case("plencP") - call swiftest_util_sort(direction * tp%plencP(1:ntp), ind) + call util_sort(direction * tp%plencP(1:ntp), ind) case("lperi", "cb_heliocentric", "rheliocentric", "index", "ipleP", "lplanetocentric") write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' case default ! Look for components in the parent class (*NOTE whm_tp does not need its own sort method, so we go straight to the swiftest_tp method) @@ -526,9 +522,9 @@ module subroutine rmvs_util_sort_rearrange_pl(self, ind) if (self%nbody == 0) return associate(pl => self, npl => self%nbody) - call swiftest_util_sort_rearrange(pl%nenc, ind, npl) - call swiftest_util_sort_rearrange(pl%tpenc1P, ind, npl) - call swiftest_util_sort_rearrange(pl%plind, ind, npl) + call util_sort_rearrange(pl%nenc, ind, npl) + call util_sort_rearrange(pl%tpenc1P, ind, npl) + call util_sort_rearrange(pl%plind, ind, npl) call swiftest_util_sort_rearrange_pl(pl,ind) end associate @@ -549,10 +545,10 @@ module subroutine rmvs_util_sort_rearrange_tp(self, ind) if (self%nbody == 0) return associate(tp => self, ntp => self%nbody) - call swiftest_util_sort_rearrange(tp%lperi, ind, ntp) - call swiftest_util_sort_rearrange(tp%plperP, ind, ntp) - call swiftest_util_sort_rearrange(tp%plencP, ind, ntp) - call swiftest_util_sort_rearrange(tp%rheliocentric, ind, ntp) + call util_sort_rearrange(tp%lperi, ind, ntp) + call util_sort_rearrange(tp%plperP, ind, ntp) + call util_sort_rearrange(tp%plencP, ind, ntp) + call util_sort_rearrange(tp%rheliocentric, ind, ntp) call swiftest_util_sort_rearrange_tp(tp,ind) end associate @@ -576,9 +572,9 @@ module subroutine rmvs_util_spill_pl(self, discards, lspill_list, ldestructive) associate(keeps => self) select type(discards) class is (rmvs_pl) - call swiftest_util_spill(keeps%nenc, discards%nenc, lspill_list, ldestructive) - call swiftest_util_spill(keeps%tpenc1P, discards%tpenc1P, lspill_list, ldestructive) - call swiftest_util_spill(keeps%plind, discards%plind, lspill_list, ldestructive) + call util_spill(keeps%nenc, discards%nenc, lspill_list, ldestructive) + call util_spill(keeps%tpenc1P, discards%tpenc1P, lspill_list, ldestructive) + call util_spill(keeps%plind, discards%plind, lspill_list, ldestructive) call whm_util_spill_pl(keeps, discards, lspill_list, ldestructive) class default @@ -607,9 +603,9 @@ module subroutine rmvs_util_spill_tp(self, discards, lspill_list, ldestructive) associate(keeps => self) select type(discards) class is (rmvs_tp) - call swiftest_util_spill(keeps%lperi, discards%lperi, lspill_list, ldestructive) - call swiftest_util_spill(keeps%plperP, discards%plperP, lspill_list, ldestructive) - call swiftest_util_spill(keeps%plencP, discards%plencP, lspill_list, ldestructive) + call util_spill(keeps%lperi, discards%lperi, lspill_list, ldestructive) + call util_spill(keeps%plperP, discards%plperP, lspill_list, ldestructive) + call util_spill(keeps%plencP, discards%plencP, lspill_list, ldestructive) call swiftest_util_spill_tp(keeps, discards, lspill_list, ldestructive) class default diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 08590c112..2b04d0697 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -78,21 +78,19 @@ module subroutine swiftest_coarray_coclone_cb(self) call coclone(self%R0) call coclone(self%dR) - do i = 1, NDIM - call coclone(self%aobl(i)) - call coclone(self%atide(i)) - call coclone(self%aoblbeg(i)) - call coclone(self%aoblend(i)) - call coclone(self%atidebeg(i)) - call coclone(self%atideend(i)) - call coclone(self%rb(i)) - call coclone(self%vb(i)) - call coclone(self%agr(i)) - call coclone(self%Ip(i)) - call coclone(self%rot(i)) - call coclone(self%L0(i)) - call coclone(self%dL(i)) - end do + call coclonevec(self%aobl) + call coclonevec(self%atide) + call coclonevec(self%aoblbeg) + call coclonevec(self%aoblend) + call coclonevec(self%atidebeg) + call coclonevec(self%atideend) + call coclonevec(self%rb) + call coclonevec(self%vb) + call coclonevec(self%agr) + call coclonevec(self%Ip) + call coclonevec(self%rot) + call coclonevec(self%L0) + call coclonevec(self%dL) return end subroutine swiftest_coarray_coclone_cb @@ -120,8 +118,6 @@ module subroutine swiftest_coarray_coclone_pl(self) call coclone(self%k2) call coclone(self%Q ) call coclone(self%tlag) - call coclone(self%k_plpl) - call coclone(self%nplpl) call coclone(self%kin) call coclone(self%lmtiny) call coclone(self%nplm) @@ -143,8 +139,6 @@ module subroutine swiftest_coarray_coclone_tp(self) ! Arguments class(swiftest_tp),intent(inout),codimension[*] :: self !! Swiftest body object - call coclone(self%k_pltp) - call coclone(self%npltp) call coclone(self%nplenc) call swiftest_coarray_coclone_body(self) @@ -214,7 +208,8 @@ module subroutine swiftest_coarray_coclone_system(self) return end subroutine swiftest_coarray_coclone_system - + + module subroutine swiftest_coarray_component_clone_info(var,src_img) !! author: David A. Minton !! @@ -225,30 +220,31 @@ module subroutine swiftest_coarray_component_clone_info(var,src_img) type(swiftest_particle_info), intent(inout) :: var integer(I4B), intent(in),optional :: src_img ! Internals - type(swiftest_particle_info),save :: tmp[*] + type(swiftest_particle_info),allocatable :: tmp[:] integer(I4B) :: img, si + allocate(tmp[*]) if (present(src_img)) then - si = src_img + si = src_img else - si = 1 + si = 1 end if - + sync all if (this_image() == si) then - do img = 1, num_images() - tmp[img] = var - end do - sync images(*) + do img = 1, num_images() + tmp[img] = var + end do + sync images(*) else - sync images(si) - var = tmp[si] + sync images(si) + var = tmp[si] end if - + return end subroutine swiftest_coarray_component_clone_info - - + + module subroutine swiftest_coarray_component_clone_info_arr1D(var,src_img) !! author: David A. Minton !! @@ -261,33 +257,35 @@ module subroutine swiftest_coarray_component_clone_info_arr1D(var,src_img) ! Internals type(swiftest_particle_info), dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + if (present(src_img)) then - si = src_img + si = src_img else - si = 1 + si = 1 end if - - sync all + isalloc = allocated(var) if (isalloc) n = size(var) sync all if (.not. isalloc[si]) return - + allocate(tmp(n[si])[*]) if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) + do img = 1, num_images() + tmp(:)[img] = var + end do + sync images(*) else - sync images(si) - if (allocated(var)) deallocate(var) - allocate(var, source=tmp) + sync images(si) + if (allocated(var)) deallocate(var) + allocate(var, source=tmp) end if - + return end subroutine swiftest_coarray_component_clone_info_arr1D @@ -400,6 +398,8 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) return end subroutine swiftest_coarray_component_collect_info_arr1D + + module subroutine swiftest_coarray_cocollect_body(self) !! author: David A. Minton @@ -410,6 +410,9 @@ module subroutine swiftest_coarray_cocollect_body(self) class(swiftest_body),intent(inout), codimension[*] :: self !! Swiftest body object integer(I4B) :: i + if (this_image() == 1) write(*,*) "Before collect " + sync all + if (allocated(self%id)) write(*,*) "Image: ",this_image(), "id: ",self%id call cocollect(self%nbody) call cocollect(self%id) call cocollect(self%info) @@ -438,6 +441,10 @@ module subroutine swiftest_coarray_cocollect_body(self) call cocollect(self%omega) call cocollect(self%capm) + if (this_image() == 1) write(*,*) "after collect " + sync all + if (allocated(self%id)) write(*,*) "Image: ",this_image(), "id: ",self%id + return end subroutine swiftest_coarray_cocollect_body @@ -450,7 +457,6 @@ module subroutine swiftest_coarray_cocollect_tp(self) ! Arguments class(swiftest_tp),intent(inout),codimension[*] :: self !! Swiftest body object - call cocollect(self%k_pltp) call cocollect(self%npltp) call cocollect(self%nplenc) call swiftest_coarray_cocollect_body(self) @@ -488,9 +494,6 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) if (this_image() == 1) then write(param%display_unit,*) " Done collecting" - do i = 1, nbody_system%tp%nbody - write(*,*) i,"mu ",nbody_system%tp%mu(i) - end do end if return @@ -520,6 +523,10 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." end if + if (this_image() == 1) write(*,*) "Before distribute " + sync all + if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "id: ",nbody_system%tp%id + ntp = nbody_system%tp%nbody sync all ntot = ntp[1] @@ -547,12 +554,15 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) call nbody_system%tp%spill(tmp, lspill_list(:), ldestructive=.true.) deallocate(tmp, cotp) + + + if (this_image() == 1) write(*,*) "After distribute " + sync all + if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "id: ",nbody_system%tp%id + if (this_image() == 1) then write(param%display_unit,*) " Done distributing" - do i = 1, nbody_system%tp%nbody - write(*,*) i,"mu ",nbody_system%tp%mu(i) - end do end if return @@ -584,7 +594,7 @@ module subroutine swiftest_coarray_initialize_system(nbody_system, param) end if return - end subroutine swiftest_coarray_initialize_system + end subroutine swiftest_coarray_initialize_system end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 9e139c986..4a34347f2 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -89,35 +89,47 @@ program swiftest_driver end if #endif -#ifdef COARRAY - if (this_image() == 1) then -#endif - call nbody_system%initialize(system_history, param) -#ifdef COARRAY - end if ! this_image() == 1 - call swiftest_coarray_initialize_system(nbody_system, param) +#ifdef COARRAY + ! The following line lets us read in the input files one image at a time + if (this_image() /= 1) sync images(this_image() - 1) +#endif + call nbody_system%initialize(system_history, param) +#ifdef COARRAY + if (this_image() == 1) then #endif ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. call nbody_system%display_run_information(param, integration_timer, phase="first") - if (param%lenergy) then - if (param%lrestart) then - call nbody_system%get_t0_values(system_history%nc, param) - else - call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum - end if - call nbody_system%conservation_report(param, lterminal=.true.) + +#ifdef COARRAY + end if ! this_image() == 1 +#endif + if (param%lenergy) then + if (param%lrestart) then + call nbody_system%get_t0_values(system_history%nc, param) + else + call nbody_system%conservation_report(param, lterminal=.false.) ! This will save the initial values of energy and momentum end if - call system_history%take_snapshot(param,nbody_system) + call nbody_system%conservation_report(param, lterminal=.true.) + end if + call system_history%take_snapshot(param,nbody_system) + +#ifdef COARRAY + if (this_image() == 1) then +#endif call nbody_system%dump(param, system_history) - -#ifdef COARRAY +#ifdef COARRAY end if ! this_image() == 1 +#endif + +#ifdef COARRAY + if (this_image() < num_images()) sync images(this_image() + 1) ! Distribute test particles to the various images call nbody_system%coarray_distribute(param) #endif do iloop = istart, nloops !> Step the nbody_system forward in time + if (this_image() == 1) write(*,*) "Image: ", this_image(), "ntp: ",nbody_system%tp%nbody call integration_timer%start() call nbody_system%step(param, nbody_system%t, dt) call integration_timer%stop() diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 71ba57b7f..68e60a04a 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1556,7 +1556,7 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) associate(n => self%nbody, tslot => nc%tslot) if (n == 0) return - call swiftest_util_sort(self%id(1:n), ind) + call util_sort(self%id(1:n), ind) do i = 1, n j = ind(i) @@ -1750,7 +1750,7 @@ module subroutine swiftest_io_netcdf_write_info_body(self, nc, param) class is (swiftest_body) associate(n => self%nbody, tslot => nc%tslot) if (n == 0) return - call swiftest_util_sort(self%id(1:n), ind) + call util_sort(self%id(1:n), ind) call nc%get_idvals() do i = 1, n @@ -1871,7 +1871,7 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i integer(I4B), intent(out) :: iostat !! IO status code character(len=*), intent(inout) :: iomsg !! Message to pass if iostat /= 0 ! Internals - logical :: tstart_set = .false. !! Is the final time set in the input file? + logical :: tstart_set = .false. !! Is the final time set in the input file? logical :: tstop_set = .false. !! Is the final time set in the input file? logical :: dt_set = .false. !! Is the step size set in the input file? integer(I4B) :: ilength, ifirst, ilast, i !! Variables used to parse input file diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 6388929ac..bbf1de296 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -1123,62 +1123,22 @@ module subroutine swiftest_user_kick_getacch_body(self, nbody_system, param, t, end subroutine swiftest_user_kick_getacch_body end interface - interface swiftest_util_append - module subroutine swiftest_util_append_arr_char_string(arr, source, nold, nsrc, lsource_mask) + interface util_append + module subroutine swiftest_util_append_arr_info(arr, source, nold, lsource_mask) implicit none - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - character(len=STRMAX), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - end subroutine swiftest_util_append_arr_char_string - - module subroutine swiftest_util_append_arr_DP(arr, source, nold, nsrc, lsource_mask) - implicit none - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - end subroutine swiftest_util_append_arr_DP - - module subroutine swiftest_util_append_arr_DPvec(arr, source, nold, nsrc, lsource_mask) - implicit none - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:,:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - end subroutine swiftest_util_append_arr_DPvec - - module subroutine swiftest_util_append_arr_I4B(arr, source, nold, nsrc, lsource_mask) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - end subroutine swiftest_util_append_arr_I4B - - module subroutine swiftest_util_append_arr_info(arr, source, nold, nsrc, lsource_mask) - implicit none - type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_particle_info), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array + type(swiftest_particle_info), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_arr_info - module subroutine swiftest_util_append_arr_kin(arr, source, nold, nsrc, lsource_mask) + module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) implicit none - type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_kinship), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array + type(swiftest_kinship), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_arr_kin - - module subroutine swiftest_util_append_arr_logical(arr, source, nold, nsrc, lsource_mask) - implicit none - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - logical, dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - end subroutine swiftest_util_append_arr_logical end interface interface @@ -1333,40 +1293,12 @@ module subroutine swiftest_util_fill_tp(self, inserts, lfill_list) end subroutine swiftest_util_fill_tp end interface - interface swiftest_util_fill - module subroutine swiftest_util_fill_arr_char_string(keeps, inserts, lfill_list) - implicit none - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - character(len=STRMAX), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - end subroutine swiftest_util_fill_arr_char_string - - module subroutine swiftest_util_fill_arr_DP(keeps, inserts, lfill_list) - implicit none - real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - end subroutine swiftest_util_fill_arr_DP - - module subroutine swiftest_util_fill_arr_DPvec(keeps, inserts, lfill_list) - implicit none - real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:,:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - end subroutine swiftest_util_fill_arr_DPvec - - module subroutine swiftest_util_fill_arr_I4B(keeps, inserts, lfill_list) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I4B), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - end subroutine swiftest_util_fill_arr_I4B - + interface util_fill module subroutine swiftest_util_fill_arr_info(keeps, inserts, lfill_list) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep type(swiftest_particle_info), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps end subroutine swiftest_util_fill_arr_info module subroutine swiftest_util_fill_arr_kin(keeps, inserts, lfill_list) @@ -1375,13 +1307,6 @@ module subroutine swiftest_util_fill_arr_kin(keeps, inserts, lfill_list) type(swiftest_kinship), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps end subroutine swiftest_util_fill_arr_kin - - module subroutine swiftest_util_fill_arr_logical(keeps, inserts, lfill_list) - implicit none - logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - logical, dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - end subroutine swiftest_util_fill_arr_logical end interface interface @@ -1526,31 +1451,7 @@ end subroutine swiftest_util_reset_kinship_pl end interface - interface swiftest_util_resize - module subroutine swiftest_util_resize_arr_char_string(arr, nnew) - implicit none - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - end subroutine swiftest_util_resize_arr_char_string - - module subroutine swiftest_util_resize_arr_DP(arr, nnew) - implicit none - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - end subroutine swiftest_util_resize_arr_DP - - module subroutine swiftest_util_resize_arr_DPvec(arr, nnew) - implicit none - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - end subroutine swiftest_util_resize_arr_DPvec - - module subroutine swiftest_util_resize_arr_I4B(arr, nnew) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - end subroutine swiftest_util_resize_arr_I4B - + interface util_resize module subroutine swiftest_util_resize_arr_info(arr, nnew) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Array to resize @@ -1562,12 +1463,6 @@ module subroutine swiftest_util_resize_arr_kin(arr, nnew) type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Array to resize integer(I4B), intent(in) :: nnew !! New size end subroutine swiftest_util_resize_arr_kin - - module subroutine swiftest_util_resize_arr_logical(arr, nnew) - implicit none - logical, dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - end subroutine swiftest_util_resize_arr_logical end interface interface @@ -1686,89 +1581,8 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar end subroutine swiftest_util_snapshot_system end interface - interface swiftest_util_sort - pure module subroutine swiftest_util_sort_i4b(arr) - implicit none - integer(I4B), dimension(:), intent(inout) :: arr - end subroutine swiftest_util_sort_i4b - - pure module subroutine swiftest_util_sort_index_i4b(arr,ind) - implicit none - integer(I4B), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind - end subroutine swiftest_util_sort_index_i4b - - pure module subroutine swiftest_util_sort_index_I4B_I8Bind(arr,ind) - implicit none - integer(I4B), dimension(:), intent(in) :: arr - integer(I8B), dimension(:), allocatable, intent(inout) :: ind - end subroutine swiftest_util_sort_index_I4b_I8Bind - - pure module subroutine swiftest_util_sort_index_I8B_I8Bind(arr,ind) - implicit none - integer(I8B), dimension(:), intent(in) :: arr - integer(I8B), dimension(:), allocatable, intent(inout) :: ind - end subroutine swiftest_util_sort_index_I8B_I8Bind - - pure module subroutine swiftest_util_sort_sp(arr) - implicit none - real(SP), dimension(:), intent(inout) :: arr - end subroutine swiftest_util_sort_sp - - pure module subroutine swiftest_util_sort_index_sp(arr,ind) - implicit none - real(SP), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind - end subroutine swiftest_util_sort_index_sp - - pure module subroutine swiftest_util_sort_dp(arr) - implicit none - real(DP), dimension(:), intent(inout) :: arr - end subroutine swiftest_util_sort_dp - - pure module subroutine swiftest_util_sort_index_dp(arr,ind) - implicit none - real(DP), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind - end subroutine swiftest_util_sort_index_dp - end interface swiftest_util_sort - - interface swiftest_util_sort_rearrange - pure module subroutine swiftest_util_sort_rearrange_arr_char_string(arr, ind, n) - implicit none - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_char_string - - pure module subroutine swiftest_util_sort_rearrange_arr_DP(arr, ind, n) - implicit none - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_DP - - pure module subroutine swiftest_util_sort_rearrange_arr_DPvec(arr, ind, n) - implicit none - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_DPvec - - pure module subroutine swiftest_util_sort_rearrange_arr_I4B(arr, ind, n) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_I4B - - pure module subroutine swiftest_util_sort_rearrange_arr_I4B_I8Bind(arr, ind, n) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_I4B_I8Bind + interface util_sort_rearrange module subroutine swiftest_util_sort_rearrange_arr_info(arr, ind, n) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array @@ -1783,20 +1597,7 @@ pure module subroutine swiftest_util_sort_rearrange_arr_kin(arr, ind, n) integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange end subroutine swiftest_util_sort_rearrange_arr_kin - pure module subroutine swiftest_util_sort_rearrange_arr_logical(arr, ind, n) - implicit none - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_logical - - pure module subroutine swiftest_util_sort_rearrange_arr_logical_I8Bind(arr, ind, n) - implicit none - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange - end subroutine swiftest_util_sort_rearrange_arr_logical_I8Bind - end interface swiftest_util_sort_rearrange + end interface util_sort_rearrange interface module subroutine swiftest_util_sort_rearrange_body(self, ind) @@ -1840,47 +1641,7 @@ end subroutine swiftest_util_sort_tp end interface - interface swiftest_util_spill - module subroutine swiftest_util_spill_arr_char_string(keeps, discards, lspill_list, ldestructive) - implicit none - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_char_string - - module subroutine swiftest_util_spill_arr_DP(keeps, discards, lspill_list, ldestructive) - implicit none - real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_DP - - module subroutine swiftest_util_spill_arr_DPvec(keeps, discards, lspill_list, ldestructive) - implicit none - real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:,:), allocatable, intent(inout) :: discards !! Array discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_DPvec - - module subroutine swiftest_util_spill_arr_I4B(keeps, discards, lspill_list, ldestructive) - implicit none - integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I4B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_I4B - - module subroutine swiftest_util_spill_arr_I8B(keeps, discards, lspill_list, ldestructive) - implicit none - integer(I8B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I8B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_I8B - + interface util_spill module subroutine swiftest_util_spill_arr_info(keeps, discards, lspill_list, ldestructive) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep @@ -1896,14 +1657,6 @@ module subroutine swiftest_util_spill_arr_kin(keeps, discards, lspill_list, ldes logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not end subroutine swiftest_util_spill_arr_kin - - module subroutine swiftest_util_spill_arr_logical(keeps, discards, lspill_list, ldestructive) - implicit none - logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - logical, dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - end subroutine swiftest_util_spill_arr_logical end interface interface @@ -1933,22 +1686,6 @@ end subroutine swiftest_util_spill_tp end interface - interface swiftest_util_unique - module subroutine swiftest_util_unique_DP(input_array, output_array, index_map) - implicit none - real(DP), dimension(:), intent(in) :: input_array !! Unsorted input array - real(DP), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) - end subroutine swiftest_util_unique_DP - - module subroutine swiftest_util_unique_I4B(input_array, output_array, index_map) - implicit none - integer(I4B), dimension(:), intent(in) :: input_array !! Unsorted input array - integer(I4B), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) - end subroutine swiftest_util_unique_I4B - end interface swiftest_util_unique - interface module subroutine swiftest_util_valid_id_system(self, param) implicit none diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 77747170b..fe46cc9e3 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -15,222 +15,97 @@ use fraggle contains - module subroutine swiftest_util_append_arr_char_string(arr, source, nold, nsrc, lsource_mask) + + module subroutine swiftest_util_append_arr_info(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of character string type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of particle information type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. implicit none ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - character(len=STRMAX), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array + type(swiftest_particle_info), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals - integer(I4B) :: nnew - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return + integer(I4B) :: nnew, nsrc, nend_orig, i + integer(I4B), dimension(:), allocatable :: idx - if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) else - call swiftest_util_resize(arr, nold + nnew) + nsrc = size(source) end if - - arr(nold + 1:nold + nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) - - return - end subroutine swiftest_util_append_arr_char_string - - - module subroutine swiftest_util_append_arr_DP(arr, source, nold, nsrc, lsource_mask) - !! author: David A. Minton - !! - !! Append a single array of double precision type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - ! Internals - integer(I4B) :: nnew - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return + if (nsrc == 0) return if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) + nend_orig = 0 + allocate(arr(nsrc)) else - call swiftest_util_resize(arr, nold + nnew) + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) end if + nnew = nend_orig + nsrc - arr(nold + 1:nold + nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) - - return - end subroutine swiftest_util_append_arr_DP - - - module subroutine swiftest_util_append_arr_DPvec(arr, source, nold, nsrc, lsource_mask) - !! author: David A. Minton - !! - !! Append a single array of double precision vector type of size (NDIM, n) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:,:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - ! Internals - integer(I4B) :: nnew - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return - - if (.not.allocated(arr)) then - allocate(arr(NDIM,nold+nnew)) + allocate(idx(nsrc)) + if (present(lsource_mask)) then + idx = pack([(i, i = 1, size(lsource_mask))], lsource_mask(:)) else - call swiftest_util_resize(arr, nold + nnew) - end if + idx = [(i, i = 1,nsrc)] + end if - arr(1, nold + 1:nold + nnew) = pack(source(1,1:nsrc), lsource_mask(1:nsrc)) - arr(2, nold + 1:nold + nnew) = pack(source(2,1:nsrc), lsource_mask(1:nsrc)) - arr(3, nold + 1:nold + nnew) = pack(source(3,1:nsrc), lsource_mask(1:nsrc)) + call swiftest_util_copy_particle_info_arr(source(:), arr(nold+1:nnew), idx) return - end subroutine swiftest_util_append_arr_DPvec + end subroutine swiftest_util_append_arr_info - module subroutine swiftest_util_append_arr_I4B(arr, source, nold, nsrc, lsource_mask) + module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of integer(I4B) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of kinship type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. implicit none ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to + type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array + type(swiftest_kinship), dimension(:), intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals - integer(I4B) :: nnew + integer(I4B) :: nnew, nsrc, nend_orig - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return - - if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) + if (present(lsource_mask)) then + nsrc = count(lsource_mask(:)) else - call swiftest_util_resize(arr, nold + nnew) + nsrc = size(source) end if - - arr(nold + 1:nold + nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) - - return - end subroutine swiftest_util_append_arr_I4B - - - module subroutine swiftest_util_append_arr_info(arr, source, nold, nsrc, lsource_mask) - !! author: David A. Minton - !! - !! Append a single array of particle information type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. - implicit none - ! Arguments - type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_particle_info), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - ! Internals - integer(I4B) :: nnew, i - integer(I4B), dimension(:), allocatable :: idx - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return + if (nsrc == 0) return if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) + nend_orig = 0 + allocate(arr(nsrc)) else - call swiftest_util_resize(arr, nold + nnew) + if (present(nold)) then + nend_orig = nold + else + nend_orig = size(arr) + end if + call util_resize(arr, nend_orig + nsrc) end if + nnew = nend_orig + nsrc - allocate(idx(nnew)) - - idx = pack([(i, i = 1, nsrc)], lsource_mask(1:nsrc)) - - call swiftest_util_copy_particle_info_arr(source(1:nsrc), arr(nold+1:nold+nnew), idx) - - return - end subroutine swiftest_util_append_arr_info - - - module subroutine swiftest_util_append_arr_kin(arr, source, nold, nsrc, lsource_mask) - !! author: David A. Minton - !! - !! Append a single array of kinship type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. - implicit none - ! Arguments - type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_kinship), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - ! Internals - integer(I4B) :: nnew - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return - - if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) + if (present(lsource_mask)) then + arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - call swiftest_util_resize(arr, nold + nnew) + arr(nold + 1:nnew) = source(1:nsrc) end if - arr(nold + 1:nold + nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) - return end subroutine swiftest_util_append_arr_kin - module subroutine swiftest_util_append_arr_logical(arr, source, nold, nsrc, lsource_mask) - !! author: David A. Minton - !! - !! Append a single array of logical type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - logical, dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in) :: nold, nsrc !! Extend of the old array and the source array, respectively - logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to - ! Internals - integer(I4B) :: nnew - - if (.not. allocated(source)) return - - nnew = count(lsource_mask(1:nsrc)) - if (nnew == 0) return - - if (.not.allocated(arr)) then - allocate(arr(nold+nnew)) - else - call swiftest_util_resize(arr, nold + nnew) - end if - - arr(nold + 1:nold + nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) - - return - end subroutine swiftest_util_append_arr_logical - module subroutine swiftest_util_append_body(self, source, lsource_mask) !! author: David A. Minton @@ -243,40 +118,36 @@ module subroutine swiftest_util_append_body(self, source, lsource_mask) class(swiftest_body), intent(in) :: source !! Source object to append logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to ! Internals - integer(I4B) :: nold, nsrc, nnew - - nold = self%nbody - nsrc = source%nbody - nnew = count(lsource_mask(1:nsrc)) - - call swiftest_util_append(self%id, source%id, nold, nsrc, lsource_mask) - call swiftest_util_append(self%info, source%info, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lmask, source%lmask, nold, nsrc, lsource_mask) - call swiftest_util_append(self%status, source%status, nold, nsrc, lsource_mask) - call swiftest_util_append(self%ldiscard, source%ldiscard, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lencounter, source%lencounter, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lcollision, source%lcollision, nold, nsrc, lsource_mask) - call swiftest_util_append(self%mu, source%mu, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rh, source%rh, nold, nsrc, lsource_mask) - call swiftest_util_append(self%vh, source%vh, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rb, source%rb, nold, nsrc, lsource_mask) - call swiftest_util_append(self%vb, source%vb, nold, nsrc, lsource_mask) - call swiftest_util_append(self%ah, source%ah, nold, nsrc, lsource_mask) - call swiftest_util_append(self%aobl, source%aobl, nold, nsrc, lsource_mask) - call swiftest_util_append(self%atide, source%atide, nold, nsrc, lsource_mask) - call swiftest_util_append(self%agr, source%agr, nold, nsrc, lsource_mask) - call swiftest_util_append(self%ir3h, source%ir3h, nold, nsrc, lsource_mask) - call swiftest_util_append(self%isperi, source%isperi, nold, nsrc, lsource_mask) - call swiftest_util_append(self%peri, source%peri, nold, nsrc, lsource_mask) - call swiftest_util_append(self%atp, source%atp, nold, nsrc, lsource_mask) - call swiftest_util_append(self%a, source%a, nold, nsrc, lsource_mask) - call swiftest_util_append(self%e, source%e, nold, nsrc, lsource_mask) - call swiftest_util_append(self%inc, source%inc, nold, nsrc, lsource_mask) - call swiftest_util_append(self%capom, source%capom, nold, nsrc, lsource_mask) - call swiftest_util_append(self%omega, source%omega, nold, nsrc, lsource_mask) - call swiftest_util_append(self%capm, source%capm, nold, nsrc, lsource_mask) - - self%nbody = nold + nnew + + + call util_append(self%id, source%id, lsource_mask=lsource_mask) + call util_append(self%info, source%info, lsource_mask=lsource_mask) + call util_append(self%lmask, source%lmask, lsource_mask=lsource_mask) + call util_append(self%status, source%status, lsource_mask=lsource_mask) + call util_append(self%ldiscard, source%ldiscard, lsource_mask=lsource_mask) + call util_append(self%lencounter, source%lencounter, lsource_mask=lsource_mask) + call util_append(self%lcollision, source%lcollision, lsource_mask=lsource_mask) + call util_append(self%mu, source%mu, lsource_mask=lsource_mask) + call util_append(self%rh, source%rh, lsource_mask=lsource_mask) + call util_append(self%vh, source%vh, lsource_mask=lsource_mask) + call util_append(self%rb, source%rb, lsource_mask=lsource_mask) + call util_append(self%vb, source%vb, lsource_mask=lsource_mask) + call util_append(self%ah, source%ah, lsource_mask=lsource_mask) + call util_append(self%aobl, source%aobl, lsource_mask=lsource_mask) + call util_append(self%atide, source%atide, lsource_mask=lsource_mask) + call util_append(self%agr, source%agr, lsource_mask=lsource_mask) + call util_append(self%ir3h, source%ir3h, lsource_mask=lsource_mask) + call util_append(self%isperi, source%isperi, lsource_mask=lsource_mask) + call util_append(self%peri, source%peri, lsource_mask=lsource_mask) + call util_append(self%atp, source%atp, lsource_mask=lsource_mask) + call util_append(self%a, source%a, lsource_mask=lsource_mask) + call util_append(self%e, source%e, lsource_mask=lsource_mask) + call util_append(self%inc, source%inc, lsource_mask=lsource_mask) + call util_append(self%capom, source%capom, lsource_mask=lsource_mask) + call util_append(self%omega, source%omega, lsource_mask=lsource_mask) + call util_append(self%capm, source%capm, lsource_mask=lsource_mask) + + self%nbody = self%nbody + count(lsource_mask(:)) return end subroutine swiftest_util_append_body @@ -295,30 +166,28 @@ module subroutine swiftest_util_append_pl(self, source, lsource_mask) select type(source) class is (swiftest_pl) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%mass, source%mass, nold, nsrc, lsource_mask) - call swiftest_util_append(self%Gmass, source%Gmass, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rhill, source%rhill, nold, nsrc, lsource_mask) - call swiftest_util_append(self%renc, source%renc, nold, nsrc, lsource_mask) - call swiftest_util_append(self%radius, source%radius, nold, nsrc, lsource_mask) - call swiftest_util_append(self%density, source%density, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rbeg, source%rbeg, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rend, source%rend, nold, nsrc, lsource_mask) - call swiftest_util_append(self%vbeg, source%vbeg, nold, nsrc, lsource_mask) - call swiftest_util_append(self%Ip, source%Ip, nold, nsrc, lsource_mask) - call swiftest_util_append(self%rot, source%rot, nold, nsrc, lsource_mask) - call swiftest_util_append(self%k2, source%k2, nold, nsrc, lsource_mask) - call swiftest_util_append(self%Q, source%Q, nold, nsrc, lsource_mask) - call swiftest_util_append(self%tlag, source%tlag, nold, nsrc, lsource_mask) - call swiftest_util_append(self%kin, source%kin, nold, nsrc, lsource_mask) - call swiftest_util_append(self%lmtiny, source%lmtiny, nold, nsrc, lsource_mask) - call swiftest_util_append(self%nplenc, source%nplenc, nold, nsrc, lsource_mask) - call swiftest_util_append(self%ntpenc, source%ntpenc, nold, nsrc, lsource_mask) - - if (allocated(self%k_plpl)) deallocate(self%k_plpl) - - call swiftest_util_append_body(self, source, lsource_mask) - end associate + call util_append(self%mass, source%mass, lsource_mask=lsource_mask) + call util_append(self%Gmass, source%Gmass, lsource_mask=lsource_mask) + call util_append(self%rhill, source%rhill, lsource_mask=lsource_mask) + call util_append(self%renc, source%renc, lsource_mask=lsource_mask) + call util_append(self%radius, source%radius, lsource_mask=lsource_mask) + call util_append(self%density, source%density, lsource_mask=lsource_mask) + call util_append(self%rbeg, source%rbeg, lsource_mask=lsource_mask) + call util_append(self%rend, source%rend, lsource_mask=lsource_mask) + call util_append(self%vbeg, source%vbeg, lsource_mask=lsource_mask) + call util_append(self%Ip, source%Ip, lsource_mask=lsource_mask) + call util_append(self%rot, source%rot, lsource_mask=lsource_mask) + call util_append(self%k2, source%k2, lsource_mask=lsource_mask) + call util_append(self%Q, source%Q, lsource_mask=lsource_mask) + call util_append(self%tlag, source%tlag, lsource_mask=lsource_mask) + call util_append(self%kin, source%kin, lsource_mask=lsource_mask) + call util_append(self%lmtiny, source%lmtiny, lsource_mask=lsource_mask) + call util_append(self%nplenc, source%nplenc, lsource_mask=lsource_mask) + call util_append(self%ntpenc, source%ntpenc, lsource_mask=lsource_mask) + + if (allocated(self%k_plpl)) deallocate(self%k_plpl) + + call swiftest_util_append_body(self, source, lsource_mask) class default write(*,*) "Invalid object passed to the append method. Source must be of class swiftest_pl or its descendents" call base_util_exit(FAILURE) @@ -341,11 +210,9 @@ module subroutine swiftest_util_append_tp(self, source, lsource_mask) select type(source) class is (swiftest_tp) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%nplenc, source%nplenc, nold, nsrc, lsource_mask) + call util_append(self%nplenc, source%nplenc, lsource_mask=lsource_mask) - call swiftest_util_append_body(self, source, lsource_mask) - end associate + call swiftest_util_append_body(self, source, lsource_mask) class default write(*,*) "Invalid object passed to the append method. Source must be of class swiftest_tp or its descendents" call base_util_exit(FAILURE) @@ -958,89 +825,6 @@ module subroutine swiftest_util_dealloc_tp(self) end subroutine swiftest_util_dealloc_tp - module subroutine swiftest_util_fill_arr_char_string(keeps, inserts, lfill_list) - !! author: David A. Minton - !! - !! Performs a fill operation on a single array of type character strings - !! This is the inverse of a spill operation - implicit none - ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - character(len=STRMAX), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - - if (.not.allocated(keeps) .or. .not.allocated(inserts)) return - - keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) - keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) - - return - end subroutine swiftest_util_fill_arr_char_string - - - module subroutine swiftest_util_fill_arr_DP(keeps, inserts, lfill_list) - !! author: David A. Minton - !! - !! Performs a fill operation on a single array of type DP - !! This is the inverse of a spill operation - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - - if (.not.allocated(keeps) .or. .not.allocated(inserts)) return - - keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) - keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) - - return - end subroutine swiftest_util_fill_arr_DP - - - module subroutine swiftest_util_fill_arr_DPvec(keeps, inserts, lfill_list) - !! author: David A. Minton - !! - !! Performs a fill operation on a single array of DP vectors with shape (NDIM, n) - !! This is the inverse of a spill operation - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:,:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - ! Internals - integer(I4B) :: i - - if (.not.allocated(keeps) .or. .not.allocated(inserts)) return - - do i = 1, NDIM - keeps(i,:) = unpack(keeps(i,:), .not.lfill_list(:), keeps(i,:)) - keeps(i,:) = unpack(inserts(i,:), lfill_list(:), keeps(i,:)) - end do - - return - end subroutine swiftest_util_fill_arr_DPvec - - - module subroutine swiftest_util_fill_arr_I4B(keeps, inserts, lfill_list) - !! author: David A. Minton - !! - !! Performs a fill operation on a single array of type I4B - !! This is the inverse of a spill operation - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I4B), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - - if (.not.allocated(keeps) .or. .not.allocated(inserts)) return - - keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) - keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) - - return - end subroutine swiftest_util_fill_arr_I4B - module subroutine swiftest_util_fill_arr_info(keeps, inserts, lfill_list) !! author: David A. Minton @@ -1070,26 +854,6 @@ module subroutine swiftest_util_fill_arr_info(keeps, inserts, lfill_list) end subroutine swiftest_util_fill_arr_info - module subroutine swiftest_util_fill_arr_logical(keeps, inserts, lfill_list) - !! author: David A. Minton - !! - !! Performs a fill operation on a single array of logicals - !! This is the inverse of a spill operation - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - logical, dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps - - if (.not.allocated(keeps) .or. .not.allocated(inserts)) return - - keeps(:) = unpack(keeps(:), .not.lfill_list(:), keeps(:)) - keeps(:) = unpack(inserts(:), lfill_list(:), keeps(:)) - - return - end subroutine swiftest_util_fill_arr_logical - - module subroutine swiftest_util_fill_arr_kin(keeps, inserts, lfill_list) !! author: David A. Minton !! @@ -1124,32 +888,32 @@ module subroutine swiftest_util_fill_body(self, inserts, lfill_list) ! For each component, pack the discarded bodies into the discard object and do the inverse with the keeps !> Fill all the common components associate(keeps => self) - call swiftest_util_fill(keeps%id, inserts%id, lfill_list) - call swiftest_util_fill(keeps%info, inserts%info, lfill_list) - call swiftest_util_fill(keeps%lmask, inserts%lmask, lfill_list) - call swiftest_util_fill(keeps%status, inserts%status, lfill_list) - call swiftest_util_fill(keeps%ldiscard, inserts%ldiscard, lfill_list) - call swiftest_util_fill(keeps%lcollision, inserts%lcollision, lfill_list) - call swiftest_util_fill(keeps%lencounter, inserts%lencounter, lfill_list) - call swiftest_util_fill(keeps%mu, inserts%mu, lfill_list) - call swiftest_util_fill(keeps%rh, inserts%rh, lfill_list) - call swiftest_util_fill(keeps%vh, inserts%vh, lfill_list) - call swiftest_util_fill(keeps%rb, inserts%rb, lfill_list) - call swiftest_util_fill(keeps%vb, inserts%vb, lfill_list) - call swiftest_util_fill(keeps%ah, inserts%ah, lfill_list) - call swiftest_util_fill(keeps%aobl, inserts%aobl, lfill_list) - call swiftest_util_fill(keeps%agr, inserts%agr, lfill_list) - call swiftest_util_fill(keeps%atide, inserts%atide, lfill_list) - call swiftest_util_fill(keeps%ir3h, inserts%ir3h, lfill_list) - call swiftest_util_fill(keeps%isperi, inserts%isperi, lfill_list) - call swiftest_util_fill(keeps%peri, inserts%peri, lfill_list) - call swiftest_util_fill(keeps%atp, inserts%atp, lfill_list) - call swiftest_util_fill(keeps%a, inserts%a, lfill_list) - call swiftest_util_fill(keeps%e, inserts%e, lfill_list) - call swiftest_util_fill(keeps%inc, inserts%inc, lfill_list) - call swiftest_util_fill(keeps%capom, inserts%capom, lfill_list) - call swiftest_util_fill(keeps%omega, inserts%omega, lfill_list) - call swiftest_util_fill(keeps%capm, inserts%capm, lfill_list) + call util_fill(keeps%id, inserts%id, lfill_list) + call util_fill(keeps%info, inserts%info, lfill_list) + call util_fill(keeps%lmask, inserts%lmask, lfill_list) + call util_fill(keeps%status, inserts%status, lfill_list) + call util_fill(keeps%ldiscard, inserts%ldiscard, lfill_list) + call util_fill(keeps%lcollision, inserts%lcollision, lfill_list) + call util_fill(keeps%lencounter, inserts%lencounter, lfill_list) + call util_fill(keeps%mu, inserts%mu, lfill_list) + call util_fill(keeps%rh, inserts%rh, lfill_list) + call util_fill(keeps%vh, inserts%vh, lfill_list) + call util_fill(keeps%rb, inserts%rb, lfill_list) + call util_fill(keeps%vb, inserts%vb, lfill_list) + call util_fill(keeps%ah, inserts%ah, lfill_list) + call util_fill(keeps%aobl, inserts%aobl, lfill_list) + call util_fill(keeps%agr, inserts%agr, lfill_list) + call util_fill(keeps%atide, inserts%atide, lfill_list) + call util_fill(keeps%ir3h, inserts%ir3h, lfill_list) + call util_fill(keeps%isperi, inserts%isperi, lfill_list) + call util_fill(keeps%peri, inserts%peri, lfill_list) + call util_fill(keeps%atp, inserts%atp, lfill_list) + call util_fill(keeps%a, inserts%a, lfill_list) + call util_fill(keeps%e, inserts%e, lfill_list) + call util_fill(keeps%inc, inserts%inc, lfill_list) + call util_fill(keeps%capom, inserts%capom, lfill_list) + call util_fill(keeps%omega, inserts%omega, lfill_list) + call util_fill(keeps%capm, inserts%capm, lfill_list) ! This is the base class, so will be the last to be called in the cascade. keeps%nbody = size(keeps%id(:)) @@ -1175,23 +939,23 @@ module subroutine swiftest_util_fill_pl(self, inserts, lfill_list) select type (inserts) ! The standard requires us to select the type of both arguments in order to access all the components class is (swiftest_pl) !> Fill components specific to the massive body class - call swiftest_util_fill(keeps%mass, inserts%mass, lfill_list) - call swiftest_util_fill(keeps%Gmass, inserts%Gmass, lfill_list) - call swiftest_util_fill(keeps%rhill, inserts%rhill, lfill_list) - call swiftest_util_fill(keeps%renc, inserts%renc, lfill_list) - call swiftest_util_fill(keeps%radius, inserts%radius, lfill_list) - call swiftest_util_fill(keeps%density, inserts%density, lfill_list) - call swiftest_util_fill(keeps%rbeg, inserts%rbeg, lfill_list) - call swiftest_util_fill(keeps%rend, inserts%rend, lfill_list) - call swiftest_util_fill(keeps%vbeg, inserts%vbeg, lfill_list) - call swiftest_util_fill(keeps%Ip, inserts%Ip, lfill_list) - call swiftest_util_fill(keeps%rot, inserts%rot, lfill_list) - call swiftest_util_fill(keeps%k2, inserts%k2, lfill_list) - call swiftest_util_fill(keeps%Q, inserts%Q, lfill_list) - call swiftest_util_fill(keeps%tlag, inserts%tlag, lfill_list) - call swiftest_util_fill(keeps%kin, inserts%kin, lfill_list) - call swiftest_util_fill(keeps%nplenc, inserts%nplenc, lfill_list) - call swiftest_util_fill(keeps%ntpenc, inserts%ntpenc, lfill_list) + call util_fill(keeps%mass, inserts%mass, lfill_list) + call util_fill(keeps%Gmass, inserts%Gmass, lfill_list) + call util_fill(keeps%rhill, inserts%rhill, lfill_list) + call util_fill(keeps%renc, inserts%renc, lfill_list) + call util_fill(keeps%radius, inserts%radius, lfill_list) + call util_fill(keeps%density, inserts%density, lfill_list) + call util_fill(keeps%rbeg, inserts%rbeg, lfill_list) + call util_fill(keeps%rend, inserts%rend, lfill_list) + call util_fill(keeps%vbeg, inserts%vbeg, lfill_list) + call util_fill(keeps%Ip, inserts%Ip, lfill_list) + call util_fill(keeps%rot, inserts%rot, lfill_list) + call util_fill(keeps%k2, inserts%k2, lfill_list) + call util_fill(keeps%Q, inserts%Q, lfill_list) + call util_fill(keeps%tlag, inserts%tlag, lfill_list) + call util_fill(keeps%kin, inserts%kin, lfill_list) + call util_fill(keeps%nplenc, inserts%nplenc, lfill_list) + call util_fill(keeps%ntpenc, inserts%ntpenc, lfill_list) if (allocated(keeps%k_plpl)) deallocate(keeps%k_plpl) @@ -1220,7 +984,7 @@ module subroutine swiftest_util_fill_tp(self, inserts, lfill_list) select type(inserts) class is (swiftest_tp) !> Spill components specific to the test particle class - call swiftest_util_fill(keeps%nplenc, inserts%nplenc, lfill_list) + call util_fill(keeps%nplenc, inserts%nplenc, lfill_list) call swiftest_util_fill_body(keeps, inserts, lfill_list) class default @@ -1706,10 +1470,10 @@ module subroutine swiftest_util_index_map_storage(self) call swiftest_util_get_vals_storage(self, idvals, tvals) - call swiftest_util_unique(idvals,self%idvals,self%idmap) + call util_unique(idvals,self%idvals,self%idmap) self%nid = size(self%idvals) - call swiftest_util_unique(tvals,self%tvals,self%tmap) + call util_unique(tvals,self%tvals,self%tmap) self%nt = size(self%tvals) return @@ -2044,16 +1808,16 @@ module subroutine swiftest_util_reset_kinship_pl(self, idx) end subroutine swiftest_util_reset_kinship_pl - module subroutine swiftest_util_resize_arr_char_string(arr, nnew) + module subroutine swiftest_util_resize_arr_info(arr, nnew) !! author: David A. Minton !! - !! Resizes an array component of type character string. nnew = 0 will deallocate. + !! Resizes an array component of type character string. Array will only be resized if has previously been allocated. Passing nnew = 0 will deallocate. implicit none ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size ! Internals - character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + type(swiftest_particle_info), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated integer(I4B) :: nold !! Old size if (nnew < 0) return @@ -2072,34 +1836,29 @@ module subroutine swiftest_util_resize_arr_char_string(arr, nnew) if (nnew == nold) return allocate(tmp(nnew)) - if (nold > 0) then - if (nnew > nold) then - tmp(1:nold) = arr(1:nold) - tmp(nold+1:nnew) = "" - else - tmp(1:nnew) = arr(1:nnew) - end if + if (nnew > nold) then + call swiftest_util_copy_particle_info_arr(arr(1:nold), tmp(1:nold)) else - tmp(1:nnew) = "" + call swiftest_util_copy_particle_info_arr(arr(1:nnew), tmp(1:nnew)) end if + call move_alloc(tmp, arr) return - end subroutine swiftest_util_resize_arr_char_string - + end subroutine swiftest_util_resize_arr_info - module subroutine swiftest_util_resize_arr_DP(arr, nnew) + + module subroutine swiftest_util_resize_arr_kin(arr, nnew) !! author: David A. Minton !! - !! Resizes an array component of double precision type. Passing nnew = 0 will deallocate. + !! Resizes an array component of type character string. Array will only be resized if has previously been allocated. Passing nnew = 0 will deallocate. implicit none ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size + type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Array to resize + integer(I4B), intent(in) :: nnew !! New size ! Internals - real(DP), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + type(swiftest_kinship), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated integer(I4B) :: nold !! Old size - real(DP), parameter :: init_val = 0.0_DP if (nnew < 0) return @@ -2114,324 +1873,98 @@ module subroutine swiftest_util_resize_arr_DP(arr, nnew) nold = 0 end if - if (nnew == nold) return - allocate(tmp(nnew)) - if (nold > 0) then - if (nnew > nold) then - tmp(1:nold) = arr(1:nold) - tmp(nold+1:nnew) = init_val - else - tmp(1:nnew) = arr(1:nnew) - end if + if (nnew > nold) then + tmp(1:nold) = arr(1:nold) else - tmp(1:nnew) = init_val + tmp(1:nnew) = arr(1:nnew) end if call move_alloc(tmp, arr) return - end subroutine swiftest_util_resize_arr_DP + end subroutine swiftest_util_resize_arr_kin - module subroutine swiftest_util_resize_arr_DPvec(arr, nnew) + module subroutine swiftest_util_resize_body(self, nnew) !! author: David A. Minton !! - !! Resizes an array component of double precision vectors of size (NDIM, n). Passing nnew = 0 will deallocate. + !! Checks the current size of a Swiftest body against the requested size and resizes it if it is too small. implicit none ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - ! Internals - real(DP), dimension(:,:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated - integer(I4B) :: nold !! Old size - real(DP), dimension(NDIM), parameter :: init_val = 0.0_DP - integer(I4B) :: i - - if (nnew < 0) return - - if (nnew == 0) then - if (allocated(arr)) deallocate(arr) - return - end if - - if (allocated(arr)) then - nold = size(arr, dim=2) - else - nold = 0 - end if - - if (nnew == nold) return - - allocate(tmp(NDIM, nnew)) - if (nold > 0) then - if (nnew > nold) then - tmp(:,1:nold) = arr(:,1:nold) - do i = nold+1, nnew - tmp(:,i) = init_val(:) - end do - else - tmp(:,1:nnew) = arr(:,1:nnew) - end if - else - do i = 1, nnew - tmp(:, i) = init_val(:) - end do - end if - call move_alloc(tmp, arr) + class(swiftest_body), intent(inout) :: self !! Swiftest body object + integer(I4B), intent(in) :: nnew !! New size neded - return + call util_resize(self%info, nnew) + call util_resize(self%id, nnew) + call util_resize(self%status, nnew) + call util_resize(self%lcollision, nnew) + call util_resize(self%lencounter, nnew) + call util_resize(self%ldiscard, nnew) + call util_resize(self%lmask, nnew) + call util_resize(self%mu, nnew) + call util_resize(self%rh, nnew) + call util_resize(self%vh, nnew) + call util_resize(self%rb, nnew) + call util_resize(self%vb, nnew) + call util_resize(self%ah, nnew) + call util_resize(self%aobl, nnew) + call util_resize(self%atide, nnew) + call util_resize(self%agr, nnew) + call util_resize(self%ir3h, nnew) + call util_resize(self%a, nnew) + call util_resize(self%e, nnew) + call util_resize(self%inc, nnew) + call util_resize(self%capom, nnew) + call util_resize(self%omega, nnew) + call util_resize(self%capm, nnew) + self%nbody = count(self%status(1:nnew) /= INACTIVE) return - end subroutine swiftest_util_resize_arr_DPvec + end subroutine swiftest_util_resize_body - module subroutine swiftest_util_resize_arr_I4B(arr, nnew) + module subroutine swiftest_util_resize_pl(self, nnew) !! author: David A. Minton !! - !! Resizes an array component of integer type. Passing nnew = 0 will deallocate. + !! Checks the current size of a Swiftest massive body against the requested size and resizes it if it is too small. implicit none ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - ! Internals - integer(I4B), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated - integer(I4B) :: nold !! Old size - integer(I4B), parameter :: init_val = -1 + class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object + integer(I4B), intent(in) :: nnew !! New size neded - if (nnew < 0) return + call swiftest_util_resize_body(self, nnew) - if (nnew == 0) then - if (allocated(arr)) deallocate(arr) - return - end if - - if (allocated(arr)) then - nold = size(arr) - else - nold = 0 - end if + call util_resize(self%mass, nnew) + call util_resize(self%Gmass, nnew) + call util_resize(self%rhill, nnew) + call util_resize(self%renc, nnew) + call util_resize(self%radius, nnew) + call util_resize(self%rbeg, nnew) + call util_resize(self%rend, nnew) + call util_resize(self%vbeg, nnew) + call util_resize(self%density, nnew) + call util_resize(self%Ip, nnew) + call util_resize(self%rot, nnew) + call util_resize(self%k2, nnew) + call util_resize(self%Q, nnew) + call util_resize(self%tlag, nnew) + call util_resize(self%kin, nnew) + call util_resize(self%lmtiny, nnew) + call util_resize(self%nplenc, nnew) + call util_resize(self%ntpenc, nnew) - if (nnew == nold) return - - allocate(tmp(nnew)) - if (nold > 0) then - if (nnew > nold) then - tmp(1:nold) = arr(1:nold) - tmp(nold+1:nnew) = init_val - else - tmp(1:nnew) = arr(1:nnew) - end if - else - tmp(1:nnew) = init_val - end if - call move_alloc(tmp, arr) + + + if (allocated(self%k_plpl)) deallocate(self%k_plpl) return - end subroutine swiftest_util_resize_arr_I4B + end subroutine swiftest_util_resize_pl - module subroutine swiftest_util_resize_arr_info(arr, nnew) + module subroutine swiftest_util_resize_tp(self, nnew) !! author: David A. Minton !! - !! Resizes an array component of type character string. Array will only be resized if has previously been allocated. Passing nnew = 0 will deallocate. - implicit none - ! Arguments - type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - ! Internals - type(swiftest_particle_info), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated - integer(I4B) :: nold !! Old size - - if (nnew < 0) return - - if (nnew == 0) then - if (allocated(arr)) deallocate(arr) - return - end if - - if (allocated(arr)) then - nold = size(arr) - else - nold = 0 - end if - - if (nnew == nold) return - - allocate(tmp(nnew)) - if (nnew > nold) then - call swiftest_util_copy_particle_info_arr(arr(1:nold), tmp(1:nold)) - else - call swiftest_util_copy_particle_info_arr(arr(1:nnew), tmp(1:nnew)) - end if - - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_resize_arr_info - - - module subroutine swiftest_util_resize_arr_kin(arr, nnew) - !! author: David A. Minton - !! - !! Resizes an array component of type character string. Array will only be resized if has previously been allocated. Passing nnew = 0 will deallocate. - implicit none - ! Arguments - type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - ! Internals - type(swiftest_kinship), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated - integer(I4B) :: nold !! Old size - - if (nnew < 0) return - - if (nnew == 0) then - if (allocated(arr)) deallocate(arr) - return - end if - - if (allocated(arr)) then - nold = size(arr) - else - nold = 0 - end if - - allocate(tmp(nnew)) - if (nnew > nold) then - tmp(1:nold) = arr(1:nold) - else - tmp(1:nnew) = arr(1:nnew) - end if - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_resize_arr_kin - - - module subroutine swiftest_util_resize_arr_logical(arr, nnew) - !! author: David A. Minton - !! - !! Resizes an array component of logical type. Passing nnew = 0 will deallocate. - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: arr !! Array to resize - integer(I4B), intent(in) :: nnew !! New size - ! Internals - logical, dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated - integer(I4B) :: nold !! Old size - logical, parameter :: init_val = .false. - - if (nnew < 0) return - - if (nnew == 0) then - if (allocated(arr)) deallocate(arr) - return - end if - - if (allocated(arr)) then - nold = size(arr) - else - nold = 0 - end if - - if (nnew == nold) return - - allocate(tmp(nnew)) - if (nold > 0) then - if (nnew > nold) then - tmp(1:nold) = arr(1:nold) - tmp(nold+1:nnew) = init_val - else - tmp(1:nnew) = arr(1:nnew) - end if - else - tmp(1:nnew) = init_val - end if - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_resize_arr_logical - - - module subroutine swiftest_util_resize_body(self, nnew) - !! author: David A. Minton - !! - !! Checks the current size of a Swiftest body against the requested size and resizes it if it is too small. - implicit none - ! Arguments - class(swiftest_body), intent(inout) :: self !! Swiftest body object - integer(I4B), intent(in) :: nnew !! New size neded - - call swiftest_util_resize(self%info, nnew) - call swiftest_util_resize(self%id, nnew) - call swiftest_util_resize(self%status, nnew) - call swiftest_util_resize(self%lcollision, nnew) - call swiftest_util_resize(self%lencounter, nnew) - call swiftest_util_resize(self%ldiscard, nnew) - call swiftest_util_resize(self%lmask, nnew) - call swiftest_util_resize(self%mu, nnew) - call swiftest_util_resize(self%rh, nnew) - call swiftest_util_resize(self%vh, nnew) - call swiftest_util_resize(self%rb, nnew) - call swiftest_util_resize(self%vb, nnew) - call swiftest_util_resize(self%ah, nnew) - call swiftest_util_resize(self%aobl, nnew) - call swiftest_util_resize(self%atide, nnew) - call swiftest_util_resize(self%agr, nnew) - call swiftest_util_resize(self%ir3h, nnew) - call swiftest_util_resize(self%a, nnew) - call swiftest_util_resize(self%e, nnew) - call swiftest_util_resize(self%inc, nnew) - call swiftest_util_resize(self%capom, nnew) - call swiftest_util_resize(self%omega, nnew) - call swiftest_util_resize(self%capm, nnew) - self%nbody = count(self%status(1:nnew) /= INACTIVE) - - return - end subroutine swiftest_util_resize_body - - - module subroutine swiftest_util_resize_pl(self, nnew) - !! author: David A. Minton - !! - !! Checks the current size of a Swiftest massive body against the requested size and resizes it if it is too small. - implicit none - ! Arguments - class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object - integer(I4B), intent(in) :: nnew !! New size neded - - call swiftest_util_resize_body(self, nnew) - - call swiftest_util_resize(self%mass, nnew) - call swiftest_util_resize(self%Gmass, nnew) - call swiftest_util_resize(self%rhill, nnew) - call swiftest_util_resize(self%renc, nnew) - call swiftest_util_resize(self%radius, nnew) - call swiftest_util_resize(self%rbeg, nnew) - call swiftest_util_resize(self%rend, nnew) - call swiftest_util_resize(self%vbeg, nnew) - call swiftest_util_resize(self%density, nnew) - call swiftest_util_resize(self%Ip, nnew) - call swiftest_util_resize(self%rot, nnew) - call swiftest_util_resize(self%k2, nnew) - call swiftest_util_resize(self%Q, nnew) - call swiftest_util_resize(self%tlag, nnew) - call swiftest_util_resize(self%kin, nnew) - call swiftest_util_resize(self%lmtiny, nnew) - call swiftest_util_resize(self%nplenc, nnew) - call swiftest_util_resize(self%ntpenc, nnew) - - - - if (allocated(self%k_plpl)) deallocate(self%k_plpl) - - return - end subroutine swiftest_util_resize_pl - - - module subroutine swiftest_util_resize_tp(self, nnew) - !! author: David A. Minton - !! - !! Checks the current size of a Swiftest test particle against the requested size and resizes it if it is too small. + !! Checks the current size of a Swiftest test particle against the requested size and resizes it if it is too small. implicit none ! Arguments class(swiftest_tp), intent(inout) :: self !! Swiftest test particle object @@ -2439,10 +1972,10 @@ module subroutine swiftest_util_resize_tp(self, nnew) call swiftest_util_resize_body(self, nnew) - call swiftest_util_resize(self%nplenc, nnew) - call swiftest_util_resize(self%isperi, nnew) - call swiftest_util_resize(self%peri, nnew) - call swiftest_util_resize(self%atp, nnew) + call util_resize(self%nplenc, nnew) + call util_resize(self%isperi, nnew) + call util_resize(self%peri, nnew) + call util_resize(self%atp, nnew) return end subroutine swiftest_util_resize_tp @@ -3154,25 +2687,25 @@ module subroutine swiftest_util_sort_body(self, sortby, ascending) associate(body => self, n => self%nbody) select case(sortby) case("id") - call swiftest_util_sort(direction * body%id(1:n), ind) + call util_sort(direction * body%id(1:n), ind) case("status") - call swiftest_util_sort(direction * body%status(1:n), ind) + call util_sort(direction * body%status(1:n), ind) case("ir3h") - call swiftest_util_sort(direction * body%ir3h(1:n), ind) + call util_sort(direction * body%ir3h(1:n), ind) case("a") - call swiftest_util_sort(direction * body%a(1:n), ind) + call util_sort(direction * body%a(1:n), ind) case("e") - call swiftest_util_sort(direction * body%e(1:n), ind) + call util_sort(direction * body%e(1:n), ind) case("inc") - call swiftest_util_sort(direction * body%inc(1:n), ind) + call util_sort(direction * body%inc(1:n), ind) case("capom") - call swiftest_util_sort(direction * body%capom(1:n), ind) + call util_sort(direction * body%capom(1:n), ind) case("mu") - call swiftest_util_sort(direction * body%mu(1:n), ind) + call util_sort(direction * body%mu(1:n), ind) case("peri") - call swiftest_util_sort(direction * body%peri(1:n), ind) + call util_sort(direction * body%peri(1:n), ind) case("atp") - call swiftest_util_sort(direction * body%atp(1:n), ind) + call util_sort(direction * body%atp(1:n), ind) case("info", "lfirst", "nbody", "ldiscard", "lcollision", "lencounter", "rh", "vh", "rb", "vb", "ah", "aobl", "atide", "agr","isperi") write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' case default @@ -3188,856 +2721,167 @@ module subroutine swiftest_util_sort_body(self, sortby, ascending) end subroutine swiftest_util_sort_body - pure module subroutine swiftest_util_sort_dp(arr) + module subroutine swiftest_util_sort_pl(self, sortby, ascending) !! author: David A. Minton !! - !! Sort input DP precision array in place into ascending numerical order using quicksort. - !! + !! Sort a Swiftest massive body object in-place. + !! sortby is a string indicating which array component to sort. implicit none ! Arguments - real(DP), dimension(:), intent(inout) :: arr + class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object + character(*), intent(in) :: sortby !! Sorting attribute + logical, intent(in) :: ascending !! Logical flag indicating whether or not the sorting should be in ascending or descending order + ! Internals + integer(I4B), dimension(:), allocatable :: ind + integer(I4B) :: direction + + if (self%nbody == 0) return - call swiftest_util_sort_qsort_DP(arr) + if (ascending) then + direction = 1 + else + direction = -1 + end if - return - end subroutine swiftest_util_sort_dp + associate(pl => self, npl => self%nbody) + select case(sortby) + case("Gmass","mass") + call util_sort(direction * pl%Gmass(1:npl), ind) + case("rhill") + call util_sort(direction * pl%rhill(1:npl), ind) + case("renc") + call util_sort(direction * pl%renc(1:npl), ind) + case("radius") + call util_sort(direction * pl%radius(1:npl), ind) + case("density") + call util_sort(direction * pl%density(1:npl), ind) + case("k2") + call util_sort(direction * pl%k2(1:npl), ind) + case("Q") + call util_sort(direction * pl%Q(1:npl), ind) + case("tlag") + call util_sort(direction * pl%tlag(1:npl), ind) + case("nplenc") + call util_sort(direction * pl%nplenc(1:npl), ind) + case("ntpenc") + call util_sort(direction * pl%ntpenc(1:npl), ind) + case("lmtiny", "nplm", "nplplm", "kin", "rbeg", "rend", "vbeg", "Ip", "rot", "k_plpl", "nplpl") + write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' + case default ! Look for components in the parent class + call swiftest_util_sort_body(pl, sortby, ascending) + return + end select + call pl%rearrange(ind) - pure module subroutine swiftest_util_sort_index_dp(arr, ind) - !! author: David A. Minton - !! - !! Sort input DP precision array by index in ascending numerical order using quick sort. - !! This algorithm works well for partially sorted arrays (which is usually the case here). - !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously - !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. - !! - implicit none - ! Arguments - real(DP), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind - ! Internals - integer(I4B) :: n, i - real(DP), dimension(:), allocatable :: tmparr + end associate - n = size(arr) - if (.not.allocated(ind)) then - allocate(ind(n)) - ind = [(i, i=1, n)] - end if - allocate(tmparr, mold=arr) - tmparr(:) = arr(ind(:)) - call swiftest_util_sort_qsort_DP(tmparr, ind) - return - end subroutine swiftest_util_sort_index_dp + end subroutine swiftest_util_sort_pl - recursive pure subroutine swiftest_util_sort_qsort_DP(arr, ind) + module subroutine swiftest_util_sort_tp(self, sortby, ascending) !! author: David A. Minton !! - !! Sort input DP precision array by index in ascending numerical order using quicksort sort. - !! + !! Sort a Swiftest test particle object in-place. + !! sortby is a string indicating which array component to sort. implicit none ! Arguments - real(DP), dimension(:), intent(inout) :: arr - integer(I4B),dimension(:),intent(out), optional :: ind - !! Internals - integer :: iq - - if (size(arr) > 1) then - if (present(ind)) then - call swiftest_util_sort_partition_DP(arr, iq, ind) - call swiftest_util_sort_qsort_DP(arr(:iq-1),ind(:iq-1)) - call swiftest_util_sort_qsort_DP(arr(iq:), ind(iq:)) - else - call swiftest_util_sort_partition_DP(arr, iq) - call swiftest_util_sort_qsort_DP(arr(:iq-1)) - call swiftest_util_sort_qsort_DP(arr(iq:)) - end if + class(swiftest_tp), intent(inout) :: self !! Swiftest test particle object + character(*), intent(in) :: sortby !! Sorting attribute + logical, intent(in) :: ascending !! Logical flag indicating whether or not the sorting should be in ascending or descending order + ! Internals + integer(I4B), dimension(:), allocatable :: ind + integer(I4B) :: direction + + if (self%nbody == 0) return + + if (ascending) then + direction = 1 + else + direction = -1 end if - return - end subroutine swiftest_util_sort_qsort_DP + associate(tp => self, ntp => self%nbody) + select case(sortby) + case("nplenc") + call util_sort(direction * tp%nplenc(1:ntp), ind) + case default ! Look for components in the parent class + call swiftest_util_sort_body(tp, sortby, ascending) + return + end select - - pure subroutine swiftest_util_sort_partition_DP(arr, marker, ind) - !! author: David A. Minton - !! - !! Partition function for quicksort on DP type - !! - implicit none - ! Arguments - real(DP), intent(inout), dimension(:) :: arr - integer(I4B), intent(inout), dimension(:), optional :: ind - integer(I4B), intent(out) :: marker - ! Internals - integer(I4B) :: i, j, itmp, narr, ipiv - real(DP) :: temp - real(DP) :: x ! pivot point + call tp%rearrange(ind) - narr = size(arr) + end associate - ! Get center as pivot, as this is likely partially sorted - ipiv = narr / 2 - x = arr(ipiv) - i = 0 - j = narr + 1 - - do - j = j - 1 - do - if (arr(j) <= x) exit - j = j - 1 - end do - i = i + 1 - do - if (arr(i) >= x) exit - i = i + 1 - end do - if (i < j) then - ! exchange A(i) and A(j) - temp = arr(i) - arr(i) = arr(j) - arr(j) = temp - if (present(ind)) then - itmp = ind(i) - ind(i) = ind(j) - ind(j) = itmp - end if - else if (i == j) then - marker = i + 1 - return - else - marker = i - return - endif - end do - return - end subroutine swiftest_util_sort_partition_DP - + end subroutine swiftest_util_sort_tp - pure module subroutine swiftest_util_sort_i4b(arr) + module subroutine swiftest_util_sort_rearrange_body(self, ind) !! author: David A. Minton !! - !! Sort input integer array in place into ascending numerical order using quick sort. - !! This algorithm works well for partially sorted arrays (which is usually the case here) - !! + !! Rearrange Swiftest body structure in-place from an index list. + !! This is a helper utility used to make polymorphic sorting work on Swiftest structures. implicit none ! Arguments - integer(I4B), dimension(:), intent(inout) :: arr + class(swiftest_body), intent(inout) :: self !! Swiftest body object + integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) - call swiftest_util_sort_qsort_I4B(arr) + associate(n => self%nbody) + call util_sort_rearrange(self%id, ind, n) + call util_sort_rearrange(self%lmask, ind, n) + call util_sort_rearrange(self%info, ind, n) + call util_sort_rearrange(self%status, ind, n) + call util_sort_rearrange(self%ldiscard, ind, n) + call util_sort_rearrange(self%lcollision, ind, n) + call util_sort_rearrange(self%lencounter, ind, n) + call util_sort_rearrange(self%rh, ind, n) + call util_sort_rearrange(self%vh, ind, n) + call util_sort_rearrange(self%rb, ind, n) + call util_sort_rearrange(self%vb, ind, n) + call util_sort_rearrange(self%ah, ind, n) + call util_sort_rearrange(self%aobl, ind, n) + call util_sort_rearrange(self%agr, ind, n) + call util_sort_rearrange(self%atide, ind, n) + call util_sort_rearrange(self%ir3h, ind, n) + call util_sort_rearrange(self%isperi, ind, n) + call util_sort_rearrange(self%peri, ind, n) + call util_sort_rearrange(self%atp, ind, n) + call util_sort_rearrange(self%mu, ind, n) + call util_sort_rearrange(self%a, ind, n) + call util_sort_rearrange(self%e, ind, n) + call util_sort_rearrange(self%inc, ind, n) + call util_sort_rearrange(self%capom, ind, n) + call util_sort_rearrange(self%omega, ind, n) + call util_sort_rearrange(self%capm, ind, n) + end associate return - end subroutine swiftest_util_sort_i4b + end subroutine swiftest_util_sort_rearrange_body - pure module subroutine swiftest_util_sort_index_I4B(arr, ind) + module subroutine swiftest_util_sort_rearrange_arr_info(arr, ind, n) !! author: David A. Minton !! - !! Sort input integer array by index in ascending numerical order using quicksort. - !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously - !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. - !! + !! Rearrange a single array of particle information type in-place from an index list. implicit none ! Arguments - integer(I4B), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind + type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against + integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange ! Internals - integer(I4B) :: n, i - integer(I4B), dimension(:), allocatable :: tmparr - - n = size(arr) - if (.not.allocated(ind)) then - allocate(ind(n)) - ind = [(i, i=1, n)] - end if - allocate(tmparr, mold=arr) - tmparr(:) = arr(ind(:)) - call swiftest_util_sort_qsort_I4B(tmparr, ind) + type(swiftest_particle_info), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - return - end subroutine swiftest_util_sort_index_I4B - - - pure module subroutine swiftest_util_sort_index_I4B_I8Bind(arr, ind) - !! author: David A. Minton - !! - !! Sort input integer array by index in ascending numerical order using quicksort. - !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously - !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. - !! - implicit none - ! Arguments - integer(I4B), dimension(:), intent(in) :: arr - integer(I8B), dimension(:), allocatable, intent(inout) :: ind - ! Internals - integer(I8B) :: n, i - integer(I4B), dimension(:), allocatable :: tmparr - - n = size(arr) - if (.not.allocated(ind)) then - allocate(ind(n)) - ind = [(i, i=1_I8B, n)] - end if - allocate(tmparr, mold=arr) - tmparr(:) = arr(ind(:)) - call swiftest_util_sort_qsort_I4B_I8Bind(tmparr, ind) - - return - end subroutine swiftest_util_sort_index_I4B_I8Bind - - - recursive pure subroutine swiftest_util_sort_qsort_I4B(arr, ind) - !! author: David A. Minton - !! - !! Sort input I4B array by index in ascending numerical order using quicksort. - !! - implicit none - ! Arguments - integer(I4B), dimension(:), intent(inout) :: arr - integer(I4B), dimension(:), intent(out), optional :: ind - ! Internals - integer(I4B) :: iq - - if (size(arr) > 1) then - if (present(ind)) then - call swiftest_util_sort_partition_I4B(arr, iq, ind) - call swiftest_util_sort_qsort_I4B(arr(:iq-1),ind(:iq-1)) - call swiftest_util_sort_qsort_I4B(arr(iq:), ind(iq:)) - else - call swiftest_util_sort_partition_I4B(arr, iq) - call swiftest_util_sort_qsort_I4B(arr(:iq-1)) - call swiftest_util_sort_qsort_I4B(arr(iq:)) - end if - end if - - return - end subroutine swiftest_util_sort_qsort_I4B - - - recursive pure subroutine swiftest_util_sort_qsort_I4B_I8Bind(arr, ind) - !! author: David A. Minton - !! - !! Sort input I4B array by index in ascending numerical order using quicksort. - !! - implicit none - ! Arguments - integer(I4B), dimension(:), intent(inout) :: arr - integer(I8B), dimension(:), intent(out), optional :: ind - ! Internals - integer(I8B) :: iq - - if (size(arr) > 1_I8B) then - if (present(ind)) then - call swiftest_util_sort_partition_I4B_I8Bind(arr, iq, ind) - call swiftest_util_sort_qsort_I4B_I8Bind(arr(:iq-1_I8B),ind(:iq-1_I8B)) - call swiftest_util_sort_qsort_I4B_I8Bind(arr(iq:), ind(iq:)) - else - call swiftest_util_sort_partition_I4B_I8Bind(arr, iq) - call swiftest_util_sort_qsort_I4B_I8Bind(arr(:iq-1_I8B)) - call swiftest_util_sort_qsort_I4B_I8Bind(arr(iq:)) - end if - end if - - return - end subroutine swiftest_util_sort_qsort_I4B_I8Bind - - - recursive pure subroutine swiftest_util_sort_qsort_I8B_I8Bind(arr, ind) - !! author: David A. Minton - !! - !! Sort input I8B array by index in ascending numerical order using quicksort. - !! - implicit none - ! Arguments - integer(I8B), dimension(:), intent(inout) :: arr - integer(I8B), dimension(:), intent(out), optional :: ind - ! Internals - integer(I8B) :: iq - - if (size(arr) > 1_I8B) then - if (present(ind)) then - call swiftest_util_sort_partition_I8B_I8Bind(arr, iq, ind) - call swiftest_util_sort_qsort_I8B_I8Bind(arr(:iq-1_I8B),ind(:iq-1_I8B)) - call swiftest_util_sort_qsort_I8B_I8Bind(arr(iq:), ind(iq:)) - else - call swiftest_util_sort_partition_I8B_I8Bind(arr, iq) - call swiftest_util_sort_qsort_I8B_I8Bind(arr(:iq-1_I8B)) - call swiftest_util_sort_qsort_I8B_I8Bind(arr(iq:)) - end if - end if - - return - end subroutine swiftest_util_sort_qsort_I8B_I8Bind - - - pure subroutine swiftest_util_sort_partition_I4B(arr, marker, ind) - !! author: David A. Minton - !! - !! Partition function for quicksort on I4B type - !! - implicit none - ! Arguments - integer(I4B), intent(inout), dimension(:) :: arr - integer(I4B), intent(inout), dimension(:), optional :: ind - integer(I4B), intent(out) :: marker - ! Internals - integer(I4B) :: i, j, itmp, narr, ipiv - integer(I4B) :: temp - integer(I4B) :: x ! pivot point - - narr = size(arr) - - ! Get center as pivot, as this is likely partially sorted - ipiv = narr / 2 - x = arr(ipiv) - i = 0 - j = narr + 1 - - do - j = j - 1 - do - if (arr(j) <= x) exit - j = j - 1 - end do - i = i + 1 - do - if (arr(i) >= x) exit - i = i + 1 - end do - if (i < j) then - ! exchange A(i) and A(j) - temp = arr(i) - arr(i) = arr(j) - arr(j) = temp - if (present(ind)) then - itmp = ind(i) - ind(i) = ind(j) - ind(j) = itmp - end if - else if (i == j) then - marker = i + 1 - return - else - marker = i - return - endif - end do - - return - end subroutine swiftest_util_sort_partition_I4B - - - pure subroutine swiftest_util_sort_partition_I4B_I8Bind(arr, marker, ind) - !! author: David A. Minton - !! - !! Partition function for quicksort on I4B type - !! - implicit none - ! Arguments - integer(I4B), intent(inout), dimension(:) :: arr - integer(I8B), intent(inout), dimension(:), optional :: ind - integer(I8B), intent(out) :: marker - ! Internals - integer(I8B) :: i, j, itmp, narr, ipiv - integer(I4B) :: temp - integer(I8B) :: x ! pivot point - - narr = size(arr) - - ! Get center as pivot, as this is likely partially sorted - ipiv = narr / 2_I8B - x = arr(ipiv) - i = 0_I8B - j = narr + 1_I8B - - do - j = j - 1_I8B - do - if (arr(j) <= x) exit - j = j - 1_I8B - end do - i = i + 1_I8B - do - if (arr(i) >= x) exit - i = i + 1_I8B - end do - if (i < j) then - ! exchange A(i) and A(j) - temp = arr(i) - arr(i) = arr(j) - arr(j) = temp - if (present(ind)) then - itmp = ind(i) - ind(i) = ind(j) - ind(j) = itmp - end if - else if (i == j) then - marker = i + 1_I8B - return - else - marker = i - return - endif - end do - - return - end subroutine swiftest_util_sort_partition_I4B_I8Bind - - - pure subroutine swiftest_util_sort_partition_I8B_I8Bind(arr, marker, ind) - !! author: David A. Minton - !! - !! Partition function for quicksort on I8B type with I8B index - !! - implicit none - ! Arguments - integer(I8B), intent(inout), dimension(:) :: arr - integer(I8B), intent(inout), dimension(:), optional :: ind - integer(I8B), intent(out) :: marker - ! Internals - integer(I8B) :: i, j, itmp, narr, ipiv - integer(I8B) :: temp - integer(I8B) :: x ! pivot point - - narr = size(arr) - - ! Get center as pivot, as this is likely partially sorted - ipiv = narr / 2_I8B - x = arr(ipiv) - i = 0_I8B - j = narr + 1_I8B - - do - j = j - 1_I8B - do - if (arr(j) <= x) exit - j = j - 1_I8B - end do - i = i + 1_I8B - do - if (arr(i) >= x) exit - i = i + 1_I8B - end do - if (i < j) then - ! exchange A(i) and A(j) - temp = arr(i) - arr(i) = arr(j) - arr(j) = temp - if (present(ind)) then - itmp = ind(i) - ind(i) = ind(j) - ind(j) = itmp - end if - else if (i == j) then - marker = i + 1_I8B - return - else - marker = i - return - endif - end do - - return - end subroutine swiftest_util_sort_partition_I8B_I8Bind - - - pure module subroutine swiftest_util_sort_sp(arr) - !! author: David A. Minton - !! - !! Sort input DP precision array in place into ascending numerical order using quicksort. - !! - implicit none - ! Arguments - real(SP), dimension(:), intent(inout) :: arr - - call swiftest_util_sort_qsort_SP(arr) - - return - end subroutine swiftest_util_sort_sp - - - pure module subroutine swiftest_util_sort_index_sp(arr, ind) - !! author: David A. Minton - !! - !! Sort input DP precision array by index in ascending numerical order using quicksort. - !! If ind is supplied already allocated, we assume it is an existing index array (e.g. a previously - !! sorted array). If it is not allocated, this subroutine swiftest_allocates it. - !! - implicit none - ! Arguments - real(SP), dimension(:), intent(in) :: arr - integer(I4B), dimension(:), allocatable, intent(inout) :: ind - ! Internals - integer(I4B) :: n, i - real(SP), dimension(:), allocatable :: tmparr - - n = size(arr) - if (.not.allocated(ind)) then - allocate(ind(n)) - ind = [(i, i=1, n)] - end if - allocate(tmparr, mold=arr) - tmparr(:) = arr(ind(:)) - call swiftest_util_sort_qsort_SP(tmparr, ind) - - return - end subroutine swiftest_util_sort_index_sp - - - recursive pure subroutine swiftest_util_sort_qsort_SP(arr, ind) - !! author: David A. Minton - !! - !! Sort input DP precision array by index in ascending numerical order using quicksort. - !! - implicit none - ! Arguments - real(SP), dimension(:), intent(inout) :: arr - integer(I4B),dimension(:),intent(out), optional :: ind - !! Internals - integer :: iq - - if (size(arr) > 1) then - if (present(ind)) then - call swiftest_util_sort_partition_SP(arr, iq, ind) - call swiftest_util_sort_qsort_SP(arr(:iq-1),ind(:iq-1)) - call swiftest_util_sort_qsort_SP(arr(iq:), ind(iq:)) - else - call swiftest_util_sort_partition_SP(arr, iq) - call swiftest_util_sort_qsort_SP(arr(:iq-1)) - call swiftest_util_sort_qsort_SP(arr(iq:)) - end if - end if - - return - end subroutine swiftest_util_sort_qsort_SP - - - pure subroutine swiftest_util_sort_partition_SP(arr, marker, ind) - !! author: David A. Minton - !! - !! Partition function for quicksort on SP type - !! - implicit none - ! Arguments - real(SP), intent(inout), dimension(:) :: arr - integer(I4B), intent(inout), dimension(:), optional :: ind - integer(I4B), intent(out) :: marker - ! Internals - integer(I4B) :: i, j, itmp, narr, ipiv - real(SP) :: temp - real(SP) :: x ! pivot point - - narr = size(arr) - - ! Get center as pivot, as this is likely partially sorted - ipiv = narr / 2 - x = arr(ipiv) - i = 0 - j = narr + 1 - - do - j = j - 1 - do - if (arr(j) <= x) exit - j = j - 1 - end do - i = i + 1 - do - if (arr(i) >= x) exit - i = i + 1 - end do - if (i < j) then - ! exchange A(i) and A(j) - temp = arr(i) - arr(i) = arr(j) - arr(j) = temp - if (present(ind)) then - itmp = ind(i) - ind(i) = ind(j) - ind(j) = itmp - end if - else if (i == j) then - marker = i + 1 - return - else - marker = i - return - endif - end do - - return - end subroutine swiftest_util_sort_partition_SP - - - module subroutine swiftest_util_sort_pl(self, sortby, ascending) - !! author: David A. Minton - !! - !! Sort a Swiftest massive body object in-place. - !! sortby is a string indicating which array component to sort. - implicit none - ! Arguments - class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object - character(*), intent(in) :: sortby !! Sorting attribute - logical, intent(in) :: ascending !! Logical flag indicating whether or not the sorting should be in ascending or descending order - ! Internals - integer(I4B), dimension(:), allocatable :: ind - integer(I4B) :: direction - - if (self%nbody == 0) return - - if (ascending) then - direction = 1 - else - direction = -1 - end if - - associate(pl => self, npl => self%nbody) - select case(sortby) - case("Gmass","mass") - call swiftest_util_sort(direction * pl%Gmass(1:npl), ind) - case("rhill") - call swiftest_util_sort(direction * pl%rhill(1:npl), ind) - case("renc") - call swiftest_util_sort(direction * pl%renc(1:npl), ind) - case("radius") - call swiftest_util_sort(direction * pl%radius(1:npl), ind) - case("density") - call swiftest_util_sort(direction * pl%density(1:npl), ind) - case("k2") - call swiftest_util_sort(direction * pl%k2(1:npl), ind) - case("Q") - call swiftest_util_sort(direction * pl%Q(1:npl), ind) - case("tlag") - call swiftest_util_sort(direction * pl%tlag(1:npl), ind) - case("nplenc") - call swiftest_util_sort(direction * pl%nplenc(1:npl), ind) - case("ntpenc") - call swiftest_util_sort(direction * pl%ntpenc(1:npl), ind) - case("lmtiny", "nplm", "nplplm", "kin", "rbeg", "rend", "vbeg", "Ip", "rot", "k_plpl", "nplpl") - write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' - case default ! Look for components in the parent class - call swiftest_util_sort_body(pl, sortby, ascending) - return - end select - - call pl%rearrange(ind) - - end associate - - return - end subroutine swiftest_util_sort_pl - - - module subroutine swiftest_util_sort_tp(self, sortby, ascending) - !! author: David A. Minton - !! - !! Sort a Swiftest test particle object in-place. - !! sortby is a string indicating which array component to sort. - implicit none - ! Arguments - class(swiftest_tp), intent(inout) :: self !! Swiftest test particle object - character(*), intent(in) :: sortby !! Sorting attribute - logical, intent(in) :: ascending !! Logical flag indicating whether or not the sorting should be in ascending or descending order - ! Internals - integer(I4B), dimension(:), allocatable :: ind - integer(I4B) :: direction - - if (self%nbody == 0) return - - if (ascending) then - direction = 1 - else - direction = -1 - end if - - associate(tp => self, ntp => self%nbody) - select case(sortby) - case("nplenc") - call swiftest_util_sort(direction * tp%nplenc(1:ntp), ind) - case default ! Look for components in the parent class - call swiftest_util_sort_body(tp, sortby, ascending) - return - end select - - call tp%rearrange(ind) - - end associate - - return - end subroutine swiftest_util_sort_tp - - - module subroutine swiftest_util_sort_rearrange_body(self, ind) - !! author: David A. Minton - !! - !! Rearrange Swiftest body structure in-place from an index list. - !! This is a helper utility used to make polymorphic sorting work on Swiftest structures. - implicit none - ! Arguments - class(swiftest_body), intent(inout) :: self !! Swiftest body object - integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) - - associate(n => self%nbody) - call swiftest_util_sort_rearrange(self%id, ind, n) - call swiftest_util_sort_rearrange(self%lmask, ind, n) - call swiftest_util_sort_rearrange(self%info, ind, n) - call swiftest_util_sort_rearrange(self%status, ind, n) - call swiftest_util_sort_rearrange(self%ldiscard, ind, n) - call swiftest_util_sort_rearrange(self%lcollision, ind, n) - call swiftest_util_sort_rearrange(self%lencounter, ind, n) - call swiftest_util_sort_rearrange(self%rh, ind, n) - call swiftest_util_sort_rearrange(self%vh, ind, n) - call swiftest_util_sort_rearrange(self%rb, ind, n) - call swiftest_util_sort_rearrange(self%vb, ind, n) - call swiftest_util_sort_rearrange(self%ah, ind, n) - call swiftest_util_sort_rearrange(self%aobl, ind, n) - call swiftest_util_sort_rearrange(self%agr, ind, n) - call swiftest_util_sort_rearrange(self%atide, ind, n) - call swiftest_util_sort_rearrange(self%ir3h, ind, n) - call swiftest_util_sort_rearrange(self%isperi, ind, n) - call swiftest_util_sort_rearrange(self%peri, ind, n) - call swiftest_util_sort_rearrange(self%atp, ind, n) - call swiftest_util_sort_rearrange(self%mu, ind, n) - call swiftest_util_sort_rearrange(self%a, ind, n) - call swiftest_util_sort_rearrange(self%e, ind, n) - call swiftest_util_sort_rearrange(self%inc, ind, n) - call swiftest_util_sort_rearrange(self%capom, ind, n) - call swiftest_util_sort_rearrange(self%omega, ind, n) - call swiftest_util_sort_rearrange(self%capm, ind, n) - end associate - - return - end subroutine swiftest_util_sort_rearrange_body - - - pure module subroutine swiftest_util_sort_rearrange_arr_char_string(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of character string in-place from an index list. - implicit none - ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary copy of arry used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_char_string - - - pure module subroutine swiftest_util_sort_rearrange_arr_DP(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of DP type in-place from an index list. - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - real(DP), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_DP - - - pure module subroutine swiftest_util_sort_rearrange_arr_DPvec(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of (NDIM,n) DP-type vectors in-place from an index list. - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - real(DP), dimension(:,:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(:,1:n) = arr(:, ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_DPvec - - - pure module subroutine swiftest_util_sort_rearrange_arr_I4B(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of integers in-place from an index list. - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - integer(I4B), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_I4B - - pure module subroutine swiftest_util_sort_rearrange_arr_I4B_I8Bind(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of integers in-place from an index list. - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - integer(I4B), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0_I8B) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_I4B_I8Bind - - - module subroutine swiftest_util_sort_rearrange_arr_info(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of particle information type in-place from an index list. - implicit none - ! Arguments - type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - type(swiftest_particle_info), dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) + if (.not. allocated(arr) .or. n <= 0) return + allocate(tmp, mold=arr) call swiftest_util_copy_particle_info_arr(arr, tmp, ind) call move_alloc(tmp, arr) return end subroutine swiftest_util_sort_rearrange_arr_info - + pure module subroutine swiftest_util_sort_rearrange_arr_kin(arr, ind, n) !! author: David A. Minton @@ -4067,48 +2911,6 @@ pure module subroutine swiftest_util_sort_rearrange_arr_kin(arr, ind, n) end subroutine swiftest_util_sort_rearrange_arr_kin - pure module subroutine swiftest_util_sort_rearrange_arr_logical(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of logicals in-place from an index list. - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - logical, dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_logical - - - pure module subroutine swiftest_util_sort_rearrange_arr_logical_I8Bind(arr, ind, n) - !! author: David A. Minton - !! - !! Rearrange a single array of logicals in-place from an index list. - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I8B), dimension(:), intent(in) :: ind !! Index to rearrange against - integer(I8B), intent(in) :: n !! Number of elements in arr and ind to rearrange - ! Internals - logical, dimension(:), allocatable :: tmp !! Temporary copy of array used during rearrange operation - - if (.not. allocated(arr) .or. n <= 0) return - allocate(tmp, mold=arr) - tmp(1:n) = arr(ind) - call move_alloc(tmp, arr) - - return - end subroutine swiftest_util_sort_rearrange_arr_logical_I8Bind - - module subroutine swiftest_util_sort_rearrange_pl(self, ind) !! author: David A. Minton !! @@ -4119,23 +2921,23 @@ module subroutine swiftest_util_sort_rearrange_pl(self, ind) integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) associate(pl => self, npl => self%nbody) - call swiftest_util_sort_rearrange(pl%mass, ind, npl) - call swiftest_util_sort_rearrange(pl%Gmass, ind, npl) - call swiftest_util_sort_rearrange(pl%rhill, ind, npl) - call swiftest_util_sort_rearrange(pl%renc, ind, npl) - call swiftest_util_sort_rearrange(pl%radius, ind, npl) - call swiftest_util_sort_rearrange(pl%density, ind, npl) - call swiftest_util_sort_rearrange(pl%rbeg, ind, npl) - call swiftest_util_sort_rearrange(pl%vbeg, ind, npl) - call swiftest_util_sort_rearrange(pl%Ip, ind, npl) - call swiftest_util_sort_rearrange(pl%rot, ind, npl) - call swiftest_util_sort_rearrange(pl%k2, ind, npl) - call swiftest_util_sort_rearrange(pl%Q, ind, npl) - call swiftest_util_sort_rearrange(pl%tlag, ind, npl) - call swiftest_util_sort_rearrange(pl%kin, ind, npl) - call swiftest_util_sort_rearrange(pl%lmtiny, ind, npl) - call swiftest_util_sort_rearrange(pl%nplenc, ind, npl) - call swiftest_util_sort_rearrange(pl%ntpenc, ind, npl) + call util_sort_rearrange(pl%mass, ind, npl) + call util_sort_rearrange(pl%Gmass, ind, npl) + call util_sort_rearrange(pl%rhill, ind, npl) + call util_sort_rearrange(pl%renc, ind, npl) + call util_sort_rearrange(pl%radius, ind, npl) + call util_sort_rearrange(pl%density, ind, npl) + call util_sort_rearrange(pl%rbeg, ind, npl) + call util_sort_rearrange(pl%vbeg, ind, npl) + call util_sort_rearrange(pl%Ip, ind, npl) + call util_sort_rearrange(pl%rot, ind, npl) + call util_sort_rearrange(pl%k2, ind, npl) + call util_sort_rearrange(pl%Q, ind, npl) + call util_sort_rearrange(pl%tlag, ind, npl) + call util_sort_rearrange(pl%kin, ind, npl) + call util_sort_rearrange(pl%lmtiny, ind, npl) + call util_sort_rearrange(pl%nplenc, ind, npl) + call util_sort_rearrange(pl%ntpenc, ind, npl) if (allocated(pl%k_plpl)) deallocate(pl%k_plpl) @@ -4157,7 +2959,7 @@ module subroutine swiftest_util_sort_rearrange_tp(self, ind) integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) associate(tp => self, ntp => self%nbody) - call swiftest_util_sort_rearrange(tp%nplenc, ind, ntp) + call util_sort_rearrange(tp%nplenc, ind, ntp) if (allocated(tp%k_pltp)) deallocate(tp%k_pltp) @@ -4168,220 +2970,6 @@ module subroutine swiftest_util_sort_rearrange_tp(self, ind) end subroutine swiftest_util_sort_rearrange_tp - module subroutine swiftest_util_spill_arr_char_string(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of type character strings - !! This is the inverse of a spill operation - implicit none - ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - ! Internals - integer(I4B) :: nspill, nkeep, nlist - character(len=STRMAX), dimension(:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(nspill)) - else if (size(discards) /= nspill) then - deallocate(discards) - allocate(discards(nspill)) - end if - - discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(nkeep)) - tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_char_string - - - module subroutine swiftest_util_spill_arr_DP(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of type DP - !! This is the inverse of a spill operation - implicit none - ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - ! Internals - integer(I4B) :: nspill, nkeep, nlist - real(DP), dimension(:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(nspill)) - else if (size(discards) /= nspill) then - deallocate(discards) - allocate(discards(nspill)) - end if - - discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(nkeep)) - tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_DP - - - module subroutine swiftest_util_spill_arr_DPvec(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of DP vectors with shape (NDIM, n) - !! This is the inverse of a spill operation - implicit none - ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep - real(DP), dimension(:,:), allocatable, intent(inout) :: discards !! Array discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - ! Internals - integer(I4B) :: i, nspill, nkeep, nlist - real(DP), dimension(:,:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(NDIM, nspill)) - else if (size(discards, dim=2) /= nspill) then - deallocate(discards) - allocate(discards(NDIM, nspill)) - end if - - do i = 1, NDIM - discards(i,:) = pack(keeps(i,1:nlist), lspill_list(1:nlist)) - end do - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(NDIM, nkeep)) - do i = 1, NDIM - tmp(i, :) = pack(keeps(i, 1:nlist), .not. lspill_list(1:nlist)) - end do - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_DPvec - - - module subroutine swiftest_util_spill_arr_I4B(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of type I4B - !! This is the inverse of a spill operation - implicit none - ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I4B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - ! Internals - integer(I4B) :: nspill, nkeep, nlist - integer(I4B), dimension(:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(nspill)) - else if (size(discards) /= nspill) then - deallocate(discards) - allocate(discards(nspill)) - end if - - discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(nkeep)) - tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_I4B - - - module subroutine swiftest_util_spill_arr_I8B(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of type I4B - !! This is the inverse of a spill operation - implicit none - ! Arguments - integer(I8B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I8B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not - ! Internals - integer(I4B) :: nspill, nkeep, nlist - integer(I8B), dimension(:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(nspill)) - else if (size(discards) /= nspill) then - deallocate(discards) - allocate(discards(nspill)) - end if - - discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(nkeep)) - tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_I8B - - module subroutine swiftest_util_spill_arr_info(keeps, discards, lspill_list, ldestructive) !! author: David A. Minton !! @@ -4472,48 +3060,6 @@ module subroutine swiftest_util_spill_arr_kin(keeps, discards, lspill_list, ldes end subroutine swiftest_util_spill_arr_kin - module subroutine swiftest_util_spill_arr_logical(keeps, discards, lspill_list, ldestructive) - !! author: David A. Minton - !! - !! Performs a spill operation on a single array of logicals - !! This is the inverse of a spill operation - implicit none - ! Arguments - logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - logical, dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or no - ! Internals - integer(I4B) :: nspill, nkeep, nlist - logical, dimension(:), allocatable :: tmp !! Array of values to keep - - nkeep = count(.not.lspill_list(:)) - nspill = count(lspill_list(:)) - nlist = size(lspill_list(:)) - - if (.not.allocated(keeps) .or. nspill == 0) return - if (.not.allocated(discards)) then - allocate(discards(nspill)) - else if (size(discards) /= nspill) then - deallocate(discards) - allocate(discards(nspill)) - end if - - discards(:) = pack(keeps(1:nlist), lspill_list(1:nlist)) - if (ldestructive) then - if (nkeep > 0) then - allocate(tmp(nkeep)) - tmp(:) = pack(keeps(1:nlist), .not. lspill_list(1:nlist)) - call move_alloc(tmp, keeps) - else - deallocate(keeps) - end if - end if - - return - end subroutine swiftest_util_spill_arr_logical - - module subroutine swiftest_util_spill_body(self, discards, lspill_list, ldestructive) !! author: David A. Minton !! @@ -4532,32 +3078,32 @@ module subroutine swiftest_util_spill_body(self, discards, lspill_list, ldestruc !> Spill all the common components associate(keeps => self) - call swiftest_util_spill(keeps%id, discards%id, lspill_list, ldestructive) - call swiftest_util_spill(keeps%info, discards%info, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lmask, discards%lmask, lspill_list, ldestructive) - call swiftest_util_spill(keeps%status, discards%status, lspill_list, ldestructive) - call swiftest_util_spill(keeps%ldiscard, discards%ldiscard, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lcollision, discards%lcollision, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lencounter, discards%lencounter, lspill_list, ldestructive) - call swiftest_util_spill(keeps%mu, discards%mu, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rh, discards%rh, lspill_list, ldestructive) - call swiftest_util_spill(keeps%vh, discards%vh, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rb, discards%rb, lspill_list, ldestructive) - call swiftest_util_spill(keeps%vb, discards%vb, lspill_list, ldestructive) - call swiftest_util_spill(keeps%ah, discards%ah, lspill_list, ldestructive) - call swiftest_util_spill(keeps%aobl, discards%aobl, lspill_list, ldestructive) - call swiftest_util_spill(keeps%agr, discards%agr, lspill_list, ldestructive) - call swiftest_util_spill(keeps%atide, discards%atide, lspill_list, ldestructive) - call swiftest_util_spill(keeps%ir3h, discards%ir3h, lspill_list, ldestructive) - call swiftest_util_spill(keeps%isperi, discards%isperi, lspill_list, ldestructive) - call swiftest_util_spill(keeps%peri, discards%peri, lspill_list, ldestructive) - call swiftest_util_spill(keeps%atp, discards%atp, lspill_list, ldestructive) - call swiftest_util_spill(keeps%a, discards%a, lspill_list, ldestructive) - call swiftest_util_spill(keeps%e, discards%e, lspill_list, ldestructive) - call swiftest_util_spill(keeps%inc, discards%inc, lspill_list, ldestructive) - call swiftest_util_spill(keeps%capom, discards%capom, lspill_list, ldestructive) - call swiftest_util_spill(keeps%omega, discards%omega, lspill_list, ldestructive) - call swiftest_util_spill(keeps%capm, discards%capm, lspill_list, ldestructive) + call util_spill(keeps%id, discards%id, lspill_list, ldestructive) + call util_spill(keeps%info, discards%info, lspill_list, ldestructive) + call util_spill(keeps%lmask, discards%lmask, lspill_list, ldestructive) + call util_spill(keeps%status, discards%status, lspill_list, ldestructive) + call util_spill(keeps%ldiscard, discards%ldiscard, lspill_list, ldestructive) + call util_spill(keeps%lcollision, discards%lcollision, lspill_list, ldestructive) + call util_spill(keeps%lencounter, discards%lencounter, lspill_list, ldestructive) + call util_spill(keeps%mu, discards%mu, lspill_list, ldestructive) + call util_spill(keeps%rh, discards%rh, lspill_list, ldestructive) + call util_spill(keeps%vh, discards%vh, lspill_list, ldestructive) + call util_spill(keeps%rb, discards%rb, lspill_list, ldestructive) + call util_spill(keeps%vb, discards%vb, lspill_list, ldestructive) + call util_spill(keeps%ah, discards%ah, lspill_list, ldestructive) + call util_spill(keeps%aobl, discards%aobl, lspill_list, ldestructive) + call util_spill(keeps%agr, discards%agr, lspill_list, ldestructive) + call util_spill(keeps%atide, discards%atide, lspill_list, ldestructive) + call util_spill(keeps%ir3h, discards%ir3h, lspill_list, ldestructive) + call util_spill(keeps%isperi, discards%isperi, lspill_list, ldestructive) + call util_spill(keeps%peri, discards%peri, lspill_list, ldestructive) + call util_spill(keeps%atp, discards%atp, lspill_list, ldestructive) + call util_spill(keeps%a, discards%a, lspill_list, ldestructive) + call util_spill(keeps%e, discards%e, lspill_list, ldestructive) + call util_spill(keeps%inc, discards%inc, lspill_list, ldestructive) + call util_spill(keeps%capom, discards%capom, lspill_list, ldestructive) + call util_spill(keeps%omega, discards%omega, lspill_list, ldestructive) + call util_spill(keeps%capm, discards%capm, lspill_list, ldestructive) nbody_old = keeps%nbody @@ -4587,24 +3133,24 @@ module subroutine swiftest_util_spill_pl(self, discards, lspill_list, ldestructi select type (discards) ! The standard requires us to select the type of both arguments in order to access all the components class is (swiftest_pl) !> Spill components specific to the massive body class - call swiftest_util_spill(keeps%mass, discards%mass, lspill_list, ldestructive) - call swiftest_util_spill(keeps%Gmass, discards%Gmass, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rhill, discards%rhill, lspill_list, ldestructive) - call swiftest_util_spill(keeps%renc, discards%renc, lspill_list, ldestructive) - call swiftest_util_spill(keeps%radius, discards%radius, lspill_list, ldestructive) - call swiftest_util_spill(keeps%density, discards%density, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rbeg, discards%rbeg, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rend, discards%rend, lspill_list, ldestructive) - call swiftest_util_spill(keeps%vbeg, discards%vbeg, lspill_list, ldestructive) - call swiftest_util_spill(keeps%Ip, discards%Ip, lspill_list, ldestructive) - call swiftest_util_spill(keeps%rot, discards%rot, lspill_list, ldestructive) - call swiftest_util_spill(keeps%k2, discards%k2, lspill_list, ldestructive) - call swiftest_util_spill(keeps%Q, discards%Q, lspill_list, ldestructive) - call swiftest_util_spill(keeps%tlag, discards%tlag, lspill_list, ldestructive) - call swiftest_util_spill(keeps%kin, discards%kin, lspill_list, ldestructive) - call swiftest_util_spill(keeps%lmtiny, discards%lmtiny, lspill_list, ldestructive) - call swiftest_util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) - call swiftest_util_spill(keeps%ntpenc, discards%ntpenc, lspill_list, ldestructive) + call util_spill(keeps%mass, discards%mass, lspill_list, ldestructive) + call util_spill(keeps%Gmass, discards%Gmass, lspill_list, ldestructive) + call util_spill(keeps%rhill, discards%rhill, lspill_list, ldestructive) + call util_spill(keeps%renc, discards%renc, lspill_list, ldestructive) + call util_spill(keeps%radius, discards%radius, lspill_list, ldestructive) + call util_spill(keeps%density, discards%density, lspill_list, ldestructive) + call util_spill(keeps%rbeg, discards%rbeg, lspill_list, ldestructive) + call util_spill(keeps%rend, discards%rend, lspill_list, ldestructive) + call util_spill(keeps%vbeg, discards%vbeg, lspill_list, ldestructive) + call util_spill(keeps%Ip, discards%Ip, lspill_list, ldestructive) + call util_spill(keeps%rot, discards%rot, lspill_list, ldestructive) + call util_spill(keeps%k2, discards%k2, lspill_list, ldestructive) + call util_spill(keeps%Q, discards%Q, lspill_list, ldestructive) + call util_spill(keeps%tlag, discards%tlag, lspill_list, ldestructive) + call util_spill(keeps%kin, discards%kin, lspill_list, ldestructive) + call util_spill(keeps%lmtiny, discards%lmtiny, lspill_list, ldestructive) + call util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) + call util_spill(keeps%ntpenc, discards%ntpenc, lspill_list, ldestructive) if (ldestructive .and. allocated(keeps%k_plpl)) deallocate(keeps%k_plpl) @@ -4634,7 +3180,7 @@ module subroutine swiftest_util_spill_tp(self, discards, lspill_list, ldestructi select type(discards) class is (swiftest_tp) !> Spill components specific to the test particle class - call swiftest_util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) + call util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) call swiftest_util_spill_body(keeps, discards, lspill_list, ldestructive) class default write(*,*) 'Error! spill method called for incompatible return type on swiftest_tp' @@ -4645,70 +3191,6 @@ module subroutine swiftest_util_spill_tp(self, discards, lspill_list, ldestructi end subroutine swiftest_util_spill_tp - module subroutine swiftest_util_unique_DP(input_array, output_array, index_map) - !! author: David A. Minton - !! - !! Takes an input unsorted integer array and returns a new array of sorted, unique values (DP version) - implicit none - ! Arguments - real(DP), dimension(:), intent(in) :: input_array !! Unsorted input array - real(DP), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) - ! Internals - real(DP), dimension(:), allocatable :: unique_array - integer(I4B) :: n - real(DP) :: lo, hi - - allocate(unique_array, mold=input_array) - allocate(index_map(size(input_array))) - lo = minval(input_array) - 1 - hi = maxval(input_array) - - n = 0 - do - n = n + 1 - lo = minval(input_array(:), mask=input_array(:) > lo) - unique_array(n) = lo - where(input_array(:) == lo) index_map(:) = n - if (lo >= hi) exit - enddo - allocate(output_array(n), source=unique_array(1:n)) - - return - end subroutine swiftest_util_unique_DP - - - module subroutine swiftest_util_unique_I4B(input_array, output_array, index_map) - !! author: David A. Minton - !! - !! Takes an input unsorted integer array and returns a new array of sorted, unique values (I4B version) - implicit none - ! Arguments - integer(I4B), dimension(:), intent(in) :: input_array !! Unsorted input array - integer(I4B), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) - ! Internals - integer(I4B), dimension(:), allocatable :: unique_array - integer(I4B) :: n, lo, hi - - allocate(unique_array, mold=input_array) - allocate(index_map, mold=input_array) - lo = minval(input_array) - 1 - hi = maxval(input_array) - - n = 0 - do - n = n + 1 - lo = minval(input_array(:), mask=input_array(:) > lo) - unique_array(n) = lo - where(input_array(:) == lo) index_map(:) = n - if (lo >= hi) exit - enddo - allocate(output_array(n), source=unique_array(1:n)) - - return - end subroutine swiftest_util_unique_I4B - module subroutine swiftest_util_valid_id_system(self, param) !! author: David A. Minton @@ -4740,11 +3222,11 @@ module subroutine swiftest_util_valid_id_system(self, param) maxid = maxval(idarr) ! Check to see if the ids are unique - call swiftest_util_unique(idarr, unique_idarr, idmap) + call util_unique(idarr, unique_idarr, idmap) if (size(unique_idarr) == nid) return ! All id values are unique ! Fix any duplicate id values and update the maxid - call swiftest_util_sort(idmap) + call util_sort(idmap) do i = 2, size(idmap) if (idmap(i) == idmap(i-1)) then maxid = maxid + 1 diff --git a/src/symba/symba_util.f90 b/src/symba/symba_util.f90 index ad1d50383..94b6aa9f5 100644 --- a/src/symba/symba_util.f90 +++ b/src/symba/symba_util.f90 @@ -25,12 +25,10 @@ module subroutine symba_util_append_pl(self, source, lsource_mask) select type(source) class is (symba_pl) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%levelg, source%levelg, nold, nsrc, lsource_mask) - call swiftest_util_append(self%levelm, source%levelm, nold, nsrc, lsource_mask) + call util_append(self%levelg, source%levelg, lsource_mask=lsource_mask) + call util_append(self%levelm, source%levelm, lsource_mask=lsource_mask) - call swiftest_util_append_pl(self, source, lsource_mask) ! Note: helio_pl does not have its own append method, so we skip back to the base class - end associate + call swiftest_util_append_pl(self, source, lsource_mask) ! Note: helio_pl does not have its own append method, so we skip back to the base class class default write(*,*) "Invalid object passed to the append method. Source must be of class symba_pl or its descendents!" call base_util_exit(FAILURE) @@ -53,12 +51,10 @@ module subroutine symba_util_append_tp(self, source, lsource_mask) select type(source) class is (symba_tp) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%levelg, source%levelg, nold, nsrc, lsource_mask) - call swiftest_util_append(self%levelm, source%levelm, nold, nsrc, lsource_mask) + call util_append(self%levelg, source%levelg, lsource_mask=lsource_mask) + call util_append(self%levelm, source%levelm, lsource_mask=lsource_mask) - call swiftest_util_append_tp(self, source, lsource_mask) ! Note: helio_tp does not have its own append method, so we skip back to the base class - end associate + call swiftest_util_append_tp(self, source, lsource_mask) ! Note: helio_tp does not have its own append method, so we skip back to the base class class default write(*,*) "Invalid object passed to the append method. Source must be of class symba_tp or its descendents!" call base_util_exit(FAILURE) @@ -134,8 +130,8 @@ module subroutine symba_util_fill_pl(self, inserts, lfill_list) associate(keeps => self) select type(inserts) class is (symba_pl) - call swiftest_util_fill(keeps%levelg, inserts%levelg, lfill_list) - call swiftest_util_fill(keeps%levelm, inserts%levelm, lfill_list) + call util_fill(keeps%levelg, inserts%levelg, lfill_list) + call util_fill(keeps%levelm, inserts%levelm, lfill_list) call swiftest_util_fill_pl(keeps, inserts, lfill_list) ! Note: helio_pl does not have its own fill method, so we skip back to the base class class default @@ -163,9 +159,9 @@ module subroutine symba_util_fill_tp(self, inserts, lfill_list) associate(keeps => self) select type(inserts) class is (symba_tp) - call swiftest_util_fill(keeps%nplenc, inserts%nplenc, lfill_list) - call swiftest_util_fill(keeps%levelg, inserts%levelg, lfill_list) - call swiftest_util_fill(keeps%levelm, inserts%levelm, lfill_list) + call util_fill(keeps%nplenc, inserts%nplenc, lfill_list) + call util_fill(keeps%levelg, inserts%levelg, lfill_list) + call util_fill(keeps%levelm, inserts%levelm, lfill_list) call swiftest_util_fill_tp(keeps, inserts, lfill_list) ! Note: helio_tp does not have its own fill method, so we skip back to the base class class default @@ -223,8 +219,8 @@ module subroutine symba_util_resize_pl(self, nnew) class(symba_pl), intent(inout) :: self !! SyMBA massive body object integer(I4B), intent(in) :: nnew !! New size neded - call swiftest_util_resize(self%levelg, nnew) - call swiftest_util_resize(self%levelm, nnew) + call util_resize(self%levelg, nnew) + call util_resize(self%levelm, nnew) call swiftest_util_resize_pl(self, nnew) @@ -240,8 +236,8 @@ module subroutine symba_util_resize_tp(self, nnew) class(symba_tp), intent(inout) :: self !! SyMBA test particle object integer(I4B), intent(in) :: nnew !! New size neded - call swiftest_util_resize(self%levelg, nnew) - call swiftest_util_resize(self%levelm, nnew) + call util_resize(self%levelg, nnew) + call util_resize(self%levelm, nnew) call swiftest_util_resize_tp(self, nnew) @@ -423,9 +419,9 @@ module subroutine symba_util_sort_pl(self, sortby, ascending) associate(pl => self, npl => self%nbody) select case(sortby) case("levelg") - call swiftest_util_sort(direction * pl%levelg(1:npl), ind) + call util_sort(direction * pl%levelg(1:npl), ind) case("levelm") - call swiftest_util_sort(direction * pl%levelm(1:npl), ind) + call util_sort(direction * pl%levelm(1:npl), ind) case default ! Look for components in the parent class call swiftest_util_sort_pl(pl, sortby, ascending) @@ -464,11 +460,11 @@ module subroutine symba_util_sort_tp(self, sortby, ascending) associate(tp => self, ntp => self%nbody) select case(sortby) case("nplenc") - call swiftest_util_sort(direction * tp%nplenc(1:ntp), ind) + call util_sort(direction * tp%nplenc(1:ntp), ind) case("levelg") - call swiftest_util_sort(direction * tp%levelg(1:ntp), ind) + call util_sort(direction * tp%levelg(1:ntp), ind) case("levelm") - call swiftest_util_sort(direction * tp%levelm(1:ntp), ind) + call util_sort(direction * tp%levelm(1:ntp), ind) case default ! Look for components in the parent class call swiftest_util_sort_tp(tp, sortby, ascending) return @@ -492,8 +488,8 @@ module subroutine symba_util_sort_rearrange_pl(self, ind) integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) associate(pl => self, npl => self%nbody) - call swiftest_util_sort_rearrange(pl%levelg, ind, npl) - call swiftest_util_sort_rearrange(pl%levelm, ind, npl) + call util_sort_rearrange(pl%levelg, ind, npl) + call util_sort_rearrange(pl%levelm, ind, npl) call swiftest_util_sort_rearrange_pl(pl,ind) end associate @@ -512,9 +508,9 @@ module subroutine symba_util_sort_rearrange_tp(self, ind) integer(I4B), dimension(:), intent(in) :: ind !! Index array used to restructure the body (should contain all 1:n index values in the desired order) associate(tp => self, ntp => self%nbody) - call swiftest_util_sort_rearrange(tp%nplenc, ind, ntp) - call swiftest_util_sort_rearrange(tp%levelg, ind, ntp) - call swiftest_util_sort_rearrange(tp%levelm, ind, ntp) + call util_sort_rearrange(tp%nplenc, ind, ntp) + call util_sort_rearrange(tp%levelg, ind, ntp) + call util_sort_rearrange(tp%levelm, ind, ntp) call swiftest_util_sort_rearrange_tp(tp,ind) end associate @@ -540,8 +536,8 @@ module subroutine symba_util_spill_pl(self, discards, lspill_list, ldestructive) associate(keeps => self) select type(discards) class is (symba_pl) - call swiftest_util_spill(keeps%levelg, discards%levelg, lspill_list, ldestructive) - call swiftest_util_spill(keeps%levelm, discards%levelm, lspill_list, ldestructive) + call util_spill(keeps%levelg, discards%levelg, lspill_list, ldestructive) + call util_spill(keeps%levelm, discards%levelm, lspill_list, ldestructive) call swiftest_util_spill_pl(keeps, discards, lspill_list, ldestructive) class default @@ -571,9 +567,9 @@ module subroutine symba_util_spill_tp(self, discards, lspill_list, ldestructive) associate(keeps => self) select type(discards) class is (symba_tp) - call swiftest_util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) - call swiftest_util_spill(keeps%levelg, discards%levelg, lspill_list, ldestructive) - call swiftest_util_spill(keeps%levelm, discards%levelm, lspill_list, ldestructive) + call util_spill(keeps%nplenc, discards%nplenc, lspill_list, ldestructive) + call util_spill(keeps%levelg, discards%levelg, lspill_list, ldestructive) + call util_spill(keeps%levelm, discards%levelm, lspill_list, ldestructive) call swiftest_util_spill_tp(keeps, discards, lspill_list, ldestructive) class default diff --git a/src/whm/whm_util.f90 b/src/whm/whm_util.f90 index 13105047e..09fa59d76 100644 --- a/src/whm/whm_util.f90 +++ b/src/whm/whm_util.f90 @@ -24,15 +24,13 @@ module subroutine whm_util_append_pl(self, source, lsource_mask) select type(source) class is (whm_pl) - associate(nold => self%nbody, nsrc => source%nbody) - call swiftest_util_append(self%eta, source%eta, nold, nsrc, lsource_mask) - call swiftest_util_append(self%muj, source%muj, nold, nsrc, lsource_mask) - call swiftest_util_append(self%ir3j, source%ir3j, nold, nsrc, lsource_mask) - call swiftest_util_append(self%xj, source%xj, nold, nsrc, lsource_mask) - call swiftest_util_append(self%vj, source%vj, nold, nsrc, lsource_mask) - - call swiftest_util_append_pl(self, source, lsource_mask) - end associate + call util_append(self%eta, source%eta, lsource_mask=lsource_mask) + call util_append(self%muj, source%muj, lsource_mask=lsource_mask) + call util_append(self%ir3j, source%ir3j, lsource_mask=lsource_mask) + call util_append(self%xj, source%xj, lsource_mask=lsource_mask) + call util_append(self%vj, source%vj, lsource_mask=lsource_mask) + + call swiftest_util_append_pl(self, source, lsource_mask) class default write(*,*) "Invalid object passed to the append method. Source must be of class whm_pl or its descendents" call base_util_exit(FAILURE) @@ -78,11 +76,11 @@ module subroutine whm_util_fill_pl(self, inserts, lfill_list) associate(keeps => self) select type(inserts) class is (whm_pl) - call swiftest_util_fill(keeps%eta, inserts%eta, lfill_list) - call swiftest_util_fill(keeps%muj, inserts%muj, lfill_list) - call swiftest_util_fill(keeps%ir3j, inserts%ir3j, lfill_list) - call swiftest_util_fill(keeps%xj, inserts%xj, lfill_list) - call swiftest_util_fill(keeps%vj, inserts%vj, lfill_list) + call util_fill(keeps%eta, inserts%eta, lfill_list) + call util_fill(keeps%muj, inserts%muj, lfill_list) + call util_fill(keeps%ir3j, inserts%ir3j, lfill_list) + call util_fill(keeps%xj, inserts%xj, lfill_list) + call util_fill(keeps%vj, inserts%vj, lfill_list) call swiftest_util_fill_pl(keeps, inserts, lfill_list) class default @@ -104,11 +102,11 @@ module subroutine whm_util_resize_pl(self, nnew) class(whm_pl), intent(inout) :: self !! WHM massive body object integer(I4B), intent(in) :: nnew !! New size neded - call swiftest_util_resize(self%eta, nnew) - call swiftest_util_resize(self%xj, nnew) - call swiftest_util_resize(self%vj, nnew) - call swiftest_util_resize(self%muj, nnew) - call swiftest_util_resize(self%ir3j, nnew) + call util_resize(self%eta, nnew) + call util_resize(self%xj, nnew) + call util_resize(self%vj, nnew) + call util_resize(self%muj, nnew) + call util_resize(self%ir3j, nnew) call swiftest_util_resize_pl(self, nnew) @@ -255,11 +253,11 @@ module subroutine whm_util_sort_pl(self, sortby, ascending) associate(pl => self, npl => self%nbody) select case(sortby) case("eta") - call swiftest_util_sort(direction * pl%eta(1:npl), ind) + call util_sort(direction * pl%eta(1:npl), ind) case("muj") - call swiftest_util_sort(direction * pl%muj(1:npl), ind) + call util_sort(direction * pl%muj(1:npl), ind) case("ir3j") - call swiftest_util_sort(direction * pl%ir3j(1:npl), ind) + call util_sort(direction * pl%ir3j(1:npl), ind) case("xj", "vj") write(*,*) 'Cannot sort by ' // trim(adjustl(sortby)) // '. Component not sortable!' case default @@ -287,11 +285,11 @@ module subroutine whm_util_sort_rearrange_pl(self, ind) if (self%nbody == 0) return associate(pl => self, npl => self%nbody) - call swiftest_util_sort_rearrange(pl%eta, ind, npl) - call swiftest_util_sort_rearrange(pl%xj, ind, npl) - call swiftest_util_sort_rearrange(pl%vj, ind, npl) - call swiftest_util_sort_rearrange(pl%muj, ind, npl) - call swiftest_util_sort_rearrange(pl%ir3j, ind, npl) + call util_sort_rearrange(pl%eta, ind, npl) + call util_sort_rearrange(pl%xj, ind, npl) + call util_sort_rearrange(pl%vj, ind, npl) + call util_sort_rearrange(pl%muj, ind, npl) + call util_sort_rearrange(pl%ir3j, ind, npl) call swiftest_util_sort_rearrange_pl(pl,ind) end associate @@ -316,11 +314,11 @@ module subroutine whm_util_spill_pl(self, discards, lspill_list, ldestructive) associate(keeps => self) select type(discards) class is (whm_pl) - call swiftest_util_spill(keeps%eta, discards%eta, lspill_list, ldestructive) - call swiftest_util_spill(keeps%muj, discards%muj, lspill_list, ldestructive) - call swiftest_util_spill(keeps%ir3j, discards%ir3j, lspill_list, ldestructive) - call swiftest_util_spill(keeps%xj, discards%xj, lspill_list, ldestructive) - call swiftest_util_spill(keeps%vj, discards%vj, lspill_list, ldestructive) + call util_spill(keeps%eta, discards%eta, lspill_list, ldestructive) + call util_spill(keeps%muj, discards%muj, lspill_list, ldestructive) + call util_spill(keeps%ir3j, discards%ir3j, lspill_list, ldestructive) + call util_spill(keeps%xj, discards%xj, lspill_list, ldestructive) + call util_spill(keeps%vj, discards%vj, lspill_list, ldestructive) call swiftest_util_spill_pl(keeps, discards, lspill_list, ldestructive) class default From 4f1593a18b7409fa0423614d14ebfd62286034c8 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 08:34:04 -0400 Subject: [PATCH 024/149] Fixed bugs in append method and now coarrays appears to work --- src/base/base_module.f90 | 32 +++++++++++++++---------------- src/coarray/coarray_collect.f90 | 11 +++-------- src/swiftest/swiftest_coarray.f90 | 25 ------------------------ src/swiftest/swiftest_driver.f90 | 1 - src/swiftest/swiftest_util.f90 | 4 ++-- 5 files changed, 21 insertions(+), 52 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 84e5a1f2e..70ba3fbbb 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -316,9 +316,9 @@ subroutine base_util_append_arr_char_string(arr, source, nold, lsource_mask) nnew = nend_orig + nsrc if (present(lsource_mask)) then - arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + arr(nend_orig + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - arr(nold + 1:nnew) = source(1:nsrc) + arr(nend_orig + 1:nnew) = source(1:nsrc) end if return @@ -359,9 +359,9 @@ subroutine base_util_append_arr_DP(arr, source, nold, lsource_mask) nnew = nend_orig + nsrc if (present(lsource_mask)) then - arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + arr(nend_orig + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - arr(nold + 1:nnew) = source(1:nsrc) + arr(nend_orig + 1:nnew) = source(1:nsrc) end if return @@ -402,11 +402,11 @@ subroutine base_util_append_arr_DPvec(arr, source, nold, lsource_mask) nnew = nend_orig + nsrc if (present(lsource_mask)) then - arr(1, nold + 1:nnew) = pack(source(1,1:nsrc), lsource_mask(1:nsrc)) - arr(2, nold + 1:nnew) = pack(source(2,1:nsrc), lsource_mask(1:nsrc)) - arr(3, nold + 1:nnew) = pack(source(3,1:nsrc), lsource_mask(1:nsrc)) + arr(1, nend_orig + 1:nnew) = pack(source(1,1:nsrc), lsource_mask(1:nsrc)) + arr(2, nend_orig + 1:nnew) = pack(source(2,1:nsrc), lsource_mask(1:nsrc)) + arr(3, nend_orig + 1:nnew) = pack(source(3,1:nsrc), lsource_mask(1:nsrc)) else - arr(:,nold + 1:nnew) = source(:,1:nsrc) + arr(:,nend_orig + 1:nnew) = source(:,1:nsrc) end if return @@ -447,9 +447,9 @@ subroutine base_util_append_arr_I4B(arr, source, nold, lsource_mask) nnew = nend_orig + nsrc if (present(lsource_mask)) then - arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + arr(nend_orig + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - arr(nold + 1:nnew) = source(1:nsrc) + arr(nend_orig + 1:nnew) = source(1:nsrc) end if return @@ -488,11 +488,11 @@ subroutine base_util_append_arr_logical(arr, source, nold, lsource_mask) call util_resize(arr, nend_orig + nsrc) end if nnew = nend_orig + nsrc - + if (present(lsource_mask)) then - arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + arr(nend_orig + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - arr(nold + 1:nnew) = source(1:nsrc) + arr(nend_orig + 1:nnew) = source(:) end if return @@ -909,7 +909,7 @@ subroutine base_util_resize_arr_logical(arr, nnew) logical, dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated integer(I4B) :: nold !! Old size logical, parameter :: init_val = .false. - + if (nnew < 0) return if (nnew == 0) then @@ -934,7 +934,7 @@ subroutine base_util_resize_arr_logical(arr, nnew) tmp(1:nnew) = arr(1:nnew) end if else - tmp(1:nnew) = init_val + tmp = init_val end if call move_alloc(tmp, arr) @@ -1885,7 +1885,7 @@ pure subroutine base_util_sort_partition_SP(arr, marker, ind) return end subroutine base_util_sort_partition_SP - + pure subroutine base_util_sort_rearrange_arr_char_string(arr, ind, n) !! author: David A. Minton !! diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 index f9a65dad1..35808d88f 100644 --- a/src/coarray/coarray_collect.f90 +++ b/src/coarray/coarray_collect.f90 @@ -50,9 +50,8 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) end do allocate(tmp(nmax)[*]) - tmp(1:n) = var(1:n) + if (isalloc) tmp(1:n) = var(1:n) - !sync all if (this_image() == di) then do img = 1, num_images() if (img /= di) then @@ -107,9 +106,8 @@ module subroutine coarray_component_collect_DP_arr2D(var,dest_img) end do allocate(tmp(NDIM,nmax)[*]) - tmp(:,1:n2) = var(:,1:n2) + if (isalloc) tmp(:,1:n2) = var(:,1:n2) - !sync all if (this_image() == di) then do img = 1, num_images() if (img /= di) then @@ -144,7 +142,6 @@ module subroutine coarray_component_collect_I4B(var,dest_img) allocate(tmp[*], source=var) - !sync all if (this_image() == di) then var = 0 do img = 1, num_images() @@ -179,7 +176,6 @@ module subroutine coarray_component_collect_I8B(var,dest_img) allocate(tmp[*], source=var) - !sync all if (this_image() == di) then var = 0 do img = 1, num_images() @@ -230,9 +226,8 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) end do allocate(tmp(nmax)[*]) - tmp(1:n) = var(1:n) + if (isalloc) tmp(1:n) = var(1:n) - !sync all if (this_image() == di) then do img = 1, num_images() if (img /= di) then diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 2b04d0697..f3643f42a 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -410,9 +410,6 @@ module subroutine swiftest_coarray_cocollect_body(self) class(swiftest_body),intent(inout), codimension[*] :: self !! Swiftest body object integer(I4B) :: i - if (this_image() == 1) write(*,*) "Before collect " - sync all - if (allocated(self%id)) write(*,*) "Image: ",this_image(), "id: ",self%id call cocollect(self%nbody) call cocollect(self%id) call cocollect(self%info) @@ -441,10 +438,6 @@ module subroutine swiftest_coarray_cocollect_body(self) call cocollect(self%omega) call cocollect(self%capm) - if (this_image() == 1) write(*,*) "after collect " - sync all - if (allocated(self%id)) write(*,*) "Image: ",this_image(), "id: ",self%id - return end subroutine swiftest_coarray_cocollect_body @@ -492,10 +485,6 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) deallocate(cotp) - if (this_image() == 1) then - write(param%display_unit,*) " Done collecting" - end if - return end subroutine swiftest_coarray_collect_system @@ -523,10 +512,6 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." end if - if (this_image() == 1) write(*,*) "Before distribute " - sync all - if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "id: ",nbody_system%tp%id - ntp = nbody_system%tp%nbody sync all ntot = ntp[1] @@ -555,16 +540,6 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) deallocate(tmp, cotp) - - if (this_image() == 1) write(*,*) "After distribute " - sync all - if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "id: ",nbody_system%tp%id - - - if (this_image() == 1) then - write(param%display_unit,*) " Done distributing" - end if - return end subroutine swiftest_coarray_distribute_system diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 4a34347f2..461979caa 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -129,7 +129,6 @@ program swiftest_driver #endif do iloop = istart, nloops !> Step the nbody_system forward in time - if (this_image() == 1) write(*,*) "Image: ", this_image(), "ntp: ",nbody_system%tp%nbody call integration_timer%start() call nbody_system%step(param, nbody_system%t, dt) call integration_timer%stop() diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index fe46cc9e3..8e6ad35e6 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -97,9 +97,9 @@ module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) nnew = nend_orig + nsrc if (present(lsource_mask)) then - arr(nold + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) + arr(nend_orig + 1:nnew) = pack(source(1:nsrc), lsource_mask(1:nsrc)) else - arr(nold + 1:nnew) = source(1:nsrc) + arr(nend_orig + 1:nnew) = source(1:nsrc) end if return From d5ee30790f6984c7e8fd01615335ccdff961e2d7 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 14:13:03 -0400 Subject: [PATCH 025/149] Fixed bugs in append methods and coarray collect procedures --- src/coarray/coarray_collect.f90 | 16 ++++--- src/swiftest/swiftest_coarray.f90 | 73 +++++++++++++------------------ src/swiftest/swiftest_io.f90 | 7 ++- src/swiftest/swiftest_util.f90 | 2 +- 4 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 index 35808d88f..bf60aa522 100644 --- a/src/coarray/coarray_collect.f90 +++ b/src/coarray/coarray_collect.f90 @@ -55,7 +55,7 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(:)[img]) + call util_append(var, tmp(1:n[img])[img]) n = n + n[img] end if end do @@ -231,7 +231,7 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(:)[img]) + call util_append(var, tmp(1:n[img])[img]) n = n + n[img] end if end do @@ -252,7 +252,7 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) integer(I4B), intent(in),optional :: dest_img ! Internals logical, dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: i,img, ti, di, ntot, istart, iend + integer(I4B) :: i,img, ti, di, ntot, istart, iend, nmax integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] @@ -271,13 +271,19 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) else n = 0 end if - allocate(tmp[*],source=var) sync all + nmax = 0 + do img = 1, num_images() + if (n[img] > nmax) nmax = n[img] + end do + + allocate(tmp(nmax)[*]) + if (isalloc) tmp(1:n) = var(1:n) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(:)[img]) + call util_append(var, tmp(1:n[img])[img]) n = n + n[img] end if end do diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index f3643f42a..9b3de3b81 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -209,7 +209,6 @@ module subroutine swiftest_coarray_coclone_system(self) end subroutine swiftest_coarray_coclone_system - module subroutine swiftest_coarray_component_clone_info(var,src_img) !! author: David A. Minton !! @@ -344,63 +343,47 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) integer(I4B), intent(in),optional :: dest_img ! Internals type(swiftest_particle_info), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, ti, di, ntot, istart, iend - integer(I4B), save :: n[*] - logical, save :: isalloc[*] - + integer(I4B) :: i,img, ti, di, ntot, istart, iend, nmax + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] + + allocate(isalloc[*]) + allocate(n[*]) + if (present(dest_img)) then - di = dest_img + di = dest_img else - di = 1 + di = 1 end if - - sync all + isalloc = allocated(var) if (isalloc) then - n = size(var) + n = size(var) else - n = 0 + n = 0 end if sync all - ntot = 0 + nmax = 0 do img = 1, num_images() - ntot = ntot + n[img] - end do - - allocate(tmp(ntot)[*]) - - ti = this_image() - - istart = 1 - iend = n - do img = 1, this_image() - 1 - istart = istart + n[img] - iend = iend + n[img] + if (n[img] > nmax) nmax = n[img] end do - - if (isalloc) then - tmp(istart:iend) = var(:) - deallocate(var) - end if - - sync all + + allocate(tmp(nmax)[*]) + if (isalloc) tmp(1:n) = var(1:n) + if (this_image() == di) then - allocate(var(ntot)) - istart = 1 - iend = n - do img = 1, num_images() - var(istart:iend) = tmp[img](istart:iend) - istart = istart + n[img] - iend = iend + n[img] - end do + do img = 1, num_images() + if (img /= di) then + call util_append(var, tmp(1:n[img])[img]) + n = n + n[img] + end if + end do end if - + return end subroutine swiftest_coarray_component_collect_info_arr1D - - module subroutine swiftest_coarray_cocollect_body(self) !! author: David A. Minton !! @@ -472,6 +455,8 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) class(swiftest_tp), allocatable, codimension[:] :: cotp character(len=NAMELEN) :: image_num_char + sync all + if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "before collecting ids: ",nbody_system%tp%id sync all if (this_image() == 1) then write(image_num_char,*) num_images() @@ -485,6 +470,8 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) deallocate(cotp) + if (this_image() == 1) write(*,*) "Image: ",this_image(), "After collecting ids: ",nbody_system%tp%id + return end subroutine swiftest_coarray_collect_system @@ -518,7 +505,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) if (ntot == 0) return allocate(lspill_list(ntot)) - num_per_image = ntot / num_images() + num_per_image = ceiling(1.0_DP * ntot / num_images()) istart = (this_image() - 1) * num_per_image + 1 if (this_image() == num_images()) then iend = ntot diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 68e60a04a..2fe0cb41a 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -183,8 +183,13 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) nbody_system%L_total_error = norm2(L_total_now(:) - nbody_system%L_total_orig(:)) / norm2(nbody_system%L_total_orig(:)) nbody_system%Mescape_error = nbody_system%GMescape / nbody_system%GMtot_orig - +#ifdef COARRAY + if (this_image() == 1) then +#endif if (lterminal) write(display_unit, EGYTERMFMT) nbody_system%L_total_error, nbody_system%E_orbit_error, nbody_system%te_error,nbody_system%Mtot_error +#ifdef COARRAY + end if ! (this_image() == 1) then +#endif if (abs(nbody_system%Mtot_error) > 100 * epsilon(nbody_system%Mtot_error)) then write(*,*) "Severe error! Mass not conserved! Halting!" diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 8e6ad35e6..a6b7566e7 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -57,7 +57,7 @@ module subroutine swiftest_util_append_arr_info(arr, source, nold, lsource_mask) idx = [(i, i = 1,nsrc)] end if - call swiftest_util_copy_particle_info_arr(source(:), arr(nold+1:nnew), idx) + call swiftest_util_copy_particle_info_arr(source(:), arr(nend_orig+1:nnew), idx) return end subroutine swiftest_util_append_arr_info From d3a98d0f81c45fb0205c7bd5a55de2910c85335c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:07:03 -0400 Subject: [PATCH 026/149] Added infrastructure to allow users to turn on or off coarrays --- python/swiftest/swiftest/simulation_class.py | 20 +++++++++++++++++ src/base/base_module.f90 | 13 +++++------ src/swiftest/swiftest_coarray.f90 | 9 ++++---- src/swiftest/swiftest_io.f90 | 23 ++++++++++++++++++++ 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 991758441..32171e72d 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -299,6 +299,11 @@ def __init__(self,read_param: bool = False, Parameter input file equivalent: `ENCOUNTER_CHECK` dask : bool, default False Use Dask to lazily load data (useful for very large datasets) + coarray : bool, default False + If true, will employ Coarrays on test particle structures to run in single program/multiple data parallel mode. + *Note" In order to use this capability, Swiftest must be compiled for Coarray support. Only certain integrators + can use Coarrays: RMVS, WHM, Helio are all compatible, but SyMBA is not, due to the way tp-pl close encounters + are handeled. verbose : bool, default True If set to True, then more information is printed by Simulation methods as they are executed. Setting to False suppresses most messages other than errors. @@ -824,6 +829,7 @@ def set_parameter(self, verbose: bool = True, **kwargs): "ephemeris_date": "MBCL", "restart": False, "encounter_save" : "NONE", + "coarray" : False, "simdir" : self.simdir } param_file = kwargs.pop("param_file",None) @@ -1065,6 +1071,7 @@ def set_feature(self, interaction_loops: Literal["TRIANGULAR", "FLAT"] | None = None, encounter_check_loops: Literal["TRIANGULAR", "SORTSWEEP"] | None = None, encounter_save: Literal["NONE", "TRAJECTORY", "CLOSEST", "BOTH"] | None = None, + coarray: bool | None = None, verbose: bool | None = None, simdir: str | os.PathLike = None, **kwargs: Any @@ -1130,6 +1137,11 @@ def set_feature(self, * "SORTSWEEP" : A Sort-Sweep algorithm is used to reduce the population of potential close encounter bodies. This algorithm is still in development, and does not necessarily speed up the encounter checking. Use with caution. + coarray : bool, default False + If true, will employ Coarrays on test particle structures to run in single program/multiple data parallel mode. + *Note" In order to use this capability, Swiftest must be compiled for Coarray support. Only certain integrators + can use Coarrays: RMVS, WHM, Helio are all compatible, but SyMBA is not, due to the way tp-pl close encounters + are handeled. tides: bool, optional Turns on tidal model (IN DEVELOPMENT - IGNORED) Yarkovsky: bool, optional @@ -1279,7 +1291,14 @@ def set_feature(self, self.driver_executable = self.binary_path / "swiftest_driver" self.param_file = Path(kwargs.pop("param_file","param.in")) + if coarray is not None: + if self.codename == "Swiftest": + self.param["COARRAY"] = coarray + update_list.append("coarray") + + self.param["TIDES"] = False + feature_dict = self.get_feature(update_list, verbose) return feature_dict @@ -1321,6 +1340,7 @@ def get_feature(self, arg_list: str | List[str] | None = None, verbose: bool | N "big_discard": "BIG_DISCARD", "interaction_loops": "INTERACTION_LOOPS", "encounter_check_loops": "ENCOUNTER_CHECK", + "coarray" : "COARRAY", "restart": "RESTART" } diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 70ba3fbbb..b47c2af7b 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -69,6 +69,7 @@ module base character(NAMELEN) :: interaction_loops = "ADAPTIVE" !! Method used to compute interaction loops. Options are "TRIANGULAR", "FLAT", or "ADAPTIVE" character(NAMELEN) :: encounter_check_plpl = "ADAPTIVE" !! Method used to compute pl-pl encounter checks. Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" character(NAMELEN) :: encounter_check_pltp = "ADAPTIVE" !! Method used to compute pl-tp encounter checks. Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" + logical :: lcoarray = .false. !! Use Coarrays for test particle parallelization. ! The following are used internally, and are not set by the user, but instead are determined by the input value of INTERACTION_LOOPS logical :: lflatten_interactions = .false. !! Use the flattened upper triangular matrix for pl-pl interaction loops @@ -2131,7 +2132,6 @@ subroutine base_coclone_param(self) ! Arguments class(base_parameters),intent(inout),codimension[*] :: self !! Collection of parameters ! Internals - integer(I4B) :: i call coclone(self%integrator) call coclone(self%param_file_name) @@ -2192,12 +2192,10 @@ subroutine base_coclone_param(self) call coclone(self%ltides ) call coclone(self%E_orbit_orig ) call coclone(self%GMtot_orig ) - do i = 1, NDIM - call coclone(self%L_total_orig(i)) - call coclone(self%L_orbit_orig(i)) - call coclone(self%L_spin_orig(i)) - call coclone(self%L_escape(i)) - end do + call coclonevec(self%L_total_orig) + call coclonevec(self%L_orbit_orig) + call coclonevec(self%L_spin_orig) + call coclonevec(self%L_escape) call coclone(self%GMescape ) call coclone(self%E_collisions ) call coclone(self%E_untracked ) @@ -2211,6 +2209,7 @@ subroutine base_coclone_param(self) call coclone(self%lyarkovsky) call coclone(self%lyorp ) call coclone(self%seed) + call coclone(self%lcoarray) return end subroutine base_coclone_param diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 9b3de3b81..cd35af401 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -455,8 +455,8 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) class(swiftest_tp), allocatable, codimension[:] :: cotp character(len=NAMELEN) :: image_num_char - sync all - if (allocated(nbody_system%tp%id)) write(*,*) "Image: ",this_image(), "before collecting ids: ",nbody_system%tp%id + if (.not.param%lcoarray) return + sync all if (this_image() == 1) then write(image_num_char,*) num_images() @@ -470,8 +470,6 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) deallocate(cotp) - if (this_image() == 1) write(*,*) "Image: ",this_image(), "After collecting ids: ",nbody_system%tp%id - return end subroutine swiftest_coarray_collect_system @@ -493,6 +491,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) class(swiftest_tp), allocatable, codimension[:] :: cotp class(swiftest_tp), allocatable :: tmp + if (.not.param%lcoarray) return sync all if (this_image() == 1) then write(image_num_char,*) num_images() @@ -543,6 +542,8 @@ module subroutine swiftest_coarray_initialize_system(nbody_system, param) class(swiftest_nbody_system), allocatable, codimension[:] :: tmp_system character(len=NAMELEN) :: image_num_char + if (.not.param%lcoarray) return + sync all if (this_image() == 1) then write(image_num_char,*) num_images() diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 2fe0cb41a..a83cdc4c2 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2071,6 +2071,9 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i case ("ENCOUNTER_SAVE") call swiftest_io_toupper(param_value) read(param_value, *) param%encounter_save + case ("COARRAY") + call swiftest_io_toupper(param_value) + if (param_value == "YES" .or. param_value == 'T') param%lcoarray = .true. case("SEED") read(param_value, *) nseeds_from_file ! Because the number of seeds can vary between compilers/systems, we need to make sure we can handle cases in which the input file has a different @@ -2317,6 +2320,26 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i param%lencounter_sas_pltp = .false. end select + + if (param%lcoarray) then +#ifdef COARRAY + if (num_images() == 1) then + write(iomsg, *) "Only one Coarray image detected. Coarrays will not be used." + param%lcoarray = .false. + end if + + select case(param%integrator) + case(INT_WHM, INT_RMVS, INT_HELIO) + case default + write(iomsg, *) "Coarray-based parallelization of test particles are not compatible with this integrator. This parameter will be ignored." + param%lcoarray = .false. + end select +#else + write(iomsg,*) "Coarray capability not detected. Swiftest must be compiled with Coarrays enabled. to use this feature." + param%lcoarray = .false. +#endif + end if + iostat = 0 call param%set_display(param%display_style) From 695371da0bae2b5631ba9aa032841f84a269ca82 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:20:00 -0400 Subject: [PATCH 027/149] Removed obsolete subroutine and fixed bugs in parameter io to make sure coarrays are turned on properly --- src/swiftest/swiftest_coarray.f90 | 31 ------------------------------- src/swiftest/swiftest_io.f90 | 1 + src/swiftest/swiftest_module.f90 | 6 ------ 3 files changed, 1 insertion(+), 37 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index cd35af401..9f255932d 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -529,35 +529,4 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) return end subroutine swiftest_coarray_distribute_system - - module subroutine swiftest_coarray_initialize_system(nbody_system, param) - !! author: David A. Minton - !! - !! Distributes test particles from image #1 out to all images. - implicit none - ! Arguments - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody system - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - ! Internals - class(swiftest_nbody_system), allocatable, codimension[:] :: tmp_system - character(len=NAMELEN) :: image_num_char - - if (.not.param%lcoarray) return - - sync all - if (this_image() == 1) then - write(image_num_char,*) num_images() - write(param%display_unit,*) " Cloning nbody system to " // trim(adjustl(image_num_char)) // " images." - end if - allocate(tmp_system[*], source=nbody_system) - call tmp_system%coclone() - if (this_image() /= 1) then - if (allocated(nbody_system)) deallocate(nbody_system) - allocate(nbody_system, source=tmp_system) - end if - - return - end subroutine swiftest_coarray_initialize_system - - end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index a83cdc4c2..8b4e5ad26 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2450,6 +2450,7 @@ module subroutine swiftest_io_param_writer(self, unit, iotype, v_list, iostat, i call io_param_writer_one("ENCOUNTER_CHECK_PLPL", param%encounter_check_plpl, unit) call io_param_writer_one("ENCOUNTER_CHECK_PLTP", param%encounter_check_pltp, unit) call io_param_writer_one("ENCOUNTER_SAVE", param%encounter_save, unit) + call io_param_writer_one("COARRAY", param%lcoarray, unit) if (param%lenergy) then call io_param_writer_one("FIRSTENERGY", param%lfirstenergy, unit) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index bbf1de296..5fac1d0ad 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -1711,12 +1711,6 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_coarray_distribute_system - - module subroutine swiftest_coarray_initialize_system(nbody_system, param) - implicit none - class(swiftest_nbody_system), allocatable, intent(inout) :: nbody_system !! Swiftest nbody system - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - end subroutine swiftest_coarray_initialize_system end interface interface coclone From 2600ef926c7e2c303f4b3b925d5386e39c455041 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:22:14 -0400 Subject: [PATCH 028/149] Helfpul io message about coarrays --- src/swiftest/swiftest_driver.f90 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 461979caa..19846f962 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -82,10 +82,15 @@ program swiftest_driver !$ write(param%display_unit,'(a,i3,/)') ' Number of threads = ', nthreads !$ if (param%log_output) write(*,'(a,i3)') ' OpenMP: Number of threads = ',nthreads #ifdef COARRAY - write(param%display_unit,*) ' Coarray parameters:' - write(param%display_unit,*) ' -------------------' - write(param%display_unit,*) ' Number of images = ', num_images() - if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() + if (param%lcoarray) then + write(param%display_unit,*) ' Coarray parameters:' + write(param%display_unit,*) ' -------------------' + write(param%display_unit,*) ' Number of images = ', num_images() + if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() + else + write(param%display_unit,*) ' Coarrays disabled.' + if (param%log_output) write(*,*) ' Coarrays disabled.' + end if end if #endif From bc84508816754b9c61e45e682a77f2492ec72ef2 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:35:15 -0400 Subject: [PATCH 029/149] Shutdown unused images when coarrays are turned off --- src/swiftest/swiftest_driver.f90 | 5 +++-- src/swiftest/swiftest_io.f90 | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 19846f962..7b8f103d9 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -38,6 +38,7 @@ program swiftest_driver param%integrator = trim(adjustl(integrator)) param%display_style = trim(adjustl(display_style)) call param%read_in(param_file_name) + if (.not.param%lcoarray .and. (this_image() /= 1)) stop ! Single image mode associate(t0 => param%t0, & tstart => param%tstart, & @@ -96,7 +97,7 @@ program swiftest_driver #ifdef COARRAY ! The following line lets us read in the input files one image at a time - if (this_image() /= 1) sync images(this_image() - 1) + if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) #endif call nbody_system%initialize(system_history, param) #ifdef COARRAY @@ -128,7 +129,7 @@ program swiftest_driver #endif #ifdef COARRAY - if (this_image() < num_images()) sync images(this_image() + 1) + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) ! Distribute test particles to the various images call nbody_system%coarray_distribute(param) #endif diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 8b4e5ad26..1fef0b3d3 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2349,7 +2349,7 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i #ifdef COARRAY end if ! this_image() == 1 - call coparam%coclone() + if (self%lcoarray) call coparam%coclone() select type(self) type is (swiftest_parameters) self = coparam From ab6b619743f4ec746eb422dc5cfd575e6507ae91 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:41:54 -0400 Subject: [PATCH 030/149] Fixed bug that was causing coarry-enabled runs to hang waiting for the parameter structures to sync up --- src/swiftest/swiftest_io.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 1fef0b3d3..8b4e5ad26 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2349,7 +2349,7 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i #ifdef COARRAY end if ! this_image() == 1 - if (self%lcoarray) call coparam%coclone() + call coparam%coclone() select type(self) type is (swiftest_parameters) self = coparam From 68c7596f7b2c40f673f71a9ceeea804adf6b45af Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 21 Apr 2023 15:49:35 -0400 Subject: [PATCH 031/149] Only call termination message on image 1 --- src/swiftest/swiftest_driver.f90 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 7b8f103d9..81f1c8c17 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -190,5 +190,11 @@ program swiftest_driver end associate - call base_util_exit(SUCCESS) +#ifdef COARRAY + if (this_image() == 1) then +#endif + call base_util_exit(SUCCESS) +#ifdef COARRAY + end if ! (this_image() == 1) +#endif end program swiftest_driver From 9cf4bcb6c96eeec9a042adc60b4da1ba4197d478 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 16:49:27 -0400 Subject: [PATCH 032/149] Improved the efficiency of coarray test particles by allowing each image to write to file independently, and to only do a distribute/collect cycle if the system gets too far out of balance --- src/swiftest/swiftest_coarray.f90 | 38 +++++++- src/swiftest/swiftest_driver.f90 | 70 ++++++--------- src/swiftest/swiftest_io.f90 | 144 +++++++++++++++++++++++------- src/swiftest/swiftest_module.f90 | 12 +++ 4 files changed, 187 insertions(+), 77 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 9f255932d..84144ae77 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -11,6 +11,37 @@ use coarray contains + + module subroutine swiftest_coarray_balance_system(nbody_system, param) + !! author: David A. Minton + !! + !! Checks whether or not the system needs to be rebalance. Rebalancing occurs when the difference between the number of test particles between the + !! image with the smallest and largest number of test particles is larger than the number of images + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + integer(I4B), codimension[:], allocatable :: ntp + integer(I4B) :: img,ntp_min, ntp_max + + allocate(ntp[*]) + ntp = nbody_system%tp%nbody + sync all + ntp_min = huge(1) + ntp_max = 0 + do img = 1, num_images() + if (ntp[img] < ntp_min) ntp_min = ntp[img] + if (ntp[img] > ntp_max) ntp_max = ntp[img] + end do + if (ntp_max - ntp_min >= num_images()) then + call nbody_system%coarray_collect(param) + call nbody_system%coarray_distribute(param) + end if + + return + end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_coclone_body(self) !! author: David A. Minton !! @@ -451,13 +482,11 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i,j - integer(I4B), codimension[*], save :: ntp class(swiftest_tp), allocatable, codimension[:] :: cotp character(len=NAMELEN) :: image_num_char if (.not.param%lcoarray) return - sync all if (this_image() == 1) then write(image_num_char,*) num_images() write(param%display_unit,*) " Collecting test particles from " // trim(adjustl(image_num_char)) // " images." @@ -486,18 +515,18 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy class(swiftest_tp), allocatable :: tp logical, dimension(:), allocatable :: lspill_list - integer(I4B), codimension[*], save :: ntp + integer(I4B), codimension[:], allocatable :: ntp character(len=NAMELEN) :: image_num_char class(swiftest_tp), allocatable, codimension[:] :: cotp class(swiftest_tp), allocatable :: tmp if (.not.param%lcoarray) return - sync all if (this_image() == 1) then write(image_num_char,*) num_images() write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." end if + allocate(ntp[*]) ntp = nbody_system%tp%nbody sync all ntot = ntp[1] @@ -529,4 +558,5 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) return end subroutine swiftest_coarray_distribute_system + end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 81f1c8c17..69b820599 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -96,20 +96,20 @@ program swiftest_driver #endif #ifdef COARRAY - ! The following line lets us read in the input files one image at a time + ! The following line lets us read in the input files one image at a time. Letting each image read the input in is faster than broadcasting all of the data if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) #endif call nbody_system%initialize(system_history, param) #ifdef COARRAY + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - if (this_image() == 1) then + ! Distribute test particles to the various images + call nbody_system%coarray_distribute(param) #endif - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. - call nbody_system%display_run_information(param, integration_timer, phase="first") -#ifdef COARRAY - end if ! this_image() == 1 -#endif + ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. + call nbody_system%display_run_information(param, integration_timer, phase="first") + if (param%lenergy) then if (param%lrestart) then call nbody_system%get_t0_values(system_history%nc, param) @@ -118,21 +118,10 @@ program swiftest_driver end if call nbody_system%conservation_report(param, lterminal=.true.) end if + call system_history%take_snapshot(param,nbody_system) - -#ifdef COARRAY - if (this_image() == 1) then -#endif - call nbody_system%dump(param, system_history) -#ifdef COARRAY - end if ! this_image() == 1 -#endif + call nbody_system%dump(param, system_history) -#ifdef COARRAY - if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - ! Distribute test particles to the various images - call nbody_system%coarray_distribute(param) -#endif do iloop = istart, nloops !> Step the nbody_system forward in time call integration_timer%start() @@ -154,40 +143,37 @@ program swiftest_driver nout = nout + 1 istep = floor(istep_out * fstep_out**nout, kind=I4B) end if + + call system_history%take_snapshot(param,nbody_system) + + if (idump == dump_cadence) then + idump = 0 + call nbody_system%dump(param, system_history) + end if #ifdef COARRAY - call nbody_system%coarray_collect(param) if (this_image() == 1) then #endif - call system_history%take_snapshot(param,nbody_system) - - if (idump == dump_cadence) then - idump = 0 - call nbody_system%dump(param, system_history) - end if - call integration_timer%report(message="Integration steps:", unit=display_unit) - call nbody_system%display_run_information(param, integration_timer) - call integration_timer%reset() +#ifdef COARRAY + end if !(this_image() == 1) +#endif + call nbody_system%display_run_information(param, integration_timer) + call integration_timer%reset() +#ifdef COARRAY + if (this_image() == 1) then +#endif if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY - end if - call nbody_system%coarray_distribute(param) + end if ! (this_image() == 1) + call nbody_system%coarray_balance(param) #endif end if end if end do ! Dump any remaining history if it exists -#ifdef COARRAY - call nbody_system%coarray_collect(param) - if (this_image() == 1) then -#endif - call nbody_system%dump(param, system_history) - call nbody_system%display_run_information(param, integration_timer, phase="last") -#ifdef COARRAY - end if ! this_image() == 1 -#endif - + call nbody_system%dump(param, system_history) + call nbody_system%display_run_information(param, integration_timer, phase="last") end associate #ifdef COARRAY diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 8b4e5ad26..a9763dc28 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -223,12 +223,26 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t type(progress_bar), save :: pbar !! Object used to print out a progress bar character(len=64) :: pbarmessage character(*), parameter :: symbacompactfmt = '(";NPLM",ES22.15,$)' +#ifdef COARRAY + character(*), parameter :: statusfmt = '("Image: ",I4, "; Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + '"; Number of active pl, tp = ", I6, ", ", I6)' + character(*), parameter :: symbastatfmt = '("Image: ",I4, "; Image: Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' +#else character(*), parameter :: statusfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, tp = ", I6, ", ", I6)' character(*), parameter :: symbastatfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' +#endif character(*), parameter :: pbarfmt = '("Time = ", ES12.5," of ",ES12.5)' +! The following will syncronize the images so that they report in order, and only write to file one at at ime + + +#ifdef COARRAY + ! The following line lets us read in the input files one image at a time + if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) +#endif phase_val = 1 if (present(phase)) then if (phase == "first") then @@ -240,36 +254,68 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t tfrac = (self%t - param%t0) / (param%tstop - param%t0) - if (phase_val == 0) then - if (param%lrestart) then - write(param%display_unit, *) " *************** Swiftest restart " // trim(adjustl(param%integrator)) // " *************** " - else - write(param%display_unit, *) " *************** Swiftest start " // trim(adjustl(param%integrator)) // " *************** " - end if - if (param%display_style == "PROGRESS") then - call pbar%reset(param%nloops) - else if (param%display_style == "COMPACT") then - write(*,*) "SWIFTEST START " // trim(adjustl(param%integrator)) +#ifdef COARRAY + if (this_image() == 1) then +#endif + if (phase_val == 0) then + if (param%lrestart) then + write(param%display_unit, *) " *************** Swiftest restart " // trim(adjustl(param%integrator)) // " *************** " + else + write(param%display_unit, *) " *************** Swiftest start " // trim(adjustl(param%integrator)) // " *************** " + end if + if (param%display_style == "PROGRESS") then + call pbar%reset(param%nloops) + else if (param%display_style == "COMPACT") then + write(param%display_unit,*) "SWIFTEST START " // trim(adjustl(param%integrator)) + end if end if - end if +#ifdef COARRAY + end if !(this_image() == 1) +#endif if (param%display_style == "PROGRESS") then - write(pbarmessage,fmt=pbarfmt) self%t, param%tstop - call pbar%update(1_I8B,message=pbarmessage) +#ifdef COARRAY + if (this_image() == 1) then +#endif + write(pbarmessage,fmt=pbarfmt) self%t, param%tstop + call pbar%update(1_I8B,message=pbarmessage) +#ifdef COARRAY + end if !(this_image() == 1) +#endif else if (param%display_style == "COMPACT") then call self%compact_output(param,integration_timer) end if if (self%pl%nplm > 0) then +#ifdef COARRAY + write(param%display_unit, symbastatfmt) this_image(),self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody +#else write(param%display_unit, symbastatfmt) self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody +#endif else +#ifdef COARRAY + write(param%display_unit, statusfmt) this_image(),self%t, tfrac, self%pl%nbody, self%tp%nbody +#else write(param%display_unit, statusfmt) self%t, tfrac, self%pl%nbody, self%tp%nbody +#endif end if - if (phase_val == -1) then - write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " - if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) - end if +#ifdef COARRAY + if (this_image() == num_images()) then +#endif + if (phase_val == -1) then + write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " + if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) + end if +#ifdef COARRAY + end if ! this_image() == num_images() + + ! Allow the other images to report + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) + + ! Wait for everyone to catch up + sync all +#endif return end subroutine swiftest_io_display_run_information @@ -330,19 +376,24 @@ module subroutine swiftest_io_dump_system(self, param, system_history) ! Dump the nbody_system history to file call system_history%dump(param) - allocate(param_restart, source=param) - param_restart%in_form = "XV" - param_restart%out_stat = 'APPEND' - param_restart%in_type = "NETCDF_DOUBLE" - param_restart%nc_in = param%outfile - param_restart%lrestart = .true. - param_restart%tstart = self%t - param_file_name = trim(adjustl(PARAM_RESTART_FILE)) - call param_restart%dump(param_file_name) - write(time_text,'(I0.20)') param%iloop - param_file_name = "param." // trim(adjustl(time_text)) // ".in" - call param_restart%dump(param_file_name) - +#ifdef COARRAY + if (this_image() == 1) then +#endif + allocate(param_restart, source=param) + param_restart%in_form = "XV" + param_restart%out_stat = 'APPEND' + param_restart%in_type = "NETCDF_DOUBLE" + param_restart%nc_in = param%outfile + param_restart%lrestart = .true. + param_restart%tstart = self%t + param_file_name = trim(adjustl(PARAM_RESTART_FILE)) + call param_restart%dump(param_file_name) + write(time_text,'(I0.20)') param%iloop + param_file_name = "param." // trim(adjustl(time_text)) // ".in" + call param_restart%dump(param_file_name) +#ifdef COARRAY + end if ! (this_image() == 1) +#endif return end subroutine swiftest_io_dump_system @@ -360,22 +411,53 @@ module subroutine swiftest_io_dump_storage(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i +#ifdef COARRAY + integer(I4B) :: img, tslot + integer(I4B), dimension(self%iframe) :: ntp_tot + integer(I4B), codimension[:], allocatable :: ntp +#endif if (self%iframe == 0) return call self%make_index_map() associate(nc => self%nc) - call nc%open(param) +#ifdef COARRAY + ! Get the sum of all test particles across snapshots from all images + allocate(ntp[*]) + ntp_tot(:) = 0 + do i = 1, self%iframe + if (allocated(self%frame(i)%item)) then + select type(nbody_system => self%frame(i)%item) + class is (swiftest_nbody_system) + ntp = nbody_system%tp%nbody + sync all + do img = 1, num_images() + ntp_tot(i) = ntp_tot(i) + ntp[img] + end do + end select + end if + end do + critical +#endif + call nc%open(param) do i = 1, self%iframe if (allocated(self%frame(i)%item)) then select type(nbody_system => self%frame(i)%item) class is (swiftest_nbody_system) call nbody_system%write_frame(nc, param) +#ifdef COARRAY + ! Record the correct number of test particles from all images + call nc%find_tslot(nbody_system%t, tslot) + call netcdf_io_check( nf90_put_var(nc%id, nc%ntp_varid, ntp_tot(i), start=[tslot]), "swiftest_io_dump_storage nf90_put_var ntp_varid" ) +#endif COARRAY end select deallocate(self%frame(i)%item) end if end do call nc%close() +#ifdef COARRAY + end critical +#endif end associate call self%reset() return diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 5fac1d0ad..73fd6e2b4 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -421,6 +421,7 @@ module swiftest procedure :: coclone => swiftest_coarray_coclone_system !! Clones the image 1 body object to all other images in the coarray structure. procedure :: coarray_collect => swiftest_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system procedure :: coarray_distribute => swiftest_coarray_distribute_system !! Distributes test particles from image #1 out to all images. + procedure :: coarray_balance => swiftest_coarray_balance_system !! Checks whether or not the test particle coarrays need to be rebalanced. #endif end type swiftest_nbody_system @@ -1700,6 +1701,17 @@ end subroutine swiftest_util_version #ifdef COARRAY interface + module subroutine swiftest_coarray_balance_system(nbody_system, param) + !! author: David A. Minton + !! + !! Checks whether or not the system needs to be rebalance. Rebalancing occurs when the image with the smallest number of test particles + !! has <90% of that of the image with the largest number of test particles. + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_collect_system(nbody_system, param) implicit none class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system From 4f2d49969244627f0675a5f32cd91e35546bd4ed Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 17:02:06 -0400 Subject: [PATCH 033/149] Improved messaging of distribute operation --- src/swiftest/swiftest_coarray.f90 | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 84144ae77..6c5ccc389 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -516,22 +516,25 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) class(swiftest_tp), allocatable :: tp logical, dimension(:), allocatable :: lspill_list integer(I4B), codimension[:], allocatable :: ntp - character(len=NAMELEN) :: image_num_char + character(len=NAMELEN) :: image_num_char, ntp_num_char class(swiftest_tp), allocatable, codimension[:] :: cotp class(swiftest_tp), allocatable :: tmp if (.not.param%lcoarray) return - if (this_image() == 1) then - write(image_num_char,*) num_images() - write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." - end if allocate(ntp[*]) ntp = nbody_system%tp%nbody sync all ntot = ntp[1] if (ntot == 0) return - + + write(image_num_char,*) num_images() + + if (this_image() == 1) then + write(ntp_num_char,*) ntot + write(param%display_unit,*) " Distributing " // trim(adjustl(ntp_num_char)) // " test particles across " // trim(adjustl(image_num_char)) // " images." + end if + allocate(lspill_list(ntot)) num_per_image = ceiling(1.0_DP * ntot / num_images()) istart = (this_image() - 1) * num_per_image + 1 @@ -553,6 +556,9 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) allocate(tmp, mold=nbody_system%tp) call nbody_system%tp%spill(tmp, lspill_list(:), ldestructive=.true.) + write(ntp_num_char,*) nbody_system%tp%nbody + write(param%display_unit,*) "Image " // trim(adjustl(image_num_char)) // " ntp: " // trim(adjustl(ntp_num_char)) + deallocate(tmp, cotp) return From d72a65b9aac77b796fb94d0d4517f61b0c08c8be Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 17:03:39 -0400 Subject: [PATCH 034/149] Fixed typo in distribution messaging --- src/swiftest/swiftest_coarray.f90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 6c5ccc389..191b06808 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -556,6 +556,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) allocate(tmp, mold=nbody_system%tp) call nbody_system%tp%spill(tmp, lspill_list(:), ldestructive=.true.) + write(image_num_char,*) this_image() write(ntp_num_char,*) nbody_system%tp%nbody write(param%display_unit,*) "Image " // trim(adjustl(image_num_char)) // " ntp: " // trim(adjustl(ntp_num_char)) From e689a29a105b35c1e8aaa5e12e97336f5fbfd29c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 17:07:17 -0400 Subject: [PATCH 035/149] More improvements to distribution messaging. Number of test particles on each image are reported in order. --- src/swiftest/swiftest_coarray.f90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 191b06808..b2f615712 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -558,7 +558,9 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(image_num_char,*) this_image() write(ntp_num_char,*) nbody_system%tp%nbody + if (this_image() /= 1) sync images(this_image() - 1) write(param%display_unit,*) "Image " // trim(adjustl(image_num_char)) // " ntp: " // trim(adjustl(ntp_num_char)) + if (this_image() < num_images()) sync images(this_image() + 1) deallocate(tmp, cotp) From 89e6768fa7135a39aac3f5515f5f54e3b0586a17 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 20:37:06 -0400 Subject: [PATCH 036/149] extra info on io read error --- src/swiftest/swiftest_io.f90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index a9763dc28..6028676a3 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1155,10 +1155,14 @@ module function swiftest_io_netcdf_read_frame_system(self, nc, param) result(ier if (npl_check /= npl) then write(*,*) "Error reading in NetCDF file: The recorded value of npl does not match the number of active massive bodies" + write(*,*) "Recorded: ",npl + write(*,*) "Active : ",npl_check end if if (ntp_check /= ntp) then write(*,*) "Error reading in NetCDF file: The recorded value of ntp does not match the number of active test particles" + write(*,*) "Recorded: ",ntp + write(*,*) "Active : ",ntp_check call base_util_exit(failure) end if From 756ae556c3a5b503f306e38d72c12e5910189959 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 23:24:58 -0400 Subject: [PATCH 037/149] adjustments to output for coarrays --- src/swiftest/swiftest_io.f90 | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 6028676a3..97ace68ed 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2428,19 +2428,30 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i iostat = 0 - call param%set_display(param%display_style) - ! Print the contents of the parameter file to standard output - if (.not.param%lrestart) call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) end associate #ifdef COARRAY end if ! this_image() == 1 call coparam%coclone() - select type(self) +#endif + select type(param => self) type is (swiftest_parameters) - self = coparam - end select +#ifdef COARRAY + param = coparam #endif + call param%set_display(param%display_style) + + if (.not.param%lrestart) then +#ifdef COARRAY + if (this_image() == 1) then +#endif + call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) +#ifdef COARRAY + end if !(this_image() == 1) +#endif + end if + ! Print the contents of the parameter file to standard output + end select return 667 continue From 63b8010151af898bb60cce76e3d6c22f9959e698 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 25 Apr 2023 10:57:14 -0400 Subject: [PATCH 038/149] Improved output logging by giving each co-image its own log file --- src/globals/globals_module.f90 | 6 +++++- src/swiftest/swiftest_coarray.f90 | 7 +++++-- src/swiftest/swiftest_driver.f90 | 9 +++++---- src/swiftest/swiftest_io.f90 | 22 ++++++++++++++++------ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/globals/globals_module.f90 b/src/globals/globals_module.f90 index fa6b18582..fd26b3404 100644 --- a/src/globals/globals_module.f90 +++ b/src/globals/globals_module.f90 @@ -102,7 +102,11 @@ module globals !> Standard file names integer(I4B), parameter :: NDUMPFILES = 2 character(*), parameter :: PARAM_RESTART_FILE = "param.restart.in" - character(*), parameter :: SWIFTEST_LOG_FILE = "swiftest.log" !! Name of file to use to log output when using "COMPACT" display style +#ifdef COARRAY + character(STRMAX) :: SWIFTEST_LOG_FILE !! Name of file to use to log output when using "COMPACT" or "PROGRESS" display style (each co-image gets its own log file) +#else + character(*), parameter :: SWIFTEST_LOG_FILE = "swiftest.log" !! Name of file to use to log output when using "COMPACT" or "PROGRESS" display style +#endif integer(I4B), parameter :: SWIFTEST_LOG_OUT = 33 !! File unit for log file when using "COMPACT" display style !> Default file names that can be changed by the user in the parameters file diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index b2f615712..9d5650cc2 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -487,9 +487,10 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) if (.not.param%lcoarray) return - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then write(image_num_char,*) num_images() write(param%display_unit,*) " Collecting test particles from " // trim(adjustl(image_num_char)) // " images." + if (param%log_output) flush(param%display_unit) end if allocate(cotp[*], source=nbody_system%tp) @@ -530,9 +531,10 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(image_num_char,*) num_images() - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then write(ntp_num_char,*) ntot write(param%display_unit,*) " Distributing " // trim(adjustl(ntp_num_char)) // " test particles across " // trim(adjustl(image_num_char)) // " images." + if (param%log_output) flush(param%display_unit) end if allocate(lspill_list(ntot)) @@ -560,6 +562,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(ntp_num_char,*) nbody_system%tp%nbody if (this_image() /= 1) sync images(this_image() - 1) write(param%display_unit,*) "Image " // trim(adjustl(image_num_char)) // " ntp: " // trim(adjustl(ntp_num_char)) + if (param%log_output) flush(param%display_unit) if (this_image() < num_images()) sync images(this_image() + 1) deallocate(tmp, cotp) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 69b820599..be55dbd6b 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -76,7 +76,7 @@ program swiftest_driver nthreads = 1 ! In the *serial* case !$ nthreads = omp_get_max_threads() ! In the *parallel* case #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif !$ write(param%display_unit,'(a)') ' OpenMP parameters:' !$ write(param%display_unit,'(a)') ' ------------------' @@ -87,13 +87,14 @@ program swiftest_driver write(param%display_unit,*) ' Coarray parameters:' write(param%display_unit,*) ' -------------------' write(param%display_unit,*) ' Number of images = ', num_images() - if (param%log_output) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() + if (param%log_output .and. this_image() == 1) write(*,'(a,i3)') ' Coarray: Number of images = ',num_images() else write(param%display_unit,*) ' Coarrays disabled.' if (param%log_output) write(*,*) ' Coarrays disabled.' end if end if #endif + if (param%log_output) flush(param%display_unit) #ifdef COARRAY ! The following line lets us read in the input files one image at a time. Letting each image read the input in is faster than broadcasting all of the data @@ -151,7 +152,7 @@ program swiftest_driver call nbody_system%dump(param, system_history) end if #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif call integration_timer%report(message="Integration steps:", unit=display_unit) #ifdef COARRAY @@ -160,7 +161,7 @@ program swiftest_driver call nbody_system%display_run_information(param, integration_timer) call integration_timer%reset() #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 97ace68ed..c36ce9b79 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -184,9 +184,13 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) nbody_system%Mescape_error = nbody_system%GMescape / nbody_system%GMtot_orig #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif - if (lterminal) write(display_unit, EGYTERMFMT) nbody_system%L_total_error, nbody_system%E_orbit_error, nbody_system%te_error,nbody_system%Mtot_error + if (lterminal) then + write(display_unit, EGYTERMFMT) nbody_system%L_total_error, nbody_system%E_orbit_error, nbody_system%te_error,nbody_system%Mtot_error + if (param%log_output) flush(display_unit) + end if + #ifdef COARRAY end if ! (this_image() == 1) then #endif @@ -255,7 +259,7 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t tfrac = (self%t - param%t0) / (param%tstop - param%t0) #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif if (phase_val == 0) then if (param%lrestart) then @@ -301,14 +305,16 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t end if #ifdef COARRAY - if (this_image() == num_images()) then + if (this_image() == num_images() .or. param%log_output) then #endif if (phase_val == -1) then write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) end if + #ifdef COARRAY end if ! this_image() == num_images() + if (param%log_output) flush(param%display_unit) ! Allow the other images to report if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) @@ -2443,9 +2449,10 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i if (.not.param%lrestart) then #ifdef COARRAY - if (this_image() == 1) then + if (this_image() == 1 .or. param%log_output) then #endif - call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) + call param%writer(unit = param%display_unit, iotype = "none", v_list = [0], iostat = iostat, iomsg = iomsg) + if (param%log_output) flush(param%display_unit) #ifdef COARRAY end if !(this_image() == 1) #endif @@ -3078,6 +3085,9 @@ module subroutine swiftest_io_set_display_param(self, display_style) self%display_unit = OUTPUT_UNIT !! stdout from iso_fortran_env self%log_output = .false. case ('COMPACT', 'PROGRESS') +#ifdef COARRAY + write(SWIFTEST_LOG_FILE,'("swiftest_coimage",I0.3,".log")') this_image() +#endif inquire(file=SWIFTEST_LOG_FILE, exist=fileExists) if (self%lrestart.and.fileExists) then open(unit=SWIFTEST_LOG_OUT, file=SWIFTEST_LOG_FILE, status="OLD", position="APPEND", err = 667, iomsg = errmsg) From 6dd5767e25a2ae297d83f740b6da6580f75ef00b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 25 Apr 2023 11:28:16 -0400 Subject: [PATCH 039/149] Improved file handling in coarray mode --- src/swiftest/swiftest_io.f90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index c36ce9b79..be96af345 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -3158,6 +3158,9 @@ module subroutine swiftest_io_write_frame_system(self, nc, param) nc%file_name = param%outfile if (lfirst) then inquire(file=param%outfile, exist=fileExists) +#ifdef COARRAY + if (this_image() /= 1) param%out_stat = 'APPEND' +#endif select case(param%out_stat) case('APPEND') From 012e05c0ff8a9c6880708f2b557cbd48b000728e Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 25 Apr 2023 19:37:24 -0400 Subject: [PATCH 040/149] Added timers to help troubleshoot speed issues with the big production runs --- src/swiftest/swiftest_driver.f90 | 22 +++++++++++++++++++-- src/swiftest/swiftest_io.f90 | 33 +++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index be55dbd6b..c40c67495 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -31,6 +31,7 @@ program swiftest_driver integer(I4B) :: nout !! Current output step integer(I4B) :: istep !! Current value of istep (used for time stretching) type(walltimer) :: integration_timer !! Object used for computing elapsed wall time + type(walltimer) :: t1, t2, t3, t4 call swiftest_io_get_args(integrator, param_file_name, display_style) @@ -105,7 +106,7 @@ program swiftest_driver if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) ! Distribute test particles to the various images - call nbody_system%coarray_distribute(param) + if (param%lcoarray) call nbody_system%coarray_distribute(param) #endif ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. @@ -132,7 +133,9 @@ program swiftest_driver nbody_system%t = t0 + iloop * dt !> Evaluate any discards or collisional outcomes + call t1%start() call nbody_system%discard(param) + call t1%stop() !> If the loop counter is at the output cadence value, append the data file with a single frame if (istep_out > 0) then @@ -145,11 +148,17 @@ program swiftest_driver istep = floor(istep_out * fstep_out**nout, kind=I4B) end if + call t2%start() call system_history%take_snapshot(param,nbody_system) + call t2%stop() if (idump == dump_cadence) then idump = 0 + call t3%start() call nbody_system%dump(param, system_history) + call t3%stop() + call t3%report(message="Dump", unit=display_unit) + call t3%reset() end if #ifdef COARRAY if (this_image() == 1 .or. param%log_output) then @@ -166,7 +175,16 @@ program swiftest_driver if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY end if ! (this_image() == 1) - call nbody_system%coarray_balance(param) + call t4%start() + if (param%lcoarray) call nbody_system%coarray_balance(param) + call t4%stop() + + call t1%report(message="Discard", unit=display_unit) + call t2%report(message="Snapshot", unit=display_unit) + call t4%report(message="Balance", unit=display_unit) + call t1%reset() + call t2%reset() + call t4%reset() #endif end if end if diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index be96af345..4d9ed3f35 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -228,16 +228,15 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t character(len=64) :: pbarmessage character(*), parameter :: symbacompactfmt = '(";NPLM",ES22.15,$)' #ifdef COARRAY - character(*), parameter :: statusfmt = '("Image: ",I4, "; Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + character(*), parameter :: co_statusfmt = '("Image: ",I4, "; Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, tp = ", I6, ", ", I6)' - character(*), parameter :: symbastatfmt = '("Image: ",I4, "; Image: Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + character(*), parameter :: co_symbastatfmt = '("Image: ",I4, "; Image: Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' -#else +#endif character(*), parameter :: statusfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, tp = ", I6, ", ", I6)' character(*), parameter :: symbastatfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' -#endif character(*), parameter :: pbarfmt = '("Time = ", ES12.5," of ",ES12.5)' ! The following will syncronize the images so that they report in order, and only write to file one at at ime @@ -292,15 +291,23 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t if (self%pl%nplm > 0) then #ifdef COARRAY - write(param%display_unit, symbastatfmt) this_image(),self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody -#else - write(param%display_unit, symbastatfmt) self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody + if (param%lcoarray) then + write(param%display_unit, co_symbastatfmt) this_image(),self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody + else +#endif + write(param%display_unit, symbastatfmt) self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody +#ifdef COARRAY + end if #endif else #ifdef COARRAY - write(param%display_unit, statusfmt) this_image(),self%t, tfrac, self%pl%nbody, self%tp%nbody -#else - write(param%display_unit, statusfmt) self%t, tfrac, self%pl%nbody, self%tp%nbody + if (param%lcoarray) then + write(param%display_unit, co_statusfmt) this_image(),self%t, tfrac, self%pl%nbody, self%tp%nbody + else +#endif + write(param%display_unit, statusfmt) self%t, tfrac, self%pl%nbody, self%tp%nbody +#ifdef COARRAY + end if #endif end if @@ -3086,7 +3093,11 @@ module subroutine swiftest_io_set_display_param(self, display_style) self%log_output = .false. case ('COMPACT', 'PROGRESS') #ifdef COARRAY - write(SWIFTEST_LOG_FILE,'("swiftest_coimage",I0.3,".log")') this_image() + if (self%lcoarray) then + write(SWIFTEST_LOG_FILE,'("swiftest_coimage",I0.3,".log")') this_image() + else + write(SWIFTEST_LOG_FILE,'("swiftest.log")') + end if #endif inquire(file=SWIFTEST_LOG_FILE, exist=fileExists) if (self%lrestart.and.fileExists) then From 9b2c7d73de1771b3978a10a572a921ea0a23bfa5 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 25 Apr 2023 20:42:58 -0400 Subject: [PATCH 041/149] Removed timers and rearranged some calls to improve speed --- src/swiftest/swiftest_coarray.f90 | 3 +-- src/swiftest/swiftest_driver.f90 | 18 ------------------ src/swiftest/swiftest_io.f90 | 13 +------------ 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 9d5650cc2..fe523f64f 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -22,10 +22,9 @@ module subroutine swiftest_coarray_balance_system(nbody_system, param) class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B), codimension[:], allocatable :: ntp + integer(I4B), codimension[*], save :: ntp integer(I4B) :: img,ntp_min, ntp_max - allocate(ntp[*]) ntp = nbody_system%tp%nbody sync all ntp_min = huge(1) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index c40c67495..e15bbc383 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -31,7 +31,6 @@ program swiftest_driver integer(I4B) :: nout !! Current output step integer(I4B) :: istep !! Current value of istep (used for time stretching) type(walltimer) :: integration_timer !! Object used for computing elapsed wall time - type(walltimer) :: t1, t2, t3, t4 call swiftest_io_get_args(integrator, param_file_name, display_style) @@ -133,9 +132,7 @@ program swiftest_driver nbody_system%t = t0 + iloop * dt !> Evaluate any discards or collisional outcomes - call t1%start() call nbody_system%discard(param) - call t1%stop() !> If the loop counter is at the output cadence value, append the data file with a single frame if (istep_out > 0) then @@ -148,17 +145,11 @@ program swiftest_driver istep = floor(istep_out * fstep_out**nout, kind=I4B) end if - call t2%start() call system_history%take_snapshot(param,nbody_system) - call t2%stop() if (idump == dump_cadence) then idump = 0 - call t3%start() call nbody_system%dump(param, system_history) - call t3%stop() - call t3%report(message="Dump", unit=display_unit) - call t3%reset() end if #ifdef COARRAY if (this_image() == 1 .or. param%log_output) then @@ -175,16 +166,7 @@ program swiftest_driver if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY end if ! (this_image() == 1) - call t4%start() if (param%lcoarray) call nbody_system%coarray_balance(param) - call t4%stop() - - call t1%report(message="Discard", unit=display_unit) - call t2%report(message="Snapshot", unit=display_unit) - call t4%report(message="Balance", unit=display_unit) - call t1%reset() - call t2%reset() - call t4%reset() #endif end if end if diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 4d9ed3f35..33bb2897c 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -242,10 +242,6 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t ! The following will syncronize the images so that they report in order, and only write to file one at at ime -#ifdef COARRAY - ! The following line lets us read in the input files one image at a time - if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) -#endif phase_val = 1 if (present(phase)) then if (phase == "first") then @@ -322,12 +318,6 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t #ifdef COARRAY end if ! this_image() == num_images() if (param%log_output) flush(param%display_unit) - - ! Allow the other images to report - if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - - ! Wait for everyone to catch up - sync all #endif return @@ -427,7 +417,7 @@ module subroutine swiftest_io_dump_storage(self, param) #ifdef COARRAY integer(I4B) :: img, tslot integer(I4B), dimension(self%iframe) :: ntp_tot - integer(I4B), codimension[:], allocatable :: ntp + integer(I4B), codimension[*], save :: ntp #endif if (self%iframe == 0) return @@ -435,7 +425,6 @@ module subroutine swiftest_io_dump_storage(self, param) associate(nc => self%nc) #ifdef COARRAY ! Get the sum of all test particles across snapshots from all images - allocate(ntp[*]) ntp_tot(:) = 0 do i = 1, self%iframe if (allocated(self%frame(i)%item)) then From 5136422e13e779ffe33bd4d0e108251716dd9713 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 25 Apr 2023 23:11:00 -0400 Subject: [PATCH 042/149] Moved balance to just after dump step so we can control how often a balance gets done --- src/swiftest/swiftest_driver.f90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index e15bbc383..931f40777 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -150,6 +150,9 @@ program swiftest_driver if (idump == dump_cadence) then idump = 0 call nbody_system%dump(param, system_history) +#ifdef COARRAY + if (param%lcoarray) call nbody_system%coarray_balance(param) +#endif end if #ifdef COARRAY if (this_image() == 1 .or. param%log_output) then @@ -166,7 +169,6 @@ program swiftest_driver if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY end if ! (this_image() == 1) - if (param%lcoarray) call nbody_system%coarray_balance(param) #endif end if end if From 7cc974c6d29d0f776dcc2615a656a6d6083c7a7f Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 26 Apr 2023 16:29:28 -0400 Subject: [PATCH 043/149] Reverted orbital element utility functions to element-wise scalar operations and got rid of .cross. , .mag. operators. These ran slow when coarrays were used (for some reason I don't entirly understand) --- src/base/base_module.f90 | 5 ++ src/collision/collision_module.f90 | 4 ++ src/rmvs/rmvs_discard.f90 | 3 ++ src/swiftest/swiftest_discard.f90 | 16 +++--- src/swiftest/swiftest_drift.f90 | 62 +++++++++++++-------- src/swiftest/swiftest_driver.f90 | 10 ++-- src/swiftest/swiftest_io.f90 | 7 +-- src/swiftest/swiftest_module.f90 | 16 +++--- src/swiftest/swiftest_orbel.f90 | 87 ++++++++++++++++-------------- src/swiftest/swiftest_util.f90 | 3 +- 10 files changed, 128 insertions(+), 85 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index b47c2af7b..8a1982c33 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -30,6 +30,11 @@ module base real(DP) :: dt = -1.0_DP !! Time step integer(I8B) :: iloop = 0_I8B !! Main loop counter integer(I8B) :: nloops = 0_I8B !! Total number of loops to execute + integer(I8B) :: istart = 0_I8B !! Starting index for loop counter + integer(I4B) :: iout = 0 !! Output cadence counter + integer(I4B) :: idump = 0 !! Dump cadence counter + integer(I4B) :: nout = 0 !! Current output step + integer(I4B) :: istep = 0 !! Current value of istep (used for time stretching) character(STRMAX) :: incbfile = CB_INFILE !! Name of input file for the central body character(STRMAX) :: inplfile = PL_INFILE !! Name of input file for massive bodies character(STRMAX) :: intpfile = TP_INFILE !! Name of input file for test particles diff --git a/src/collision/collision_module.f90 b/src/collision/collision_module.f90 index 86b3ff76c..82bec250b 100644 --- a/src/collision/collision_module.f90 +++ b/src/collision/collision_module.f90 @@ -19,7 +19,11 @@ module collision public character(len=*), parameter :: COLLISION_OUTFILE = 'collisions.nc' !! Name of NetCDF output file for collision information +#ifdef COARRAY + character(len=STRMAX) :: COLLISION_LOG_OUT !! Name of log file for collision diagnostic information (each co-image gets its own) +#else character(len=*), parameter :: COLLISION_LOG_OUT = "collisions.log" !! Name of log file for collision diagnostic information +#endif !>Symbolic names for collisional outcomes from collresolve_resolve: integer(I4B), parameter :: COLLRESOLVE_REGIME_MERGE = 1 diff --git a/src/rmvs/rmvs_discard.f90 b/src/rmvs/rmvs_discard.f90 index 7e0cb9905..bb6c3063e 100644 --- a/src/rmvs/rmvs_discard.f90 +++ b/src/rmvs/rmvs_discard.f90 @@ -54,6 +54,9 @@ module subroutine rmvs_discard_tp(self, nbody_system, param) call swiftest_discard_tp(tp, nbody_system, param) end associate + + return + end subroutine rmvs_discard_tp end submodule s_rmvs_discard \ No newline at end of file diff --git a/src/swiftest/swiftest_discard.f90 b/src/swiftest/swiftest_discard.f90 index da9208f6a..dd2f07eab 100644 --- a/src/swiftest/swiftest_discard.f90 +++ b/src/swiftest/swiftest_discard.f90 @@ -131,7 +131,7 @@ subroutine swiftest_discard_cb_tp(tp, nbody_system, param) ! Internals integer(I4B) :: i real(DP) :: energy, vb2, rb2, rh2, rmin2, rmax2, rmaxu2 - character(len=STRMAX) :: idstr, timestr + character(len=STRMAX) :: idstr, timestr, message associate(ntp => tp%nbody, cb => nbody_system%cb, Gmtot => nbody_system%Gmtot) rmin2 = max(param%rmin * param%rmin, cb%radius * cb%radius) @@ -144,8 +144,9 @@ subroutine swiftest_discard_cb_tp(tp, nbody_system, param) tp%status(i) = DISCARDED_RMAX write(idstr, *) tp%id(i) write(timestr, *) nbody_system%t - write(*, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & + write(message, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & " too far from the central body at t = " // trim(adjustl(timestr)) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) tp%ldiscard(i) = .true. tp%lmask(i) = .false. call tp%info(i)%set_value(status="DISCARDED_RMAX", discard_time=nbody_system%t, discard_rh=tp%rh(:,i), & @@ -154,8 +155,9 @@ subroutine swiftest_discard_cb_tp(tp, nbody_system, param) tp%status(i) = DISCARDED_RMIN write(idstr, *) tp%id(i) write(timestr, *) nbody_system%t - write(*, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & + write(message, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & " too close to the central body at t = " // trim(adjustl(timestr)) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) tp%ldiscard(i) = .true. tp%lmask(i) = .false. call tp%info(i)%set_value(status="DISCARDED_RMIN", discard_time=nbody_system%t, discard_rh=tp%rh(:,i), & @@ -168,8 +170,9 @@ subroutine swiftest_discard_cb_tp(tp, nbody_system, param) tp%status(i) = DISCARDED_RMAXU write(idstr, *) tp%id(i) write(timestr, *) nbody_system%t - write(*, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & + write(message, *) "Particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstr)) // ")" // & " is unbound and too far from barycenter at t = " // trim(adjustl(timestr)) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) tp%ldiscard(i) = .true. tp%lmask(i) = .false. call tp%info(i)%set_value(status="DISCARDED_RMAXU", discard_time=nbody_system%t, discard_rh=tp%rh(:,i), & @@ -254,7 +257,7 @@ subroutine swiftest_discard_pl_tp(tp, nbody_system, param) integer(I4B) :: i, j, isp real(DP) :: r2min, radius real(DP), dimension(NDIM) :: dx, dv - character(len=STRMAX) :: idstri, idstrj, timestr + character(len=STRMAX) :: idstri, idstrj, timestr, message associate(ntp => tp%nbody, pl => nbody_system%pl, npl => nbody_system%pl%nbody, t => nbody_system%t, dt => param%dt) do i = 1, ntp @@ -271,9 +274,10 @@ subroutine swiftest_discard_pl_tp(tp, nbody_system, param) write(idstri, *) tp%id(i) write(idstrj, *) pl%id(j) write(timestr, *) nbody_system%t - write(*, *) "Test particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstri)) // ")" & + write(message, *) "Test particle " // trim(adjustl(tp%info(i)%name)) // " (" // trim(adjustl(idstri)) // ")" & // " too close to massive body " // trim(adjustl(pl%info(j)%name)) // " (" // trim(adjustl(idstrj)) // ")" & // " at t = " // trim(adjustl(timestr)) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) tp%ldiscard(i) = .true. call tp%info(i)%set_value(status="DISCARDED_PLR", discard_time=nbody_system%t, discard_rh=tp%rh(:,i), & discard_vh=tp%vh(:,i), discard_body_id=pl%id(j)) diff --git a/src/swiftest/swiftest_drift.f90 b/src/swiftest/swiftest_drift.f90 index b7811f88c..4c7302908 100644 --- a/src/swiftest/swiftest_drift.f90 +++ b/src/swiftest/swiftest_drift.f90 @@ -1,10 +1,10 @@ -!! Copyright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh +!! Coryright 2022 - David Minton, Carlisle Wishard, Jennifer Pouplin, Jake Elliott, & Dana Singh !! This file is part of Swiftest. !! Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License !! as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. !! Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty !! of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -!! You should have received a copy of the GNU General Public License along with Swiftest. +!! You should have received a cory of the GNU General Public License along with Swiftest. !! If not, see: https://www.gnu.org/licenses. submodule (swiftest) s_swiftest_drift @@ -104,7 +104,7 @@ module subroutine swiftest_drift_all(mu, x, v, n, param, dt, lmask, iflag) end subroutine swiftest_drift_all - pure elemental module subroutine swiftest_drift_one(mu, px, py, pz, vx, vy, vz, dt, iflag) + pure elemental module subroutine swiftest_drift_one(mu, rx, ry, rz, vx, vy, vz, dt, iflag) !! author: The Purdue Swiftest Team - David A. Minton, Carlisle A. Wishard, Jennifer L.L. Pouplin, and Jacob R. Elliott !! !! Perform Danby drift for one body, redoing drift with smaller substeps if original accuracy is insufficient @@ -114,18 +114,18 @@ pure elemental module subroutine swiftest_drift_one(mu, px, py, pz, vx, vy, vz, implicit none ! Arguments real(DP), intent(in) :: mu !! G * (Mcb + m), G = gravitational constant, Mcb = mass of central body, m = mass of body to drift - real(DP), intent(inout) :: px, py, pz, vx, vy, vz !! Position and velocity of body to drift + real(DP), intent(inout) :: rx, ry, rz, vx, vy, vz !! Position and velocity of body to drift real(DP), intent(in) :: dt !! Step size integer(I4B), intent(out) :: iflag !! iflag : error status flag for Danby drift (0 = OK, nonzero = ERROR) ! Internals integer(I4B) :: i real(DP) :: dttmp - call swiftest_drift_dan(mu, px, py, pz, vx, vy, vz, dt, iflag) + call swiftest_drift_dan(mu, rx, ry, rz, vx, vy, vz, dt, iflag) if (iflag /= 0) then dttmp = 0.1_DP * dt do i = 1, 10 - call swiftest_drift_dan(mu, px, py, pz, vx, vy, vz, dttmp, iflag) + call swiftest_drift_dan(mu, rx, ry, rz, vx, vy, vz, dttmp, iflag) if (iflag /= 0) exit end do end if @@ -134,7 +134,7 @@ pure elemental module subroutine swiftest_drift_one(mu, px, py, pz, vx, vy, vz, end subroutine swiftest_drift_one - pure subroutine swiftest_drift_dan(mu, px0, py0, pz0, vx0, vy0, vz0, dt0, iflag) + pure subroutine swiftest_drift_dan(mu, rx0, ry0, rz0, vx0, vy0, vz0, dt0, iflag) !! author: David A. Minton !! !! Perform Kepler drift, solving Kepler's equation in appropriate variables @@ -144,23 +144,21 @@ pure subroutine swiftest_drift_dan(mu, px0, py0, pz0, vx0, vy0, vz0, dt0, iflag) implicit none ! Arguments real(DP), intent(in) :: mu !! G * (m1 + m2), G = gravitational constant, m1 = mass of central body, m2 = mass of body to drift - real(DP), intent(inout) :: px0, py0, pz0 !! position of body to drift + real(DP), intent(inout) :: rx0, ry0, rz0 !! position of body to drift real(DP), intent(inout) :: vx0, vy0, vz0 !! velocity of body to drift real(DP), intent(in) :: dt0 !! time step integer(I4B), intent(out) :: iflag !! error status flag for Kepler drift (0 = OK, nonzero = NO CONVERGENCE) ! Internals - real(DP) :: dt, f, g, fdot, gdot, c1, c2, c3, u, alpha, fp, r0 + real(DP) :: rx, ry, rz, vx, vy, vz, dt + real(DP) :: f, g, fdot, gdot, c1, c2, c3, u, alpha, fp, r0 real(DP) :: v0s, a, asq, en, dm, ec, es, esq, xkep, fchk, s, c - real(DP), dimension(NDIM) :: x, v, x0, v0 ! Executable code iflag = 0 dt = dt0 - x0 = [px0, py0, pz0] - v0 = [vx0, vy0, vz0] - r0 = sqrt(dot_product(x0(:), x0(:))) - v0s = dot_product(v0(:), v0(:)) - u = dot_product(x0(:), v0(:)) + r0 = sqrt(rx0*rx0 + ry0*ry0 + rz0*rz0) + v0s = vx0*vx0 + vy0*vy0 + vz0*vz0 + u = rx0*vx0 + ry0*vy0 + rz0*vz0 alpha = 2 * mu / r0 - v0s if (alpha > 0.0_DP) then a = mu / alpha @@ -186,10 +184,19 @@ pure subroutine swiftest_drift_dan(mu, px0, py0, pz0, vx0, vy0, vz0, dt0, iflag) g = dt + (s - xkep) / en fdot = -(a / (r0 * fp)) * en * s gdot = (c - 1.0_DP) / fp + 1.0_DP - x(:) = x0(:) * f + v0(:) * g - v(:) = x0(:) * fdot + v0(:) * gdot - px0 = x(1); py0 = x(2); pz0 = x(3) - vx0 = v(1); vy0 = v(2); vz0 = v(3) + rx = rx0 * f + vx0 * g + ry = ry0 * f + vy0 * g + rz = rz0 * f + vz0 * g + vx = rx0 * fdot + vx0 * gdot + vy = ry0 * fdot + vy0 * gdot + vz = rz0 * fdot + vz0 * gdot + + rx0 = rx + ry0 = ry + rz0 = rz + vx0 = vx + vy0 = vy + vz0 = vz iflag = 0 return end if @@ -201,10 +208,19 @@ pure subroutine swiftest_drift_dan(mu, px0, py0, pz0, vx0, vy0, vz0, dt0, iflag) g = dt - mu * c3 fdot = -mu / (fp * r0) * c1 gdot = 1.0_DP - mu / fp * c2 - x(:) = x0(:) * f + v0(:) * g - v(:) = x0(:) * fdot + v0(:) * gdot - px0 = x(1); py0 = x(2); pz0 = x(3) - vx0 = v(1); vy0 = v(2); vz0 = v(3) + rx = rx0 * f + vx0 * g + ry = ry0 * f + vy0 * g + rz = rz0 * f + vz0 * g + vx = rx0 * fdot + vx0 * gdot + vy = ry0 * fdot + vy0 * gdot + vz = rz0 * fdot + vz0 * gdot + + rx0 = rx + ry0 = ry + rz0 = rz + vx0 = vx + vy0 = vy + vz0 = vz end if return diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 931f40777..d8e2ae219 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -25,11 +25,6 @@ program swiftest_driver character(len=:), allocatable :: integrator !! Integrator type code (see globals for symbolic names) character(len=:), allocatable :: param_file_name !! Name of the file containing user-defined parameters character(len=:), allocatable :: display_style !! Style of the output display {"STANDARD", "COMPACT", "PROGRESS"}). Default is "STANDARD" - integer(I8B) :: istart !! Starting index for loop counter - integer(I4B) :: iout !! Output cadence counter - integer(I4B) :: idump !! Dump cadence counter - integer(I4B) :: nout !! Current output step - integer(I4B) :: istep !! Current value of istep (used for time stretching) type(walltimer) :: integration_timer !! Object used for computing elapsed wall time call swiftest_io_get_args(integrator, param_file_name, display_style) @@ -45,6 +40,11 @@ program swiftest_driver dt => param%dt, & tstop => param%tstop, & iloop => param%iloop, & + istart => param%istart, & + iout => param%iout, & + idump => param%idump, & + nout => param%nout, & + istep => param%istep, & nloops => param%nloops, & istep_out => param%istep_out, & fstep_out => param%fstep_out, & diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 33bb2897c..32bc6b2d8 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2300,9 +2300,6 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i ! Calculate the G for the nbody_system units param%GU = GC / (param%DU2M**3 / (param%MU2KG * param%TU2S**2)) - ! A minimal log of collision outcomes is stored in the following log file - ! More complete data on collisions is stored in the NetCDF output files - call swiftest_io_log_start(param, COLLISION_LOG_OUT, "Collision logfile") if ((param%encounter_save /= "NONE") .and. & (param%encounter_save /= "TRAJECTORY") .and. & @@ -2451,7 +2448,11 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i if (param%log_output) flush(param%display_unit) #ifdef COARRAY end if !(this_image() == 1) + write(COLLISION_LOG_OUT,'("collision_coimage",I0.3,".log")') this_image() #endif + ! A minimal log of collision outcomes is stored in the following log file + ! More complete data on collisions is stored in the NetCDF output files + call swiftest_io_log_start(param, COLLISION_LOG_OUT, "Collision logfile") end if ! Print the contents of the parameter file to standard output end select diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 73fd6e2b4..50cb16e23 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -521,11 +521,11 @@ module subroutine swiftest_drift_body(self, nbody_system, param, dt) real(DP), intent(in) :: dt !! Stepsize end subroutine swiftest_drift_body - pure elemental module subroutine swiftest_drift_one(mu, px, py, pz, vx, vy, vz, dt, iflag) + pure elemental module subroutine swiftest_drift_one(mu, rx, ry, rz, vx, vy, vz, dt, iflag) !$omp declare simd(swiftest_drift_one) implicit none real(DP), intent(in) :: mu !! G * (Mcb + m), G = gravitational constant, Mcb = mass of central body, m = mass of body to drift - real(DP), intent(inout) :: px, py, pz, vx, vy, vz !! Position and velocity of body to drift + real(DP), intent(inout) :: rx, ry, rz, vx, vy, vz !! Position and velocity of body to drift real(DP), intent(in) :: dt !! Step size integer(I4B), intent(out) :: iflag !! iflag : error status flag for Danby drift (0 = OK, nonzero = ERROR) end subroutine swiftest_drift_one @@ -1027,22 +1027,22 @@ pure module subroutine swiftest_orbel_scget(angle, sx, cx) real(DP), intent(out) :: sx, cx end subroutine swiftest_orbel_scget - pure elemental module subroutine swiftest_orbel_xv2aeq(mu, px, py, pz, vx, vy, vz, a, e, q) + pure elemental module subroutine swiftest_orbel_xv2aeq(mu, rx, ry, rz, vx, vy, vz, a, e, q) !$omp declare simd(swiftest_orbel_xv2aeq) implicit none real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: e !! eccentricity real(DP), intent(out) :: q !! periapsis end subroutine swiftest_orbel_xv2aeq - pure module subroutine swiftest_orbel_xv2aqt(mu, px, py, pz, vx, vy, vz, a, q, capm, tperi) + pure module subroutine swiftest_orbel_xv2aqt(mu, rx, ry, rz, vx, vy, vz, a, q, capm, tperi) !$omp declare simd(swiftest_orbel_xv2aqt) implicit none real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: q !! periapsis @@ -1050,10 +1050,10 @@ pure module subroutine swiftest_orbel_xv2aqt(mu, px, py, pz, vx, vy, vz, a, q, c real(DP), intent(out) :: tperi !! time of pericenter passage end subroutine swiftest_orbel_xv2aqt - pure module subroutine swiftest_orbel_xv2el(mu, px, py, pz, vx, vy, vz, a, e, inc, capom, omega, capm, varpi, lam, f, cape, capf) + pure module subroutine swiftest_orbel_xv2el(mu, rx, ry, rz, vx, vy, vz, a, e, inc, capom, omega, capm, varpi, lam, f, cape, capf) implicit none real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: e !! eccentricity diff --git a/src/swiftest/swiftest_orbel.f90 b/src/swiftest/swiftest_orbel.f90 index 8e351e911..a3ec6f424 100644 --- a/src/swiftest/swiftest_orbel.f90 +++ b/src/swiftest/swiftest_orbel.f90 @@ -690,7 +690,7 @@ real(DP) pure function swiftest_orbel_fhybrid(e,n) end function swiftest_orbel_fhybrid - pure elemental module subroutine swiftest_orbel_xv2aeq(mu, px, py, pz, vx, vy, vz, a, e, q) + pure elemental module subroutine swiftest_orbel_xv2aeq(mu, rx, ry, rz, vx, vy, vz, a, e, q) !! author: David A. Minton !! !! Compute semimajor axis, eccentricity, and pericentric distance from relative Cartesian position and velocity @@ -700,25 +700,27 @@ pure elemental module subroutine swiftest_orbel_xv2aeq(mu, px, py, pz, vx, vy, v implicit none !! Arguments real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: e !! eccentricity real(DP), intent(out) :: q !! periapsis ! Internals integer(I4B) :: iorbit_type - real(DP) :: r, v2, h2, energy, fac - real(DP), dimension(NDIM) :: hvec, x, v + real(DP) :: hx, hy, hz, r, v2, h2, energy, fac a = 0.0_DP e = 0.0_DP q = 0.0_DP - x = [px, py, pz] - v = [vx, vy, vz] - r = .mag.x(:) - v2 = dot_product(v(:), v(:)) - hvec(:) = x(:) .cross. v(:) - h2 = dot_product(hvec(:), hvec(:)) + + r = sqrt(rx*rx + ry*ry + rz*rz) + v2 = vx*vx + vy*vy + vz*vz + + hx = ry*vz - rz*vy + hy = rz*vx - rx*vz + hz = rx*vy - ry*vx + h2 = hx*hx + hy*hy + hz*hz + if (h2 == 0.0_DP) return energy = 0.5_DP * v2 - mu / r if (abs(energy * r / mu) < sqrt(TINYVALUE)) then @@ -755,7 +757,7 @@ pure elemental module subroutine swiftest_orbel_xv2aeq(mu, px, py, pz, vx, vy, v end subroutine swiftest_orbel_xv2aeq - pure module subroutine swiftest_orbel_xv2aqt(mu, px, py, pz, vx, vy, vz, a, q, capm, tperi) + pure module subroutine swiftest_orbel_xv2aqt(mu, rx, ry, rz, vx, vy, vz, a, q, capm, tperi) !! author: David A. Minton !! !! Compute semimajor axis, pericentric distance, mean anomaly, and time to nearest pericenter passage from @@ -767,7 +769,7 @@ pure module subroutine swiftest_orbel_xv2aqt(mu, px, py, pz, vx, vy, vz, a, q, c implicit none ! Arguments real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: q !! periapsis @@ -775,21 +777,21 @@ pure module subroutine swiftest_orbel_xv2aqt(mu, px, py, pz, vx, vy, vz, a, q, c real(DP), intent(out) :: tperi !! time of pericenter passage ! Internals integer(I4B) :: iorbit_type - real(DP) :: r, v2, h2, rdotv, energy, fac, w, face, cape, e, tmpf, capf, mm - real(DP), dimension(NDIM) :: hvec, x, v + real(DP) :: hx, hy, hz, r, v2, h2, rdotv, energy, fac, w, face, cape, e, tmpf, capf, mm a = 0.0_DP q = 0.0_DP capm = 0.0_DP tperi = 0.0_DP - x = [px, py, pz] - v = [vx, vy, vz] - r = sqrt(dot_product(x(:), x(:))) - v2 = dot_product(v(:), v(:)) - hvec(:) = x(:) .cross. v(:) - h2 = dot_product(hvec(:), hvec(:)) + hx = ry*vz - rz*vy + hy = rz*vx - rx*vz + hz = rx*vy - ry*vx + h2 = hx*hx + hy*hy + hz*hz if (h2 == 0.0_DP) return - rdotv = dot_product(x(:), v(:)) + + r = sqrt(rx*rx + ry*ry + rz*rz) + v2 = vx*vx + vy*vy + vz*vz + rdotv = rx*vx + ry*vy + rz*vz energy = 0.5_DP * v2 - mu / r if (abs(energy * r / mu) < sqrt(TINYVALUE)) then iorbit_type = PARABOLA @@ -897,7 +899,7 @@ module subroutine swiftest_orbel_xv2el_vec(self, cb) end subroutine swiftest_orbel_xv2el_vec - pure module subroutine swiftest_orbel_xv2el(mu, px, py, pz, vx, vy, vz, a, e, inc, capom, omega, capm, varpi, lam, f, cape, capf) + pure module subroutine swiftest_orbel_xv2el(mu, rx, ry, rz, vx, vy, vz, a, e, inc, capom, omega, capm, varpi, lam, f, cape, capf) !! author: David A. Minton !! !! Compute osculating orbital elements from relative Cartesian position and velocity @@ -915,7 +917,7 @@ pure module subroutine swiftest_orbel_xv2el(mu, px, py, pz, vx, vy, vz, a, e, in implicit none ! Arguments real(DP), intent(in) :: mu !! Gravitational constant - real(DP), intent(in) :: px,py,pz !! Position vector + real(DP), intent(in) :: rx,ry,rz !! Position vector real(DP), intent(in) :: vx,vy,vz !! Velocity vector real(DP), intent(out) :: a !! semimajor axis real(DP), intent(out) :: e !! eccentricity @@ -930,8 +932,7 @@ pure module subroutine swiftest_orbel_xv2el(mu, px, py, pz, vx, vy, vz, a, e, in real(DP), intent(out) :: capf !! hyperbolic anomaly (hyperbolic orbits) ! Internals integer(I4B) :: iorbit_type - real(DP) :: r, v2, h2, h, rdotv, energy, fac, u, w, cw, sw, face, tmpf, sf, cf, rdot, h_over_r2 - real(DP), dimension(NDIM) :: hvec, x, v + real(DP) :: hx, hy, hz, r, v2, h2, h, rdotv, energy, fac, u, w, cw, sw, face, tmpf, sf, cf, rdot, h_over_r2 a = 0.0_DP e = 0.0_DP @@ -944,29 +945,37 @@ pure module subroutine swiftest_orbel_xv2el(mu, px, py, pz, vx, vy, vz, a, e, in f = 0.0_DP cape = 0.0_DP capf = 0.0_DP - x = [px, py, pz] - v = [vx, vy, vz] - r = .mag. x(:) - v2 = dot_product(v(:), v(:)) - hvec = x(:) .cross. v(:) - h2 = dot_product(hvec(:), hvec(:)) - h = .mag. hvec(:) + + hx = ry*vz - rz*vy + hy = rz*vx - rx*vz + hz = rx*vy - ry*vx + h2 = hx*hx + hy*hy +hz*hz + h = sqrt(h2) + if(hz>h) then ! Hal's fix + hz = h + hx = 0.0_DP + hy = 0.0_DP + endif if (h2 <= 10 * tiny(0.0_DP)) return - rdotv = dot_product(x(:), v(:)) + h = SQRT(h2) + + r = sqrt(rx*rx + ry*ry + rz*rz) + v2 = vx*vx + vy*vy + vz*vz + rdotv = rx*vx + ry*vy + rz*vz energy = 0.5_DP * v2 - mu / r - fac = hvec(3) / h + fac = hz / h if (fac < -1.0_DP) then inc = PI else if (fac < 1.0_DP) then inc = acos(fac) end if - fac = sqrt(hvec(1)**2 + hvec(2)**2) / h + fac = sqrt(hx**2 + hy**2) / h if (fac**2 < TINYVALUE) then - u = atan2(py, px) - if (hvec(3) < 0.0_DP) u = -u + u = atan2(ry, rx) + if (hz < 0.0_DP) u = -u else - capom = atan2(hvec(1), -hvec(2)) - u = atan2(pz / sin(inc), px * cos(capom) + py * sin(capom)) + capom = atan2(hx, -hy) + u = atan2(rz / sin(inc), rx * cos(capom) + ry * sin(capom)) end if if (capom < 0.0_DP) capom = capom + TWOPI if (u < 0.0_DP) u = u + TWOPI diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index a6b7566e7..0737aab77 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1514,8 +1514,9 @@ module subroutine swiftest_util_peri(n,m, r, v, atp, q, isperi) integer(I4B) :: i real(DP), dimension(n) :: e !! Temporary, just to make use of the xv2aeq subroutine real(DP) :: vdotr + character(len=NAMELEN) :: message - do concurrent(i = 1:n) + do i = 1,n vdotr = dot_product(r(:,i),v(:,i)) if (isperi(i) == -1) then if (vdotr >= 0.0) then From 6617319bacfce533d70637c40d0452bc5f4fad42 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 28 Apr 2023 17:38:19 -0400 Subject: [PATCH 044/149] Added new variables to pyton io --- python/swiftest/swiftest/io.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py index 6a376d387..0c0007488 100644 --- a/python/swiftest/swiftest/io.py +++ b/python/swiftest/swiftest/io.py @@ -32,7 +32,8 @@ "ENCOUNTER_CHECK", "TSTART", "DUMP_CADENCE", - "ENCOUNTER_SAVE") + "ENCOUNTER_SAVE", + "COARRAY") # This list defines features that are booleans, so must be converted to/from string when writing/reading from file bool_param = ["RESTART", @@ -46,7 +47,8 @@ "ENERGY", "GR", "YARKOVSKY", - "YORP"] + "YORP", + "COARRAY"] int_param = ["ISTEP_OUT", "DUMP_CADENCE"] float_param = ["T0", "TSTART", "TSTOP", "DT", "CHK_RMIN", "CHK_RMAX", "CHK_EJECT", "CHK_QMIN", "DU2M", "MU2KG", @@ -59,7 +61,7 @@ # This defines Xarray Dataset variables that are strings, which must be processed due to quirks in how NetCDF-Fortran # handles strings differently than Python's Xarray. -string_varnames = ["name", "particle_type", "status", "origin_type", "stage", "regime"] +string_varnames = ["name", "particle_type", "origin_type", "stage", "regime"] char_varnames = ["space"] int_varnames = ["id", "ntp", "npl", "nplm", "discard_body_id", "collision_id", "status"] From 1d2ce5663470751bfb4257bf90da6fd81a7c1a5c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 28 Apr 2023 17:38:40 -0400 Subject: [PATCH 045/149] Explicitly set mass, radius, etc. to 0 for test particles --- src/swiftest/swiftest_io.f90 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 33bb2897c..2469c2eb4 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1710,6 +1710,17 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) ! call netcdf_io_check( nf90_put_var(nc%id, nc%k2_varid, self%k2(j), start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body k2_varid" ) ! call netcdf_io_check( nf90_put_var(nc%id, nc%Q_varid, self%Q(j), start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body Q_varid" ) ! end if + class is (swiftest_tp) + call netcdf_io_check( nf90_put_var(nc%id, nc%Gmass_varid, 0.0_DP, start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body Gmass_varid" ) + call netcdf_io_check( nf90_put_var(nc%id, nc%mass_varid, 0.0_DP, start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body mass_varid" ) + if (param%lrhill_present) then + call netcdf_io_check( nf90_put_var(nc%id, nc%rhill_varid, 0.0_DP, start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body rhill_varid" ) + end if + if (param%lclose) call netcdf_io_check( nf90_put_var(nc%id, nc%radius_varid, 0.0_DP, start=[idslot, tslot]), "netcdf_io_write_frame_body nf90_put_var body radius_varid" ) + if (param%lrotation) then + call netcdf_io_check( nf90_put_var(nc%id, nc%Ip_varid, [0.0_DP,0.0_DP,0.0_DP], start=[1,idslot, tslot], count=[NDIM,1,1]), "netcdf_io_write_frame_body nf90_put_var body Ip_varid" ) + call netcdf_io_check( nf90_put_var(nc%id, nc%rot_varid, [0.0_DP,0.0_DP,0.0_DP], start=[1,idslot, tslot], count=[NDIM,1,1]), "netcdf_io_write_frame_body nf90_put_var body rotx_varid" ) + end if end select end do From 22cf76752a1fa07d3922928dbde110647ef290e7 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sun, 30 Apr 2023 18:02:04 -0400 Subject: [PATCH 046/149] Added deallocate statements to get memory issues under control --- src/coarray/coarray_clone.f90 | 37 +++++++++++++++++++++---------- src/coarray/coarray_collect.f90 | 28 ++++++++++++++++------- src/swiftest/swiftest_coarray.f90 | 20 ++++++++++++----- 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/src/coarray/coarray_clone.f90 b/src/coarray/coarray_clone.f90 index 4e9a34591..9f7e1ea1a 100644 --- a/src/coarray/coarray_clone.f90 +++ b/src/coarray/coarray_clone.f90 @@ -41,6 +41,7 @@ module subroutine coarray_component_clone_char(var,src_img) sync images(si) var = tmp[si] end if + deallocate(tmp) return end subroutine coarray_component_clone_char @@ -78,6 +79,7 @@ module subroutine coarray_component_clone_DP(var,src_img) var = tmp[si] end if + deallocate(tmp) return end subroutine coarray_component_clone_DP @@ -123,6 +125,7 @@ module subroutine coarray_component_clone_DP_arr1D(var,src_img) if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if + deallocate(isalloc,n,tmp) return end subroutine coarray_component_clone_DP_arr1D @@ -173,6 +176,8 @@ module subroutine coarray_component_clone_DP_arr2D(var,src_img) allocate(var, source=tmp) end if + deallocate(isalloc,n1,n2,tmp) + return end subroutine coarray_component_clone_DP_arr2D @@ -199,10 +204,9 @@ module subroutine coarray_component_clone_DP_vec1D(var,src_img) end if allocate(tmp(NDIM)[*]) - sync all if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var(:) + tmp(:)[img] = var(:) end do sync images(*) else @@ -257,6 +261,8 @@ module subroutine coarray_component_clone_DP_vec2D(var,src_img) allocate(var, source=tmp) end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_clone_DP_vec2D @@ -282,10 +288,9 @@ module subroutine coarray_component_clone_I4B(var,src_img) si = 1 end if - sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -293,6 +298,8 @@ module subroutine coarray_component_clone_I4B(var,src_img) var = tmp[si] end if + deallocate(tmp) + return end subroutine coarray_component_clone_I4B @@ -328,7 +335,7 @@ module subroutine coarray_component_clone_I4B_arr1D(var,src_img) allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var + tmp(:)[img] = var end do sync images(*) else @@ -337,6 +344,8 @@ module subroutine coarray_component_clone_I4B_arr1D(var,src_img) allocate(var, source=tmp) end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_clone_I4B_arr1D @@ -361,10 +370,9 @@ module subroutine coarray_component_clone_I8B(var,src_img) si = 1 end if - sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -372,6 +380,8 @@ module subroutine coarray_component_clone_I8B(var,src_img) var = tmp[si] end if + deallocate(tmp) + return end subroutine coarray_component_clone_I8B @@ -397,10 +407,9 @@ module subroutine coarray_component_clone_lgt(var,src_img) si = 1 end if - sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -408,6 +417,7 @@ module subroutine coarray_component_clone_lgt(var,src_img) var = tmp[si] end if + deallocate(tmp) return end subroutine coarray_component_clone_lgt @@ -444,7 +454,7 @@ module subroutine coarray_component_clone_lgt_arr1D(var,src_img) allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var + tmp(:)[img] = var end do sync images(*) else @@ -453,6 +463,8 @@ module subroutine coarray_component_clone_lgt_arr1D(var,src_img) allocate(var, source=tmp) end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_clone_lgt_arr1D @@ -477,10 +489,9 @@ module subroutine coarray_component_clone_QP(var,src_img) si = 1 end if - !sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -488,6 +499,8 @@ module subroutine coarray_component_clone_QP(var,src_img) var = tmp[si] end if + deallocate(tmp) + return end subroutine coarray_component_clone_QP diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 index bf60aa522..0f85bf41a 100644 --- a/src/coarray/coarray_collect.f90 +++ b/src/coarray/coarray_collect.f90 @@ -54,13 +54,15 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() - if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) - n = n + n[img] - end if + if (img /= di) then + call util_append(var, tmp(1:n[img])[img]) + n = n + n[img] + end if end do end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_collect_DP_arr1D @@ -117,6 +119,8 @@ module subroutine coarray_component_collect_DP_arr2D(var,dest_img) end do end if + deallocate(isalloc,n1,n2,tmp) + return end subroutine coarray_component_collect_DP_arr2D @@ -151,6 +155,8 @@ module subroutine coarray_component_collect_I4B(var,dest_img) var = 0 end if + deallocate(tmp) + return end subroutine coarray_component_collect_I4B @@ -185,6 +191,8 @@ module subroutine coarray_component_collect_I8B(var,dest_img) var = 0 end if + deallocate(tmp) + return end subroutine coarray_component_collect_I8B @@ -230,13 +238,15 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() - if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) - n = n + n[img] - end if + if (img /= di) then + call util_append(var, tmp(1:n[img])[img]) + n = n + n[img] + end if end do end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_collect_I4B_arr1D @@ -289,6 +299,8 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) end do end if + deallocate(isalloc,n,tmp) + return end subroutine coarray_component_collect_lgt_arr1D diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index fe523f64f..a95f8adc8 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -258,8 +258,7 @@ module subroutine swiftest_coarray_component_clone_info(var,src_img) else si = 1 end if - - sync all + if (this_image() == si) then do img = 1, num_images() tmp[img] = var @@ -269,6 +268,8 @@ module subroutine swiftest_coarray_component_clone_info(var,src_img) sync images(si) var = tmp[si] end if + + deallocate(tmp) return end subroutine swiftest_coarray_component_clone_info @@ -314,6 +315,8 @@ module subroutine swiftest_coarray_component_clone_info_arr1D(var,src_img) if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if + + deallocate(isalloc,n,tmp) return end subroutine swiftest_coarray_component_clone_info_arr1D @@ -331,8 +334,8 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) ! Internals type(swiftest_kinship), dimension(:), codimension[:], allocatable :: tmp integer(I4B) :: img, si - integer(I4B), save :: n[*] - logical, save :: isalloc[*] + integer(I4B), allocatable :: n[:] + logical, allocatable :: isalloc[:] if (present(src_img)) then si = src_img @@ -340,7 +343,8 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) si = 1 end if - sync all + allocate(isalloc[*]) + allocate(n[*]) isalloc = allocated(var) if (isalloc) n = size(var) sync all @@ -357,6 +361,8 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if + + deallocate(isalloc,n,tmp) return end subroutine swiftest_coarray_component_clone_kin_arr1D @@ -409,6 +415,8 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) end if end do end if + + deallocate(isalloc,n,tmp) return end subroutine swiftest_coarray_component_collect_info_arr1D @@ -564,7 +572,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) if (param%log_output) flush(param%display_unit) if (this_image() < num_images()) sync images(this_image() + 1) - deallocate(tmp, cotp) + deallocate(ntp, lspill_list, tmp, cotp) return end subroutine swiftest_coarray_distribute_system From 92816831f090d9e2d8784bfd0f14bf4dab18fcee Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sat, 6 May 2023 08:29:51 -0400 Subject: [PATCH 047/149] Fixed a bunch of issues with coarray file i/o --- cmake/Modules/SetFortranFlags.cmake | 6 +- src/coarray/coarray_collect.f90 | 34 +++-- src/netcdf_io/netcdf_io_implementations.f90 | 25 ++-- src/swiftest/swiftest_coarray.f90 | 141 +++++++++++++++++++- src/swiftest/swiftest_io.f90 | 93 ++++++------- src/swiftest/swiftest_module.f90 | 15 ++- src/swiftest/swiftest_util.f90 | 2 +- 7 files changed, 241 insertions(+), 75 deletions(-) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index fd2ea1aee..403bc6ed0 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -320,9 +320,9 @@ SET_COMPILE_FLAG(FASTMATH_FLAGS "${FASTMATH_FLAGS}" ##################### # Enables the optimization reports to be generated SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-pg -qopt-report=5 -traceback -p -g3" # Intel - "/Qopt-report:5 /traceback -g3" # Windows Intel - "-pg -fbacktrace" + Fortran "-O2 -pg -qopt-report=5 -traceback -p -g3" # Intel + "/O2 /Qopt-report:5 /traceback -g3" # Windows Intel + "-O2 -pg -fbacktrace" ) # Sanitize diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 index 0f85bf41a..566113a07 100644 --- a/src/coarray/coarray_collect.f90 +++ b/src/coarray/coarray_collect.f90 @@ -14,7 +14,7 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) !! author: David A. Minton !! - !! Collects components of a coarray derived type from all images and combines them into destination image component . The default destination image is 1 + !! Collects components of a coarray derived type from all images and combines them into destination image component. The default destination image is 1 !! real(DP) 1D allocatable array version implicit none ! Arguments @@ -55,10 +55,14 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) - n = n + n[img] + call util_append(var, tmp(1:n[img])[img]) + n = n + n[img] end if end do + sync images(*) + else + sync images(di) + if (allocated(var)) deallocate(var) end if deallocate(isalloc,n,tmp) @@ -112,11 +116,15 @@ module subroutine coarray_component_collect_DP_arr2D(var,dest_img) if (this_image() == di) then do img = 1, num_images() - if (img /= di) then - call util_append(var, tmp(:,:)[img]) - n2 = n2 + n2[img] - end if + if (img /= di) then + call util_append(var, tmp(:,:)[img]) + n2 = n2 + n2[img] + end if end do + sync images(*) + else + sync images(di) + if (allocated(var)) deallocate(var) end if deallocate(isalloc,n1,n2,tmp) @@ -149,7 +157,7 @@ module subroutine coarray_component_collect_I4B(var,dest_img) if (this_image() == di) then var = 0 do img = 1, num_images() - var = var + tmp[img] + var = var + tmp[img] end do else var = 0 @@ -185,7 +193,7 @@ module subroutine coarray_component_collect_I8B(var,dest_img) if (this_image() == di) then var = 0 do img = 1, num_images() - var = var + tmp[img] + var = var + tmp[img] end do else var = 0 @@ -243,6 +251,10 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) n = n + n[img] end if end do + sync images(*) + else + sync images(di) + if (allocated(var)) deallocate(var) end if deallocate(isalloc,n,tmp) @@ -297,6 +309,10 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) n = n + n[img] end if end do + sync images(*) + else + sync images(di) + if (allocated(var)) deallocate(var) end if deallocate(isalloc,n,tmp) diff --git a/src/netcdf_io/netcdf_io_implementations.f90 b/src/netcdf_io/netcdf_io_implementations.f90 index c59acd0fc..7b1de6b34 100644 --- a/src/netcdf_io/netcdf_io_implementations.f90 +++ b/src/netcdf_io/netcdf_io_implementations.f90 @@ -37,9 +37,12 @@ module subroutine netcdf_io_close(self) implicit none ! Arguments class(netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset + character(namelen) :: message if (self%lfile_is_open) then - call netcdf_io_check( nf90_close(self%id), "netcdf_io_close" ) + write(message,*) this_image() + message = "netcdf_io_close on image " // trim(adjustl(message)) + call netcdf_io_check( nf90_close(self%id), message) self%lfile_is_open = .false. end if @@ -68,13 +71,17 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) if (self%max_tslot > 0) then allocate(tvals(self%max_tslot)) call netcdf_io_check( nf90_get_var(self%id, self%time_varid, tvals(:), start=[1]), "netcdf_io_find_tslot get_var" ) + where(tvals(:) /= tvals(:)) tvals(:) = huge(1.0_DP) else allocate(tvals(1)) - tvals(1) = -huge(1.0_DP) + tvals(1) = huge(1.0_DP) end if - tslot = findloc(tvals, t, dim=1) - if (tslot == 0) tslot = self%max_tslot + 1 + tslot = 1 + do + if ((t <= tvals(tslot)) .or. (tslot > self%max_tslot)) exit + tslot = tslot + 1 + end do self%max_tslot = max(self%max_tslot, tslot) self%tslot = tslot @@ -99,16 +106,16 @@ module subroutine netcdf_io_find_idslot(self, id, idslot) if (.not.allocated(self%idvals)) call self%get_idvals() self%max_idslot = size(self%idvals) - idslot = findloc(self%idvals, id, dim=1) - if (idslot == 0) then - self%max_idslot = self%max_idslot + 1 - idslot = self%max_idslot + idslot = id + 1 + if (idslot > self%max_idslot) then ! Update the idvals array allocate(idvals(idslot)) - idvals(1:idslot-1) = self%idvals(1:idslot-1) + idvals(:) = NF90_FILL_INT + idvals(1:self%max_idslot) = self%idvals(1:self%max_idslot) idvals(idslot) = id call move_alloc(idvals, self%idvals) + self%max_idslot = idslot end if self%idslot = idslot diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index a95f8adc8..e35d3d838 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -81,6 +81,134 @@ module subroutine swiftest_coarray_coclone_body(self) return end subroutine swiftest_coarray_coclone_body + module subroutine swiftest_coarray_coclone_nc(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_netcdf_parameters),intent(inout),codimension[*] :: self !! Swiftest body object + + call coclone(self%file_name) + call coclone(self%lfile_is_open) + call coclone(self%out_type) + call coclone(self%id) + call coclone(self%tslot) + call coclone(self%max_tslot) + call coclone(self%idvals) + call coclone(self%idslot) + call coclone(self%max_idslot) + call coclone(self%str_dimname) + call coclone(self%str_dimid) + call coclone(self%time_dimname) + call coclone(self%time_dimid) + call coclone(self%time_varid) + call coclone(self%name_dimname) + call coclone(self%name_dimid) + call coclone(self%name_varid) + call coclone(self%space_dimname) + call coclone(self%space_dimid) + call coclone(self%space_varid) + call coclone(self%id_varname) + call coclone(self%id_varid) + call coclone(self%status_varname) + call coclone(self%status_varid) + call coclone(self%ptype_varname) + call coclone(self%ptype_varid) + call coclone(self%npl_varname) + call coclone(self%npl_varid) + call coclone(self%ntp_varname) + call coclone(self%ntp_varid) + call coclone(self%nplm_varname) + call coclone(self%nplm_varid) + call coclone(self%a_varname) + call coclone(self%a_varid) + call coclone(self%e_varname) + call coclone(self%e_varid) + call coclone(self%inc_varname) + call coclone(self%inc_varid) + call coclone(self%capom_varname) + call coclone(self%capom_varid) + call coclone(self%omega_varname) + call coclone(self%omega_varid) + call coclone(self%capm_varname) + call coclone(self%capm_varid) + call coclone(self%varpi_varname) + call coclone(self%varpi_varid) + call coclone(self%lam_varname) + call coclone(self%lam_varid) + call coclone(self%f_varname) + call coclone(self%f_varid) + call coclone(self%cape_varname) + call coclone(self%cape_varid) + call coclone(self%rh_varname) + call coclone(self%rh_varid) + call coclone(self%vh_varname) + call coclone(self%vh_varid) + call coclone(self%gr_pseudo_vh_varname) + call coclone(self%gr_pseudo_vh_varid) + call coclone(self%Gmass_varname) + call coclone(self%Gmass_varid) + call coclone(self%mass_varname) + call coclone(self%mass_varid) + call coclone(self%rhill_varname) + call coclone(self%rhill_varid) + call coclone(self%radius_varname) + call coclone(self%radius_varid) + call coclone(self%Ip_varname) + call coclone(self%Ip_varid) + call coclone(self%rot_varname) + call coclone(self%rot_varid) + call coclone(self%j2rp2_varname) + call coclone(self%j2rp2_varid) + call coclone(self%j4rp4_varname) + call coclone(self%j4rp4_varid) + call coclone(self%k2_varname) + call coclone(self%k2_varid) + call coclone(self%q_varname) + call coclone(self%Q_varid) + call coclone(self%ke_orb_varname) + call coclone(self%KE_orb_varid) + call coclone(self%ke_spin_varname) + call coclone(self%KE_spin_varid) + call coclone(self%pe_varname) + call coclone(self%PE_varid) + call coclone(self%be_varname) + call coclone(self%BE_varid) + call coclone(self%te_varname) + call coclone(self%TE_varid) + call coclone(self%L_orbit_varname) + call coclone(self%L_orbit_varid) + call coclone(self%L_spin_varname) + call coclone(self%L_spin_varid) + call coclone(self%L_escape_varname) + call coclone(self%L_escape_varid) + call coclone(self%E_collisions_varname) + call coclone(self%E_collisions_varid) + call coclone(self%E_untracked_varname) + call coclone(self%E_untracked_varid) + call coclone(self%GMescape_varname) + call coclone(self%GMescape_varid) + call coclone(self%origin_type_varname) + call coclone(self%origin_type_varid) + call coclone(self%origin_time_varname) + call coclone(self%origin_time_varid) + call coclone(self%collision_id_varname) + call coclone(self%collision_id_varid) + call coclone(self%origin_rh_varname) + call coclone(self%origin_rh_varid) + call coclone(self%origin_vh_varname) + call coclone(self%origin_vh_varid) + call coclone(self%discard_time_varname) + call coclone(self%discard_time_varid) + call coclone(self%discard_rh_varname) + call coclone(self%discard_rh_varid) + call coclone(self%discard_vh_varname) + call coclone(self%discard_vh_varid) + call coclone(self%discard_body_id_varname) + call coclone(self%lpseudo_vel_exists) + return + end subroutine swiftest_coarray_coclone_nc module subroutine swiftest_coarray_coclone_cb(self) !! author: David A. Minton @@ -178,13 +306,13 @@ end subroutine swiftest_coarray_coclone_tp module subroutine swiftest_coarray_coclone_system(self) !! author: David A. Minton - !! - !! Broadcasts the image 1 object to all other images in a coarray + !! + !! Broadcasts the image 1 object to all other images in a coarray implicit none ! Arguments class(swiftest_nbody_system),intent(inout),codimension[*] :: self !! Swiftest body object ! Internals - integer(I4B) :: i, img + integer(I4B) :: i call self%cb%coclone() call self%pl%coclone() @@ -414,6 +542,10 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) n = n + n[img] end if end do + sync images(*) + else + sync images(di) + if (allocated(var)) deallocate(var) end if deallocate(isalloc,n,tmp) @@ -520,8 +652,7 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy - class(swiftest_tp), allocatable :: tp + integer(I4B) :: istart, iend, ntot, num_per_image, ncopy logical, dimension(:), allocatable :: lspill_list integer(I4B), codimension[:], allocatable :: ntp character(len=NAMELEN) :: image_num_char, ntp_num_char diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 6eeb64e2b..69453e214 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -414,53 +414,45 @@ module subroutine swiftest_io_dump_storage(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i -#ifdef COARRAY - integer(I4B) :: img, tslot - integer(I4B), dimension(self%iframe) :: ntp_tot - integer(I4B), codimension[*], save :: ntp -#endif if (self%iframe == 0) return call self%make_index_map() associate(nc => self%nc) #ifdef COARRAY - ! Get the sum of all test particles across snapshots from all images - ntp_tot(:) = 0 - do i = 1, self%iframe - if (allocated(self%frame(i)%item)) then - select type(nbody_system => self%frame(i)%item) - class is (swiftest_nbody_system) - ntp = nbody_system%tp%nbody - sync all - do img = 1, num_images() - ntp_tot(i) = ntp_tot(i) + ntp[img] - end do - end select - end if - end do - - critical + critical #endif call nc%open(param) +#ifdef COARRAY + end critical +#endif do i = 1, self%iframe + ! Writing files is more efficient if we write out the common frames from each image before going to the next frame +#ifdef COARRAY + if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) +#endif if (allocated(self%frame(i)%item)) then select type(nbody_system => self%frame(i)%item) class is (swiftest_nbody_system) call nbody_system%write_frame(nc, param) -#ifdef COARRAY - ! Record the correct number of test particles from all images - call nc%find_tslot(nbody_system%t, tslot) - call netcdf_io_check( nf90_put_var(nc%id, nc%ntp_varid, ntp_tot(i), start=[tslot]), "swiftest_io_dump_storage nf90_put_var ntp_varid" ) -#endif COARRAY end select deallocate(self%frame(i)%item) end if +#ifdef COARRAY + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) +#endif end do - call nc%close() #ifdef COARRAY - end critical + sync all + if (this_image() == 1) then +#endif + call nc%close() +#ifdef COARRAY + else + nc%lfile_is_open = .false. + end if #endif end associate + call self%reset() return end subroutine swiftest_io_dump_storage @@ -938,7 +930,6 @@ module subroutine swiftest_io_netcdf_open(self, param, readonly) end if associate(nc => self) - write(errmsg,*) "swiftest_io_netcdf_open nf90_open ",trim(adjustl(nc%file_name)) call netcdf_io_check( nf90_open(nc%file_name, mode, nc%id), errmsg) self%lfile_is_open = .true. @@ -1061,7 +1052,7 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) real(DP), dimension(:,:), allocatable :: rh integer(I4B), dimension(:), allocatable :: body_status logical, dimension(:), allocatable :: lvalid - integer(I4B) :: idmax, status + integer(I4B) :: idmax, status,i call netcdf_io_check( nf90_inquire_dimension(self%id, self%name_dimid, len=idmax), "swiftest_io_netcdf_get_valid_masks nf90_inquire_dimension name_dimid" ) @@ -1069,7 +1060,6 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) allocate(tpmask(idmax)) allocate(plmask(idmax)) allocate(lvalid(idmax)) - associate(tslot => self%tslot) call netcdf_io_check( nf90_get_var(self%id, self%Gmass_varid, Gmass, start=[1,tslot], count=[idmax,1]), "swiftest_io_netcdf_get_valid_masks nf90_getvar Gmass_varid" ) @@ -1097,7 +1087,8 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) end if end if - plmask(:) = Gmass(:) == Gmass(:) + plmask(:) = (Gmass(:) == Gmass(:)) + where(plmask(:)) plmask(:) = Gmass(:) > 0.0_DP tpmask(:) = .not. plmask(:) plmask(1) = .false. ! This is the central body @@ -1634,10 +1625,11 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, idslot, old_mode + integer(I4B) :: i, j, idslot, old_mode, ntp integer(I4B), dimension(:), allocatable :: ind real(DP), dimension(NDIM) :: vh !! Temporary variable to store heliocentric velocity values when converting from pseudovelocity in GR-enabled runs real(DP) :: a, e, inc, omega, capom, capm, varpi, lam, f, cape, capf + logical, dimension(:), allocatable :: tpmask, plmask call self%write_info(nc, param) @@ -1721,12 +1713,20 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) call netcdf_io_check( nf90_put_var(nc%id, nc%Ip_varid, [0.0_DP,0.0_DP,0.0_DP], start=[1,idslot, tslot], count=[NDIM,1,1]), "netcdf_io_write_frame_body nf90_put_var body Ip_varid" ) call netcdf_io_check( nf90_put_var(nc%id, nc%rot_varid, [0.0_DP,0.0_DP,0.0_DP], start=[1,idslot, tslot], count=[NDIM,1,1]), "netcdf_io_write_frame_body nf90_put_var body rotx_varid" ) end if - end select end do end associate end select end select +#ifdef COARRAY + select type(self) + class is (swiftest_tp) + call nc%get_valid_masks(plmask, tpmask) + ntp = count(tpmask(:)) + call netcdf_io_check( nf90_put_var(nc%id, nc%ntp_varid, ntp, start=[nc%tslot]), "netcdf_io_write_frame_body nf90_put_var ntp_varid" ) + end select +#endif + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode), "netcdf_io_write_frame_body nf90_set_fill old_mode" ) return @@ -1782,8 +1782,14 @@ module subroutine swiftest_io_netcdf_write_frame_system(self, nc, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters call self%write_hdr(nc, param) - call self%cb%write_frame(nc, param) - call self%pl%write_frame(nc, param) +#ifdef COARRAY + if (this_image() == 1) then +#endif + call self%cb%write_frame(nc, param) + call self%pl%write_frame(nc, param) +#ifdef COARRAY + end if ! this_image() == 1 +#endif call self%tp%write_frame(nc, param) return @@ -1802,13 +1808,15 @@ module subroutine swiftest_io_netcdf_write_hdr_system(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i,tslot, idmax - integer(I4B), dimension(:), allocatable :: body_status + logical, dimension(:), allocatable :: tpmask, plmask + integer(I4B) :: tslot call nc%find_tslot(self%t, tslot) call netcdf_io_check( nf90_put_var(nc%id, nc%time_varid, self%t, start=[tslot]), "netcdf_io_write_hdr_system nf90_put_var time_varid" ) call netcdf_io_check( nf90_put_var(nc%id, nc%npl_varid, self%pl%nbody, start=[tslot]), "netcdf_io_write_hdr_system nf90_put_var npl_varid" ) +#ifndef COARRAY call netcdf_io_check( nf90_put_var(nc%id, nc%ntp_varid, self%tp%nbody, start=[tslot]), "netcdf_io_write_hdr_system nf90_put_var ntp_varid" ) +#endif if (param%lmtiny_pl) call netcdf_io_check( nf90_put_var(nc%id, nc%nplm_varid, self%pl%nplm, start=[tslot]), "netcdf_io_write_hdr_system nf90_put_var nplm_varid" ) if (param%lenergy) then @@ -1825,10 +1833,6 @@ module subroutine swiftest_io_netcdf_write_hdr_system(self, nc, param) call netcdf_io_check( nf90_put_var(nc%id, nc%GMescape_varid, self%GMescape, start=[tslot]), "netcdf_io_write_hdr_system nf90_put_var GMescape_varid" ) end if - ! Set the status flag to INACTIVE by default - call netcdf_io_check( nf90_inquire_dimension(nc%id, nc%name_dimid, len=idmax), "netcdf_io_get_t0_values_system name_dimid" ) - call netcdf_io_check( nf90_put_var(nc%id, nc%status_varid, [(INACTIVE, i=1,idmax)], start=[1,tslot], count=[idmax,1]), "netcdf_io_write_info_body nf90_put_var status_varid" ) - return end subroutine swiftest_io_netcdf_write_hdr_system @@ -3148,7 +3152,7 @@ module subroutine swiftest_io_toupper(string) end subroutine swiftest_io_toupper - module subroutine swiftest_io_write_frame_system(self, nc, param) + module subroutine swiftest_io_initialize_output_file_system(self, nc, param) !! author: The Purdue Swiftest Team - David A. Minton, Carlisle A. Wishard, Jennifer L.L. Pouplin, and Jacob R. Elliott !! !! Write a frame (header plus records for each massive body and active test particle) to output binary file @@ -3194,7 +3198,6 @@ module subroutine swiftest_io_write_frame_system(self, nc, param) lfirst = .false. end if - call swiftest_io_netcdf_write_frame_system(self, nc, param) end associate return @@ -3202,6 +3205,6 @@ module subroutine swiftest_io_write_frame_system(self, nc, param) 667 continue write(*,*) "Error writing nbody_system frame: " // trim(adjustl(errmsg)) call base_util_exit(FAILURE) - end subroutine swiftest_io_write_frame_system + end subroutine swiftest_io_initialize_output_file_system end submodule s_swiftest_io diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 50cb16e23..6555617bb 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -57,6 +57,9 @@ module swiftest procedure :: get_valid_masks => swiftest_io_netcdf_get_valid_masks !! Gets logical masks indicating which bodies are valid pl and tp type at the current time procedure :: open => swiftest_io_netcdf_open !! Opens a NetCDF file and does the variable inquiries to activate variable ids procedure :: flush => swiftest_io_netcdf_flush !! Flushes a NetCDF file by closing it then opening it again +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_nc +#endif end type swiftest_netcdf_parameters @@ -406,12 +409,13 @@ module swiftest procedure :: write_hdr => swiftest_io_netcdf_write_hdr_system !! Write a header for an output frame in NetCDF format procedure :: read_particle_info => swiftest_io_netcdf_read_particle_info_system !! Read in particle metadata from file procedure :: read_in => swiftest_io_read_in_system !! Reads the initial conditions for an nbody system - procedure :: write_frame => swiftest_io_write_frame_system !! Write a frame of input data from file + procedure :: write_frame => swiftest_io_netcdf_write_frame_system !! Write a frame of input data from file procedure :: obl_pot => swiftest_obl_pot_system !! Compute the contribution to the total gravitational potential due solely to the oblateness of the central body procedure :: dealloc => swiftest_util_dealloc_system !! Deallocates all allocatables and resets all values to defaults. Acts as a base for a finalizer procedure :: get_energy_and_momentum => swiftest_util_get_energy_and_momentum_system !! Calculates the total nbody_system energy and momentum procedure :: get_idvals => swiftest_util_get_idvalues_system !! Returns an array of all id values in use in the nbody_system procedure :: rescale => swiftest_util_rescale_system !! Rescales the nbody_system into a new set of units + procedure :: initialize_output_file => swiftest_io_initialize_output_file_system !! Write a frame of input data from file procedure :: initialize => swiftest_util_setup_initialize_system !! Initialize the nbody_system from input files procedure :: init_particle_info => swiftest_util_setup_initialize_particle_info_system !! Initialize the nbody_system from input files ! procedure :: step_spin => tides_step_spin_system !! Steps the spins of the massive & central bodies due to tides. @@ -894,12 +898,12 @@ module subroutine swiftest_io_toupper(string) character(*), intent(inout) :: string !! String to make upper case end subroutine swiftest_io_toupper - module subroutine swiftest_io_write_frame_system(self, nc, param) + module subroutine swiftest_io_initialize_output_file_system(self, nc, param) implicit none class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody_system object class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - end subroutine swiftest_io_write_frame_system + end subroutine swiftest_io_initialize_output_file_system module subroutine swiftest_kick_getacch_int_pl(self, param) implicit none @@ -1764,6 +1768,11 @@ module subroutine swiftest_coarray_coclone_cb(self) class(swiftest_cb),intent(inout),codimension[*] :: self !! Swiftest cb object end subroutine swiftest_coarray_coclone_cb + module subroutine swiftest_coarray_coclone_nc(self) + implicit none + class(swiftest_netcdf_parameters),intent(inout),codimension[*] :: self !! Swiftest body object + end subroutine swiftest_coarray_coclone_nc + module subroutine swiftest_coarray_coclone_pl(self) implicit none class(swiftest_pl),intent(inout),codimension[*] :: self !! Swiftest pl object diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 0737aab77..879f6c8f3 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2379,7 +2379,7 @@ module subroutine swiftest_util_setup_initialize_system(self, system_history, pa ! Write initial conditions to file nc%file_name = param%outfile - call nbody_system%write_frame(nc, param) + call nbody_system%initialize_output_file(nc, param) call nc%close() end associate From e1bc876c263fd40c6a2148d4af131f763f0fc180 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sat, 6 May 2023 08:36:48 -0400 Subject: [PATCH 048/149] shifted sync to end of loop --- src/swiftest/swiftest_io.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 69453e214..ba0559721 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -439,10 +439,10 @@ module subroutine swiftest_io_dump_storage(self, param) end if #ifdef COARRAY if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) + sync all #endif end do #ifdef COARRAY - sync all if (this_image() == 1) then #endif call nc%close() From 678e812e8569fb9b3462975b0ac48d0efdc767ee Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 10 May 2023 09:46:55 -0400 Subject: [PATCH 049/149] Fixed issue that was causing a segfault when an append operation was done using an unallocated source array --- src/base/base_module.f90 | 20 +++++++++++++++----- src/swiftest/swiftest_module.f90 | 4 ++-- src/swiftest/swiftest_util.f90 | 8 ++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 8a1982c33..c2b91e6bf 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -295,12 +295,14 @@ subroutine base_util_append_arr_char_string(arr, source, nold, lsource_mask) implicit none ! Arguments character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - character(len=STRMAX), dimension(:), intent(in) :: source !! Array to append + character(len=STRMAX), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else @@ -338,12 +340,14 @@ subroutine base_util_append_arr_DP(arr, source, nold, lsource_mask) implicit none ! Arguments real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:), intent(in) :: source !! Array to append + real(DP), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else @@ -381,12 +385,14 @@ subroutine base_util_append_arr_DPvec(arr, source, nold, lsource_mask) implicit none ! Arguments real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:,:), intent(in) :: source !! Array to append + real(DP), dimension(:,:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else @@ -426,12 +432,14 @@ subroutine base_util_append_arr_I4B(arr, source, nold, lsource_mask) implicit none ! Arguments integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), intent(in) :: source !! Array to append + integer(I4B), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else @@ -469,12 +477,14 @@ subroutine base_util_append_arr_logical(arr, source, nold, lsource_mask) implicit none ! Arguments logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - logical, dimension(:), intent(in) :: source !! Array to append + logical, dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 6555617bb..c86f86dd1 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -1132,7 +1132,7 @@ end subroutine swiftest_user_kick_getacch_body module subroutine swiftest_util_append_arr_info(arr, source, nold, lsource_mask) implicit none type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_particle_info), dimension(:), intent(in) :: source !! Array to append + type(swiftest_particle_info), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_arr_info @@ -1140,7 +1140,7 @@ end subroutine swiftest_util_append_arr_info module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) implicit none type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_kinship), dimension(:), intent(in) :: source !! Array to append + type(swiftest_kinship), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to end subroutine swiftest_util_append_arr_kin diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 879f6c8f3..31b1dc3dd 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -23,13 +23,15 @@ module subroutine swiftest_util_append_arr_info(arr, source, nold, lsource_mask) implicit none ! Arguments type(swiftest_particle_info), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_particle_info), dimension(:), intent(in) :: source !! Array to append + type(swiftest_particle_info), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig, i integer(I4B), dimension(:), allocatable :: idx + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else @@ -70,12 +72,14 @@ module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) implicit none ! Arguments type(swiftest_kinship), dimension(:), allocatable, intent(inout) :: arr !! Destination array - type(swiftest_kinship), dimension(:), intent(in) :: source !! Array to append + type(swiftest_kinship), dimension(:), allocatable, intent(in) :: source !! Array to append integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig + if (.not.allocated(source)) return + if (present(lsource_mask)) then nsrc = count(lsource_mask(:)) else From 1da23063ed25572568024ed90f2b476d96f6fe2a Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 10 May 2023 11:03:36 -0400 Subject: [PATCH 050/149] Fixed issue where oblateness acceleration variable was being allocated when settting up bodies for non-oblate central bodies. --- src/base/base_module.f90 | 6 ++++++ src/swiftest/swiftest_util.f90 | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index c2b91e6bf..01c111661 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -1054,6 +1054,7 @@ subroutine base_util_spill_arr_char_string(keeps, discards, lspill_list, ldestru nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then @@ -1096,6 +1097,7 @@ subroutine base_util_spill_arr_DP(keeps, discards, lspill_list, ldestructive) nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then @@ -1138,6 +1140,7 @@ subroutine base_util_spill_arr_DPvec(keeps, discards, lspill_list, ldestructive) nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(NDIM, nspill)) else if (size(discards, dim=2) /= nspill) then @@ -1184,6 +1187,7 @@ subroutine base_util_spill_arr_I4B(keeps, discards, lspill_list, ldestructive) nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then @@ -1226,6 +1230,7 @@ subroutine base_util_spill_arr_I8B(keeps, discards, lspill_list, ldestructive) nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then @@ -1268,6 +1273,7 @@ subroutine base_util_spill_arr_logical(keeps, discards, lspill_list, ldestructiv nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 31b1dc3dd..be98690c3 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2426,7 +2426,6 @@ module subroutine swiftest_util_setup_body(self, n, param) allocate(self%vb(NDIM, n)) allocate(self%ah(NDIM, n)) allocate(self%ir3h(n)) - allocate(self%aobl(NDIM, n)) allocate(self%isperi(n)) allocate(self%peri(n)) allocate(self%atp(n)) @@ -2465,11 +2464,14 @@ module subroutine swiftest_util_setup_body(self, n, param) self%vb(:,:) = 0.0_DP self%ah(:,:) = 0.0_DP self%ir3h(:) = 0.0_DP - self%aobl(:,:) = 0.0_DP self%isperi(:) = 1 self%peri(:) = 0.0_DP self%atp(:) = 0.0_DP + if (param%loblatecb) then + allocate(self%aobl(NDIM, n)) + self%aobl(:,:) = 0.0_DP + end if if (param%ltides) then allocate(self%atide(NDIM, n)) self%atide(:,:) = 0.0_DP @@ -2996,6 +2998,7 @@ module subroutine swiftest_util_spill_arr_info(keeps, discards, lspill_list, lde nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then @@ -3043,6 +3046,7 @@ module subroutine swiftest_util_spill_arr_kin(keeps, discards, lspill_list, ldes nlist = size(lspill_list(:)) if (.not.allocated(keeps) .or. nspill == 0) return + if (size(keeps) < nkeep) return if (.not.allocated(discards)) then allocate(discards(nspill)) else if (size(discards) /= nspill) then From 912b0524d7542704cc5f9f0ef567cfceacacc305 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 11 May 2023 10:30:48 -0400 Subject: [PATCH 051/149] Fixed bug that caused out of array bounds error --- src/netcdf_io/netcdf_io_implementations.f90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/netcdf_io/netcdf_io_implementations.f90 b/src/netcdf_io/netcdf_io_implementations.f90 index 7b1de6b34..40c561183 100644 --- a/src/netcdf_io/netcdf_io_implementations.f90 +++ b/src/netcdf_io/netcdf_io_implementations.f90 @@ -62,6 +62,7 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) integer(I4B), intent(out) :: tslot !! The index of the time slot where this data belongs ! Internals real(DP), dimension(:), allocatable :: tvals + integer(I4B) :: i if (.not.self%lfile_is_open) return @@ -75,11 +76,12 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) else allocate(tvals(1)) tvals(1) = huge(1.0_DP) + self%max_tslot = 1 end if tslot = 1 - do - if ((t <= tvals(tslot)) .or. (tslot > self%max_tslot)) exit + do i = 1, self%max_tslot + if (t <= tvals(tslot)) exit tslot = tslot + 1 end do self%max_tslot = max(self%max_tslot, tslot) From 704ce2d93fb07235b5d0b747e33895fbc815b6e3 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 11 May 2023 11:06:26 -0400 Subject: [PATCH 052/149] Added intermediate local allocatable array to the coarray_collect methods --- src/coarray/coarray_collect.f90 | 16 ++++++++++++---- src/swiftest/swiftest_coarray.f90 | 4 +++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/coarray/coarray_collect.f90 b/src/coarray/coarray_collect.f90 index 566113a07..0c27e110d 100644 --- a/src/coarray/coarray_collect.f90 +++ b/src/coarray/coarray_collect.f90 @@ -25,6 +25,7 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) integer(I4B) :: i,img, ti, di, istart, iend, nmax integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] + real(DP), dimension(:), allocatable :: vari1 allocate(isalloc[*]) allocate(n[*]) @@ -55,7 +56,8 @@ module subroutine coarray_component_collect_DP_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) + allocate(vari1, source=tmp(1:n[img])[img]) + call util_append(var, vari1) n = n + n[img] end if end do @@ -85,6 +87,7 @@ module subroutine coarray_component_collect_DP_arr2D(var,dest_img) integer(I4B) :: i, img, ti, di, ntot, istart, iend, nmax integer(I4B), allocatable :: n1[:], n2[:] logical, allocatable :: isalloc[:] + real(DP), dimension(:,:), allocatable :: vari1 allocate(n1[*]) allocate(n2[*]) @@ -117,7 +120,8 @@ module subroutine coarray_component_collect_DP_arr2D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(:,:)[img]) + allocate(vari1,source=tmp(:,1:n2[img])[img]) + call util_append(var, vari1) n2 = n2 + n2[img] end if end do @@ -219,6 +223,7 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) integer(I4B) :: i,img, ti, di, istart, iend, nmax integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] + integer(I4B), dimension(:), allocatable :: vari1 allocate(isalloc[*]) allocate(n[*]) @@ -247,7 +252,8 @@ module subroutine coarray_component_collect_I4B_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) + allocate(vari1, source=tmp(1:n[img])[img]) + call util_append(var, vari1) n = n + n[img] end if end do @@ -277,6 +283,7 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) integer(I4B) :: i,img, ti, di, ntot, istart, iend, nmax integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] + logical, dimension(:), allocatable :: vari1 allocate(isalloc[*]) allocate(n[*]) @@ -305,7 +312,8 @@ module subroutine coarray_component_collect_lgt_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) + allocate(vari1, source=tmp(1:n[img])[img]) + call util_append(var, vari1) n = n + n[img] end if end do diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index e35d3d838..e97488a0f 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -510,6 +510,7 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) integer(I4B) :: i,img, ti, di, ntot, istart, iend, nmax integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] + type(swiftest_particle_info), dimension(:), allocatable :: vari1 allocate(isalloc[*]) allocate(n[*]) @@ -538,7 +539,8 @@ module subroutine swiftest_coarray_component_collect_info_arr1D(var,dest_img) if (this_image() == di) then do img = 1, num_images() if (img /= di) then - call util_append(var, tmp(1:n[img])[img]) + allocate(vari1, source=tmp(1:n[img])[img]) + call util_append(var, vari1) n = n + n[img] end if end do From 4a4440c66bc90323c451b5cad21ad647991d33f4 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 12 May 2023 20:05:04 -0400 Subject: [PATCH 053/149] Fixed problem with writing files in coarray mode --- src/swiftest/swiftest_io.f90 | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index ba0559721..d63a1a95d 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -414,22 +414,20 @@ module subroutine swiftest_io_dump_storage(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i + type(walltimer) :: iotimer if (self%iframe == 0) return call self%make_index_map() associate(nc => self%nc) #ifdef COARRAY - critical + sync all + if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) + write(param%display_unit,*) "File output started" + call iotimer%start() #endif call nc%open(param) -#ifdef COARRAY - end critical -#endif do i = 1, self%iframe ! Writing files is more efficient if we write out the common frames from each image before going to the next frame -#ifdef COARRAY - if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) -#endif if (allocated(self%frame(i)%item)) then select type(nbody_system => self%frame(i)%item) class is (swiftest_nbody_system) @@ -437,19 +435,13 @@ module subroutine swiftest_io_dump_storage(self, param) end select deallocate(self%frame(i)%item) end if -#ifdef COARRAY - if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - sync all -#endif end do -#ifdef COARRAY - if (this_image() == 1) then -#endif - call nc%close() -#ifdef COARRAY - else - nc%lfile_is_open = .false. - end if + call nc%close() +#ifdef COARRAY + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) + call iotimer%stop() + call iotimer%report(message="File output :", unit=param%display_unit) + sync all #endif end associate From 172f437d048d4af30a7c6c3ffa592cf9827c61c1 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 10:14:02 -0400 Subject: [PATCH 054/149] Fixed J2 and J4 bug --- python/swiftest/swiftest/init_cond.py | 4 ++-- python/swiftest/swiftest/simulation_class.py | 10 +++++----- src/swiftest/swiftest_io.f90 | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/python/swiftest/swiftest/init_cond.py b/python/swiftest/swiftest/init_cond.py index 30fb46c6d..a4c47548c 100644 --- a/python/swiftest/swiftest/init_cond.py +++ b/python/swiftest/swiftest/init_cond.py @@ -277,8 +277,8 @@ def vec2xr(param: Dict, **kwargs: Any): space_coords = np.array(["x","y","z"]) vector_vars = ["rh","vh","Ip","rot"] - scalar_vars = ["name","a","e","inc","capom","omega","capm","Gmass","radius","rhill","J2","J4"] - time_vars = ["rh","vh","Ip","rot","a","e","inc","capom","omega","capm","Gmass","radius","rhill","J2","J4"] + scalar_vars = ["name","a","e","inc","capom","omega","capm","Gmass","radius","rhill","j2rp2","j4rp4"] + time_vars = ["rh","vh","Ip","rot","a","e","inc","capom","omega","capm","Gmass","radius","rhill","j2rp2","j4rp4"] # Check for valid keyword arguments kwargs = {k:kwargs[k] for k,v in kwargs.items() if v is not None} diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 32171e72d..2e9e17206 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -2223,9 +2223,9 @@ def add_solar_system_body(self, values = list(np.hsplit(np.array(body_list[0],dtype=np.dtype(object)),17)) else: values = list(np.squeeze(np.hsplit(np.array(body_list,np.dtype(object)),17))) - keys = ["id","name","a","e","inc","capom","omega","capm","rh","vh","Gmass","radius","rhill","Ip","rot","J2","J4"] + keys = ["id","name","a","e","inc","capom","omega","capm","rh","vh","Gmass","radius","rhill","Ip","rot","j2rp2","j4rp4"] kwargs = dict(zip(keys,values)) - scalar_floats = ["a","e","inc","capom","omega","capm","Gmass","radius","rhill","J2","J4"] + scalar_floats = ["a","e","inc","capom","omega","capm","Gmass","radius","rhill","j2rp2","j4rp4"] vector_floats = ["rh","vh","Ip","rot"] scalar_ints = ["id"] @@ -2434,7 +2434,7 @@ def add_body(self, Hill's radius values if these are massive bodies rot: (3) or (n,3) array-like of float, optional Rotation rate vectors if these are massive bodies with rotation enabled. - Ip: (3) or (n,3) array-like of flaot, optional + Ip: (3) or (n,3) array-like of float, optional Principal axes moments of inertia vectors if these are massive bodies with rotation enabled. Returns @@ -2451,7 +2451,7 @@ def input_to_array(val,t,n=None): elif t == "i": t = np.int64 elif t == "s": - t = np.str + t = str if val is None: return None, n @@ -2551,7 +2551,7 @@ def input_to_array_3d(val,n=None): Gmass = self.GU * mass dsnew = init_cond.vec2xr(self.param, name=name, a=a, e=e, inc=inc, capom=capom, omega=omega, capm=capm, id=id, - Gmass=Gmass, radius=radius, rhill=rhill, Ip=Ip, rh=rh, vh=vh,rot=rot, J2=J2, J4=J4, time=time) + Gmass=Gmass, radius=radius, rhill=rhill, Ip=Ip, rh=rh, vh=vh,rot=rot, j2rp2=J2, j4rp4=J4, time=time) dsnew = self._combine_and_fix_dsnew(dsnew) self.save(verbose=False) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index d63a1a95d..8cb720ec0 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -2930,6 +2930,15 @@ module subroutine swiftest_io_read_in_system(self, nc, param) if (.not.param%loblatecb) then if (allocated(self%pl%aobl)) deallocate(self%pl%aobl) if (allocated(self%tp%aobl)) deallocate(self%tp%aobl) + else + if (self%pl%nbody > 0) then + if (.not. allocated(self%pl%aobl)) allocate(self%pl%aobl(NDIM,self%pl%nbody)) + self%pl%aobl(:,:) = 0.0_DP + end if + if (self%tp%nbody > 0) then + if (.not. allocated(self%tp%aobl)) allocate(self%tp%aobl(NDIM,self%tp%nbody)) + self%tp%aobl(:,:) = 0.0_DP + end if end if return From a0df181876fe99672a2417f1c373c22e6f154aef Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 12:07:33 -0400 Subject: [PATCH 055/149] Fixed bugs related to conversion from Swiftest to Swifter --- python/swiftest/swiftest/io.py | 28 +++++++++++--------- python/swiftest/swiftest/simulation_class.py | 11 ++++---- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py index 0c0007488..f7517e5af 100644 --- a/python/swiftest/swiftest/io.py +++ b/python/swiftest/swiftest/io.py @@ -16,6 +16,7 @@ import sys import tempfile import re +import os # This defines features that are new in Swiftest and not in Swifter (for conversion between param.in files) newfeaturelist = ("RESTART", @@ -30,9 +31,14 @@ "SEED", "INTERACTION_LOOPS", "ENCOUNTER_CHECK", + "ENCOUNTER_CHECK_PLPL", + "ENCOUNTER_CHECK_PLTP", "TSTART", "DUMP_CADENCE", "ENCOUNTER_SAVE", + "MIN_GMFRAG", + "NFRAG_REDUCTION", + "COLLISION_MODEL", "COARRAY") # This list defines features that are booleans, so must be converted to/from string when writing/reading from file @@ -1227,7 +1233,7 @@ def swiftest_xr2infile(ds, param, in_type="NETCDF_DOUBLE", infile_name=None,fram print(f"{in_type} is an unknown file type") -def swifter_xr2infile(ds, param, framenum=-1): +def swifter_xr2infile(ds, param, simdir=os.getcwd, framenum=-1): """ Writes a set of Swifter input files from a single frame of a Swiftest xarray dataset @@ -1266,7 +1272,7 @@ def swifter_xr2infile(ds, param, framenum=-1): if param['IN_TYPE'] == 'ASCII': # Swiftest Central body file - plfile = open(param['PL_IN'], 'w') + plfile = open(os.path.join(simdir,param['PL_IN']), 'w') print(pl.id.count().values + 1, file=plfile) print(cb.id.values[0], GMSun, file=plfile) print('0.0 0.0 0.0', file=plfile) @@ -1279,21 +1285,20 @@ def swifter_xr2infile(ds, param, framenum=-1): print(i.values, pli['Gmass'].values, file=plfile) if param['CHK_CLOSE']: print(pli['radius'].values, file=plfile) - print(pli['rh'].values[0,0], pli['ry'].values[0,1], pli['rh'].values[0,2], file=plfile) - print(pli['vh'].values[0,0], pli['vh'].values[0,1], pli['vh'].values[0,2], file=plfile) + print(pli['rh'].values[0], pli['rh'].values[1], pli['rh'].values[2], file=plfile) + print(pli['vh'].values[0], pli['vh'].values[1], pli['vh'].values[2], file=plfile) plfile.close() # TP file - tpfile = open(param['TP_IN'], 'w') + tpfile = open(os.path.join(simdir,param['TP_IN']), 'w') print(tp.id.count().values, file=tpfile) for i in tp.id: tpi = tp.sel(id=i) print(i.values, file=tpfile) - print(tpi['rh'].values[0,0], tpi['ry'].values[0,1], tpi['rh'].values[0,2], file=tpfile) - print(tpi['vh'].values[0,0], tpi['vh'].values[0,1], tpi['vh'].values[0,2], file=tpfile) + print(tpi['rh'].values[0], tpi['rh'].values[1], tpi['rh'].values[2], file=tpfile) + print(tpi['vh'].values[0], tpi['vh'].values[1], tpi['vh'].values[2], file=tpfile) tpfile.close() else: - # Now make Swiftest files print(f"{param['IN_TYPE']} is an unknown input file type") return @@ -1848,6 +1853,8 @@ def swiftest2swifter_param(swiftest_param, J2=0.0, J4=0.0): swifter_param['C'] = swiftest.einsteinC * np.longdouble(TU2S) / np.longdouble(DU2M) for key in newfeaturelist: tmp = swifter_param.pop(key, None) + if "ISTEP_DUMP" not in swifter_param: + swifter_param["ISTEP_DUMP"] = swifter_param["ISTEP_OUT"] swifter_param['J2'] = J2 swifter_param['J4'] = J4 if swifter_param['OUT_STAT'] == "REPLACE": @@ -1858,11 +1865,6 @@ def swiftest2swifter_param(swiftest_param, J2=0.0, J4=0.0): swifter_param['OUT_TYPE'] = 'REAL4' if swifter_param['OUT_FORM'] == 'XVEL': swifter_param['OUT_FORM'] = 'XV' - IN_FORM = swifter_param.pop("IN_FORM", None) - INTERACTION_LOOPS = swifter_param.pop("INTERACTION_LOOPS", None) - ENCOUNTER_CHECK = swifter_param.pop("ENCOUNTER_CHECK", None) - ENCOUNTER_CHECK_PLPL = swifter_param.pop("ENCOUNTER_CHECK_PLPL", None) - ENCOUNTER_CHECK_PLTP = swifter_param.pop("ENCOUNTER_CHECK_PLTP", None) swifter_param['! VERSION'] = "Swifter parameter file converted from Swiftest" return swifter_param diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 2e9e17206..59f4e6669 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -2984,12 +2984,11 @@ def save(self, else: shutil.copy2(self.binary_source, self.driver_executable) elif codename == "Swifter": - if codename == "Swiftest": - swifter_param = io.swiftest2swifter_param(param) - else: - swifter_param = param - io.swifter_xr2infile(self.data, swifter_param, framenum) - self.write_param(param_file, param=swifter_param,**kwargs) + swifter_param = io.swiftest2swifter_param(param) + if "rhill" in self.data: + swifter_param['RHILL_PRESENT'] = 'YES' + io.swifter_xr2infile(self.data, swifter_param, self.simdir, framenum) + self.write_param(codename=codename,param_file=param_file,param=swifter_param,**kwargs) else: warnings.warn(f'Saving to {codename} not supported',stacklevel=2) From fb15c36443b0eb0d9c0d3f9bf1833b9f7e459013 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 12:14:00 -0400 Subject: [PATCH 056/149] Added Swifter vs Swiftest comparison for RMVS --- examples/.gitignore | 1 + examples/rmvs_swifter_comparison/.gitignore | 3 ++ .../1pl_1tp_encounter/.gitignore | 3 ++ .../1pl_1tp_encounter/init_cond.py | 38 +++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 examples/rmvs_swifter_comparison/.gitignore create mode 100644 examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore create mode 100755 examples/rmvs_swifter_comparison/1pl_1tp_encounter/init_cond.py diff --git a/examples/.gitignore b/examples/.gitignore index 90ecccf87..dbdf17577 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -9,3 +9,4 @@ !solar_impact !Swifter_Swiftest !swiftest_performance +!rmvs_swifter_comparison diff --git a/examples/rmvs_swifter_comparison/.gitignore b/examples/rmvs_swifter_comparison/.gitignore new file mode 100644 index 000000000..ecf4c57c8 --- /dev/null +++ b/examples/rmvs_swifter_comparison/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!1pl_1tp_encounter diff --git a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore new file mode 100644 index 000000000..80c7e7bd6 --- /dev/null +++ b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!init_cond.py diff --git a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/init_cond.py b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/init_cond.py new file mode 100755 index 000000000..9d997f720 --- /dev/null +++ b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/init_cond.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import numpy as np +import swiftest + +# Simple initial conditions of a circular planet with one test particle in a close encounter state +# Simulation start, stop, and output cadence times +t_0 = 0 # simulation start time +deltaT = 0.25 * swiftest.JD2S / swiftest.YR2S # simulation step size +end_sim = 0.15 +t_print = deltaT #output interval to print results + +iout = int(np.ceil(t_print / deltaT)) + +radius = np.double(4.25875607065041e-05) +Gmass = np.double(0.00012002693582795244940133) +apl = np.longdouble(1.0) +atp = np.longdouble(1.01) +vpl = np.longdouble(2 * np.pi) +vtp = np.longdouble(2 * np.pi / np.sqrt(atp)) + +p_pl = np.array([apl, 0.0, 0.0], dtype=np.double) +v_pl = np.array([0.0, vpl, 0.0], dtype=np.double) + +p_tp = np.array([atp, 0.0, 0.0], dtype=np.double) +v_tp = np.array([0.0, vtp, 0.0], dtype=np.double) + +rhill = np.double(apl * 0.0100447248332378922085) + +sim = swiftest.Simulation(simdir="swiftest_sim",init_cond_format="XV", general_relativity=False, integrator="RMVS") +sim.clean() +sim.add_solar_system_body(["Sun"]) +sim.add_body(name=["Planet"], rh=[p_pl], vh=[v_pl], Gmass=[Gmass], radius=[radius], rhill=[rhill]) +sim.add_body(name=["TestParticle"], rh=[p_tp], vh=[v_tp]) +sim.set_parameter(tstart=t_0, tstop=end_sim, dt=deltaT, istep_out=iout, dump_cadence=0) +sim.save() + +sim.set_parameter(simdir="swifter_sim",codename="Swifter",init_cond_file_type="ASCII",output_format="XV",output_file_type="REAL8") +sim.save() \ No newline at end of file From 1620fdf736b7324174be2e60eee473221937be3c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 15:06:51 -0400 Subject: [PATCH 057/149] Cleaned up bugs that were preventing Swifter input file from being read properly --- python/swiftest/swiftest/init_cond.py | 6 +- python/swiftest/swiftest/io.py | 304 ++++--------------- python/swiftest/swiftest/simulation_class.py | 245 +++++++-------- 3 files changed, 179 insertions(+), 376 deletions(-) diff --git a/python/swiftest/swiftest/init_cond.py b/python/swiftest/swiftest/init_cond.py index a4c47548c..8c0227d09 100644 --- a/python/swiftest/swiftest/init_cond.py +++ b/python/swiftest/swiftest/init_cond.py @@ -233,7 +233,7 @@ def vec2xr(param: Dict, **kwargs: Any): Parameters ---------- param : dict - Swiftest paramuration parameters. + Swiftest simulation parameters. name : str or array-like of str, optional Name or names of Bodies. If none passed, name will be "Body" id : int or array-like of int, optional @@ -252,9 +252,9 @@ def vec2xr(param: Dict, **kwargs: Any): capm : float or array-like of float, optional mean anomaly for param['IN_FORM'] == "EL" rh : (n,3) array-like of float, optional - Position vector array. This can be used instead of passing v1, v2, and v3 sepearately for "XV" input format + Position vector array. vh : (n,3) array-like of float, optional - Velocity vector array. This can be used instead of passing v4, v5, and v6 sepearately for "XV" input format + Velocity vector array. Gmass : float or array-like of float, optional G*mass values if these are massive bodies (only one of mass or Gmass can be passed) radius : float or array-like of float, optional diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py index f7517e5af..8f4e12078 100644 --- a/python/swiftest/swiftest/io.py +++ b/python/swiftest/swiftest/io.py @@ -28,6 +28,7 @@ "YARKOVSKY", "YORP", "IN_FORM", + "NC_IN", "SEED", "INTERACTION_LOOPS", "ENCOUNTER_CHECK", @@ -483,6 +484,7 @@ def swifter_stream(f, param): tlab : string list Labels for the tvec data """ + while True: # Loop until you read the end of file try: # Read single-line header @@ -492,32 +494,52 @@ def swifter_stream(f, param): t = record[0] npl = record[1][0] - 1 ntp = record[2][0] - - pvec = np.empty((6, npl)) + + if param['OUT_FORM'] == 'XV': + rpvec = np.empty((npl,3)) + vpvec = np.empty((npl,3)) + rtvec = np.empty((ntp,3)) + vtvec = np.empty((ntp,3)) + elif param['OUT_FORM'] == 'EL': + elempvec = np.empty((npl,6)) + elemtvec = np.empty((ntp,6)) plid = np.empty(npl, dtype='int') - tvec = np.empty((6, ntp)) tpid = np.empty(ntp, dtype='int') if npl > 0: GMpl = np.empty(npl) Rpl = np.empty(npl) for i in range(npl): # Read single-line pl frame for - record = f.read_record(' 0: for i in range(ntp): - record = f.read_record(' 0: - plid = f.read_ints() - dtstr = f'({npl[0]},)a{NAMELEN}' - names = f.read_record(np.dtype(dtstr)) - plnames = [] - for i in range(npl[0]): - plnames.append(np.char.strip(str(names[i], encoding='utf-8'))) - p1 = f.read_reals(np.float64) - p2 = f.read_reals(np.float64) - p3 = f.read_reals(np.float64) - p4 = f.read_reals(np.float64) - p5 = f.read_reals(np.float64) - p6 = f.read_reals(np.float64) - if param['OUT_FORM'] == 'XVEL': - p7 = f.read_reals(np.float64) - p8 = f.read_reals(np.float64) - p9 = f.read_reals(np.float64) - p10 = f.read_reals(np.float64) - p11 = f.read_reals(np.float64) - p12 = f.read_reals(np.float64) - GMpl = f.read_reals(np.float64) - if param['RHILL_PRESENT']: - rhill = f.read_reals(np.float64) - Rpl = f.read_reals(np.float64) - if param['ROTATION']: - Ipplx = f.read_reals(np.float64) - Ipply = f.read_reals(np.float64) - Ipplz = f.read_reals(np.float64) - rotplx = f.read_reals(np.float64) - rotply = f.read_reals(np.float64) - rotplz = f.read_reals(np.float64) - if param['TIDES']: - k2pl = f.read_reals(np.float64) - Qpl = f.read_reals(np.float64) - if ntp[0] > 0: - tpid = f.read_ints() - dtstr = f'({ntp[0]},)a{NAMELEN}' - names = f.read_record(np.dtype(dtstr)) - tpnames = [] - for i in range(npl[0]): - tpnames.append(np.char.strip(str(names[i], encoding='utf-8'))) - t1 = f.read_reals(np.float64) - t2 = f.read_reals(np.float64) - t3 = f.read_reals(np.float64) - t4 = f.read_reals(np.float64) - t5 = f.read_reals(np.float64) - t6 = f.read_reals(np.float64) - if param['OUT_FORM'] == 'XVEL': - t7 = f.read_reals(np.float64) - t8 = f.read_reals(np.float64) - t9 = f.read_reals(np.float64) - t10 = f.read_reals(np.float64) - t11 = f.read_reals(np.float64) - t12 = f.read_reals(np.float64) - clab, plab, tlab, infolab_float, infolab_int, infolab_str = make_swiftest_labels(param) - - if npl > 0: - pvec = np.vstack([p1, p2, p3, p4, p5, p6]) - if param['OUT_FORM'] == 'XVEL': - pvec = np.vstack([pvec, p7, p8, p9, p10, p11, p12]) - pvec = np.vstack([pvec, GMpl, Rpl]) - else: - if param['OUT_FORM'] == 'XVEL': - pvec = np.empty((14, 0)) - else: - pvec = np.empty((8, 0)) - plid = np.empty(0) - plnames = np.empty(0) - if ntp > 0: - tvec = np.vstack([t1, t2, t3, t4, t5, t6]) - if param['OUT_FORM'] == 'XVEL': - tvec = np.vstack([tvec, t7, t8, t9, t10, t11, t12]) - else: - if param['OUT_FORM'] == 'XVEL': - tvec = np.empty((12, 0)) - else: - tvec = np.empty((6, 0)) - tpid = np.empty(0) - tpnames = np.empty(0) - cvec = np.array([Mcb, Rcb, J2cb, J4cb]) - if param['RHILL_PRESENT']: - if npl > 0: - pvec = np.vstack([pvec, rhill]) - if param['ROTATION']: - cvec = np.vstack([cvec, Ipcbx, Ipcby, Ipcbz, rotcbx, rotcby, rotcbz]) - if npl > 0: - pvec = np.vstack([pvec, Ipplx, Ipply, Ipplz, rotplx, rotply, rotplz]) - if param['TIDES']: - cvec = np.vstack([cvec, k2cb, Qcb]) - if npl > 0: - pvec = np.vstack([pvec, k2pl, Qpl]) - yield t, cbid, cbnames, cvec.T, clab, \ - npl, plid, plnames, pvec.T, plab, \ - ntp, tpid, tpnames, tvec.T, tlab - + if param['OUT_FORM'] == 'XV': + yield t, npl, [plid, rpvec, vpvec, GMpl, Rpl], plab, \ + ntp, [tpid, rtvec, vtvec], tlab + elif param['OUT_FORM'] == 'EL': + yield t, npl, [plid, elempvec, GMpl, Rpl], plab, \ + ntp, [tpid, elemtvec], tlab def swifter2xr(param, verbose=True): """ @@ -783,9 +573,16 @@ def swifter2xr(param, verbose=True): dims = ['time', 'id','vec'] pl = [] tp = [] + ds = [] with FortranFile(param['BIN_OUT'], 'r') as f: - for t, npl, plid, pvec, plab, \ - ntp, tpid, tvec, tlab in swifter_stream(f, param): + for t, npl, pvec, plab, \ + ntp, tvec, tlab in swifter_stream(f, param): + + pvec_args = dict(zip(plab,pvec)) + plds = swiftest.init_cond.vec2xr(param,**pvec_args) + if ntp > 0: + tvec_args = dict(zip(tlab,tvec)) + # Prepare frames by adding an extra axis for the time coordinate plframe = np.expand_dims(pvec, axis=0) tpframe = np.expand_dims(tvec, axis=0) @@ -806,6 +603,7 @@ def swifter2xr(param, verbose=True): tpds = tpda.to_dataset(dim='vec') if verbose: print('\nCreating Dataset') ds = xr.combine_by_coords([plds, tpds]) + #if param['OUT_FORM'] = 'XV': if verbose: print(f"Successfully converted {ds.sizes['time']} output frames.") return ds @@ -1848,9 +1646,9 @@ def swiftest2swifter_param(swiftest_param, J2=0.0, J4=0.0): DU2M = swifter_param.pop("DU2M", 1.0) TU2S = swifter_param.pop("TU2S", 1.0) GR = swifter_param.pop("GR", None) - if GR is not None: - if GR: - swifter_param['C'] = swiftest.einsteinC * np.longdouble(TU2S) / np.longdouble(DU2M) + # if GR is not None: + # if GR: + # swifter_param['C'] = swiftest.einsteinC * np.longdouble(TU2S) / np.longdouble(DU2M) for key in newfeaturelist: tmp = swifter_param.pop(key, None) if "ISTEP_DUMP" not in swifter_param: diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 59f4e6669..2ff9c8454 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -345,7 +345,6 @@ def __init__(self,read_param: bool = False, # Parameters are set in reverse priority order. First the defaults, then values from a pre-existing input file, # then using the arguments passed via **kwargs. - #-------------------------- # Lowest Priority: Defaults: #-------------------------- @@ -371,7 +370,7 @@ def __init__(self,read_param: bool = False, self.read_encounters = read_encounters if read_param or read_data: - if self.read_param(read_init_cond = True, dask=dask): + if self.read_param(read_init_cond = True, dask=dask, **kwargs): # We will add the parameter file to the kwarg list. This will keep the set_parameter method from # overriding everything with defaults when there are no arguments passed to Simulation() kwargs['param_file'] = self.param_file @@ -851,8 +850,8 @@ def set_parameter(self, verbose: bool = True, **kwargs): # Setters returning parameter dictionary values param_dict = {} - param_dict.update(self.set_unit_system(**kwargs)) param_dict.update(self.set_integrator(**kwargs)) + param_dict.update(self.set_unit_system(**kwargs)) param_dict.update(self.set_simulation_time(**kwargs)) param_dict.update(self.set_init_cond_files(**kwargs)) param_dict.update(self.set_output_files(**kwargs)) @@ -955,6 +954,11 @@ def set_integrator(self, self.binary_path = "NOT IMPLEMENTED FOR THIS CODE" self.driver_executable = None update_list.append("driver_executable") + + if self.codename == "Swifter": + J2 = self.param.pop("J2",0.0) + J4 = self.param.pop("J4",0.0) + self.param = io.swiftest2swifter_param(self.param, J2, J4) if integrator is not None: valid_integrator = ["symba","rmvs","whm","helio"] @@ -1172,113 +1176,21 @@ def set_feature(self, if close_encounter_check is not None: self.param["CHK_CLOSE"] = close_encounter_check update_list.append("close_encounter_check") - - if general_relativity is not None: - self.param["GR"] = general_relativity - update_list.append("general_relativity") - - fragmentation_models = ["FRAGGLE"] - if collision_model is not None: - collision_model = collision_model.upper() - fragmentation = collision_model in fragmentation_models - if self.codename != "Swiftest" and self.integrator != "symba" and fragmentation: - warnings.warn("Fragmentation is only available on Swiftest SyMBA.",stacklevel=2) - self.param['COLLISION_MODEL'] = "MERGE" - else: - self.param['COLLISION_MODEL'] = collision_model - update_list.append("collision_model") - if fragmentation: - if "MIN_GMFRAG" not in self.param and minimum_fragment_mass is None and minimum_fragment_gmass is None: - warnings.warn("Minimum fragment mass is not set. Set it using minimum_fragment_gmass or minimum_fragment_mass",stacklevel=2) - else: - update_list.append("minimum_fragment_gmass") - - if minimum_fragment_gmass is not None and minimum_fragment_mass is not None: - warnings.warn("Only set either minimum_fragment_mass or minimum_fragment_gmass, but not both!",stacklevel=2) - - if minimum_fragment_gmass is not None: - self.param["MIN_GMFRAG"] = minimum_fragment_gmass - if "minmum_fragment_gmass" not in update_list: - update_list.append("minimum_fragment_gmass") - elif minimum_fragment_mass is not None: - self.param["MIN_GMFRAG"] = minimum_fragment_mass * self.GU - if "minimum_fragment_gmass" not in update_list: - update_list.append("minimum_fragment_gmass") - - if nfrag_reduction is not None: - self.param["NFRAG_REDUCTION"] = nfrag_reduction - update_list.append("nfrag_reduction") - - if rotation is not None: - self.param['ROTATION'] = rotation - update_list.append("rotation") - - if self.param['COLLISION_MODEL'] == "FRAGGLE" and not self.param['ROTATION']: - self.param['ROTATION'] = True - update_list.append("rotation") - - if compute_conservation_values is not None: - self.param["ENERGY"] = compute_conservation_values - update_list.append("compute_conservation_values") - + + if rhill_present is not None: + self.param["RHILL_PRESENT"] = rhill_present + update_list.append("rhill_present") + if extra_force is not None: self.param["EXTRA_FORCE"] = extra_force update_list.append("extra_force") - if big_discard is not None: if self.codename != "Swifter": self.param["BIG_DISCARD"] = False else: self.param["BIG_DISCARD"] = big_discard - update_list.append("big_discard") - - if rhill_present is not None: - self.param["RHILL_PRESENT"] = rhill_present - update_list.append("rhill_present") - - if restart is not None: - self.param["RESTART"] = restart - update_list.append("restart") - - if interaction_loops is not None: - valid_vals = ["TRIANGULAR", "FLAT"] - interaction_loops = interaction_loops.upper() - if interaction_loops not in valid_vals: - msg = f"{interaction_loops} is not a valid option for interaction loops." - msg += f"\nMust be one of {valid_vals}" - warnings.warn(msg,stacklevel=2) - if "INTERACTION_LOOPS" not in self.param: - self.param["INTERACTION_LOOPS"] = valid_vals[0] - else: - self.param["INTERACTION_LOOPS"] = interaction_loops - update_list.append("interaction_loops") - - if encounter_check_loops is not None: - valid_vals = ["TRIANGULAR", "SORTSWEEP"] - encounter_check_loops = encounter_check_loops.upper() - if encounter_check_loops not in valid_vals: - msg = f"{encounter_check_loops} is not a valid option for interaction loops." - msg += f"\nMust be one of {valid_vals}" - warnings.warn(msg,stacklevel=2) - if "ENCOUNTER_CHECK" not in self.param: - self.param["ENCOUNTER_CHECK"] = valid_vals[0] - else: - self.param["ENCOUNTER_CHECK"] = encounter_check_loops - update_list.append("encounter_check_loops") - - if encounter_save is not None: - valid_vals = ["NONE", "TRAJECTORY", "CLOSEST", "BOTH"] - encounter_save = encounter_save.upper() - if encounter_save not in valid_vals: - msg = f"{encounter_save} is not a valid option for encounter_save." - msg += f"\nMust be one of {valid_vals}" - warnings.warn(msg,stacklevel=2) - if "ENCOUNTER_SAVE" not in self.param: - self.param["ENCOUNTER_SAVE"] = valid_vals[0] - else: - self.param["ENCOUNTER_SAVE"] = encounter_save - update_list.append("encounter_save") + update_list.append("big_discard") if simdir is not None: self.simdir = Path(simdir) @@ -1291,15 +1203,106 @@ def set_feature(self, self.driver_executable = self.binary_path / "swiftest_driver" self.param_file = Path(kwargs.pop("param_file","param.in")) - if coarray is not None: - if self.codename == "Swiftest": - self.param["COARRAY"] = coarray - update_list.append("coarray") - - - self.param["TIDES"] = False + if self.codename == "Swiftest": + if general_relativity is not None: + self.param["GR"] = general_relativity + update_list.append("general_relativity") + + fragmentation_models = ["FRAGGLE"] + if collision_model is not None: + collision_model = collision_model.upper() + fragmentation = collision_model in fragmentation_models + if self.codename != "Swiftest" and self.integrator != "symba" and fragmentation: + warnings.warn("Fragmentation is only available on Swiftest SyMBA.",stacklevel=2) + self.param['COLLISION_MODEL'] = "MERGE" + else: + self.param['COLLISION_MODEL'] = collision_model + update_list.append("collision_model") + if fragmentation: + if "MIN_GMFRAG" not in self.param and minimum_fragment_mass is None and minimum_fragment_gmass is None: + warnings.warn("Minimum fragment mass is not set. Set it using minimum_fragment_gmass or minimum_fragment_mass",stacklevel=2) + else: + update_list.append("minimum_fragment_gmass") + + if minimum_fragment_gmass is not None and minimum_fragment_mass is not None: + warnings.warn("Only set either minimum_fragment_mass or minimum_fragment_gmass, but not both!",stacklevel=2) + + if minimum_fragment_gmass is not None: + self.param["MIN_GMFRAG"] = minimum_fragment_gmass + if "minmum_fragment_gmass" not in update_list: + update_list.append("minimum_fragment_gmass") + elif minimum_fragment_mass is not None: + self.param["MIN_GMFRAG"] = minimum_fragment_mass * self.GU + if "minimum_fragment_gmass" not in update_list: + update_list.append("minimum_fragment_gmass") + + if nfrag_reduction is not None: + self.param["NFRAG_REDUCTION"] = nfrag_reduction + update_list.append("nfrag_reduction") + + if rotation is not None: + self.param['ROTATION'] = rotation + update_list.append("rotation") + + if self.param['COLLISION_MODEL'] == "FRAGGLE" and not self.param['ROTATION']: + self.param['ROTATION'] = True + update_list.append("rotation") + + if compute_conservation_values is not None: + self.param["ENERGY"] = compute_conservation_values + update_list.append("compute_conservation_values") + + if restart is not None: + self.param["RESTART"] = restart + update_list.append("restart") + + if interaction_loops is not None: + valid_vals = ["TRIANGULAR", "FLAT"] + interaction_loops = interaction_loops.upper() + if interaction_loops not in valid_vals: + msg = f"{interaction_loops} is not a valid option for interaction loops." + msg += f"\nMust be one of {valid_vals}" + warnings.warn(msg,stacklevel=2) + if "INTERACTION_LOOPS" not in self.param: + self.param["INTERACTION_LOOPS"] = valid_vals[0] + else: + self.param["INTERACTION_LOOPS"] = interaction_loops + update_list.append("interaction_loops") + + if encounter_check_loops is not None: + valid_vals = ["TRIANGULAR", "SORTSWEEP"] + encounter_check_loops = encounter_check_loops.upper() + if encounter_check_loops not in valid_vals: + msg = f"{encounter_check_loops} is not a valid option for interaction loops." + msg += f"\nMust be one of {valid_vals}" + warnings.warn(msg,stacklevel=2) + if "ENCOUNTER_CHECK" not in self.param: + self.param["ENCOUNTER_CHECK"] = valid_vals[0] + else: + self.param["ENCOUNTER_CHECK"] = encounter_check_loops + update_list.append("encounter_check_loops") + + if encounter_save is not None: + valid_vals = ["NONE", "TRAJECTORY", "CLOSEST", "BOTH"] + encounter_save = encounter_save.upper() + if encounter_save not in valid_vals: + msg = f"{encounter_save} is not a valid option for encounter_save." + msg += f"\nMust be one of {valid_vals}" + warnings.warn(msg,stacklevel=2) + if "ENCOUNTER_SAVE" not in self.param: + self.param["ENCOUNTER_SAVE"] = valid_vals[0] + else: + self.param["ENCOUNTER_SAVE"] = encounter_save + update_list.append("encounter_save") - + if coarray is not None: + if self.codename == "Swiftest": + self.param["COARRAY"] = coarray + update_list.append("coarray") + + self.param["TIDES"] = False + + feature_dict = self.get_feature(update_list, verbose) return feature_dict @@ -1902,11 +1905,12 @@ def set_unit_system(self, if all(key in self.param for key in ["MU2KG","DU2M","TU2S"]): self.GU = constants.GC * self.param["TU2S"] ** 2 * self.param["MU2KG"] / self.param["DU2M"] ** 3 - if recompute_unit_values and \ - MU2KG_old != self.param['MU2KG'] or \ - DU2M_old != self.param['DU2M'] or \ - TU2S_old != self.param['TU2S']: - self.update_param_units(MU2KG_old, DU2M_old, TU2S_old) + if "MU2KG" in self.param and "DU2M" in self.param and "TU2S" in self.param: + if recompute_unit_values and \ + MU2KG_old != self.param['MU2KG'] or \ + DU2M_old != self.param['DU2M'] or \ + TU2S_old != self.param['TU2S']: + self.update_param_units(MU2KG_old, DU2M_old, TU2S_old) unit_dict = self.get_unit_system(update_list, verbose) @@ -2348,14 +2352,14 @@ def _get_instance_var(self, arg_list: str | List[str], valid_arg: Dict, verbose: Parameters ---------- arg_list: str | List[str] - A single string or list of strings containing the names of the the instance variable to get. + A single string or list of strings containing the names of the the instance variable to get. valid_arg: dict - A dictionary where the key is the parameter argument and the value is the equivalent instance variable value. + A dictionary where the key is the parameter argument and the value is the equivalent instance variable value. verbose: bool, optional - If passed, it will override the Simulation object's verbose flag + If passed, it will override the Simulation object's verbose flag **kwargs - A dictionary of additional keyword argument. This allows this method to be called by the more general - get_parameter method, which takes all possible Simulation parameters as arguments, so these are ignored. + A dictionary of additional keyword argument. This allows this method to be called by the more general + get_parameter method, which takes all possible Simulation parameters as arguments, so these are ignored. Returns ------- @@ -2622,7 +2626,8 @@ def read_param(self, codename: Literal["Swiftest", "Swifter", "Swift"] | None = None, read_init_cond : bool | None = None, verbose: bool | None = None, - dask: bool = False): + dask: bool = False, + **kwargs : Any): """ Reads in an input parameter file and stores the values in the param dictionary. From 12bc34ee1425ffb43ccd36ccf65a0c3f5e332a78 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 15:25:14 -0400 Subject: [PATCH 058/149] Improved the reading of Swifter files using the vec2xr method --- python/swiftest/swiftest/init_cond.py | 10 +++++---- python/swiftest/swiftest/io.py | 31 +++++++++------------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/python/swiftest/swiftest/init_cond.py b/python/swiftest/swiftest/init_cond.py index 8c0227d09..efaa6429f 100644 --- a/python/swiftest/swiftest/init_cond.py +++ b/python/swiftest/swiftest/init_cond.py @@ -282,10 +282,12 @@ def vec2xr(param: Dict, **kwargs: Any): # Check for valid keyword arguments kwargs = {k:kwargs[k] for k,v in kwargs.items() if v is not None} - if "rot" not in kwargs and "Gmass" in kwargs: - kwargs['rot'] = np.zeros((len(kwargs['Gmass']),3)) - if "Ip" not in kwargs and "Gmass" in kwargs: - kwargs['Ip'] = np.full((len(kwargs['Gmass']),3), 0.4) + + if "rotation" in param and param['rotation'] == True: + if "rot" not in kwargs and "Gmass" in kwargs: + kwargs['rot'] = np.zeros((len(kwargs['Gmass']),3)) + if "Ip" not in kwargs and "Gmass" in kwargs: + kwargs['Ip'] = np.full((len(kwargs['Gmass']),3), 0.4) if "time" not in kwargs: kwargs["time"] = np.array([0.0]) diff --git a/python/swiftest/swiftest/io.py b/python/swiftest/swiftest/io.py index 8f4e12078..32a10f34d 100644 --- a/python/swiftest/swiftest/io.py +++ b/python/swiftest/swiftest/io.py @@ -573,37 +573,26 @@ def swifter2xr(param, verbose=True): dims = ['time', 'id','vec'] pl = [] tp = [] - ds = [] with FortranFile(param['BIN_OUT'], 'r') as f: for t, npl, pvec, plab, \ ntp, tvec, tlab in swifter_stream(f, param): + sys.stdout.write('\r' + f"Reading in time {t[0]:.3e}") + sys.stdout.flush() + pvec_args = dict(zip(plab,pvec)) - plds = swiftest.init_cond.vec2xr(param,**pvec_args) + pl.append(swiftest.init_cond.vec2xr(param,time=t,**pvec_args)) if ntp > 0: tvec_args = dict(zip(tlab,tvec)) + tp.append(swiftest.init_cond.vec2xr(param,time=t,**tvec_args)) - # Prepare frames by adding an extra axis for the time coordinate - plframe = np.expand_dims(pvec, axis=0) - tpframe = np.expand_dims(tvec, axis=0) - - # Create xarray DataArrays out of each body type - plxr = xr.DataArray(plframe, dims=dims, coords={'time': t, 'id': plid, 'vec': plab}) - tpxr = xr.DataArray(tpframe, dims=dims, coords={'time': t, 'id': tpid, 'vec': tlab}) - - pl.append(plxr) - tp.append(tpxr) - sys.stdout.write('\r' + f"Reading in time {t[0]:.3e}") - sys.stdout.flush() - - plda = xr.concat(pl, dim='time') - tpda = xr.concat(tp, dim='time') + plds = xr.concat(pl, dim='time') + if len(tp) > 0: + tpds = xr.concat(tp, dim='time') - plds = plda.to_dataset(dim='vec') - tpds = tpda.to_dataset(dim='vec') if verbose: print('\nCreating Dataset') - ds = xr.combine_by_coords([plds, tpds]) - #if param['OUT_FORM'] = 'XV': + if len(tp) > 0: + ds = xr.combine_by_coords([plds, tpds]) if verbose: print(f"Successfully converted {ds.sizes['time']} output frames.") return ds From 36edbd384801eddb86df6d787475aaa7f6ff5b17 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 15:45:50 -0400 Subject: [PATCH 059/149] Added a Swiftest vs. Swifter plotting script --- .../1pl_1tp_encounter/.gitignore | 2 + .../swiftest_vs_swifter.ipynb | 159 ++++++++++++++++++ .../1pl_1tp_encounter/swiftest_vs_swifter.py | 9 + 3 files changed, 170 insertions(+) create mode 100644 examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.ipynb create mode 100644 examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.py diff --git a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore index 80c7e7bd6..89c2a1c6c 100644 --- a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore +++ b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/.gitignore @@ -1,3 +1,5 @@ * !.gitignore !init_cond.py +!swiftest_vs_swifter.py +!swiftest_vs_swifter.ipynb diff --git a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.ipynb b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.ipynb new file mode 100644 index 000000000..5f5f7b2ba --- /dev/null +++ b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import swiftest" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Swifter file /home/daminton/git_debug/swiftest/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swifter_sim/param.in\n", + "Reading in time 1.355e-01\n", + "Creating Dataset\n", + "Successfully converted 199 output frames.\n", + "Swifter simulation data stored as xarray DataSet .data\n" + ] + } + ], + "source": [ + "swiftersim = swiftest.Simulation(simdir=\"swifter_sim\", read_data=True, codename=\"Swifter\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Swiftest file /home/daminton/git_debug/swiftest/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_sim/param.in\n", + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 1 output frames.\n", + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 221 output frames.\n", + "Swiftest simulation data stored as xarray DataSet .data\n", + "Reading initial conditions file as .init_cond\n", + "Finished reading Swiftest dataset files.\n" + ] + } + ], + "source": [ + "swiftestsim = swiftest.Simulation(simdir=\"swiftest_sim\",read_data=True)\n", + "swiftestsim.data = swiftestsim.data.swap_dims({\"name\" : \"id\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "swiftdiff = swiftestsim.data - swiftersim.data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAEiCAYAAAC4D9oGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxNElEQVR4nO3de1RVdf7/8dfhLsrFS4IUInZRHLsJI2KR1jioZWXaN9MirbT8zhQptUyzlZcmHKvVmJqWDnmZb6nzzZyxGXNkKqxGROWLZsnPzPCWEGkF3gLEz++PhjMeQQTc5xw2PB9rnbU6n/PZe7/3aa838HJfHMYYIwAAAAAAAIv4eLsAAAAAAADQvBA2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2AAAAAAAASxE2oMV55513dPXVV6tVq1Zq3769BgwYoBMnTkiSxowZo6FDh2rGjBnq2LGjQkND9eijj6qiosK5/Pr163XjjTcqPDxc7du315AhQ7R3716XbRw6dEj33nuv2rVrp9atWyshIUG5ubnOz9977z3Fx8crKChIXbt21YwZM3T69Gm37K8xRgMGDNCgQYNkjJEk/fjjj+rcubOmTp3qlm0CsF5L61379u2Tj4+Ptm3b5jI+b948xcTEOPsZgKarpfUtSerSpYscDkeNF9ASETagRSkqKtLIkSP10EMPqaCgQNnZ2Ro2bJjLL60ffPCBCgoK9NFHH2nFihVas2aNZsyY4fz8xIkTSk9P19atW/XBBx/Ix8dHd911l86cOSNJOn78uPr166fDhw9r7dq12rFjhyZNmuT8/B//+Ifuv/9+paWladeuXXrjjTe0dOlSvfDCC+et+6233lKbNm3qfL311lu1LutwOLRs2TJt2bJFc+fOlSSNHz9eERERmj59+sV+pQA8oCX2ri5dumjAgAFasmSJy/iSJUs0ZswYfnkHmriW2LckaevWrSoqKlJRUZEOHTqkPn36KDk5+WK/TsCeDNCC5OXlGUlm3759tX4+evRo065dO3PixAnn2MKFC02bNm1MVVVVrcuUlJQYSWbnzp3GGGPeeOMNExISYo4ePVrr/OTkZJORkeEy9qc//cl06tTpvHWXlZWZPXv21PkqKyurc9///Oc/m8DAQDNlyhQTHBxsdu/eXed8AE1HS+1dq1atMm3btjU//fSTMcaY7du3G4fDYQoLC8+7DICmoaX2rbOlpaWZmJgYU1JSUq/5QHND2IAW5fTp0+ZXv/qVCQkJMXfffbdZtGiR+f77752fjx492tx8880uy2zfvt3lh+VXX31lRo4caWJjY01ISIhp3bq1kWT+/ve/G2OM+e///m9z0003nbeG4OBgExQUZFq3bu18BQUFGUkuP3DdYeTIkUaSWbhwoVu3A8BaLbV3lZeXm0suucSsWLHCGPPzL+633HKLW7YFwFottW9Ve+ONN0zr1q3N9u3b3bodoCnjMgq0KL6+vsrKytL777+vHj16aN68eerWrZsKCwsvuGz1Kbu33367jh49qsWLFys3N9d5XWD1NYatWrWqcz1nzpzRjBkztH37dudr586d2rNnj4KCgmpd5mJP6ZOkkydPKi8vT76+vtqzZ88F9xdA09FSe1dAQIBSU1O1ZMkSVVRU6O2339ZDDz10wX0G4H0ttW9JUnZ2th5//HEtX75c11577QX3F2iu/LxdAOBpDodDN9xwg2644QY999xziomJ0Zo1a5Seni5J2rFjh06dOuX8AbZ582a1adNGl112mY4ePaqCggK98cYbzuvvPv30U5f1X3PNNfrjH/+o77//Xu3ataux/V69emn37t264oor6l3zHXfcocTExDrnRERE1Pn5k08+KR8fH73//vu69dZbddttt+mWW26pdw0AvKul9q6xY8eqZ8+eWrBggSorKzVs2LB6bx+Ad7XEvvXVV19p+PDheuaZZ+hXaPEIG9Ci5Obm6oMPPlBKSoo6duyo3Nxcfffdd4qLi3POqaio0MMPP6xnn31W+/fv17Rp0/TYY4/Jx8dHbdu2Vfv27bVo0SJ16tRJBw4c0OTJk122MXLkSGVkZGjo0KGaNWuWOnXqpPz8fEVFRSkpKUnPPfechgwZoujoaP3Xf/2XfHx89Nlnn2nnzp363e9+V2vdISEhCgkJafR+//3vf9ebb76pnJwc9erVS5MnT9bo0aP12WefqW3bto1eLwDPaKm9S5Li4uLUp08fPf3003rooYcu+C+ZAJqGlti3Tp06pdtvv13XXXedHnnkERUXFzs/i4yMbNQ6AVvz9nUcgCft2rXLDBw40FxyySUmMDDQXHXVVWbevHnOz0ePHm3uvPNO89xzz5n27dubNm3amLFjxzpvTmaMMVlZWSYuLs4EBgaaa665xmRnZxtJZs2aNc45+/btM8OHDzehoaEmODjYJCQkmNzcXOfn69evN3379jWtWrUyoaGhpnfv3mbRokVu2eeSkhITERHhcoOkyspK07t3b3PPPfe4ZZsArNUSe9fZMjMzjSSzZcsWt28LgDVaYt8qLCw0kmp9AS2RwxgeVA1UGzNmjH788Uf95S9/8XYpAFBvzb13vfDCC1q5cqV27tzp7VIAWKS59y0AEjeIBAAATdLx48e1detWzZs3T2lpad4uBwAANABhAwAAaJIee+wx3XjjjerXrx9PoQAAwGa4jAIAAAAAAFiKMxsAAAAAAIClCBsAAAAAAIClCBsAAAAAAICl/LxdQHNw5swZHT58WCEhIXI4HN4uB0AzZIzRsWPHFBUVJR+fi8+J6VsA3I2+BcCOrO5dLRlhgwUOHz6s6Ohob5cBoAU4ePCgLrvssoteD30LgKfQtwDYkVW9qyUjbLBASEiIpJ8PyNDQUC9XA6A5KisrU3R0tLPfXCz6FgB3o28BsCOre1dLRthggepT+UJDQ/nhB8CtrDp1mL4FwFPoWwDsiMu1Lh4XoQAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEvZLmxYsGCBYmNjFRQUpPj4eH3yySd1zt+4caPi4+MVFBSkrl276vXXXz/v3JUrV8rhcGjo0KEWVw0AAAAAQMthq7Bh1apVmjBhgqZOnar8/HwlJydr8ODBOnDgQK3zCwsLdeuttyo5OVn5+fl65plnlJaWptWrV9eYu3//fj311FNKTk52924AAAAAANCs2SpseOWVV/Twww9r7NixiouL05w5cxQdHa2FCxfWOv/1119X586dNWfOHMXFxWns2LF66KGH9PLLL7vMq6qq0n333acZM2aoa9euntgVAAAAAACaLduEDRUVFcrLy1NKSorLeEpKijZt2lTrMjk5OTXmDxw4UNu2bVNlZaVzbObMmbrkkkv08MMPW184AAAAAAAtjJ+3C6ivI0eOqKqqShERES7jERERKi4urnWZ4uLiWuefPn1aR44cUadOnfSvf/1LmZmZ2r59e71rKS8vV3l5ufN9WVlZ/XcEALyAvgXAbuhbAGBvtjmzoZrD4XB5b4ypMXah+dXjx44d0/3336/FixerQ4cO9a5h1qxZCgsLc76io6MbsAcA4Hn0LQB2Q98CAHuzTdjQoUMH+fr61jiLoaSkpMbZC9UiIyNrne/n56f27dtr79692rdvn26//Xb5+fnJz89Py5cv19q1a+Xn56e9e/fWut4pU6aotLTU+Tp48KA1OwkAbkLfAmA39C0AsDfbXEYREBCg+Ph4ZWVl6a677nKOZ2Vl6c4776x1maSkJL333nsuYxs2bFBCQoL8/f3VvXt37dy50+XzZ599VseOHdOrr7563gQ9MDBQgYGBF7lHAOA59C0AdkPfAgB7s03YIEnp6elKTU1VQkKCkpKStGjRIh04cEDjx4+X9HMC/s0332j58uWSpPHjx2v+/PlKT0/XuHHjlJOTo8zMTK1YsUKSFBQUpJ49e7psIzw8XJJqjAMAAAAAgPqxVdgwYsQIHT16VDNnzlRRUZF69uypdevWKSYmRpJUVFSkAwcOOOfHxsZq3bp1mjhxol577TVFRUVp7ty5Gj58uLd2AQAAAACAZs9hqu+YiEYrKytTWFiYSktLFRoa6u1yADRDVvcZ+hYAd6NvAbAjeo11bHODSAAAAAAAYA+EDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFKEDQAAAAAAwFK2CxsWLFig2NhYBQUFKT4+Xp988kmd8zdu3Kj4+HgFBQWpa9euev31110+X7x4sZKTk9W2bVu1bdtWAwYM0JYtW9y5CwAAAAAANGu2ChtWrVqlCRMmaOrUqcrPz1dycrIGDx6sAwcO1Dq/sLBQt956q5KTk5Wfn69nnnlGaWlpWr16tXNOdna2Ro4cqY8++kg5OTnq3LmzUlJS9M0333hqtwAAAAAAaFYcxhjj7SLqKzExUb169dLChQudY3FxcRo6dKhmzZpVY/7TTz+ttWvXqqCgwDk2fvx47dixQzk5ObVuo6qqSm3bttX8+fP1wAMP1KuusrIyhYWFqbS0VKGhoQ3cKwC4MKv7DH0LgLvRtwDYEb3GOn7eLqC+KioqlJeXp8mTJ7uMp6SkaNOmTbUuk5OTo5SUFJexgQMHKjMzU5WVlfL396+xzMmTJ1VZWal27dqdt5by8nKVl5c735eVlTVkVwDA4+hbAOyGvgUA9mabyyiOHDmiqqoqRUREuIxHRESouLi41mWKi4trnX/69GkdOXKk1mUmT56sSy+9VAMGDDhvLbNmzVJYWJjzFR0d3cC9AQDPom8BsBv6FgDYm23ChmoOh8PlvTGmxtiF5tc2LkkvvviiVqxYoXfffVdBQUHnXeeUKVNUWlrqfB08eLAhuwAAHkffAmA39C0AsDfbXEbRoUMH+fr61jiLoaSkpMbZC9UiIyNrne/n56f27du7jL/88svKyMjQP//5T11zzTV11hIYGKjAwMBG7AUAeAd9C4Dd0LcAwN5sc2ZDQECA4uPjlZWV5TKelZWlvn371rpMUlJSjfkbNmxQQkKCy/0aXnrpJT3//PNav369EhISrC8eAAAAAIAWxDZhgySlp6frj3/8o958800VFBRo4sSJOnDggMaPHy/p59Ptzn6CxPjx47V//36lp6eroKBAb775pjIzM/XUU08557z44ot69tln9eabb6pLly4qLi5WcXGxjh8/7vH9AwAAAACgObDNZRSSNGLECB09elQzZ85UUVGRevbsqXXr1ikmJkaSVFRUpAMHDjjnx8bGat26dZo4caJee+01RUVFae7cuRo+fLhzzoIFC1RRUaG7777bZVvTpk3T9OnTPbJfAAAAAAA0Jw5TfcdENBrPYgXgbjyvHoDd0LcA2BG9xjq2uowCAAAAAAA0fYQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUoQNAAAAAADAUn7eLgAAAAAAgObGGKPTp0+rqqrK26W4ha+vr/z8/ORwOGr9nLABAAAAAAALVVRUqKioSCdPnvR2KW4VHBysTp06KSAgoMZnhA0AAAAAAFjkzJkzKiwslK+vr6KiohQQEHDef/23K2OMKioq9N1336mwsFBXXnmlfHxc79JA2AAAAAAAgEUqKip05swZRUdHKzg42NvluE2rVq3k7++v/fv3q6KiQkFBQS6fc4NIAAAAAAAsdu6/9DdHde1j8997AAAAAADgUYQNAAAAAADYSP/+/TVhwoTzft6lSxfNmTPHY/XUhns2AAAAAABgI++++678/f29XUadCBsAAAAAALCRdu3aebuEC+IyCgAAAAAAbOTsyyhKSkp0++23q1WrVoqNjdVbb73l3eL+jTMbAAAAAACwqTFjxujgwYP68MMPFRAQoLS0NJWUlHi7LMIGAAAAAADs6Msvv9T777+vzZs3KzExUZKUmZmpuLg4L1fGZRQAAAAAANhSQUGB/Pz8lJCQ4Bzr3r27wsPDvVfUvxE2AAAAAABgQ8YYSZLD4fByJTURNgAAAAAAYENxcXE6ffq0tm3b5hzbvXu3fvzxR+8V9W+EDQAAAAAA2FC3bt00aNAgjRs3Trm5ucrLy9PYsWPVqlUrb5dG2AAAAAAAgF0tWbJE0dHR6tevn4YNG6ZHHnlEHTt29HZZPI0CAAAAAAA7yc7Odv53ZGSk/va3v7l8npqa6uGKarqoMxsqKip06NAhHThwwOXlTgsWLFBsbKyCgoIUHx+vTz75pM75GzduVHx8vIKCgtS1a1e9/vrrNeasXr1aPXr0UGBgoHr06KE1a9a4q3wAAAAAAJq9RoUNe/bsUXJyslq1aqWYmBjFxsYqNjZWXbp0UWxsrNU1Oq1atUoTJkzQ1KlTlZ+fr+TkZA0ePPi8AUdhYaFuvfVWJScnKz8/X88884zS0tK0evVq55ycnByNGDFCqamp2rFjh1JTU3XPPfcoNzfXbfsBAAAAAEBz5jDVz8pogBtuuEF+fn6aPHmyOnXqVOMxG9dee61lBZ4tMTFRvXr10sKFC51jcXFxGjp0qGbNmlVj/tNPP621a9eqoKDAOTZ+/Hjt2LFDOTk5kqQRI0aorKxM77//vnPOoEGD1LZtW61YsaJedZWVlSksLEylpaUKDQ294Hxz5oxOnTxWr3UDaKb8g9UqwK/ejylqaJ+xen30LQDyD5YcDrXy961X76JvAWgSvPA7108//aTCwkLnGfnNWV372qh7Nmzfvl15eXnq3r27JQXWR0VFhfLy8jR58mSX8ZSUFG3atKnWZXJycpSSkuIyNnDgQGVmZqqyslL+/v7KycnRxIkTa8yZM2fOeWspLy9XeXm5831ZWVmD9uXUyWMKfrlzg5YB0LzE/fSm8mbeqeAAz9w6h74F4GLF/fSmTilIu2YO9Ejvom8BsIKnf+fCfzTqMooePXroyJEjVtdSpyNHjqiqqkoREREu4xERESouLq51meLi4lrnnz592ln/+eacb52SNGvWLIWFhTlf0dHRjdklAPAY+hYAu6FvAYC91TveOTtNnj17tiZNmqSMjAxdffXV8vf3d5lrxaly53Pu6S/GmDpPialt/rnjDV3nlClTlJ6e7nxfVlbWoB+ArYJDdPIp995IE0DTlucfrFb+vh7bHn0LwMXKO+syCk+gbwGwgqd/58J/1DtsCA8Pd/kD3BijX/3qVy5zqv9Ir6qqsq7Cf+vQoYN8fX1rnHFQUlJS48yEapGRkbXO9/PzU/v27eucc751SlJgYKACAwMbsxuSJIePj4LbhDV6eQBoKPoWALuhbwGAvdU7bPjoo48k/Xz9XEZGhkaOHOnRezYEBAQoPj5eWVlZuuuuu5zjWVlZuvPOO2tdJikpSe+9957L2IYNG5SQkOA8GyMpKUlZWVku923YsGGD+vbt64a9AAAAAACg+at32NCvXz/nf99333265ZZbdOWVV7qlqPNJT09XamqqEhISlJSUpEWLFunAgQMaP368pJ9Pt/vmm2+0fPlyST8/eWL+/PlKT0/XuHHjlJOTo8zMTJenTDzxxBO66aabNHv2bN15553661//qn/+85/69NNPPbpvAAAAAAA0F426QeQDDzygzMxMq2u5oBEjRmjOnDmaOXOmrrvuOn388cdat26dYmJiJElFRUU6cOA/1+bFxsZq3bp1ys7O1nXXXafnn39ec+fO1fDhw51z+vbtq5UrV2rJkiW65pprtHTpUq1atUqJiYke3z8AAAAAAJoDh6m+Y2IDPP7441q+fLmuuOIKJSQkqHXr1i6fv/LKK5YVaAdWP0caAM7l7efVA0BD0bcA2JEVveann35SYWGhYmNjFRQUZHGF7vXxxx/rpZdeUl5enoqKirRmzRoNHTr0vPPr2tdGPWz0888/V69evSRJX375pctndT3FAQAAAAAANE0nTpzQtddeqwcffNDlioDGaFTYUH2zSAAAAAAA0DwMHjxYgwcPtmRdjQobAAAAAABA/RhjdKqyyuPbbeXv67WrDwgbAAAAAABwo1OVVerx3D88vt1dMwcqOMA7f/Y36mkUAAAAAAAA58OZDQAAAAAAuFErf1/tmjnQK9v1FsIGAAAAAADcyOFweO1yBm9pWXsLAAAAAABqdfz4cX311VfO94WFhdq+fbvatWunzp07N2hdhA0AAAAAAEDbtm3TzTff7Hyfnp4uSRo9erSWLl3aoHURNgAAAAAAAPXv31/GGEvWxdMoAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAACApQgbAAAAAABo4WbNmqVf/vKXCgkJUceOHTV06FDt3r270esjbAAAAAAAoIXbuHGjfvvb32rz5s3KysrS6dOnlZKSohMnTjRqfX4W1wcAAAAAAGxm/fr1Lu+XLFmijh07Ki8vTzfddFOD10fYAAAAAACAOxkjVZ70/Hb9gyWHo1GLlpaWSpLatWvXqOUJGwAAAAAAcKfKk1JGlOe3+8xhKaB1gxczxig9PV033nijevbs2ahNEzYAAAAAAACnxx57TJ999pk+/fTTRq/DNmHDDz/8oLS0NK1du1aSdMcdd2jevHkKDw8/7zLGGM2YMUOLFi3SDz/8oMTERL322mv6xS9+IUn6/vvvNW3aNG3YsEEHDx5Uhw4dNHToUD3//PMKCwvzxG4BAAAAAJo7/+CfzzLwxnYb6PHHH9fatWv18ccf67LLLmv0pm0TNowaNUqHDh1y3rTikUceUWpqqt57773zLvPiiy/qlVde0dKlS3XVVVfpd7/7nX79619r9+7dCgkJ0eHDh3X48GG9/PLL6tGjh/bv36/x48fr8OHDeueddzy1awAAAACA5szhaNTlDJ5kjNHjjz+uNWvWKDs7W7GxsRe1PluEDQUFBVq/fr02b96sxMRESdLixYuVlJSk3bt3q1u3bjWWMcZozpw5mjp1qoYNGyZJWrZsmSIiIvT222/r0UcfVc+ePbV69WrnMpdffrleeOEF3X///Tp9+rT8/Gzx9QAAAAAAcFF++9vf6u2339Zf//pXhYSEqLi4WJIUFhamVq1aNXh9PlYX6A45OTkKCwtzBg2S1KdPH4WFhWnTpk21LlNYWKji4mKlpKQ4xwIDA9WvX7/zLiP9fMfN0NBQggYAAAAAQIuxcOFClZaWqn///urUqZPztWrVqkatzxZ/URcXF6tjx441xjt27OhMW2pbRpIiIiJcxiMiIrR///5alzl69Kief/55Pfroo3XWU15ervLycuf7srKyOucDgLfRtwDYDX0LADzLGGPp+rx6ZsP06dPlcDjqfG3btk2S5Kjl2aDGmFrHz3bu5+dbpqysTLfddpt69OihadOm1bnOWbNmKSwszPmKjo6+0K4CgFfRtwDYDX0LAOzNq2HDY489poKCgjpfPXv2VGRkpL799tsay3/33Xc1zlyoFhkZKUk1znwoKSmpscyxY8c0aNAgtWnTRmvWrJG/v3+ddU+ZMkWlpaXO18GDBxuy2wDgcfQtAHZD3wIAe/PqZRQdOnRQhw4dLjgvKSlJpaWl2rJli3r37i1Jys3NVWlpqfr27VvrMrGxsYqMjFRWVpauv/56SVJFRYU2btyo2bNnO+eVlZVp4MCBCgwM1Nq1axUUFHTBegIDAxUYGFifXQSAJoG+BcBu6FsAYG+2uEFkXFycBg0apHHjxmnz5s3avHmzxo0bpyFDhrg8iaJ79+5as2aNpJ8vn5gwYYIyMjK0Zs0aff755xozZoyCg4M1atQoST+f0ZCSkqITJ04oMzNTZWVlKi4uVnFxsaqqqryyrwAAAAAA2J0tbhApSW+99ZbS0tKcT5e44447NH/+fJc5u3fvVmlpqfP9pEmTdOrUKf3mN7/RDz/8oMTERG3YsEEhISGSpLy8POXm5kqSrrjiCpd1FRYWqkuXLm7cIwAAAAAAmifbhA3t2rXT//zP/9Q559y7ZzocDk2fPl3Tp0+vdX7//v0tv+MmAAAAAAAt4W/NuvbRFpdRAAAAAABgB9UPHDh58qSXK3G/6n2s7SELtjmzAQAAAACAps7X11fh4eEqKSmRJAUHB8vhcHi5KmsZY3Ty5EmVlJQoPDxcvr6+NeYQNgAAAAAAYKHIyEhJcgYOzVV4eLhzX89F2AAAAAAAgIUcDoc6deqkjh07qrKy0tvluIW/v3+tZzRUI2wAAAAAAMANfH196/yDvDnjBpEAAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBShA0AAAAAAMBStgkbfvjhB6WmpiosLExhYWFKTU3Vjz/+WOcyxhhNnz5dUVFRatWqlfr3768vvvjivHMHDx4sh8Ohv/zlL9bvAAAAAAAALYRtwoZRo0Zp+/btWr9+vdavX6/t27crNTW1zmVefPFFvfLKK5o/f762bt2qyMhI/frXv9axY8dqzJ0zZ44cDoe7ygcAAAAAoMXw83YB9VFQUKD169dr8+bNSkxMlCQtXrxYSUlJ2r17t7p161ZjGWOM5syZo6lTp2rYsGGSpGXLlikiIkJvv/22Hn30UefcHTt26JVXXtHWrVvVqVMnz+wUAAAAAADNlC3ObMjJyVFYWJgzaJCkPn36KCwsTJs2bap1mcLCQhUXFyslJcU5FhgYqH79+rksc/LkSY0cOVLz589XZGRkveopLy9XWVmZywsAmjL6FgC7oW8BgL3ZImwoLi5Wx44da4x37NhRxcXF511GkiIiIlzGIyIiXJaZOHGi+vbtqzvvvLPe9cyaNct574iwsDBFR0fXe1kA8Ab6FgC7oW8BgL15NWyYPn26HA5Hna9t27ZJUq33UzDGXPA+C+d+fvYya9eu1Ycffqg5c+Y0qO4pU6aotLTU+Tp48GCDlgcAT6NvAbAb+hYA2JtX79nw2GOP6d57761zTpcuXfTZZ5/p22+/rfHZd999V+PMhWrVl0QUFxe73IehpKTEucyHH36ovXv3Kjw83GXZ4cOHKzk5WdnZ2bWuOzAwUIGBgXXWDQBNCX0LgN3QtwDA3rwaNnTo0EEdOnS44LykpCSVlpZqy5Yt6t27tyQpNzdXpaWl6tu3b63LxMbGKjIyUllZWbr++uslSRUVFdq4caNmz54tSZo8ebLGjh3rstzVV1+tP/zhD7r99tsvZtcAAAAAAGixbPE0iri4OA0aNEjjxo3TG2+8IUl65JFHNGTIEJcnUXTv3l2zZs3SXXfdJYfDoQkTJigjI0NXXnmlrrzySmVkZCg4OFijRo2S9PPZD7XdFLJz586KjY31zM4BAAAAANDM2CJskKS33npLaWlpzqdL3HHHHZo/f77LnN27d6u0tNT5ftKkSTp16pR+85vf6IcfflBiYqI2bNigkJAQj9YOAAAAAEBL4jDGGG8XYXdlZWUKCwtTaWmpQkNDvV0OgGbI6j5D3wLgbvQtAHZEr7GOLR59CQAAAAAA7IOwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWIqwAQAAAAAAWMrP2wU0B8YYSVJZWZmXKwHQXFX3l+p+c7HoWwDcjb4FwI6s7l0tGWGDBY4dOyZJio6O9nIlAJq7Y8eOKSwszJL1SPQtAO5H3wJgR1b1rpbMYYhsLtqZM2d0+PBhhYSEyOFw1GuZsrIyRUdH6+DBgwoNDXVzhdaidu+xc/3UfnGMMTp27JiioqLk43PxV8DRt+zFzvVTu/d4u3761sWzc/3U7j12rr8p1G5172rJOLPBAj4+PrrssssatWxoaKjtmkA1avceO9dP7Y1nZbpO37InO9dP7d7jzfrpW9awc/3U7j12rt/btXNGgzWIagAAAAAAgKUIGwAAAAAAgKUIG7wkMDBQ06ZNU2BgoLdLaTBq9x4710/t9mfn78HOtUv2rp/avcfu9VvB7t+Bneundu+xc/12rh01cYNIAAAAAABgKc5sAAAAAAAAliJsAAAAAAAAliJsAAAAAAAAliJsaKQFCxYoNjZWQUFBio+P1yeffFLn/I0bNyo+Pl5BQUHq2rWrXn/99RpzVq9erR49eigwMFA9evTQmjVrLnq7nqh98eLFSk5OVtu2bdW2bVsNGDBAW7ZscZkzffp0ORwOl1dkZKTXa1+6dGmNuhwOh3766aeL2q6n6u/fv3+t9d92223OOd747ouKijRq1Ch169ZNPj4+mjBhQq3zmuIxX5/aPXnMW4m+9R+e/n9o595F33LlqWPeHfXbsXfRt/6DvuW92j3Ztxpaf1PrXfQtuDBosJUrVxp/f3+zePFis2vXLvPEE0+Y1q1bm/3799c6/+uvvzbBwcHmiSeeMLt27TKLFy82/v7+5p133nHO2bRpk/H19TUZGRmmoKDAZGRkGD8/P7N58+ZGb9dTtY8aNcq89tprJj8/3xQUFJgHH3zQhIWFmUOHDjnnTJs2zfziF78wRUVFzldJSUm963ZX7UuWLDGhoaEudRUVFV3Udj1Z/9GjR13q/vzzz42vr69ZsmSJc443vvvCwkKTlpZmli1bZq677jrzxBNP1JjTVI/5+tTuqWPeSvQt7/Qtd9Xvqd5F33LlqWPeXfXbrXfRt+hbTeW791Tfakz9Tal30bdwLsKGRujdu7cZP368y1j37t3N5MmTa50/adIk0717d5exRx991PTp08f5/p577jGDBg1ymTNw4EBz7733Nnq7nqr9XKdPnzYhISFm2bJlzrFp06aZa6+9tt511sYdtS9ZssSEhYVZul2r1tOY7/4Pf/iDCQkJMcePH3eOeeO7P1u/fv1q/eHRVI/5+tR+Lncd81aib3mnbxlj795F33LlqWP+YtfTXHoXfYu+daHteqr2c7mrbxlj795F38K5uIyigSoqKpSXl6eUlBSX8ZSUFG3atKnWZXJycmrMHzhwoLZt26bKyso651SvszHb9VTt5zp58qQqKyvVrl07l/E9e/YoKipKsbGxuvfee/X111/Xq2531378+HHFxMTosssu05AhQ5Sfn39R2/V0/WfLzMzUvffeq9atW7uMe/q7r4+mesw3hjuOeSvRt7zTt9xdv7t7F32rJk8c81au50Kacu+ib9G36rNdT9Z+Nnf0rcbWXx9N9bhvjKbct1ATYUMDHTlyRFVVVYqIiHAZj4iIUHFxca3LFBcX1zr/9OnTOnLkSJ1zqtfZmO16qvZzTZ48WZdeeqkGDBjgHEtMTNTy5cv1j3/8Q4sXL1ZxcbH69u2ro0ePerX27t27a+nSpVq7dq1WrFihoKAg3XDDDdqzZ0+jt+vJ+s+2ZcsWff755xo7dqzLuDe++/poqsd8Y7jjmLcSfcs7fcud9Xuid9G3avLEMW/lei6kKfcu+hZ9qz7b9VTtZ3NX32ps/fXRVI/7xmjKfQs1+Xm7ALtyOBwu740xNcYuNP/c8fqss6HbrW8tF1t7tRdffFErVqxQdna2goKCnOODBw92/vfVV1+tpKQkXX755Vq2bJnS09O9VnufPn3Up08f5+c33HCDevXqpXnz5mnu3LmN3q6n6j9bZmamevbsqd69e7uMe+u7t2qd3jjmG8Ldx7yV6Fve6VvuqN+TvYu+1fB1eutnRkPYpXfRt+hbTe27d3ffakz9Vq2T37lgNc5saKAOHTrI19e3RkJXUlJSI8mrFhkZWet8Pz8/tW/fvs451etszHY9VXu1l19+WRkZGdqwYYOuueaaOmtp3bq1rr76amea7e3aq/n4+OiXv/ylsy4rvndP1H/y5EmtXLmyRspeG0989/XRVI/5hnDnMW8l+pZ3+pYn6q/mjt5F36rJE8e8les5Hzv0LvoWfas+2/V07e7sW42tvz6a6nHfEHboW6iJsKGBAgICFB8fr6ysLJfxrKws9e3bt9ZlkpKSaszfsGGDEhIS5O/vX+ec6nU2Zrueql2SXnrpJT3//PNav369EhISLlhLeXm5CgoK1KlTJ6/XfjZjjLZv3+6sy4rv3RP1//nPf1Z5ebnuv//+C9biie++PprqMV9f7j7mrUTf8k7fcnf9Z3NH76Jv1eSJY97K9dTGLr2LvkXfqs92PV27O/tWY+uvj6Z63NeXXfoWamH9PSebv+rHumRmZppdu3aZCRMmmNatW5t9+/YZY4yZPHmySU1Ndc6vfqTOxIkTza5du0xmZmaNR+r861//Mr6+vub3v/+9KSgoML///e/P+0ia823XW7XPnj3bBAQEmHfeecflkTPHjh1zznnyySdNdna2+frrr83mzZvNkCFDTEhIiNdrnz59ulm/fr3Zu3evyc/PNw8++KDx8/Mzubm5ln7v7qq/2o033mhGjBhR63a98d0bY0x+fr7Jz8838fHxZtSoUSY/P9988cUXzs+b6jFfn9o9dcxbib7lnb7lrvo91bvoW97pW+6q3269i75F32oq3301d/etxtRvTNPpXfQtnIuwoZFee+01ExMTYwICAkyvXr3Mxo0bnZ+NHj3a9OvXz2V+dna2uf76601AQIDp0qWLWbhwYY11/u///q/p1q2b8ff3N927dzerV69u0Ha9VXtMTIyRVOM1bdo055wRI0aYTp06GX9/fxMVFWWGDRvm0ki8VfuECRNM586dTUBAgLnkkktMSkqK2bRpU4O26836jTFm9+7dRpLZsGFDrdv01ndf2zERExPjMqepHvMXqt2Tx7yV6Fv/4en/h3buXfStGJc5njrm3VG/HXsXfes/6Fveq90Yz/WtxtTflHoXfQtncxjz7zugAAAAAAAAWIB7NgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAAAAAAAEsRNgAWy87OlsPh0I8//ujtUgCgXuhbAOyGvgU0fQ5jjPF2EYCd9e/fX9ddd53mzJkjSaqoqND333+viIgIORwO7xYHALWgbwGwG/oWYD9+3i4AaG4CAgIUGRnp7TIAoN7oWwDshr4FNH1cRgFchDFjxmjjxo169dVX5XA45HA4tHTpUpfT+pYuXarw8HD97W9/U7du3RQcHKy7775bJ06c0LJly9SlSxe1bdtWjz/+uKqqqpzrrqio0KRJk3TppZeqdevWSkxMVHZ2tnd2FECzQd8CYDf0LcCeOLMBuAivvvqqvvzyS/Xs2VMzZ86UJH3xxRc15p08eVJz587VypUrdezYMQ0bNkzDhg1TeHi41q1bp6+//lrDhw/XjTfeqBEjRkiSHnzwQe3bt08rV65UVFSU1qxZo0GDBmnnzp268sorPbqfAJoP+hYAu6FvAfZE2ABchLCwMAUEBCg4ONh5Kt//+3//r8a8yspKLVy4UJdffrkk6e6779af/vQnffvtt2rTpo169Oihm2++WR999JFGjBihvXv3asWKFTp06JCioqIkSU899ZTWr1+vJUuWKCMjw3M7CaBZoW8BsBv6FmBPhA2ABwQHBzt/8ElSRESEunTpojZt2riMlZSUSJL+7//+T8YYXXXVVS7rKS8vV/v27T1TNIAWjb4FwG7oW0DTQtgAeIC/v7/Le4fDUevYmTNnJElnzpyRr6+v8vLy5Ovr6zLv7B+YAOAu9C0AdkPfApoWwgbgIgUEBLjcaMgK119/vaqqqlRSUqLk5GRL1w0A9C0AdkPfAuyHp1EAF6lLly7Kzc3Vvn37dOTIEWdafjGuuuoq3XfffXrggQf07rvvqrCwUFu3btXs2bO1bt06C6oG0JLRtwDYDX0LsB/CBuAiPfXUU/L19VWPHj10ySWX6MCBA5asd8mSJXrggQf05JNPqlu3brrjjjuUm5ur6OhoS9YPoOWibwGwG/oWYD8OY4zxdhEAAAAAAKD54MwGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgKcIGAAAAAABgqf8PkSwGwQz1DMEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff['rh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBwAAAEiCAYAAABa08F/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4oUlEQVR4nO3de3hU1bnH8d/kSgJhIEACKSHBHiFQUBEEglCwcgIqoqIVxEbAAvVYRMFLRVsuWol6rAevaBEBlYpWmxZvUY41tBYSLhKrkhMQuQoxgJJw0SQk6/xhM2XIhZnJntvO9/M888jsWXuvdybjC+vNWms7jDFGAAAAAAAAFooIdgAAAAAAAMB+KDgAAAAAAADLUXAAAAAAAACWo+AAAAAAAAAsR8EBAAAAAABYjoIDAAAAAACwHAUHAAAAAABgOQoOAAAAAADAchQcAAAAAACA5Sg4AAAAAAAAy1FwAAAAAAAAlqPgAAAAAAAALEfBAbb02muvqW/fvoqLi1OHDh00cuRIHT9+XJI0efJkXXnllVqwYIGSkpLUtm1b/eIXv1BVVZXr/Ly8PA0dOlTt2rVThw4dNGbMGO3YscOtj3379mnChAlKTExU69atNWDAABUWFrpef+ONN9S/f3+1atVKZ511lhYsWKCTJ0/65f0aYzRy5EiNHj1axhhJ0pEjR9StWzfde++9fukTgP+0tBy2a9cuRUREaNOmTW7Hn3jiCaWlpbnyGoDw0dLymCSlp6fL4XDUewAtGQUH2M6BAwd03XXX6cYbb1RxcbHy8/M1btw4t3+wvv/++youLtYHH3ygl19+Wbm5uVqwYIHr9ePHj2v27NnauHGj3n//fUVEROiqq65SbW2tJOnYsWMaPny49u/fr9WrV+vjjz/WXXfd5Xr93Xff1c9+9jPNnDlTW7du1bPPPqvly5frgQceaDTulStXqk2bNk0+Vq5c2eC5DodDK1as0IYNG/T4449Lkm666SYlJydr/vz5zf1IAQRQS8xh6enpGjlypJYtW+Z2fNmyZZo8eTL/YAfCTEvMY5K0ceNGHThwQAcOHNC+ffs0ePBgDRs2rLkfJxDeDDy2du1aM2bMGNOlSxcjyeTm5oZEf1u3bjWXX365adu2rWnTpo0ZNGiQ2b17t19jC2WbN282ksyuXbsafH3SpEkmMTHRHD9+3HVs8eLFpk2bNqampqbBc8rKyowk88knnxhjjHn22WdNQkKCOXz4cIPthw0bZhYuXOh27MUXXzRdunRpNO6Kigqzffv2Jh8VFRVNvvdXX33VxMbGmjlz5pj4+HhTUlLSZHsAoael5rBXXnnFtG/f3nz33XfGGGOKioqMw+EwO3fubPQcAKGppeaxU82cOdOkpaWZsrIyj9oDdhUVvFJH+Dl+/LjOPfdcTZkyRVdffXVI9Ldjxw4NHTpUP//5z7VgwQI5nU4VFxerVatWfo8vVJ177rm6+OKL1bdvX40aNUpZWVm65ppr1L59e7c28fHxrueZmZk6duyY9u7dq7S0NO3YsUO/+c1vVFBQoEOHDrmq5Xv27FGfPn1UVFSkfv36KTExscEYNm/erI0bN7pV0WtqavTdd9/pxIkTbn3XSUhIUEJCQrPe+09/+lPl5uYqJydHixcvVo8ePZp1PQCB11Jz2JVXXqkZM2YoNzdXEyZM0PPPP6+LLrpI6enpPl8TQHC01DxW5/e//72WLl2qf/zjH+rUqVOzrweEM5ZUeOGSSy7Rb3/7W40bN67B16uqqnTXXXfpBz/4gVq3bq1BgwYpPz/fb/1J0r333qtLL71UDz/8sPr166ezzjpLl112mZKSknzuN9xFRkZqzZo1euedd9S7d2898cQT6tmzp3bu3HnGc+um7V5++eU6fPiwlixZosLCQtd6wLq1hXFxcU1ep7a2VgsWLFBRUZHr8cknn2j79u2NFoOaO41Pkk6cOKHNmzcrMjJS27dvP+P7BRB6WmoOi4mJUXZ2tpYtW6aqqir94Q9/0I033njG9wwg9LTUPCZJ+fn5uuWWW/TCCy/o3HPPPeP7BeyOGQ4WmjJlinbt2qVVq1YpJSVFubm5Gj16tD755BOdffbZlvdXW1urt956S3fddZdGjRqlLVu2qHv37pozZ46uvPJKy/sLJw6HQxdeeKEuvPBCzZ07V2lpacrNzdXs2bMlSR9//LG+/fZb119WBQUFatOmjbp27arDhw+ruLhYzz77rGvd3Ycffuh2/XPOOUfPPfecvv766wYr6+eff75KSkr0H//xHx7HPHbsWA0aNKjJNsnJyU2+fvvttysiIkLvvPOOLr30Ul122WX6yU9+4nEMAEJDS81hU6dOVZ8+ffT000+rurq6yYI7gNDWEvPY559/rquvvlr33HMP+QuoE+w1HeFKp+2p8PnnnxuHw2G+/PJLt3YXX3yxmTNnjuX9GWPMgQMHjCQTHx9vHn30UbNlyxaTk5NjHA6Hyc/Pb3af4aqgoMA88MADZuPGjWb37t3m1VdfNTExMebtt982xny/brBNmzbmuuuuM5999pl5++23TXJysrn77ruNMcbU1NSYDh06mJ/97Gdm+/bt5v333zcXXHCB28+gsrLS9OjRwwwbNsx8+OGHZseOHea1114z69atM8YYk5eXZ6Kiosy8efPMp59+arZu3WpWrVpl7r33Xr+97zfffNPExMSYzZs3G2OM+fWvf226du1qvv76a7/1CcB6LTWH1RkyZIiJiYkxN910k9/7AuAfLTGPnThxwmRkZJif/OQnZv/+/ebAgQOuB9CSUXDw0ekFgFdffdVIMq1bt3Z7REVFmWuvvdYYY8zOnTuNpCYfv/zlLz3qzxhjvvzySyPJXHfddW7HL7/8cjNhwgRL32842bp1qxk1apTp1KmTiY2NNT169DBPPPGE6/VJkyaZK664wsydO9d06NDBtGnTxkydOtW1UZkxxqxZs8b06tXLxMbGmnPOOcfk5+fX+xns2rXLXH311aZt27YmPj7eDBgwwBQWFrpez8vLM0OGDDFxcXGmbdu2ZuDAgeb3v/+9X95zWVmZSU5Odtscqbq62gwcOND1/QMQHlpiDjvV0qVLjSSzYcMGv/cFwD9aYh5r6t/5QEvmMIabW/vC4XAoNzfXtXThlVde0fXXX6/PPvtMkZGRbm3btGmjzp07q7q6ut79g0/Xvn37Bqdqnd6f9P0attatW2vevHn69a9/7Tr+q1/9Sh9++KH+8Y9/+P4GbWzy5Mk6cuSI/vznPwc7FADwmt1z2AMPPKBVq1bpk08+CXYoAPzE7nkMwL+xh4NF+vXrp5qaGpWVlTV6v93o6GhlZGRY1mdMTIwuuOAClZSUuB3ftm2b0tLSLOsHAAB/O3bsmIqLi/XEE0/o/vvvD3Y4AADAAhQcvHDs2DF9/vnnruc7d+5UUVGREhMT1aNHD11//fW64YYb9Lvf/U79+vXToUOH9Ne//lV9+/bVpZdeaml/3bp1kyTdeeedGj9+vH784x/roosuUl5ent54441m3R0DAIBAmzFjhl5++WVdeeWV3J0CAACbYEmFF/Lz83XRRRfVOz5p0iQtX75c1dXV+u1vf6sXXnhBX375pTp06KDMzEwtWLBAffv2tby/Os8//7xycnK0b98+9ezZUwsWLNAVV1zhdX8AAAAAAFiFggMAAAAAALBcRLADAAAAAAAA9kPBAQAAAAAAWI5NI8+gtrZW+/fvV0JCghwOR7DDAWBzxhgdPXpUKSkpiohofk2YHAYg0MhjAMKZ1TmspaPgcAb79+9XampqsMMA0MLs3btXXbt2bfZ1yGEAgoU8BiCcWZXDWjoKDmeQkJAg6fsvXNu2bYMcDQC7q6ioUGpqqiv3NBc5DECgkccAhDOrc1hLR8HhDOqm7rVt25a/5AAEjFXThslhAIKFPAYgnLGEyxosSgEAAAAAAJaj4AAAAAAAACwXdgWHp59+Wt27d1erVq3Uv39//f3vf2+y/dq1a9W/f3+1atVKZ511lp555pkARQoAAAAAQMsVVgWHV155RbfddpvuvfdebdmyRcOGDdMll1yiPXv2NNh+586duvTSSzVs2DBt2bJF99xzj2bOnKnXX389wJEDAAAAANCyhFXB4dFHH9XPf/5zTZ06Vb169dKiRYuUmpqqxYsXN9j+mWeeUbdu3bRo0SL16tVLU6dO1Y033qhHHnkkwJEDAAAAANCyhM1dKqqqqrR582bdfffdbsezsrK0bt26Bs9Zv369srKy3I6NGjVKS5cuVXV1taKjoy2P09ScVOW29y2/rsf9t+8uk3iW5ydUn1DEvkKpttZ/QQEtWG3K+VJce8VFR4bFbsemtlbfnjga7DAAhJLoeMXFRIVFDpPIYwBOE2Y5zG7CpuBw6NAh1dTUKDk52e14cnKySktLGzyntLS0wfYnT57UoUOH1KVLl3rnVFZWqrKy0vW8oqLCqzi//e6E4l+51qtzrHTSRCiz8kkdVDuP2j8VvUiXRW7wb1BACzah6tcqqO2trfeNUnyM/1Nus3PYiaOKf6Sb1WEBCGO9vntem++7IiA5TCKPAbBWoHMY3IXdp356ZcoY02S1qqH2DR2vk5OTowULFjQnQn1Sm96M832X4diraEeNkhzf6KBp59E5XR2HJEm7apN1VHF+jA5omY6bVgHtr/k5DACCizwGAPbhMHUj8BBXVVWl+Ph4/fGPf9RVV13lOn7rrbeqqKhIa9eurXfOj3/8Y/Xr10+PPfaY61hubq6uvfZanThxosElFQ1V1VNTU1VeXq62bdueMU5jjL6trvH27Vmi1ZPnKKLiS3035X9V26WfZ+c8/xNFlH6s78a/qtofXuznCIGWy9MlFRUVFXI6nR7nnNM1O4cxFRnA6bycjkweAxBSApzD4C5sZjjExMSof//+WrNmjVvBYc2aNbriiisaPCczM1NvvPGG27H33ntPAwYMaHT/htjYWMXGxvocp8PhCN50Hcf3e4C2ioyQPI7h+70bWkVHenEOgFDV7BwWEaH4Nk4LIwIA75DHAMA+wuouFbNnz9Zzzz2n559/XsXFxZo1a5b27Nmjm266SZI0Z84c3XDDDa72N910k3bv3q3Zs2eruLhYzz//vJYuXao77rgjWG/Bv1xVOy8mrdQ1dYTVVwEAAAAAEOLC6lfa48eP1+HDh3XffffpwIED6tOnj95++22lpaVJkg4cOKA9e/a42nfv3l1vv/22Zs2apaeeekopKSl6/PHHdfXVVwfrLfhXXdHAeHHHibq2FBwAAAAAABYKq4KDJN188826+eabG3xt+fLl9Y4NHz5cH330kZ+jChEUHAAAAAAAIYJRpp1QcAAAAAAAhAhGmbbyrz0cvLrxiGsTB6uDAQAAAAC0YBQc7IQZDgAAAACAEMEo004oOAAAAAAAQgSjTDuh4AAAAAAACBGMMu3EUbeHgy8FB/ZwAAAAAABYh4KDnbiKBl5sGunaM5KCAwAAAADAOhQc7MS1pMKbggNLKgAAAAAA1mOUaSfs4QAAAAAACBGMMu2kOQUHsaQCAAAAAGAdCg62UrdppBdLKuo2cWCGAwAAAADAQowy7YQlFQAAAACAEMEo004oOAAAAAAAQgSjTDuh4AAAAAAACBGMMu3EUbeHgy8FBzaNBAAAAABYh4KDnbhmKXixaWRdU2Y4AAAAAAAsxCjTThw+3KWCGQ4AAAAAAD+g4GAn7OEAAAAAAAgRjDLtpDkFBzHDAQAAAABgHQoOtuLDkoq6TRyY4QAAAAAAsBCjTDthSQUAAAAAIEQwyrQTCg4AAAAAgBDBKNNOKDgAAAAAAEJE2Iwyv/nmG2VnZ8vpdMrpdCo7O1tHjhxptH11dbV+9atfqW/fvmrdurVSUlJ0ww03aP/+/YELOtCaVXBg00gAAAAAgHXCpuAwceJEFRUVKS8vT3l5eSoqKlJ2dnaj7U+cOKGPPvpIv/nNb/TRRx/pT3/6k7Zt26axY8cGMOoAcxUNPNw08tTNJZnhAAAAAACwUFSwA/BEcXGx8vLyVFBQoEGDBkmSlixZoszMTJWUlKhnz571znE6nVqzZo3bsSeeeEIDBw7Unj171K1bt4DEHlB1BQdPZzhQcAAAAAAA+ElYjDLXr18vp9PpKjZI0uDBg+V0OrVu3TqPr1NeXi6Hw6F27dr5IcoQ4FpS4ekMh1MKEyypAAAAAABYKCxmOJSWliopKane8aSkJJWWlnp0je+++0533323Jk6cqLZt2zbarrKyUpWVla7nFRUV3gccLN7u4eDWjoIDYAdhncMAQOQxALCToM5wmD9/vhwOR5OPTZs2SZIcDfwG3hjT4PHTVVdXa8KECaqtrdXTTz/dZNucnBzXxpROp1Opqam+vbmg8HZJxakzHMJisguAMwjvHAYA5DEAsJOgjjJnzJih4uLiJh99+vRR586d9dVXX9U7/+DBg0pOTm6yj+rqal177bXauXOn1qxZ0+TsBkmaM2eOysvLXY+9e/c26z0GlLdLKsQeDoDdhHUOAwCRxwDAToK6pKJjx47q2LHjGdtlZmaqvLxcGzZs0MCBAyVJhYWFKi8v15AhQxo9r67YsH37dn3wwQfq0KHDGfuKjY1VbGys528ilDRnSQUFB8AWwjqHAYDIYwBgJ2ExyuzVq5dGjx6tadOmqaCgQAUFBZo2bZrGjBnjdoeKjIwM5ebmSpJOnjypa665Rps2bdLKlStVU1Oj0tJSlZaWqqqqKlhvxb8oOAAAAAAAQkTYjDJXrlypvn37KisrS1lZWTrnnHP04osvurUpKSlReXm5JGnfvn1avXq19u3bp/POO09dunRxPby5s0VYaVbBgU0jAQAAAADWCYu7VEhSYmKiXnrppSbbmFP2LkhPT3d73iK4igae3haTPRwAAAAAAP7BKNNOHNylAgAAAAAQGhhl2onXSyqY4QAAAAAA8A9GmXbi7W0x2cMBAAAAAOAnFBxsxdclFRQbAAAAAADWouBgJ97OcKjbXJLlFAAAAAAAizHStBNfb4tJwQEAAAAAYDFGmnZCwQEAAAAAECIYadqJzwUH9nAAAAAAAFiLgoOduAoHnt6lgj0cAAAAAAD+wUjTTlhSAQAAAAAIEYw07cTh420xKTgAAAAAACzGSNNOvL0tpqsdezgAAAAAAKxFwcFWfJ3hQMEBAAAAAGAtCg524u0MB7FpJAAAAADAPxhp2gmbRgIAAAAAQgQjTTuh4AAAAAAACBGMNO3E54IDezgAAAAAAKxFwcFOXIUDL+9SwQwHAAAAAIDFGGnaCUsqAAAAAAAhgpGmnTh8vS0mXwMAAAAAgLUYadqJ1zMc6pZesIcDAAAAAMBaFBzsxFVw8HQPBzaNBAAAAAD4BwUHW6lbUuFhwUFsGgkAAAAA8I+wGWl+8803ys7OltPplNPpVHZ2to4cOeLx+b/4xS/kcDi0aNEiv8UYdGwaCQAAAAAIEWEz0pw4caKKioqUl5envLw8FRUVKTs726Nz//znP6uwsFApKSl+jjLIKDgAAAAAAEJEVLAD8ERxcbHy8vJUUFCgQYMGSZKWLFmizMxMlZSUqGfPno2e++WXX2rGjBl69913ddlllwUq5ODwueDAHg4AAAAAAGuFxa+2169fL6fT6So2SNLgwYPldDq1bt26Rs+rra1Vdna27rzzTv3oRz8KRKjBxW0xAQAAAAAhIixmOJSWliopKane8aSkJJWWljZ63kMPPaSoqCjNnDnT474qKytVWVnpel5RUeFdsMHkKhx4epcKNo0E7CascxgAiDwGAHYS1JHm/Pnz5XA4mnxs2rRJkuRoYNq/MabB45K0efNmPfbYY1q+fHmjbRqSk5Pj2pjS6XQqNTXVtzcXDOzhALR4YZ3DAEDkMQCwk6CONGfMmKHi4uImH3369FHnzp311Vdf1Tv/4MGDSk5ObvDaf//731VWVqZu3bopKipKUVFR2r17t26//Xalp6c3GtOcOXNUXl7ueuzdu9eqt+t/vi6pEHs4AHYR1jkMAEQeAwA7CeqSio4dO6pjx45nbJeZmany8nJt2LBBAwcOlCQVFhaqvLxcQ4YMafCc7OxsjRw50u3YqFGjlJ2drSlTpjTaV2xsrGJjY714FyHENcPB2yUVFBwAuwjrHAYAIo8BgJ2ExR4OvXr10ujRozVt2jQ9++yzkqTp06drzJgxbneoyMjIUE5Ojq666ip16NBBHTp0cLtOdHS0Onfu3ORdLcJb3QwHDwsOYg8HAAAAAIB/hEXBQZJWrlypmTNnKisrS5I0duxYPfnkk25tSkpKVF5eHozwQgN7OAAAAABAyKipqVF1dXWww/Cb6OhoRUZGNvp62BQcEhMT9dJLLzXZxpzhN/u7du2yMKIQRMEBAAAAAELCsWPHtG/fvjOOU8OZw+FQ165d1aZNmwZfD5uCAzzgc8GBPRwAAAAAwCo1NTXat2+f4uPj1alTJ6/unBgujDE6ePCg9u3bp7PPPrvBmQ4UHOzE17tUMMMBAAAAACxTXV0tY4w6deqkuLi4YIfjN506ddKuXbtUXV3dYMGBkaaduAoH3t6lgq8BAAAAAFjNjjMbTnWm98dI007YwwEAAAAAECIYadqJrwUH2bvqBgAAAAB2M2LECN12222Nvp6enq5FixYFLJ6GsIeDnbj2cPB0SQWbRgIAAABAOPrTn/6k6OjoYIfRJAoOtuJlwcF1GhNdAAAAACCcJCYmBjuEM2KkaSfs4QAAAAAALcKpSyrKysp0+eWXKy4uTt27d9fKlSuDG9y/MMPBTnwuOLCkAgAAAADC1eTJk7V371799a9/VUxMjGbOnKmysrJgh0XBwVaY4QAAAAAALcq2bdv0zjvvqKCgQIMGDZIkLV26VL169QpyZCypsBfXppEUHAAAAACgJSguLlZUVJQGDBjgOpaRkaF27doFL6h/YaRpJ67Cgad3qTCnnQcAAAAACCfmX+M6RwgulWekaScsqQAAAACAFqVXr146efKkNm3a5DpWUlKiI0eOBC+of2GkaSe+FhwUepUwAAAAAMCZ9ezZU6NHj9a0adNUWFiozZs3a+rUqYqLiwt2aBQcbMXnPRwoOAAAAABAuFq2bJlSU1M1fPhwjRs3TtOnT1dSUlKww/L+LhXHjx/Xgw8+qPfff19lZWWqrXUf3H7xxReWBQdv1RUcPG3PHg4AAAAAEI7y8/Ndf+7cubPefPNNt9ezs7MDHFF9Xhccpk6dqrVr1yo7O1tdunQJyY0pWiyvl1RQcAAAAAAA+IfXBYd33nlHb731li688EJ/xIPm8HnTSIpGAAAAAABref2r7fbt2ysxMdEfsaC5uEsFAAAAACBEeD3SvP/++zV37lydOHHCH/GgOSg4AAAAAABChEdLKvr16+e2V8Pnn3+u5ORkpaenKzo62q3tRx99ZG2E8JzrZ+ThrpHs4QAAAAAA8BOPCg5XXnmln8OAJZjhAAAAAAAIER4VHObNm+f685QpU3T99dfr4osv5g4VocbXgoP4OQIAAAAArOX1r7YPHz6sMWPGqGvXrrrjjjtUVFTkh7Dq++abb5SdnS2n0ymn06ns7GwdOXLkjOcVFxdr7NixcjqdSkhI0ODBg7Vnzx7/BxwMzHAAAAAAAIQIr0eaq1evVmlpqebNm6dNmzapf//+6t27txYuXKhdu3b5IcTvTZw4UUVFRcrLy1NeXp6KioqUnZ3d5Dk7duzQ0KFDlZGRofz8fH388cf6zW9+o1atWvktzuD610wF4+keDtwWEwAAAADgHx4tqThdu3btNH36dE2fPl379u3Tyy+/rOeff15z587VyZMnrY5RxcXFysvLU0FBgQYNGiRJWrJkiTIzM1VSUqKePXs2eN69996rSy+9VA8//LDr2FlnnWV5fCHD4WXBQWwaCQAAAADwj2aNNKurq7Vp0yYVFhZq165dSk5OtiouN+vXr5fT6XQVGyRp8ODBcjqdWrduXYPn1NbW6q233lKPHj00atQoJSUladCgQfrzn//slxhDgtdLKuoKDsxwAAAAAABIf/vb33T55ZcrJSVFDoejWWNonwoOH3zwgaZNm6bk5GRNmjRJCQkJeuONN7R3716fA2lKaWmpkpKS6h1PSkpSaWlpg+eUlZXp2LFjevDBBzV69Gi99957uuqqqzRu3DitXbu20b4qKytVUVHh9ggb7OEAtHhhncMAQOQxAAi248eP69xzz9WTTz7Z7Gt5vaSia9euOnz4sEaNGqVnn31Wl19+uc97IsyfP18LFixoss3GjRslqcE7YhhjGr1TRm3t94PpK664QrNmzZIknXfeeVq3bp2eeeYZDR8+vMHzcnJyzhhTyKLgALR4YZ3DAEDkMQAItksuuUSXXHKJJdfyuuAwd+5c/fSnP1X79u2b3fmMGTM0YcKEJtukp6frn//8p7766qt6rx08eLDRZRwdO3ZUVFSUevfu7Xa8V69e+vDDDxvtb86cOZo9e7breUVFhVJTU5uMMWS4ii+ebhrJHg6A3YR1DgMAkccA2JMxRt9W1wSl77joyEZ/Ue9vXhccpk+fblnnHTt2VMeOHc/YLjMzU+Xl5dqwYYMGDhwoSSosLFR5ebmGDBnS4DkxMTG64IILVFJS4nZ827ZtSktLa7Sv2NhYxcbGevEuQggzHIAWL6xzGACIPAbAnr6trlHvue8Gpe+t941SfIxP94totrAYafbq1UujR4/WtGnTVFBQoIKCAk2bNk1jxoxxu0NFRkaGcnNzXc/vvPNOvfLKK1qyZIk+//xzPfnkk3rjjTd08803B+Nt+J+vBQexaSQAAAAAwFrBKXP4YOXKlZo5c6aysrIkSWPHjq23iUVJSYnKy8tdz6+66io988wzysnJ0cyZM9WzZ0+9/vrrGjp0aEBjDxhmOAAAAABAyImLjtTW+0YFre9gCZuCQ2Jiol566aUm2xhTf++CG2+8UTfeeKO/wgox/5qp0MDn0CBXwYEZDgAAAADgLw6HI2jLGoKp5b1jO3N4WXAQm0YCAAAAAP7t2LFj+vzzz13Pd+7cqaKiIiUmJqpbt25eXYuCg514vaSiruDADAcAAAAAgLRp0yZddNFFrud1dw6aNGmSli9f7tW1KDjYCXs4AAAAAACaYcSIEQ1uV+ALRpp2QsEBAAAAABAiGGnaiWtphKebRrKHAwAAAADAPxhp2gkzHAAAAAAAIYKRpp34WnAQm0YCAAAAAKxFwcFOmOEAAAAAAAgRjDRt5ZSZCp7sKkrBAQAAAADgJ4w07eTUwoFHtzGp2zSSJRUAAAAAAGtRcLCTUwsHniyrcM1woOAAAAAAALAWBQc7cZvh4EnBgdtiAgAAAAD8g5GmnXhdcGAPBwAAAACAfzDStBO3pRGebBrJDAcAAAAAwPdycnJ0wQUXKCEhQUlJSbryyitVUlLi8/UYadoJMxwAAAAAAD5au3atfvnLX6qgoEBr1qzRyZMnlZWVpePHj/t0vSiL40Mw+VpwEJtGAgAAAEBLl5eX5/Z82bJlSkpK0ubNm/XjH//Y6+tRcLATZjgAAAAAQOgxRqo+EZy+o+N9vjNheXm5JCkxMdGn8yk42AkFBwAAAAAIPdUnpIUpwen7nv1STGuvTzPGaPbs2Ro6dKj69OnjU9cUHGzllKqV8WDTyLqNJX2sdgEAAAAA7GnGjBn65z//qQ8//NDna1BwsBO3GQ6e3KWiboYDBQcAAAAA8Jvo+O9nGgSrby/dcsstWr16tf72t7+pa9euPndNwcFOTi0ceLSkgttiAgAAAIDfORw+LWsINGOMbrnlFuXm5io/P1/du3dv1vUoONiJw6Hvl1UY9nAAAAAAAHjll7/8pf7whz/oL3/5ixISElRaWipJcjqdiouL8/p6jDTtpm6WAwUHAAAAAIAXFi9erPLyco0YMUJdunRxPV555RWfrhc2I81vvvlG2dnZcjqdcjqdys7O1pEjR5o859ixY5oxY4a6du2quLg49erVS4sXLw5MwMHiKh54socDSyoAAAAAAN8zxjT4mDx5sk/XC5uR5sSJE1VUVKS8vDzl5eWpqKhI2dnZTZ4za9Ys5eXl6aWXXlJxcbFmzZqlW265RX/5y18CFHUQ1BUPvJnhIDaNBAAAAABYKywKDsXFxcrLy9Nzzz2nzMxMZWZmasmSJXrzzTdVUlLS6Hnr16/XpEmTNGLECKWnp2v69Ok699xztWnTpgBGH2C+FByY4QAAAAAAsFhYjDTXr18vp9OpQYMGuY4NHjxYTqdT69ata/S8oUOHavXq1fryyy9ljNEHH3ygbdu2adSoUYEIOzgoOAAAAAAAQkBY3KWitLRUSUlJ9Y4nJSW5ds1syOOPP65p06apa9euioqKUkREhJ577jkNHTq00XMqKytVWVnpel5RUdG84AOubtNID/ZwEHs4AHYT/jkMQEtHHgMA+wjqSHP+/PlyOBxNPuqWPzgc9fcZMMY0eLzO448/roKCAq1evVqbN2/W7373O91888363//930bPycnJcW1M6XQ6lZqa2vw3Gkg+zXBgDwfALsI+hwFo8chjAGAfQZ3hMGPGDE2YMKHJNunp6frnP/+pr776qt5rBw8eVHJycoPnffvtt7rnnnuUm5uryy67TJJ0zjnnqKioSI888ohGjhzZ4Hlz5szR7NmzXc8rKirC6y86V8HBm7tUUHAA7CLscxiAFo88BsBOjEczz8PXmd5fUAsOHTt2VMeOHc/YLjMzU+Xl5dqwYYMGDhwoSSosLFR5ebmGDBnS4DnV1dWqrq5WRIT7JI7IyEjV1jb+2//Y2FjFxsZ68S5CTF3xgD0cgBYp7HMYgBaPPAbADiIjIyVJVVVViouLC3I0/lNVVSXp3+/3dGGxh0OvXr00evRoTZs2Tc8++6wkafr06RozZox69uzpapeRkaGcnBxdddVVatu2rYYPH64777xTcXFxSktL09q1a/XCCy/o0UcfDdZb8T8KDgAAAAAQVFFRUYqPj9fBgwcVHR1d7xfhdlBbW6uDBw8qPj5eUVENlxbCouAgSStXrtTMmTOVlZUlSRo7dqyefPJJtzYlJSUqLy93PV+1apXmzJmj66+/Xl9//bXS0tL0wAMP6Kabbgpo7AHlKh54saRCLKkAAAAAAKs4HA516dJFO3fu1O7du4Mdjt9ERESoW7duje6tGDYFh8TERL300ktNtjl9/Ujnzp21bNkyf4YVergtJgAAAAAEXUxMjM4++2zXsgM7iomJaXL2RtgUHOAhCg4AAAAAEBIiIiLUqlWrYIcRNIw07YaCAwAAAAAgBDDStJ26TSM9uf1K3W0x+RoAAAAAAKzFSNNufJrhwKaRAAAAAABrUXCwG1fBwZO7VFBwAAAAAAD4BwUHu6krHng0w4ElFQAAAAAA/2CkaTdeFRzYNBIAAAAA4B+MNO3GVTzwZElFXRuWVAAAAAAArEXBwW64LSYAAAAAIAQw0rQbCg4AAAAAgBDASNNuKDgAAAAAAEIAI03bqds00oM9HMRdKgAAAAAA/sFI0258muHAppEAAAAAAGtRcLAbllQAAAAAAEIAI027cXixpKKuDTMcAAAAAAAWo+BgN8xwAAAAAACEAEaaduOareDFDAcxwwEAAAAAYC0KDnbDDAcAAAAAQAhgpGk3FBwAAAAAACGAkabdUHAAAAAAAIQARpq2U3eXCgoOAAAAAIDgYaRpN64ZDh5sGiluiwkAAAAA8A8KDnbDkgoAAAAAQAgIm5HmAw88oCFDhig+Pl7t2rXz6BxjjObPn6+UlBTFxcVpxIgR+uyzz/wbaLA5vFlSUTfDIWy+BgAAAACAMBE2I82qqir99Kc/1X/91395fM7DDz+sRx99VE8++aQ2btyozp076z//8z919OhRP0YaZN4sqXDNcGBJBQAAAADAWmFTcFiwYIFmzZqlvn37etTeGKNFixbp3nvv1bhx49SnTx+tWLFCJ06c0B/+8Ac/RxtEruKBJwWHujYUHAAAAAAA1gqbgoO3du7cqdLSUmVlZbmOxcbGavjw4Vq3bl0QI/Mz9nAAAAAAAISAqGAH4C+lpaWSpOTkZLfjycnJ2r17d6PnVVZWqrKy0vW8oqLCPwH6CwUHoEUL+xwGoMUjjwGAfQR1pDl//nw5HI4mH5s2bWpWH47T9icwxtQ7dqqcnBw5nU7XIzU1tVn9BxwFB6BFC/scBqDFI48BgH0EdaQ5Y8YMFRcXN/no06ePT9fu3LmzpH/PdKhTVlZWb9bDqebMmaPy8nLXY+/evT71Hzze3KWCggNgN+GfwwC0dOQxALCPoC6p6Nixozp27OiXa3fv3l2dO3fWmjVr1K9fP0nf3+li7dq1euihhxo9LzY2VrGxsX6JKSC8uUtF3caS3KUCsI2wz2EAWjzyGADYR9j8anvPnj0qKirSnj17VFNTo6KiIhUVFenYsWOuNhkZGcrNzZX0/VKK2267TQsXLlRubq4+/fRTTZ48WfHx8Zo4cWKw3ob/saQCAAAAABACwmbTyLlz52rFihWu53WzFj744AONGDFCklRSUqLy8nJXm7vuukvffvutbr75Zn3zzTcaNGiQ3nvvPSUkJAQ09oByeLik4tQZEBQcAAAAAAAWC5uCw/Lly7V8+fIm25jTlhE4HA7Nnz9f8+fP919gocbTGQ6nvk7BAQAAAABgMUaaduPpfgwe7fEAAAAAAIBvKDjYDTMcAAAAAAAhgJGm3VBwAAAAAACEAEaadkPBAQAAAAAQAhhp2o6nd6mg4AAAAAAA8B9GmnbjmuFwpk0hT70tpocbTQIAAAAA4CEKDnbDkgoAAAAAQAhgpGk3FBwAAAAAACGAkabdODzdw+HUJRV8DQAAAAAA1mKkaTeu/RjOsIeD2x4P7OEAAAAAALAWBQe78XTTSLclFRQcAAAAAADWigp2ALBYXcHhoxelnX9rvF1NVd0JFBwAAAAAAJaj4GA3bVO+/2/5nu8fnrYHAAAAAMBCFBzsZshMqfM5UtVxz9p3HeDfeAAAAAAALRIFB7uJipV6jAp2FAAAAACAFo5NIwEAAAAAgOUoOAAAAAAAAMtRcAAAAAAAAJaj4AAAAAAAACxHwQEAAAAAAFiOggMAAAAAALAct8U8A2OMJKmioiLIkQBoCepyTV3uaS5yGIBAI48BCGdW57CWjoLDGRw9elSSlJqaGuRIALQkR48eldPptOQ6EjkMQOCRxwCEM6tyWEvnMJRumlRbW6v9+/crISFBDofDo3MqKiqUmpqqvXv3qm3btn6O0HrEH1zhHH84xy6FRvzGGB09elQpKSmKiGj+qjdyGPEHGvEHVyjETx5rvnCOP5xjl4g/2EIhfqtzWEvHDIcziIiIUNeuXX06t23btmH5P3od4g+ucI4/nGOXgh+/ldV0chjxBwvxB1ew4yePWSOc4w/n2CXiD7Zgx8/MButQsgEAAAAAAJaj4AAAAAAAACxHwcEPYmNjNW/ePMXGxgY7FJ8Qf3CFc/zhHLsU/vFbJdw/B+IPLuIPrnCP3yrh/jmEc/zhHLtE/MEW7vGjPjaNBAAAAAAAlmOGAwAAAAAAsBwFBwAAAAAAYDkKDgAAAAAAwHIUHBrw9NNPq3v37mrVqpX69++vv//97022X7t2rfr3769WrVrprLPO0jPPPFOvzeuvv67evXsrNjZWvXv3Vm5ubrP7DVT8S5Ys0bBhw9S+fXu1b99eI0eO1IYNG9zazJ8/Xw6Hw+3RuXPnkIh/+fLl9WJzOBz67rvvmtVvoOIfMWJEg/FfdtllrjZWff7exH7gwAFNnDhRPXv2VEREhG677bYG24Xqd9+T+AP93bcKOYwcFkqffyBzmLfxk8e+Rx7zvd9AxR/InyM5jBxGDoOlDNysWrXKREdHmyVLlpitW7eaW2+91bRu3drs3r27wfZffPGFiY+PN7feeqvZunWrWbJkiYmOjjavvfaaq826detMZGSkWbhwoSkuLjYLFy40UVFRpqCgwOd+Axn/xIkTzVNPPWW2bNliiouLzZQpU4zT6TT79u1ztZk3b5750Y9+ZA4cOOB6lJWVeRW7v+JftmyZadu2rVtsBw4caFa/gYz/8OHDbnF/+umnJjIy0ixbtszVxorP39vYd+7caWbOnGlWrFhhzjvvPHPrrbfWaxPK331P4g/kd98q5DByWKh9/oHKYb7ETx77HnnMt34DGX+gfo7kMHIYOQxWo+BwmoEDB5qbbrrJ7VhGRoa5++67G2x/1113mYyMDLdjv/jFL8zgwYNdz6+99lozevRotzajRo0yEyZM8LnfQMZ/upMnT5qEhASzYsUK17F58+aZc88916tYG+KP+JctW2acTqel/Vp1HV8+///5n/8xCQkJ5tixY65jVnz+zfkMhg8f3uBfEqH83T9VY/Gfzp/ffauQw8hhnvRr1XVCKYcZQx4jj/0becw35DBymLf9noochoawpOIUVVVV2rx5s7KystyOZ2Vlad26dQ2es379+nrtR40apU2bNqm6urrJNnXX9KXfQMZ/uhMnTqi6ulqJiYlux7dv366UlBR1795dEyZM0BdffOFx7P6O/9ixY0pLS1PXrl01ZswYbdmypVn9Bjr+Uy1dulQTJkxQ69at3Y435/O36jM4XSh/933hr+++VchhDcd/OnJY4OM/lT9ymK/xeyKUv/++II+5twmFn2Oo5DFyWOPxn4oc1jByGBpDweEUhw4dUk1NjZKTk92OJycnq7S0tMFzSktLG2x/8uRJHTp0qMk2ddf0pd9Axn+6u+++Wz/4wQ80cuRI17FBgwbphRde0LvvvqslS5aotLRUQ4YM0eHDh4Mef0ZGhpYvX67Vq1fr5ZdfVqtWrXThhRdq+/btPvcbyPhPtWHDBn366aeaOnWq2/Hmfv5WfQanC+Xvvi/89d23Cjms4fhPRw4LbPyn8lcO8zV+T4Ty998X5LGGrxlO/x9J/vk5ksMajv9U5LDGkcPQmKhgBxCKHA6H23NjTL1jZ2p/+nFPrultv97E09z46zz88MN6+eWXlZ+fr1atWrmOX3LJJa4/9+3bV5mZmfrhD3+oFStWaPbs2UGNf/DgwRo8eLDr9QsvvFDnn3++nnjiCT3++OM+9xuo+E+1dOlS9enTRwMHDnQ7btXnb9Vn4O01g/XZeyMQ332rkMPIYZ70G6j4T+XvHOZL/FZdkzxmLfJY8PIYOYwc1px+yWE4HTMcTtGxY0dFRkbWq8KVlZXVq9bV6dy5c4Pto6Ki1KFDhybb1F3Tl34DGX+dRx55RAsXLtR7772nc845p8lYWrdurb59+7qq16EQf52IiAhdcMEFrtjC5fM/ceKEVq1aVa+q3hBvP3+rPoPThfJ33xv+/u5bhRzWcPx1yGHBjd+fOczX+D0Ryt9/b5DHQvfnGCp5jBzWdPzksKaRw9AYCg6niImJUf/+/bVmzRq342vWrNGQIUMaPCczM7Ne+/fee08DBgxQdHR0k23qrulLv4GMX5L++7//W/fff7/y8vI0YMCAM8ZSWVmp4uJidenSJSTiP5UxRkVFRa7YwuHzl6RXX31VlZWV+tnPfnbGWLz9/K36DE4Xyt99TwXiu28VcljD8UvksFCI3585zNf4PRHK339PkcdC++cYKnmMHNZ0/OSwppHD0Cjr96EMb3W3c1m6dKnZunWrue2220zr1q3Nrl27jDHG3H333SY7O9vVvu52OrNmzTJbt241S5curXc7nX/84x8mMjLSPPjgg6a4uNg8+OCDjd6OprF+gxn/Qw89ZGJiYsxrr73mdruZo0ePutrcfvvtJj8/33zxxRemoKDAjBkzxiQkJIRE/PPnzzd5eXlmx44dZsuWLWbKlCkmKirKFBYWetxvMOOvM3ToUDN+/PgG+7Xi8/c2dmOM2bJli9myZYvp37+/mThxotmyZYv57LPPXK+H8nffk/gD+d23CjmMHBZqn38df+cwX+I3hjxmDHnM036DGX+gfo7kMHIYOQxWo+DQgKeeesqkpaWZmJgYc/7555u1a9e6Xps0aZIZPny4W/v8/HzTr18/ExMTY9LT083ixYvrXfOPf/yj6dmzp4mOjjYZGRnm9ddf96rfYMaflpZmJNV7zJs3z9Vm/PjxpkuXLiY6OtqkpKSYcePGuSWLYMZ/2223mW7dupmYmBjTqVMnk5WVZdatW+dVv8GM3xhjSkpKjCTz3nvvNdinVZ+/t7E39L1IS0tzaxPK3/0zxR/o775VyGHksFD6/I0JXA7zJX7yGHnM036DGX8gf47kMHIYOQxWchjzr51RAAAAAAAALMIeDgAAAAAAwHIUHAAAAAAAgOUoOAAAAAAAAMtRcAAAAAAAAJaj4AAAAAAAACxHwQEAAAAAAFiOggMAAAAAALAcBQcAAAAAAGA5Cg6Al/Lz8+VwOHTkyJFghwIAPiGPAQhn5DAgfDiMMSbYQQChbMSIETrvvPO0aNEiSVJVVZW+/vprJScny+FwBDc4APAAeQxAOCOHAeErKtgBAOEmJiZGnTt3DnYYAOAz8hiAcEYOA8IHSyqAJkyePFlr167VY489JofDIYfDoeXLl7tN41u+fLnatWunN998Uz179lR8fLyuueYaHT9+XCtWrFB6errat2+vW265RTU1Na5rV1VV6a677tIPfvADtW7dWoMGDVJ+fn5w3igA2yKPAQhn5DAgvDHDAWjCY489pm3btqlPnz667777JEmfffZZvXYnTpzQ448/rlWrVuno0aMaN26cxo0bp3bt2untt9/WF198oauvvlpDhw7V+PHjJUlTpkzRrl27tGrVKqWkpCg3N1ejR4/WJ598orPPPjug7xOAfZHHAIQzchgQ3ig4AE1wOp2KiYlRfHy8a+re//3f/9VrV11drcWLF+uHP/yhJOmaa67Riy++qK+++kpt2rRR7969ddFFF+mDDz7Q+PHjtWPHDr388svat2+fUlJSJEl33HGH8vLytGzZMi1cuDBwbxKArZHHAIQzchgQ3ig4ABaIj493/QUnScnJyUpPT1ebNm3cjpWVlUmSPvroIxlj1KNHD7frVFZWqkOHDoEJGgBOQR4DEM7IYUBoouAAWCA6OtrtucPhaPBYbW2tJKm2tlaRkZHavHmzIiMj3dqd+hcjAAQKeQxAOCOHAaGJggNwBjExMW4bDFmhX79+qqmpUVlZmYYNG2bptQHgdOQxAOGMHAaEL+5SAZxBenq6CgsLtWvXLh06dMhVGW+OHj166Prrr9cNN9ygP/3pT9q5c6c2btyohx56SG+//bYFUQPAv5HHAIQzchgQvig4AGdwxx13KDIyUr1791anTp20Z88eS667bNky3XDDDbr99tvVs2dPjR07VoWFhUpNTbXk+gBQhzwGIJyRw4Dw5TDGmGAHAQAAAAAA7IUZDgAAAAAAwHIUHAAAAAAAgOUoOAAAAAAAAMtRcAAAAAAAAJaj4AAAAAAAACxHwQEAAAAAAFiOggMAAAAAALAcBQcAAAAAAGA5Cg4AAAAAAMByFBwAAAAAAIDlKDgAAAAAAADLUXAAAAAAAACW+38PTmcaR2/46AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff['vh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (My debug_env Kernel)", + "language": "python", + "name": "debug_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.py b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.py new file mode 100644 index 000000000..dce7c1358 --- /dev/null +++ b/examples/rmvs_swifter_comparison/1pl_1tp_encounter/swiftest_vs_swifter.py @@ -0,0 +1,9 @@ +import swiftest +import numpy as np + +swiftersim = swiftest.Simulation(simdir="swifter_sim", read_data=True, codename="Swifter") +swiftestsim = swiftest.Simulation(simdir="swiftest_sim",read_data=True) +swiftestsim.data = swiftestsim.data.swap_dims({"name" : "id"}) +swiftdiff = swiftestsim.data - swiftersim.data +swiftdiff['rh'].plot(x="time",hue="id",col="space") +swiftdiff['vh'].plot(x="time",hue="id",col="space") \ No newline at end of file From fed8c213ea075e32a1948064f44352bb3ea311db Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 16:31:19 -0400 Subject: [PATCH 060/149] More change to clean up Swifter file saving --- python/swiftest/swiftest/simulation_class.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 2ff9c8454..804eee651 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -1441,13 +1441,19 @@ def ascii_file_input_error_msg(codename): if "IN_FORM" in self.param: init_cond_format = self.param['IN_FORM'] else: - init_cond_format = "EL" + if self.codename.title() == "Swiftest": + init_cond_format = "EL" + else: + init_cond_format = "XV" if init_cond_file_type is None: if "IN_TYPE" in self.param: init_cond_file_type = self.param['IN_TYPE'] else: - init_cond_file_type = "NETCDF_DOUBLE" + if self.codename.title() == "Swiftest": + init_cond_file_type = "NETCDF_DOUBLE" + else: + init_cond_file_type = "ASCII" if self.codename.title() == "Swiftest": init_cond_keys = ["CB", "PL", "TP"] @@ -1664,10 +1670,11 @@ def set_output_files(self, else: self.param['BIN_OUT'] = output_file_name - if output_format != "XV" and self.codename != "Swiftest": - warnings.warn(f"{output_format} is not compatible with {self.codename}. Setting to XV",stacklevel=2) - output_format = "XV" - self.param["OUT_FORM"] = output_format + if output_format is not None: + if output_format != "XV" and self.codename != "Swiftest": + warnings.warn(f"{output_format} is not compatible with {self.codename}. Setting to XV",stacklevel=2) + output_format = "XV" + self.param["OUT_FORM"] = output_format if self.restart: self.param["OUT_STAT"] = "APPEND" From ca9eaa4cab1e9b03c6a64ae5b0350c3962d090c1 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 17:45:10 -0400 Subject: [PATCH 061/149] Put back some of the original operations from the Swifter code in the drift functions --- src/swiftest/swiftest_drift.f90 | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/swiftest/swiftest_drift.f90 b/src/swiftest/swiftest_drift.f90 index 4c7302908..5c0427c62 100644 --- a/src/swiftest/swiftest_drift.f90 +++ b/src/swiftest/swiftest_drift.f90 @@ -251,22 +251,22 @@ pure subroutine swiftest_drift_kepmd(dm, es, ec, x, s, c) ! executable code fac1 = 1.0_DP / (1.0_DP - ec) q = fac1 * dm - fac2 = es**2 * fac1 - ec / 3.0_DP + fac2 = es*es*fac1 - ec / 3.0_DP x = q * (1.0_DP - 0.5_DP * fac1 * q * (es - q * fac2)) - y = x**2 + y = x*x s = x * (a0 - y * (a1 - y * (a2 - y * (a3 - y * (a4 - y))))) / a0 - c = sqrt(1.0_DP - s**2) + c = sqrt(1.0_DP - s*s) f = x - ec * s + es * (1.0_DP - c) - dm fp = 1.0_DP - ec * c + es * s fpp = ec * s + es * c fppp = ec * c - es * s dx = -f / fp - dx = -f / (fp + dx * fpp * 0.5_DP) - dx = -f / (fp + dx * fpp * 0.5_DP + dx**2* fppp * SIXTH) + dx = -f / (fp + dx * fpp/2.0_DP) + dx = -f / (fp + dx * fpp/2.0_DP + dx*dx * fppp * SIXTH) x = x + dx - y = x**2 + y = x*x s = x * (a0 - y * (a1 - y * (a2 - y * (a3 - y * (a4 - y))))) / a0 - c = sqrt(1.0_DP - s**2) + c = sqrt(1.0_DP - s*s) return end subroutine swiftest_drift_kepmd @@ -352,18 +352,18 @@ pure subroutine swiftest_drift_kepu_guess(dt, r0, mu, alpha, u, s) if (alpha > 0.0_DP) then if (dt / r0 <= thresh) then - s = dt / r0 - (dt**2 * u) / (2 * r0**3) + s = dt/r0 - (dt*dt*u)/(2.0_DP*r0*r0*r0) else - a = mu / alpha - en = sqrt(mu / a**3) - ec = 1.0_DP - r0 / a - es = u / (en * a**2) - e = sqrt(ec**2 + es**2) - y = en * dt - es + a = mu/alpha + en = sqrt(mu/(a*a*a)) + ec = 1.0_DP - r0/a + es = u/(en*a*a) + e = sqrt(ec*ec + es*es) + y = en*dt - es call swiftest_orbel_scget(y, sy, cy) - sigma = sign(1.0_DP, es * cy + ec * sy) - x = y + sigma * danbyk * e - s = x / sqrt(alpha) + sigma = sign(1.0_DP, es*cy + ec*sy) + x = y + sigma*DANBYK*e + s = x/sqrt(alpha) end if else call swiftest_drift_kepu_p3solve(dt, r0, mu, alpha, u, s, iflag) @@ -408,16 +408,16 @@ pure subroutine swiftest_drift_kepu_lag(s, dt, r0, mu, alpha, u, fp, c1, c2, c3, do nc = 0, ncmax x = s * s * alpha call swiftest_drift_kepu_stumpff(x, c0, c1, c2, c3) - c1 = c1 * s - c2 = c2 * s**2 - c3 = c3 * s**3 + c1 = c1*s + c2 = c2*s*s + c3 = c3*s*s*s f = r0 * c1 + u * c2 + mu * c3 - dt fp = r0 * c0 + u * c1 + mu * c2 fpp = (-r0 * alpha + mu) * c1 + u * c0 - ds = -ln * f / (fp + sign(1.0_DP, fp) * sqrt(abs((ln - 1)**2 * fp**2 - (ln - 1) * ln * f * fpp))) + ds = -ln*f/(fp + sign(1.0_DP, fp)*sqrt(abs((ln - 1.0_DP)*(ln - 1.0_DP)*fp*fp - (ln - 1.0_DP)*ln*f*fpp))) s = s + ds fdt = f / dt - if (fdt**2 < DANBYB**2) then + if (fdt*fdt < DANBYB*DANBYB) then iflag = 0 return end if @@ -454,23 +454,23 @@ pure subroutine swiftest_drift_kepu_new(s, dt, r0, mu, alpha, u, fp, c1, c2, c3, real(DP) :: x, c0, ds, f, fpp, fppp, fdt do nc = 0, 6 - x = s**2 * alpha + x = s*s*alpha call swiftest_drift_kepu_stumpff(x, c0, c1, c2, c3) - c1 = c1 * s - c2 = c2 * s**2 - c3 = c3 * s**3 - f = r0 * c1 + u * c2 + mu * c3 - dt - fp = r0 * c0 + u * c1 + mu * c2 - fpp = (-r0 * alpha + mu) * c1 + u * c0 - fppp = (-r0 * alpha + mu) * c0 - u * alpha * c1 - ds = -f / fp - ds = -f / (fp + ds * fpp * 0.5_DP) - ds = -f / (fp + ds * fpp * 0.5_DP + ds**2 * fppp * SIXTH) + c1 = c1*s + c2 = c2*s*s + c3 = c3*s*s*s + f = r0*c1 + u*c2 + mu*c3 - dt + fp = r0*c0 + u*c1 + mu*c2 + fpp = (-r0*alpha + mu)*c1 + u*c0 + fppp = (-r0*alpha + mu)*c0 - u*alpha*c1 + ds = -f/fp + ds = -f/(fp + ds*fpp/2.0_DP) + ds = -f/(fp + ds*fpp/2.0_DP + ds*ds*fppp/6.0_DP) s = s + ds - fdt = f / dt - if (fdt**2 < DANBYB**2) then - iflag = 0 - return + fdt = f/dt + if (fdt*fdt < DANBYB*DANBYB) then + iflag = 0 + return end if end do iflag = 1 @@ -503,9 +503,9 @@ pure subroutine swiftest_drift_kepu_p3solve(dt, r0, mu, alpha, u, s, iflag) a2 = 0.5_DP * u / denom a1 = r0 / denom a0 = -dt / denom - q = (a1 - a2**2 * THIRD) * THIRD - r = (a1 * a2 - 3 * a0) * SIXTH - (a2 * THIRD)**3 - sq2 = q**3 + r**2 + q = (a1 - a2*a2 * THIRD) * THIRD + r = (a1*a2 - 3 * a0) * SIXTH - (a2*a2*a2)/27.0_DP + sq2 = q*q*q + r*r if (sq2 >= 0.0_DP) then sq = sqrt(sq2) if ((r + sq) <= 0.0_DP) then @@ -565,9 +565,9 @@ pure subroutine swiftest_drift_kepu_stumpff(x, c0, c1, c2, c3) if (n /= 0) then do i = n, 1, -1 c3 = (c2 + c0 * c3) / 4.0_DP - c2 = c1**2 / 2.0_DP + c2 = c1*c1 / 2.0_DP c1 = c0 * c1 - c0 = 2 * c0**2 - 1.0_DP + c0 = 2 * c0*c0 - 1.0_DP x = x * 4 end do end if From 9adbcfebe9fe4068cde97d49b559924d222de35f Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 15 May 2023 22:28:36 -0400 Subject: [PATCH 062/149] Restored some floating point operations to make them closer to the Swifter version --- src/swiftest/swiftest_kick.f90 | 2 +- src/swiftest/swiftest_util.f90 | 2 +- src/whm/whm_kick.f90 | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/swiftest/swiftest_kick.f90 b/src/swiftest/swiftest_kick.f90 index 430679b43..751702fa2 100644 --- a/src/swiftest/swiftest_kick.f90 +++ b/src/swiftest/swiftest_kick.f90 @@ -430,7 +430,7 @@ pure module subroutine swiftest_kick_getacch_int_one_tp(rji2, xr, yr, zr, GMpl, ! Internals real(DP) :: fac - fac = GMpl * sqrt(1.0_DP / (rji2*rji2*rji2)) + fac = GMpl / (rji2*sqrt(rji2)) ax = ax - fac * xr ay = ay - fac * yr az = az - fac * zr diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index be98690c3..3fc664af9 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -2609,7 +2609,7 @@ module subroutine swiftest_util_snapshot_system(self, param, nbody_system, t, ar end select end select - call nbody_system%pl%set_rhill(nbody_system%cb) + if (.not.param%lrhill_present) call nbody_system%pl%set_rhill(nbody_system%cb) ! Take a minimal snapshot wihout all of the extra storage objects allocate(snapshot, mold=nbody_system) diff --git a/src/whm/whm_kick.f90 b/src/whm/whm_kick.f90 index 6469809e5..403678ed6 100644 --- a/src/whm/whm_kick.f90 +++ b/src/whm/whm_kick.f90 @@ -124,14 +124,13 @@ function whm_kick_getacch_ah0(mu, rhp, n) result(ah0) ! Result real(DP), dimension(NDIM) :: ah0 ! Internals - real(DP) :: fac, r2, ir3h, irh + real(DP) :: fac, r2, ir3h integer(I4B) :: i ah0(:) = 0.0_DP do i = 1, n r2 = dot_product(rhp(:, i), rhp(:, i)) - irh = 1.0_DP / sqrt(r2) - ir3h = irh / r2 + ir3h = 1.0_DP / (r2 * sqrt(r2)) fac = mu(i) * ir3h ah0(:) = ah0(:) - fac * rhp(:, i) end do From 9bb229b7b34f398cc802c8d5bab4ec810acb02d6 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 16 May 2023 11:07:36 -0400 Subject: [PATCH 063/149] Restructured oblateness acceleration to be in functional form so I can call it from the outer encounters like the original RMVS did --- src/swiftest/swiftest_module.f90 | 18 ++++++--- src/swiftest/swiftest_obl.f90 | 63 ++++++++++++++++++-------------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index c86f86dd1..f7be030d1 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -146,7 +146,6 @@ module swiftest procedure :: read_in => swiftest_io_read_in_body !! Read in body initial conditions from an ascii file procedure :: write_frame => swiftest_io_netcdf_write_frame_body !! I/O routine for writing out a single frame of time-series data for all bodies in the nbody_system in NetCDF format procedure :: write_info => swiftest_io_netcdf_write_info_body !! Dump contents of particle information metadata to file - procedure :: accel_obl => swiftest_obl_acc_body !! Compute the barycentric accelerations of bodies due to the oblateness of the central body procedure :: el2xv => swiftest_orbel_el2xv_vec !! Convert orbital elements to position and velocity vectors procedure :: xv2el => swiftest_orbel_xv2el_vec !! Convert position and velocity vectors to orbital elements procedure :: setup => swiftest_util_setup_body !! A constructor that sets the number of bodies and allocates all allocatable arrays @@ -995,11 +994,18 @@ pure module subroutine swiftest_kick_getacch_int_one_tp(rji2, xr, yr, zr, Gmpl, real(DP), intent(inout) :: ax, ay, az !! Acceleration vector components of test particle end subroutine swiftest_kick_getacch_int_one_tp - module subroutine swiftest_obl_acc_body(self, nbody_system) - implicit none - class(swiftest_body), intent(inout) :: self !! Swiftest body object - class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object - end subroutine swiftest_obl_acc_body + module subroutine swiftest_obl_acc(n, GMcb, j2rp2, j4rp4, rh, lmask, aobl, GMpl, aoblcb) + implicit none + integer(I4B), intent(in) :: n !! Number of bodies + real(DP), intent(in) :: GMcb !! Central body G*Mass + real(DP), intent(in) :: j2rp2 !! J2 * R**2 for the central body + real(DP), intent(in) :: j4rp4 !! J4 * R**4 for the central body + real(DP), dimension(:,:), intent(in) :: rh !! Heliocentric positions of bodies + logical, dimension(:), intent(in) :: lmask !! Logical mask of bodies to compute aobl + real(DP), dimension(:,:), intent(out) :: aobl !! Barycentric acceleration of bodies due to central body oblateness + real(DP), dimension(:), intent(in), optional :: GMpl !! Masses of input bodies if they are not test particles + real(DP), dimension(:), intent(out), optional :: aoblcb !! Barycentric acceleration of central body (only needed if input bodies are massive) + end subroutine swiftest_obl_acc module subroutine swiftest_obl_acc_pl(self, nbody_system) implicit none diff --git a/src/swiftest/swiftest_obl.f90 b/src/swiftest/swiftest_obl.f90 index 2b87b7264..6bd7480fb 100644 --- a/src/swiftest/swiftest_obl.f90 +++ b/src/swiftest/swiftest_obl.f90 @@ -9,7 +9,7 @@ submodule (swiftest) s_swiftest_obl contains - module subroutine swiftest_obl_acc_body(self, nbody_system) + module subroutine swiftest_obl_acc(n, GMcb, j2rp2, j4rp4, rh, lmask, aobl, GMpl, aoblcb) !! author: David A. Minton !! !! Compute the barycentric accelerations of bodies due to the oblateness of the central body @@ -19,33 +19,46 @@ module subroutine swiftest_obl_acc_body(self, nbody_system) !! Adapted from Hal Levison's Swift routine obl_acc.f and obl_acc_tp.f implicit none ! Arguments - class(swiftest_body), intent(inout) :: self !! Swiftest body object - class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object + integer(I4B), intent(in) :: n !! Number of bodies + real(DP), intent(in) :: GMcb !! Central body G*Mass + real(DP), intent(in) :: j2rp2 !! J2 * R**2 for the central body + real(DP), intent(in) :: j4rp4 !! J4 * R**4 for the central body + real(DP), dimension(:,:), intent(in) :: rh !! Heliocentric positions of bodies + logical, dimension(:), intent(in) :: lmask !! Logical mask of bodies to compute aobl + real(DP), dimension(:,:), intent(out) :: aobl !! Barycentric acceleration of bodies due to central body oblateness + real(DP), dimension(:), intent(in), optional :: GMpl !! Masses of input bodies if they are not test particles + real(DP), dimension(:), intent(out), optional :: aoblcb !! Barycentric acceleration of central body (only needed if input bodies are massive) ! Internals integer(I4B) :: i real(DP) :: r2, irh, rinv2, t0, t1, t2, t3, fac1, fac2 - if (self%nbody == 0) return - - associate(n => self%nbody, cb => nbody_system%cb) - self%aobl(:,:) = 0.0_DP - do concurrent(i = 1:n, self%lmask(i)) - r2 = dot_product(self%rh(:, i), self%rh(:, i)) - irh = 1.0_DP / sqrt(r2) - rinv2 = irh**2 - t0 = -cb%Gmass * rinv2 * rinv2 * irh - t1 = 1.5_DP * cb%j2rp2 - t2 = self%rh(3, i) * self%rh(3, i) * rinv2 - t3 = 1.875_DP * cb%j4rp4 * rinv2 - fac1 = t0 * (t1 - t3 - (5 * t1 - (14.0_DP - 21.0_DP * t2) * t3) * t2) - fac2 = 2 * t0 * (t1 - (2.0_DP - (14.0_DP * t2 / 3.0_DP)) * t3) - self%aobl(:, i) = fac1 * self%rh(:, i) - self%aobl(3, i) = fac2 * self%rh(3, i) + self%aobl(3, i) + if (n == 0) return + + aobl(:,:) = 0.0_DP + do concurrent(i = 1:n, lmask(i)) + r2 = dot_product(rh(:, i), rh(:, i)) + irh = 1.0_DP / sqrt(r2) + rinv2 = irh**2 + t0 = -GMcb * rinv2 * rinv2 * irh + t1 = 1.5_DP * j2rp2 + t2 = rh(3, i) * rh(3, i) * rinv2 + t3 = 1.875_DP * j4rp4 * rinv2 + fac1 = t0 * (t1 - t3 - (5 * t1 - (14.0_DP - 21.0_DP * t2) * t3) * t2) + fac2 = 2 * t0 * (t1 - (2.0_DP - (14.0_DP * t2 / 3.0_DP)) * t3) + aobl(:, i) = fac1 * rh(:, i) + aobl(3, i) = fac2 * rh(3, i) + aobl(3, i) + end do + + if (present(GMpl) .and. present(aoblcb)) then + aoblcb(:) = 0.0_DP + do i = n, 1, -1 + if (lmask(i)) aoblcb(:) = aoblcb(:) - GMpl(i) * aobl(:, i) / GMcb end do - end associate + end if + return - end subroutine swiftest_obl_acc_body + end subroutine swiftest_obl_acc module subroutine swiftest_obl_acc_pl(self, nbody_system) @@ -65,11 +78,7 @@ module subroutine swiftest_obl_acc_pl(self, nbody_system) if (self%nbody == 0) return associate(pl => self, npl => self%nbody, cb => nbody_system%cb) - call swiftest_obl_acc_body(pl, nbody_system) - cb%aobl(:) = 0.0_DP - do i = npl, 1, -1 - if (pl%lmask(i)) cb%aobl(:) = cb%aobl(:) - pl%Gmass(i) * pl%aobl(:, i) / cb%Gmass - end do + call swiftest_obl_acc(npl, cb%Gmass, cb%j2rp2, cb%j4rp4, pl%rh, pl%lmask, pl%aobl, pl%Gmass, cb%aobl) do concurrent(i = 1:npl, pl%lmask(i)) pl%ah(:, i) = pl%ah(:, i) + pl%aobl(:, i) - cb%aobl(:) @@ -99,7 +108,7 @@ module subroutine swiftest_obl_acc_tp(self, nbody_system) if (self%nbody == 0) return associate(tp => self, ntp => self%nbody, cb => nbody_system%cb) - call swiftest_obl_acc_body(tp, nbody_system) + call swiftest_obl_acc(ntp, cb%Gmass, cb%j2rp2, cb%j4rp4, tp%rh, tp%lmask, tp%aobl) if (nbody_system%lbeg) then aoblcb = cb%aoblbeg else From 6f0c12fcd0b3fa89757e437221035a90c3a79e3b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 16 May 2023 11:28:01 -0400 Subject: [PATCH 064/149] Put in the oblateness accelerations for central body for test particles in outer encounter of RMVS --- src/rmvs/rmvs_step.f90 | 4 ++++ src/rmvs/rmvs_util.f90 | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rmvs/rmvs_step.f90 b/src/rmvs/rmvs_step.f90 index bcb38797a..11f7d2756 100644 --- a/src/rmvs/rmvs_step.f90 +++ b/src/rmvs/rmvs_step.f90 @@ -202,6 +202,10 @@ subroutine rmvs_step_out(cb, pl, tp, nbody_system, param, t, dt) call tp%step(nbody_system, param, outer_time, dto) tp%lfirst = lfirsttp else + if (param%loblatecb) then + call swiftest_obl_acc(npl, cb%Gmass, cb%j2rp2, cb%j4rp4, pl%rbeg, pl%lmask, pl%outer(outer_index-1)%aobl, pl%Gmass, cb%aoblbeg) + call swiftest_obl_acc(npl, cb%Gmass, cb%j2rp2, cb%j4rp4, pl%rend, pl%lmask, pl%outer(outer_index)%aobl, pl%Gmass, cb%aoblend) + end if call tp%step(nbody_system, param, outer_time, dto) end if do j = 1, npl diff --git a/src/rmvs/rmvs_util.f90 b/src/rmvs/rmvs_util.f90 index 224ba2cd1..6b26bf12f 100644 --- a/src/rmvs/rmvs_util.f90 +++ b/src/rmvs/rmvs_util.f90 @@ -298,13 +298,15 @@ module subroutine rmvs_util_setup_pl(self, n, param) allocate(pl%outer(i)%v(NDIM, n)) pl%outer(i)%x(:,:) = 0.0_DP pl%outer(i)%v(:,:) = 0.0_DP + allocate(pl%outer(i)%aobl(NDIM, n)) + pl%outer(i)%aobl(:,:) = 0.0_DP end do do i = 0, NTPHENC allocate(pl%inner(i)%x(NDIM, n)) allocate(pl%inner(i)%v(NDIM, n)) - allocate(pl%inner(i)%aobl(NDIM, n)) pl%inner(i)%x(:,:) = 0.0_DP pl%inner(i)%v(:,:) = 0.0_DP + allocate(pl%inner(i)%aobl(NDIM, n)) pl%inner(i)%aobl(:,:) = 0.0_DP end do ! if (param%ltides) then From dc41783bca3df9dd6d3cc155c7ab9e1fa0351268 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 16 May 2023 15:53:32 -0400 Subject: [PATCH 065/149] Added new example of rmvs vs. swifter comparison with multiple planets and encountering test particles --- examples/rmvs_swifter_comparison/.gitignore | 1 + .../8pl_16tp_encounters/.gitignore | 5 + .../8pl_16tp_encounters/init_cond.py | 32 ++++ .../swiftest_vs_swifter.ipynb | 159 ++++++++++++++++++ .../swiftest_vs_swifter.py | 9 + 5 files changed, 206 insertions(+) create mode 100644 examples/rmvs_swifter_comparison/8pl_16tp_encounters/.gitignore create mode 100755 examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py create mode 100644 examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb create mode 100644 examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.py diff --git a/examples/rmvs_swifter_comparison/.gitignore b/examples/rmvs_swifter_comparison/.gitignore index ecf4c57c8..4d901ba7d 100644 --- a/examples/rmvs_swifter_comparison/.gitignore +++ b/examples/rmvs_swifter_comparison/.gitignore @@ -1,3 +1,4 @@ * !.gitignore !1pl_1tp_encounter +!8pl_16tp_encounters diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/.gitignore b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/.gitignore new file mode 100644 index 000000000..89c2a1c6c --- /dev/null +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/.gitignore @@ -0,0 +1,5 @@ +* +!.gitignore +!init_cond.py +!swiftest_vs_swifter.py +!swiftest_vs_swifter.ipynb diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py new file mode 100755 index 000000000..d59c068d6 --- /dev/null +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import numpy as np +import swiftest + +tstart = 0.0 +dt = 1.0 +tstop = 365.25e2 +tstep_out = 100*dt + +sim = swiftest.Simulation(simdir="swiftest_sim",init_cond_format="XV",output_format="XV",general_relativity=False, integrator="RMVS", rhill_present=True, MU="Msun", DU="AU", TU="d") +sim.clean() +sim.add_solar_system_body(["Sun","Mercury","Venus","Earth","Mars","Jupiter","Saturn","Uranus","Neptune"]) + +plname = sim.init_cond['name'].where(sim.init_cond['name'] != "Sun", drop=True) +pl = sim.init_cond.sel(name=plname) + +for i,n in enumerate(pl.name): + pli = pl.sel(name=n) + rstart = 2 * pli['radius'].data[0] # Start the test particles at a multiple of the planet radius away + vstart = 1.5 * np.sqrt(2 * pli['Gmass'].data[0]) / rstart # Start the test particle velocities at a multiple of the escape speed + rstart_vec = np.array([rstart / np.sqrt(2.0), rstart / np.sqrt(2.0), 0.0]) + vstart_vec = np.array([vstart, 0.0, 0.0]) + rp = pli['rh'].data[0] + vp = pli['vh'].data[0] + sim.add_body(name=[f"TestParticle{100+i}",f"TestParticle{200+i}"],rh=[rp+rstart_vec, rp-rstart_vec],vh=[vp+vstart_vec, vp-vstart_vec]) + + +sim.set_parameter(tstart=tstart, tstop=tstop, dt=dt, tstep_out=tstep_out, dump_cadence=0) +sim.save() + +sim.set_parameter(simdir="swifter_sim",codename="Swifter",init_cond_file_type="ASCII",output_file_type="REAL8") +sim.save() diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb new file mode 100644 index 000000000..920def237 --- /dev/null +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import swiftest" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Swifter file /home/daminton/git_debug/swiftest/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swifter_sim/param.in\n", + "Reading in time 2.330e+04\n", + "Creating Dataset\n", + "Successfully converted 234 output frames.\n", + "Swifter simulation data stored as xarray DataSet .data\n" + ] + } + ], + "source": [ + "swiftersim = swiftest.Simulation(simdir=\"swifter_sim\", read_data=True, codename=\"Swifter\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading Swiftest file /home/daminton/git_debug/swiftest/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_sim/param.in\n", + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 1 output frames.\n", + "\n", + "Creating Dataset from NetCDF file\n", + "Successfully converted 367 output frames.\n", + "Swiftest simulation data stored as xarray DataSet .data\n", + "Reading initial conditions file as .init_cond\n", + "Finished reading Swiftest dataset files.\n" + ] + } + ], + "source": [ + "swiftestsim = swiftest.Simulation(simdir=\"swiftest_sim\",read_data=True)\n", + "swiftestsim.data = swiftestsim.data.swap_dims({\"name\" : \"id\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "swiftdiff = swiftestsim.data - swiftersim.data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCYAAAIjCAYAAADfthq2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABeJklEQVR4nO3deXxU5d3///fJvhACCYQkQiC1yBZEBEQUISgNREARdykG0FZvA5TGumFV3EDbW3+ofEXxpoALS1sNUKsgVhOkCEIwBQURNAgKGEAIZN+u3x80I4EAyWQmZ2byej4e0zpn+1xnQq6Tec91nbGMMUYAAAAAAAA28LO7AQAAAAAAoPkimAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYJsLsBAAAAAAD4qurqapWXl9vdDNsEBgbK39//rNsQTAAAAAAA4Abl5eXKy8tTdXW13U2xVatWrRQbGyvLsupcTzABAAAAAICLGWO0f/9++fv7q0OHDvLza353UjDGqLi4WPn5+ZKkuLi4OrcjmAAAAAAAwMUqKytVXFys+Ph4hYWF2d0c24SGhkqS8vPzFRMTU+e0juYX2QAAAAAA4GZVVVWSpKCgIJtbYr+aYKaioqLO9QQTAAAAAAC4yZnuq9CcnOs1IJgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAMAHJCcna+rUqWdc36lTJ82aNavJ2lNffCsHAAAAAAA+4J133lFgYKDdzWgwggkAAAAAAHxAVFSU3U1wClM5AAAAAADwASdP5cjPz9eoUaMUGhqqxMREvfXWW/Y27iwYMQEAAAAAgI8ZP3689u7dq48++khBQUGaMmWK8vPz7W5WnQgmAAAAAADwIV9//bXef/99rV+/Xv3795ckzZs3T926dbO5ZXVjKgcAAAAAAD5k+/btCggIUN++fR3LunbtqlatWtnXqLMgmAAAAAAAwIcYYyRJlmXZ3JL6IZgAAAAAAMCHdOvWTZWVldq0aZNj2Y4dO3T06FH7GnUWBBMAAAAAAPiQLl26aPjw4frNb36jDRs2KCcnR3feeadCQ0PtblqdCCYAAAAAAPAx8+fPV4cOHTR48GCNGTNGv/3tbxUTE2N3s+pkmZrJJwAAAAAAwCVKS0uVl5enxMREhYSE2N0cW53rtWDEBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAcFizZo1GjRql+Ph4WZalZcuWubUewQQAAAAAAHAoKipSr169NHv27CapF9AkVQAAAAAAaMaMMSqpqLKldmigvyzLqvf2qampSk1NdWOLaiOYAAAAAADAzUoqqtT90VW21N72xDCFBXnu23+mcgAAAAAAANt4bmQCAAAAAICPCA3017YnhtlW25MRTAAAAAAA4GaWZXn0dAo7MZUDAAAAAADYhrgGAAAAAAA4FBYWateuXY7neXl5ys3NVVRUlBISElxej2ACAAAAAAA4bNq0SUOGDHE8z8jIkCSlpaVpwYIFLq9HMAEAAAAAABySk5NljGmyetxjAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAASJJmzpypfv36KSIiQjExMRo9erR27Njh1poEEwAAAAAAQJKUnZ2t9PR0rV+/XqtXr1ZlZaVSUlJUVFTktpoBbjsyAAAAAADwKitXrqz1fP78+YqJiVFOTo4GDRrklpoEEwAAAAAAuJsxUkWxPbUDwyTLcmrXgoICSVJUVJQrW1QLwQQAAAAAAO5WUSzNiLen9rR9UlB4g3czxigjI0MDBw5UUlKSGxp2AsEEAAAAAAA4zaRJk7RlyxatXbvWrXUIJgAAAAAAcLfAsBMjF+yq3UCTJ0/WihUrtGbNGrVv394NjfoZwQQAAAAAAO5mWU5Np2hqxhhNnjxZmZmZysrKUmJiottrEkwAAAAAAABJUnp6uhYtWqTly5crIiJCBw4ckCRFRkYqNDTULTX93HJUAAAAAADgdebMmaOCggIlJycrLi7O8Vi6dKnbajJiAgAAAAAASDoxlaOpMWICAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAABIkubMmaMLL7xQLVu2VMuWLTVgwAC9//77bq1JMAEAAAAAACRJ7du31zPPPKNNmzZp06ZNuvLKK3Xttdfqyy+/dFtNyxhj3HZ0AAAAAACaodLSUuXl5SkxMVEhISF2N6dRoqKi9Oc//1l33HGHU/uf67UIaGwDAQAAAADA2RljVFJZYkvt0IBQWZbV4P2qqqr0t7/9TUVFRRowYIAbWnYCwQQAAAAAAG5WUlmi/ov621J7w20bFBYYVu/tt27dqgEDBqi0tFQtWrRQZmamunfv7rb2cY8JAAAAAADg0KVLF+Xm5mr9+vX6n//5H6WlpWnbtm1uq8c9JgAAAAAAcLFT76vgjVM5agwdOlTnn3++Xn31Vaf25x4TAAAAAADYzLKsBk2n8CTGGJWVlbnt+AQTAAAAAABAkjRt2jSlpqaqQ4cOOn78uJYsWaKsrCytXLnSbTUJJgAAAAAAgCTpxx9/1Lhx47R//35FRkbqwgsv1MqVK/WrX/3KbTUJJgAAAAAAgCRp3rx5TV6Tb+UAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAB1mjlzpizL0tSpU91Wg2ACAAAAAACcZuPGjZo7d64uvPBCt9YhmAAAAAAAALUUFhZq7Nixeu2119S6dWu31gpw69EBAAAAAICMMTIlJbbUtkJDZVlWg/ZJT0/XiBEjNHToUD311FNuatkJBBMAAAAAALiZKSnRjov72FK7y+YcWWFh9d5+yZIl2rx5szZu3OjGVv2MYAIAAAAAAEiS9u7dq9/97nf64IMPFBIS0iQ1LWOMaZJKAAAAAAA0E6WlpcrLy1NiYqJCQkK8ZirHsmXLdN1118nf39+xrKqqSpZlyc/PT2VlZbXW1cepr8WpGDEBAAAAAICbWZbVoOkUdrnqqqu0devWWssmTJigrl276oEHHmhwKFEfBBMAAAAAAECSFBERoaSkpFrLwsPDFR0dfdpyV+HrQgEAAAAAgG0YMQEAAAAAAM4oKyvLrcdnxAQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAAJAkTZ8+XZZl1XrExsa6tWaAW48OAAAAAAC8So8ePfThhx86nvv7+7u1HsEEAAAAAABwCAgIcPsoiVr1mqwSAAAAAADNlDFGleXVttQOCPKTZVn13n7nzp2Kj49XcHCw+vfvrxkzZugXv/iF+9rntiMDAAAAAABJUmV5teb+LtuW2r99YbACg+s3HaN///56/fXXdcEFF+jHH3/UU089pcsuu0xffvmloqOj3dI+ggkAAAAAACBJSk1Ndfx3z549NWDAAJ1//vlauHChMjIy3FKTYAIAAAAAADcLCPLTb18YbFttZ4WHh6tnz57auXOnC1tUG8EEAAAAAABuZllWvadTeJKysjJt375dV1xxhdtqOB+bAAAAAAAAn/KHP/xB2dnZysvL04YNG3TDDTfo2LFjSktLc1tNRkwAAAAAAABJ0vfff69bb71Vhw4dUtu2bXXppZdq/fr16tixo9tqEkwAAAAAAABJ0pIlS5q8JlM5AAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgm4BH+/ve/q2fPngoNDVV0dLSGDh2qoqIiSdL48eM1evRoPf7444qJiVHLli111113qby83LH/ypUrNXDgQLVq1UrR0dEaOXKkvvnmm1o1vv/+e91yyy2KiopSeHi4+vbtqw0bNjjW/+Mf/1CfPn0UEhKiX/ziF3r88cdVWVnplvM1xmjo0KEaPny4jDGSpKNHjyohIUEPP/ywW2oCsE9z6+N2794tPz8/bdq0qdbyl156SR07dnT0ewB8S3Pr6ySpU6dOsizrtAfg7X744Qf9+te/VnR0tMLCwnTRRRcpJyfHbfUIJmC7/fv369Zbb9XEiRO1fft2ZWVlacyYMbX+cP3Xv/6l7du36+OPP9bixYuVmZmpxx9/3LG+qKhIGRkZ2rhxo/71r3/Jz89P1113naqrqyVJhYWFGjx4sPbt26cVK1boP//5j+6//37H+lWrVunXv/61pkyZom3btunVV1/VggUL9PTTT5+x3W+99ZZatGhx1sdbb71V576WZWnhwoX67LPP9OKLL0qS7r77brVr107Tp09v7EsKwIM0xz6uU6dOGjp0qObPn19r+fz58zV+/Hj+aAd8UHPs6yRp48aN2r9/v/bv36/vv/9el156qa644orGvpyArY4cOaLLL79cgYGBev/997Vt2zY999xzatWqlfuKGh+RnZ1tRo4caeLi4owkk5mZaXu9tLQ0I6nWo3///m5tlzfKyckxkszu3bvrXJ+WlmaioqJMUVGRY9mcOXNMixYtTFVVVZ375OfnG0lm69atxhhjXn31VRMREWEOHz5c5/ZXXHGFmTFjRq1lb7zxhomLiztju48dO2Z27tx51sexY8fOeu5//etfTXBwsHnooYdMWFiY2bFjx1m3B+B9mmsft3TpUtO6dWtTWlpqjDEmNzfXWJZl8vLyzrgPAO/VXPu6k02ZMsV07NjR5Ofn12t7+L6SkhKzbds2U1JSYndTGuSBBx4wAwcOdOkxz/VaBLgv8mhaRUVF6tWrlyZMmKDrr7/eY+oNHz681idGQUFBbm+bt+nVq5euuuoq9ezZU8OGDVNKSopuuOEGtW7dutY2YWFhjucDBgxQYWGh9u7dq44dO+qbb77RI488ovXr1+vQoUOO5HzPnj1KSkpSbm6uevfuraioqDrbkJOTo40bN9ZK1KuqqlRaWqri4uJatWtEREQoIiKiUed+4403KjMzUzNnztScOXN0wQUXNOp4ADxPc+3jRo8erUmTJikzM1O33HKL/vKXv2jIkCHq1KmT08cE4Lmaa19XY+7cuZo3b57+/e9/q23bto0+HnyTMUaVZWW21A4IDq73iMUVK1Zo2LBhuvHGG5Wdna3zzjtP99xzj37zm9+4r31uO3ITS01NVWpq6hnXl5eX649//KPeeustHT16VElJSXr22WeVnJzslno1goODFRsb61SN5sLf31+rV6/WunXr9MEHH+ill17Sww8/rA0bNigxMfGs+9b8co0aNUodOnTQa6+9pvj4eFVXVyspKckxbzE0NPSsx6murtbjjz+uMWPGnLYuJCSkzn3eeust3XXXXWc97quvvqqxY8eecX1xcbFycnLk7++vnTt3nvVYALxTc+3jgoKCNG7cOM2fP19jxozRokWLNGvWrLMeD4D3aq59nSRlZWVp8uTJWrx4sXr16nXWY6F5qywr04tpN9hSe8rCvyvwDL8Hp/r22281Z84cZWRkaNq0afrss880ZcoUBQcH6/bbb3dL+3wmmDiXCRMmaPfu3VqyZIni4+OVmZmp4cOHa+vWrercubPb6mZlZSkmJkatWrXS4MGD9fTTTysmJsZt9byVZVm6/PLLdfnll+vRRx9Vx44dlZmZqYyMDEnSf/7zH5WUlDguSOvXr1eLFi3Uvn17HT58WNu3b9err77qmNO3du3aWse/8MIL9X//93/66aef6kzZL774Yu3YsUO//OUv693ma665Rv379z/rNu3atTvr+nvvvVd+fn56//33dfXVV2vEiBG68sor690GAN6hufZxd955p5KSkvTyyy+roqKizjcLAHxHc+zrdu3apeuvv17Tpk2jj4PPqK6uVt++fTVjxgxJUu/evfXll19qzpw5BBON8c0332jx4sX6/vvvFR8fL0n6wx/+oJUrV2r+/PmOF9zVUlNTdeONN6pjx47Ky8vTI488oiuvvFI5OTkKDg52S01vtGHDBv3rX/9SSkqKYmJitGHDBh08eFDdunVzbFNeXq477rhDf/zjH/Xdd9/pscce06RJk+Tn56fWrVsrOjpac+fOVVxcnPbs2aMHH3ywVo1bb71VM2bM0OjRozVz5kzFxcXp888/V3x8vAYMGKBHH31UI0eOVIcOHXTjjTfKz89PW7Zs0datW/XUU0/V2e7GDv375z//qb/85S/69NNPdfHFF+vBBx9UWlqatmzZUmvYIwDv1lz7OEnq1q2bLr30Uj3wwAOaOHHiOT/tBOC9mmNfV1JSolGjRumiiy7Sb3/7Wx04cMCxjhHTqEtAcLCmLPy7bbXrKy4uTt27d6+1rFu3bnr77bdd3ayfufSOFh5Cp9yM8q9//auRZMLDw2s9AgICzE033WSMMSYvL++0G1We+khPT69XvTPZt2+fCQwMNG+//bYrTtNnbNu2zQwbNsy0bdvWBAcHmwsuuMC89NJLjvVpaWnm2muvNY8++qiJjo42LVq0MHfeeafjhmrGGLN69WrTrVs3ExwcbC688EKTlZV12s9l9+7d5vrrrzctW7Y0YWFhpm/fvmbDhg2O9StXrjSXXXaZCQ0NNS1btjSXXHKJmTt3rlvOOT8/37Rr167WDZoqKirMJZdc4vg3CcA3NMc+7mTz5s0zksxnn33m9loA7NMc+7qzvX8AjPHem1/eeuutp938curUqWbAgAFOH/Ncr4VljO99mbhlWcrMzNTo0aMlSUuXLtXYsWP15Zdfyt/fv9a2LVq0UGxsrCoqKk77nuRTtW7dus6hXKfWO5vOnTvrzjvv1AMPPFDv82nuxo8fr6NHj2rZsmV2NwUAXM7X+7inn35aS5Ys0datW+1uCgAb+XpfB9SltLRUeXl5SkxMPON9TjzRxo0bddlll+nxxx/XTTfdpM8++0y/+c1vNHfu3LPeb+VszvVaNIupHL1791ZVVZXy8/PP+L3CgYGB6tq1q1vbcfjwYe3du1dxcXFurQMAgN0KCwu1fft2vfTSS3ryySftbg4AAKinfv36KTMzUw899JCeeOIJJSYmatasWU6HEvXhM8FEYWGhdu3a5Xiel5en3NxcRUVF6YILLtDYsWN1++2367nnnlPv3r116NAhffTRR+rZs6euvvpql9ZLSEhQYWGhpk+fruuvv15xcXHavXu3pk2bpjZt2ui6665zyTkDAOCpJk2apMWLF2v06NGaOHGi3c0BAAANMHLkSI0cObLJ6vnMVI6srCwNGTLktOVpaWlasGCBKioq9NRTT+n111/XDz/8oOjoaA0YMECPP/64evbs6fJ6JSUlGj16tD7//HMdPXpUcXFxGjJkiJ588kl16NDBqXMEAAAAAHgHb53K4Q7nei18JpgAAAAAAMBTEEz87FyvhZ8NbQIAAAAAAJDk5feYqK6u1r59+xQRESHLsuxuDgDIGKPjx48rPj5efn6Nz37p5wB4Gvo5AL7O1f0czs2rg4l9+/ZxvwYAHmnv3r1q3759o49DPwfAU9HPAfB1rurncG5eHUxERERIOvEPpmXLlja3BgCkY8eOqUOHDo7+qbHo5wB4Gvo5AL7O1f0czs2rg4ma4X4tW7bkQgbAo7hqODL9HABPRT8HwNcxvazpMGEGAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAABIkjp16iTLsk57pKenu62mV9/8EgAAAAAAuM7GjRtVVVXleP7FF1/oV7/6lW688Ua31SSYAAAA8EHHjh1TSUnJObfz9/dXmzZtmqBFAABv0LZt21rPn3nmGZ1//vkaPHiw22oSTAAAAPiY7777TvPnz6/Xtq1atdLUqVPd2yAAgIwxMhXVttS2Av2c+vrT8vJyvfnmm8rIyHDr16cSTAAAAPiY/Px8SSdGQwQHB59127CwsKZoEgA0e6aiWvseXWdL7fgnLpMV5N/g/ZYtW6ajR49q/Pjxrm/USQgmAAAAfEx19YlP5Lp06aKbbrrJ5tYAALzVvHnzlJqaqvj4eLfWIZgAAADwMTXBhJ8fX8AGAJ7CCvRT/BOX2Va7ob777jt9+OGHeuedd9zQotoIJgAAAHyMMUYSwQQAeBLLspyaTmGX+fPnKyYmRiNGjHB7La5WAAAAPqZmxIQ7b1QGAPBd1dXVmj9/vtLS0hQQ4P7xDAQTAAAAPoapHACAxvjwww+1Z88eTZw4sUnqMZUDAADAxzCVAwDQGCkpKY5rSVPgagUAAOBjmMoBAPAmBBMAAAA+hqkcAABvwtUKAADAxzCVAwDgTbhaAQAA+BimcgAAvAnBBAAAgI9hxAQAwJtwtQIAAPAx3GMCAOBNuFoBAAD4GKZyAAC8CcEEAACAj2EqBwDAm3C1AgAA8DFM5QAAeBOuVgAAAD6GqRwAAG9CMAEAAOBjmMoBAHBWZWWl/vjHPyoxMVGhoaH6xS9+oSeeeMIRertDgNuODAAAAFswlQMA4Kxnn31Wr7zyihYuXKgePXpo06ZNmjBhgiIjI/W73/3OLTUJJgAAAHwMUzkAwPMYY1RRUWFL7cDAwHpfEz799FNde+21GjFihCSpU6dOWrx4sTZt2uS29hFMAAAA+BimcgCA56moqNCMGTNsqT1t2jQFBQXVa9uBAwfqlVde0ddff60LLrhA//nPf7R27VrNmjXLbe0jmAAAAPAxjJgAADjrgQceUEFBgbp27Sp/f39VVVXp6aef1q233uq2mgQTAAAAPoZ7TACA5wkMDNS0adNsq11fS5cu1ZtvvqlFixapR48eys3N1dSpUxUfH6+0tDS3tI9gAgAAwMcwlQMAPI9lWfWeTmGn++67Tw8++KBuueUWSVLPnj313XffaebMmW4LJmy/Wv3www/69a9/rejoaIWFhemiiy5STk6O3c0CAADwWkzlAAA4q7i4+LRg29/f33e/LvTIkSO6/PLLNWTIEL3//vuKiYnRN998o1atWtnZLAAAAK/GVA4AgLNGjRqlp59+WgkJCerRo4c+//xzPf/885o4caLbatoaTDz77LPq0KGD5s+f71jWqVOnM25fVlamsrIyx/Njx465s3kA0OTo5wC4gidP5aCfAwDP9tJLL+mRRx7RPffco/z8fMXHx+uuu+7So48+6raatl6tVqxYob59++rGG29UTEyMevfurddee+2M28+cOVORkZGOR4cOHZqwtQDgfvRzAFzBk6dy0M8BgGeLiIjQrFmz9N1336mkpETffPONnnrqKbfeH8PWYOLbb7/VnDlz1LlzZ61atUp33323pkyZotdff73O7R966CEVFBQ4Hnv37m3iFgOAe9HPAXAFT57KQT8HADiVrVM5qqur1bdvX82YMUOS1Lt3b3355ZeaM2eObr/99tO2Dw4OVnBwcFM3EwCaDP0cAFfw5Kkc9HMAgFPZerWKi4tT9+7day3r1q2b9uzZY1OLAAAAvJ8nT+UAAOBUtgYTl19+uXbs2FFr2ddff62OHTva1CIAAADv58lTOQAAOJWtV6vf//73Wr9+vWbMmKFdu3Zp0aJFmjt3rtLT0+1sFgAAgFfz5KkcAACcytarVb9+/ZSZmanFixcrKSlJTz75pGbNmqWxY8fa2SwAAACvxlQOAIA3sfXml5I0cuRIjRw50u5mAAAA+AymcgAAvAlXKwAAAB/DVA4AgDfhagUAAOBjmMoBAPAmBBMAAAA+hqkcAABvwtUKAADAxzCVAwDQGMePH9fUqVPVsWNHhYaG6rLLLtPGjRvdVo+rFQAAgI9hKgcAoDHuvPNOrV69Wm+88Ya2bt2qlJQUDR06VD/88INb6tn+rRwAAABwLaZyAIDnMcaourrEltp+fqH1DqtLSkr09ttva/ny5Ro0aJAkafr06Vq2bJnmzJmjp556yuXtI5gAAADwMUzlAADPU11doqzsnrbUTh68Vf7+YfXatrKyUlVVVQoJCam1PDQ0VGvXrnVH85jKAQAA4GuYygEAcFZERIQGDBigJ598Uvv27VNVVZXefPNNbdiwQfv373dLTUZMAAAA+BimcgCA5/HzC1Xy4K221W6IN954QxMnTtR5550nf39/XXzxxbrtttu0efNmt7SPYAIAAMDHMJUDADyPZVn1nk5ht/PPP1/Z2dkqKirSsWPHFBcXp5tvvlmJiYluqcfVCgAAwMcwlQMA4Arh4eGKi4vTkSNHtGrVKl177bVuqcOICQAAAB9SE0pIjJgAADhn1apVMsaoS5cu2rVrl+677z516dJFEyZMcEs9rlYAAAA+pGYah0QwAQBwTkFBgdLT09W1a1fdfvvtGjhwoD744AMFBga6pR4jJgAAAHzIySMmmMoBAHDGTTfdpJtuuqnJ6hGjAwAA+BCmcgAAvA1XKwAAAB/CVA4AgLfhagUAAOBDmMoBAPA2BBMAAAA+hKkcAABvw9UKAADAh9RM5bAsixETAACvQDABAADgQ2pGTBBKAAC8BcEEAACAD6kZMcE0DgCAt+CKBQAA4ENqRkwQTAAAvAVXLAAAAB/CVA4AgLchmAAAAPAhTOUAAHgbrlgAAAA+hKkcAIDGWrNmjUaNGqX4+HhZlqVly5bVWm+M0fTp0xUfH6/Q0FAlJyfryy+/dLoeVywAAAAfwlQOAEBjFRUVqVevXpo9e3ad6//0pz/p+eef1+zZs7Vx40bFxsbqV7/6lY4fP+5UvYDGNBYAAACehakcAOCZjDEq/m943NTC/PwaFFinpqYqNTW1znXGGM2aNUsPP/ywxowZI0lauHCh2rVrp0WLFumuu+5qcPsIJgAAAHwIUzkAwDMVV1fr/DVbban9zaCeCvf3d8mx8vLydODAAaWkpDiWBQcHa/DgwVq3bp1TwQRXLAAAAB/CVA4AgDsdOHBAktSuXbtay9u1a+dY11CMmAAAAPAhTOUAAM8U5uenbwb1tK22q50agBtjnA7FCSYAAAB8CCMmAMAzWZblsukUdoqNjZV0YuREXFycY3l+fv5poyjqiygdAADAh3CPCQCAOyUmJio2NlarV692LCsvL1d2drYuu+wyp47JiAkAAAAfwlQOAEBjFRYWateuXY7neXl5ys3NVVRUlBISEjR16lTNmDFDnTt3VufOnTVjxgyFhYXptttuc6oewQQAAIAPYSoHAKCxNm3apCFDhjieZ2RkSJLS0tK0YMEC3X///SopKdE999yjI0eOqH///vrggw8UERHhVD2CCQAAAB/CVA4AQGMlJyc7RuDVxbIsTZ8+XdOnT3dJPa5YAAAAPoSpHAAAb8MVCwAAwIcwlQMA4G0IJgAAAHwIUzkAAN6GKxYAAIAPYSoHAMDbeMwVa+bMmbIsS1OnTrW7KQAAAF6LqRwAAG/jEcHExo0bNXfuXF144YV2NwUAAMCrMZUDAOBtbL9iFRYWauzYsXrttdfUunXrs25bVlamY8eO1XoAgC+hnwPQWJ4+lYN+DgBwKtuvWOnp6RoxYoSGDh16zm1nzpypyMhIx6NDhw5N0EIAaDr0cwAay9OnctDPAQBOZWswsWTJEm3evFkzZ86s1/YPPfSQCgoKHI+9e/e6uYUA0LTo5wA0lqdP5aCfAwCcKsCuwnv37tXvfvc7ffDBBwoJCanXPsHBwQoODnZzywDAPvRzABrL06dy0M8BAE5l2xUrJydH+fn56tOnjwICAhQQEKDs7Gy9+OKLCggIUFVVlV1NAwAA8FqePpUDAOD51qxZo1GjRik+Pl6WZWnZsmW11r/zzjsaNmyY2rRpI8uylJub26h6tgUTV111lbZu3arc3FzHo2/fvho7dqxyc3Pl7+9vV9MAAAC8lqdP5QAAeL6ioiL16tVLs2fPPuP6yy+/XM8884xL6tk2lSMiIkJJSUm1loWHhys6Ovq05QAAAKgfT5/KAQDNlTFGJRX2zAwIDfRv0Ei61NRUpaamnnH9uHHjJEm7d+9ubNMk2RhMAAAAwPWYygEAnqmkokrdH11lS+1tTwxTWJDnvv33qJZlZWXZ3QQAAACvxlQOAIC38ahgAgAAAI3DVA4A8Eyhgf7a9sQw22p7MoIJAAAAH8JUDgDwTJZlefR0CjvxqgBoclVVVfriiy9UVFRkd1Pq5dJLL+WTR8BL7Nu3z2U34vJW33//vSRGTAAAvAfBBIAmt2PHDmVmZtrdjHrr37+/3U0AUE9vvfWW14Se7hYYGGh3EwAAXqqwsFC7du1yPM/Ly1Nubq6ioqKUkJCgn376SXv27NG+ffsknfj7XpJiY2MVGxvb4HoEEwCaXHFxsSSpZcuW6tSpk72NAeBTakKJbt26Nes35kFBQerTp4/dzQAAeKlNmzZpyJAhjucZGRmSpLS0NC1YsEArVqzQhAkTHOtvueUWSdJjjz2m6dOnN7gewQSAJlcz//m8887TmDFjbG4NAF9Rc9NHSRo5cqTCw8NtbA0AAN4rOTm51nX1VOPHj9f48eNdVq9RwUR5ebny8/MdbzJqJCQkNKpRAHwbX2UHwB1O/nuEGz8CAOA9nAomdu7cqYkTJ2rdunW1lhtjZFmWqqqqXNI4AL6Jr7ID4A4nBxP0LwAAeA+ngonx48crICBA7777ruLi4vhUAkCD8FV2ANzh5CGnBBMAAHgPp4KJ3Nxc5eTkqGvXrq5uD4BmgBETANyBqRwAAHgnp94VdO/eXYcOHXJ1WwA0E9xjAoA7MJUDAADvVO+r9rFjxxyPZ599Vvfff7+ysrJ0+PDhWuuOHTvmzvYC8AFM5QDgDkzlAADAO9V7KkerVq1qvYkwxuiqq66qtQ03vwRQH0zlAOAOTOUAAMA71TuY+PjjjyVJZWVlmjFjhm699VbuMQHAKUzlAOAO9C0AAHinegcTgwcPdvz32LFjdeWVV6pz585uaRQA38ZUDgDuwGgsAAC8k1NX7ttvv13z5s1zdVsANBO8eQDgDoSeAAB4J6e+LrS8vFz/93//p9WrV6tv374KDw+vtf755593SeMA+CaGWwNwB/oWAABcY82aNfrzn/+snJwc7d+/X5mZmRo9erQkqaKiQn/84x/13nvv6dtvv1VkZKSGDh2qZ555RvHx8U7VcyqY+OKLL3TxxRdLkr7++uta6/iUAsC58KkmAHdgNBYAAK5RVFSkXr16acKECbr++utrrSsuLtbmzZv1yCOPqFevXjpy5IimTp2qa665Rps2bXKqnlPBRM2NMAHAGbx5AOAOhJ4AAI9mjFRRbE/twDCpAdfH1NRUpaam1rkuMjJSq1evrrXspZde0iWXXKI9e/YoISGhwc1zKpgAgMbgzQMAdyD0BAB4tIpiaYZzUx0abdo+KSj83Ns5qaCgQJZlqVWrVk7tz5UbQJNjHjgAd6BvAQCg6ZWWlurBBx/UbbfdppYtWzp1DEZMAGhyfKoJwB0YjQUA8GiBYSdGLthV2w0qKip0yy23qLq6Wi+//LLTxyGYANDkePMAwB0IPQEAHs2y3DqdoqlVVFTopptuUl5enj766COnR0tIBBMAbMBwawDuQN8CAEDTqAkldu7cqY8//ljR0dGNOh7BBIAmx6eaANyB0VgAALhGYWGhdu3a5Xiel5en3NxcRUVFKT4+XjfccIM2b96sd999V1VVVTpw4IAkKSoqSkFBQQ2uRzABoMnx5gGAOxB6AgDgGps2bdKQIUMczzMyMiRJaWlpmj59ulasWCFJuuiii2rt9/HHHys5ObnB9QgmADQ5hlsDcAdCTwAAXCM5OdkR+NflbOucwbsCAE2OTzUBuAOhJwAA3okrN4Amx6eaANyB0BMAAO/ElRtAk+NTTQDuQOgJAIB34l0BgCbHp5oA3IHQEwAA78SVG0CT41NNAO5A6AkAgHfiyg2gyfGpJgB3IPQEAMA78a4AQJPjU00A7kDoCQCAd+LKDaDJ8akmAHcg9AQAwDtx5QbQ5PhUE4A7EHoCAOCdeFcAoMnxqSYAdyD0BADAO3HlBtDk+FQTgDsQegIA4Bpr1qzRqFGjFB8fL8uytGzZslrrp0+frq5duyo8PFytW7fW0KFDtWHDBqfrceUG0OT4VBOAOxB6AgDgGkVFRerVq5dmz55d5/oLLrhAs2fP1tatW7V27Vp16tRJKSkpOnjwoFP1AhrT2MaaOXOm3nnnHX311VcKDQ3VZZddpmeffVZdunSxs1kA3IxPNQG4A6EnAMCTGWNUUlliS+3QgNAGBfepqalKTU094/rbbrut1vPnn39e8+bN05YtW3TVVVc1uH22BhPZ2dlKT09Xv379VFlZqYcfflgpKSnatm2bwsPD7WwaADfiU00A7kDoCQDwZCWVJeq/qL8ttTfctkFhgWFuOXZ5ebnmzp2ryMhI9erVy6lj2BpMrFy5stbz+fPnKyYmRjk5ORo0aNBp25eVlamsrMzx/NixY25vIwDX483DmdHPAc4j9PQO9HMA4Bveffdd3XLLLSouLlZcXJxWr16tNm3aOHUsW4OJUxUUFEiSoqKi6lw/c+ZMPf74403ZJABuwHDrM6OfA5xH3+Id6OcANFehAaHacJvzN4hsbG1XGzJkiHJzc3Xo0CG99tpruummm7RhwwbFxMQ0+Fgec+U2xigjI0MDBw5UUlJSnds89NBDKigocDz27t3bxK0E4Ap8qnlm9HOA8xiN5R3o5wA0V5ZlKSwwzJaHO/7uDg8P1y9/+UtdeumlmjdvngICAjRv3jynjuUxIyYmTZqkLVu2aO3atWfcJjg4WMHBwU3YKgDuwJuHM6OfA5xH6Okd6OcAwDcZY2pN1WsIjwgmJk+erBUrVmjNmjVq37693c0B4GYMtwbgDoSeAAC4RmFhoXbt2uV4npeXp9zcXEVFRSk6OlpPP/20rrnmGsXFxenw4cN6+eWX9f333+vGG290qp6twYQxRpMnT1ZmZqaysrKUmJhoZ3MANEBZWZkqKioavF9QUBCfagJwC0JPAABcY9OmTRoyZIjjeUZGhiQpLS1Nr7zyir766istXLhQhw4dUnR0tPr166dPPvlEPXr0cKqercFEenq6Fi1apOXLlysiIkIHDhyQJEVGRio01PU35wDgGnl5eXrzzTdVVVXV4H0DAgJUWVkpiTcPAFyL0BMAANdITk52jESsyzvvvOPSera+K5gzZ44KCgqUnJysuLg4x2Pp0qV2NgvAOezdu9epUEKSI5SQCCYAuBZTOQAA8E62T+UA4H1qQom+fftq5MiR9d7v7bff1tatWx3P+VQTgCsxlQMAAO/ElRtAg9UEE/7+/g3a79TtefMAwJWYygEAgHfiXQGABqsJJgICGjboimACgDsxlQMAAO/ElRtAg9XcJ6KxIyb4VBOAKzFiAgAA70QwAaDBnJ3KceoICz7VBOBK3GMCAADvxJUbQIMxlQOAJ2IqBwAA3okrN4AGYyoHAE/EVA4AALwTwQSABmMqBwBPxFQOAAC8E1duAA3mqqkcfKoJwJWYygEAgHfiyg2gwVwxlcOyLIIJAC7FVA4AAFxjzZo1GjVqlOLj42VZlpYtW3bGbe+66y5ZlqVZs2Y5XY9gAkCDuWIqB59oAnA1pnIAAOAaRUVF6tWrl2bPnn3W7ZYtW6YNGzYoPj6+UfUaNg4bAOSaqRy8cQDgakzlAAB4MmOMTEmJLbWt0NAGjShMTU1VamrqWbf54YcfNGnSJK1atUojRoxoVPsIJgA0mKumcgCAKzGVAwDgyUxJiXZc3MeW2l0258gKC3PZ8aqrqzVu3Djdd9996tGjR6OPx0cKABqMqRwAPBFTOQAAaBrPPvusAgICNGXKFJccjxETABrMFVM5+EQTgKsxlQMA4Mms0FB12ZxjW21XycnJ0QsvvKDNmze77G96ggkADeaKqRy8cQDgakzlAAB4MsuyXDqdwi6ffPKJ8vPzlZCQ4FhWVVWle++9V7NmzdLu3bsbfEyCCQAN5uxUDoIJAO7EVA4AANxv3LhxGjp0aK1lw4YN07hx4zRhwgSnjkkwAaDBXHGPCT7RBOBqTOUAAMA1CgsLtWvXLsfzvLw85ebmKioqSgkJCYqOjq61fWBgoGJjY9WlSxen6hFMAGgwvi4UgCdiKgcAAK6xadMmDRkyxPE8IyNDkpSWlqYFCxa4vB7BBIAGMcZwjwkAHompHAAAuEZycrJjJGJ9OHNfiZMRTACSKioq9MYbb+jw4cN2N8UlwsPDddttt6lVq1YuP3bNH/4SUzkAeBamcgAA4J0IJgBJ+/bt0549e+xuhssUFRXpm2++UZ8+fVx+7JppHBJTOQB4FqZyAADgnQgmAEllZWWSpLZt2+qGG26wuTWN8+GHH2rnzp2Oc3K1mmkcUuOmcvDGAYCrMWICAADvRDAB6OdgIjw8XO3atbO5NY0TGRkpSSovL3fL8U8eMdHQP/5PHmFx8pQQAHAF7jEBAIB34soN6Oc38UFBQTa3pPFqzsFdIyZO/kaOho56OHnERENupgMA9cFUDgAAvBPBBKCf38QHBwfb3JLGqzkHd42YcPYbOU7dhxETAFyNqRwAAHgnrtyAfg4mGDFxbjUjJpwJJk7+FJNgAoCrMZUDAADvxJUb0M+jC3xpxERTTOVoDKZyAHA1pnIAAOCdCCYAMZWjIRozleNkjJgA4GpM5QAAwDtx5QbEzS8bojFTOU5GMAHA1RgxAQCAdyKYAOSbIyaYygGgueEeEwAAuMaaNWs0atQoxcfHy7IsLVu2rNb68ePHy7KsWo9LL73U6XpcuQH5VjBRM2LCXVM5GDEBwFMxlQMAANcoKipSr169NHv27DNuM3z4cO3fv9/xeO+995yu17iPPAEf4UtTOdw9YoJ7TADwVEzlAAB4MmOMKsvt+Rs4IMivQdfH1NRUpaamnnWb4OBgxcbGNrZpkggmAEm+NWKi5hwqKytVVVXV6ADhVEzlAOCJjDGMmAAAeLTK8mrN/V22LbV/+8JgBQa79n1BVlaWYmJi1KpVKw0ePFhPP/20YmJinDoWV25AvhVMnDzqwx3TOZjKAcATnRx2EkwAAOBeqampeuutt/TRRx/pueee08aNG3XllVc6PWqbEROAfGsqR0BAgPz9/VVVVaWysjKFhoa69PhM5QDgiU7uU5jKAQDwRAFBfvrtC4Ntq+1KN998s+O/k5KS1LdvX3Xs2FH//Oc/NWbMmAYfj2ACzV5VVZXjzbYvjJiQTgQsJSUlbh0x0dipHADgSicHE4yYAAB4IsuyXD6dwlPExcWpY8eO2rlzp1P7884Czd7Jw418YcSEdCJgKSkpccsNMF01lQMAXImpHAAkqeLHH3Vk8WJVHzt+zm39WkYoZupU9zcKaAYOHz6svXv3Ki4uzqn9CSbQ7NWMKvD39/eZUQDu/GYOV03lAABXYioH4PmOvfeeDr40W9WlpW6rUfXTTzL1/PsnoF07ggngDAoLC7Vr1y7H87y8POXm5ioqKkpRUVGaPn26rr/+esXFxWn37t2aNm2a2rRpo+uuu86per7xLgxoBF+68WWNmpEfTOUA0FwwlaP5qfjhBxX84x8ylVV2NwX1YMrKdHj+fOm/H3C4U0ivC9Xi8svPuZ1feAu3twXwVps2bdKQIUMczzMyMiRJaWlpmjNnjrZu3arXX39dR48eVVxcnIYMGaKlS5cqIiLCqXq2v7N4+eWX9ec//1n79+9Xjx49NGvWLF1xxRV2NwvNiC/d+LKGO0dMMJUDgCc6eSoHIyaah33THlbxhg12NwMN1PLqVEVNvMNtx/cLDlLQ+efLIqAEGiU5ObnWtfVUq1atcmk9W4OJpUuXaurUqXr55Zd1+eWX69VXX1Vqaqq2bdumhIQEt9Q89OOPWvuvt91ybNSPv58l+UvykL8bjx4tVGXlQRUfP65//H1OvfYJk7+soBayrLovesFW4M+nFxggnfJHcqC/JeMfouAAy/EHtPELOW07ZxUXFqhaVfp2xzb5l5W45Jg19v+wV9Wq0qED+7Vl/boG71+tnz/Zcmb/YDXsZfIPkAKCghr1RqXdL3soOCTE6f2b0uH8fH3y0d/tbkaz5u9nnejfvPxv4jAFyAo+Sz+ngJ9/r4Lq2c/5u66fO1VxWcmJ/sWytDVnvVtqNJVgq+n7udhOPRTkJSMHC44c0X+WLVHbDRtkLEvFl/aV/Dzkj4omYaT/vlewdKY3DXUtN2dZfY7tT35es9g6+f/OsO0pz6tbtdShyxKU981HqjZG5r+nYlQtY4ys/74JMv9deuJN0X8ftdbV/JeRdVI1Y6pPbLe5Zm3t+ka1WxYkf1lWoE7/o9TIT5Yi/cJPe1VOFugnWVXlalH5kyxLKo7uoYqIjmfdByc05O85/4ATgbN/YGCz+XuuObI1mHj++ed1xx136M4775QkzZo1S6tWrdKcOXM0c+bM07YvKyur9QnwsWPHGlxz+dv/p8tefNP5RsMn9XViny437JdfwJlTxHPpVvoXpe1+Q4HGPUMaIyXt3v65drvp2Ae2f64D7zm3b43V2z93VZPOqufEr+Qf6PzPSvqrEpL6uKw9Z9PYfu6Tj/6uLtNfcnWz0Ex5ej93qpr+ZfW2zU1Sz5M0vp/7mxK6XOyy9pxNY/u5nPUfKffDl5QqaWNn6X+Tm+ZaAhcp+szuFjhs2L1XYWf5RLg+TvRza/7bz+W6pF2omzf9PYeGs+3znPLycuXk5CglJaXW8pSUFK1bV/enqDNnzlRkZKTj0aFDh4bXLW2aP44AwBmu6OcAwJM1up+rqNCgL068OVl1cXMaKQEAvssyZ5s44kb79u3Teeedp3//+9+67LLLHMtnzJihhQsXaseOHaftU1fC3qFDBxUUFKhly5b1qrt9yxat+/e7jT8BOM+yTkRiHvS3hCVLkS0iFBRYv6zO06dyVFVX6cChQ6qock8QFxwQqHZt2sjvDOd/NlXVVTp8pEDRrSPl79fw+1R4+lSOY8eOKTIyskH90ska288dzs/X+0sXqKzM9Tc+Rf1ZlmQsNayfK62WCk66JAdKirLvXi5hxk/yD5XO8HvurwBZNYO4AwJ06skGWVK1X5CC/CzH72yVdXp/6GptwqLUwkumJJxJsBr2T8ffTwoIbEQ/F2ApdlDvek/lsLufKzhyRC/+f4+p/bf7tPHibrX+TVl1vXL1fFka9OpZfqqWv8wpr/nJ0wp+XtjQn4tVx3+e63prqVz+KpOfglStln6nXv/r6pD+O+3sXGdunemJOcOulqPJNT+Pn//3xJKaf6snL699hLpeg9pPigMCtSaqgwoDnLtHWGJRge75/itZCjqt/s/lfl5e3jpSCqxdK9Dvv/2cv+Xurs02P+Yf1N79+11+XP/qqv/OwDrxt7BlSYH+gSqvrJBkFBocosrKKpVVlEtW9X//mfg5fQ05r107/Trtt03291yN0tJS5eXlKTExUSHNfBrJuV4L229+eepF1BhzxgtrcHBwo785oduFF6rbhRc26hgA4C6N7eeiY2L068n3u7BFaCqlu47q0P9tdTwP6RytNrd3t7FFgHs0tp+LbN1ajzzxoiRpgqsa5QMOHz6sl146MZUvMDDQJce0LEsDBgyodWd+T1JljEqqqs+9YR38LEth/s59rWFzU1FRUeubj05VUlKi7777ThUVFY5lLVq0UEJCggICAv4bPJz+qEtNnZpvVyovLz/rDRjry7Isn7rRvS+yLZho06aN/P39deDAgVrL8/Pz1a5dO5taBQCAPfwjg876HADOJjo6Wt27d9e2bdtqvUFsrLVr1+qSSy5RePjZbwRpB3/LUosAviXM3c4VdAUHB6tVq1YuqXXq1z0TJjQftgUTQUFB6tOnj1avXq3rrvs5rVy9erWuvfZau5oFAIAt/FsGnfU5AJzLDTfcoGPHjrnkE2ZJ+utf/6r9+/dr3bp16tmzp2N5y5YtFRYW5pIaACDZPJUjIyND48aNU9++fTVgwADNnTtXe/bs0d13321nswAAaHJ+wQGygv1lyk58pa5/S+++TwKApufn5+eyT64l6ZJLLtHy5cv173//W//+978dy6+55hpdfHHTfIsLgObB1mDi5ptv1uHDh/XEE09o//79SkpK0nvvvaeOHfn+XwBA8+PfMkiVB0sc/w0AdkpKStKWLVt08ODBWssDAmy/TR0AH2N7r3LPPffonnvusbsZAADYjmACgCcJDAxUWlqa3c0A0Aw0/Lv+AACAW5w8fYOpHAAAwC5r1qzRqFGjFB8fL8uytGzZstO22b59u6655hpFRkYqIiJCl156qfbs2eNUPYIJAAA8RM0oCSvQT1YId5oHAAD2KCoqUq9evTR79uw613/zzTcaOHCgunbtqqysLP3nP//RI488opCQEKfq2T6VAwAAnOD332DCv2XQGb/jHQAAeCdjjCrLymypHRAc3KC/LVJTU5WamnrG9Q8//LCuvvpq/elPf3Is+8UvfuF8+5zeEwAAuFRg2xNfvxfQJtTmlgAAAFerLCvTi2k32FJ7ysK/K9DJ0Qynqq6u1j//+U/df//9GjZsmD7//HMlJibqoYce0ujRo506JlM5AADwEMG/bKWo27qq1XW/tLspAAAAdcrPz1dhYaGeeeYZDR8+XB988IGuu+46jRkzRtnZ2U4dkxETAAB4CMvPUtiFbe1uBgAAcIOA4GBNWfh322q7SnV1tSTp2muv1e9//3tJ0kUXXaR169bplVde0eDBgxvePpe1DgAAAAAA1MmyLJdNp7BTmzZtFBAQoO7du9da3q1bN61du9apYzKVAwAAAAAA1EtQUJD69eunHTt21Fr+9ddfq2PHjk4dkxETAAAAAADAobCwULt27XI8z8vLU25urqKiopSQkKD77rtPN998swYNGqQhQ4Zo5cqV+sc//qGsrCyn6hFMAAAAAAAAh02bNmnIkCGO5xkZGZKktLQ0LViwQNddd51eeeUVzZw5U1OmTFGXLl309ttva+DAgU7VI5gAAAAAAAAOycnJMsacdZuJEydq4sSJLqnHPSYAAAAAAIBtvHrERE2Cc+zYMZtbAgAn1PRH50qY64t+DoCnoZ8D4Otc3c/h3Lw6mDh+/LgkqUOHDja3BABqO378uCIjI11yHIl+DoDnoZ8D4Otc1c/h3Lw6mIiPj9fevXsVEREhy7Lqvd+xY8fUoUMH7d27Vy1btnRjC+3DOfqO5nCevnSOxhgdP35c8fHxLjmeM/2cL72eZ9MczrM5nKPUPM7Tl86Rfq7pNIfz5Bx9hy+dp6v7OZybVwcTfn5+at++vdP7t2zZ0ut/ac6Fc/QdzeE8feUcXZmsN6af85XX81yaw3k2h3OUmsd5+so50s81reZwnpyj7/CV82SkRNPi5pcAAAAAAMA2BBMAAAAAAMA2zTKYCA4O1mOPPabg4GC7m+I2nKPvaA7n2RzOsSk1l9ezOZxnczhHqXmcZ3M4x6bUXF7P5nCenKPvaC7nCfewDN+BAgAAAACAS5WWliovL0+JiYkKCQmxuzm2Otdr0SxHTAAAAAAAAM9AMAEAAAAAABzWrFmjUaNGKT4+XpZladmyZbXWW5ZV5+PPf/6zU/UIJgAAAAAAgENRUZF69eql2bNn17l+//79tR5/+ctfZFmWrr/+eqfqBTSmsQAAAAAA4NyMMTIV1bbUtgL9ZFlWvbdPTU1VamrqGdfHxsbWer58+XINGTJEv/jFL5xqX7MbMfHyyy87brjRp08fffLJJ3Y3qU7Tp08/bVjMyT98Y4ymT5+u+Ph4hYaGKjk5WV9++WWtY5SVlWny5Mlq06aNwsPDdc011+j777+vtc2RI0c0btw4RUZGKjIyUuPGjdPRo0fddl7nGhLUlOe1Z88ejRo1SuHh4WrTpo2mTJmi8vJyt5/j+PHjT/vZXnrppV51jjNnzlS/fv0UERGhmJgYjR49Wjt27Ki1jS/8LL0V/Rz9XA36OefRz3k2+jn7+rnm0MfV5zzp57znPD2JqajWvkfX2fJwZyDy448/6p///KfuuOMOp4/RrIKJpUuXaurUqXr44Yf1+eef64orrlBqaqr27Nljd9Pq1KNHj1rDY7Zu3epY96c//UnPP/+8Zs+erY0bNyo2Nla/+tWvdPz4ccc2U6dOVWZmppYsWaK1a9eqsLBQI0eOVFVVlWOb2267Tbm5uVq5cqVWrlyp3NxcjRs3zm3ndK4hQU11XlVVVRoxYoSKioq0du1aLVmyRG+//bbuvfdet5+jJA0fPrzWz/a9996rtd7TzzE7O1vp6elav369Vq9ercrKSqWkpKioqMixjS/8LL0R/Rz9XA36ucahn/Nc9HP29nPNoY+rz3lK9HPecp5wv4ULFyoiIkJjxoxx/iCmGbnkkkvM3XffXWtZ165dzYMPPmhTi87sscceM7169apzXXV1tYmNjTXPPPOMY1lpaamJjIw0r7zyijHGmKNHj5rAwECzZMkSxzY//PCD8fPzMytXrjTGGLNt2zYjyaxfv96xzaeffmokma+++soNZ1WbJJOZmel43pTn9d577xk/Pz/zww8/OLZZvHixCQ4ONgUFBW47R2OMSUtLM9dee+0Z9/G2czTGmPz8fCPJZGdnG2N882fpLejn6Ofo5+jnfB39nOf0c82hj6vrPI2hn/Pmn2dTKikpMdu2bTMlJSXGmBOvaVVZpS2P6upqp8+jrt+Bk3Xp0sVMmjSpQa/FqZrNiIny8nLl5OQoJSWl1vKUlBStW7fOplad3c6dOxUfH6/ExETdcsst+vbbbyVJeXl5OnDgQK1zCQ4O1uDBgx3nkpOTo4qKilrbxMfHKykpybHNp59+qsjISPXv39+xzaWXXqrIyEhbXpOmPK9PP/1USUlJio+Pd2wzbNgwlZWVKScnx63nKUlZWVmKiYnRBRdcoN/85jfKz893rPPGcywoKJAkRUVFSWpeP0tPQj9HP+dJvxv0c955np6Ofs6z+7nm9ntBP+ed52kny7LkF+Rvy6Mh95doiE8++UQ7duzQnXfe2ajjNJtg4tChQ6qqqlK7du1qLW/Xrp0OHDhgU6vOrH///nr99de1atUqvfbaazpw4IAuu+wyHT582NHes53LgQMHFBQUpNatW591m5iYmNNqx8TE2PKaNOV5HThw4LQ6rVu3VlBQkNvPPTU1VW+99ZY++ugjPffcc9q4caOuvPJKlZWVOdrmTedojFFGRoYGDhyopKQkR+2aNp/tHLzpPL0B/Vzd29DP/Yx+zjn0c56Dfq7ubTyln2tOvxf0c77184Tz5s2bpz59+qhXr16NOk6z+1aOU5MiY4zb0qPGOPkOqD179tSAAQN0/vnna+HChY4b6zhzLqduU9f2dr8mTXVedp37zTff7PjvpKQk9e3bVx07dtQ///nPs87L8tRznDRpkrZs2aK1a9eets7Xf5aein7O8/99+PrvBv2c7/wsPRX9nGf/+2gOvxf0c77188TpCgsLtWvXLsfzvLw85ebmKioqSgkJCZKkY8eO6W9/+5uee+65RtdrNiMm2rRpI39//9MSt/z8/NPSOU8UHh6unj17aufOnY67OZ/tXGJjY1VeXq4jR46cdZsff/zxtFoHDx605TVpyvOKjY09rc6RI0dUUVHR5OceFxenjh07aufOnY62ecs5Tp48WStWrNDHH3+s9u3bO5Y315+l3ejn6t6Gfu5n9HMNRz/nWejn6t7GU/q55vx7QT/n+eeJhtm0aZN69+6t3r17S5IyMjLUu3dvPfroo45tlixZImOMbr311kbXazbBRFBQkPr06aPVq1fXWr569WpddtllNrWq/srKyrR9+3bFxcUpMTFRsbGxtc6lvLxc2dnZjnPp06ePAgMDa22zf/9+ffHFF45tBgwYoIKCAn322WeObTZs2KCCggJbXpOmPK8BAwboiy++0P79+x3bfPDBBwoODlafPn3cep6nOnz4sPbu3au4uDhJ3nGOxhhNmjRJ77zzjj766CMlJibWWt9cf5Z2o5+jn/PU3w36Oc89T29DP+fZ/Vxz/r2gn/Pc84RzkpOTZYw57bFgwQLHNr/97W9VXFysyMjIxhc8660zfcySJUtMYGCgmTdvntm2bZuZOnWqCQ8PN7t377a7aae59957TVZWlvn222/N+vXrzciRI01ERISjrc8884yJjIw077zzjtm6dau59dZbTVxcnDl27JjjGHfffbdp3769+fDDD83mzZvNlVdeaXr16mUqKysd2wwfPtxceOGF5tNPPzWffvqp6dmzpxk5cqTbzuv48ePm888/N59//rmRZJ5//nnz+eefm++++65Jz6uystIkJSWZq666ymzevNl8+OGHpn379ue8m2xjz/H48ePm3nvvNevWrTN5eXnm448/NgMGDDDnnXeeV53j//zP/5jIyEiTlZVl9u/f73gUFxc7tvGFn6U3op+jn6tBP9c49HOei37O3n6uOfRx5zpP+jnv+3na5VzfRNGcnOu1aFbBhDHG/L//9/9Mx44dTVBQkLn44osdX4fjaW6++WYTFxdnAgMDTXx8vBkzZoz58ssvHeurq6vNY489ZmJjY01wcLAZNGiQ2bp1a61jlJSUmEmTJpmoqCgTGhpqRo4cafbs2VNrm8OHD5uxY8eaiIgIExERYcaOHWuOHDnitvP6+OOPjaTTHmlpaU1+Xt99950ZMWKECQ0NNVFRUWbSpEmmtLTUredYXFxsUlJSTNu2bU1gYKBJSEgwaWlpp7Xf08+xrvOTZObPn+/Yxhd+lt6Kfo5+rgb9nPPo5zwb/Zx9/Vxz6OPOdZ70c97387QLwcTPzvVaWMYY0/hxFwAAAAAAoEZpaany8vKUmJiokJAQu5tjq3O9Fs3mHhMAAAAAAMDzEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAACHNWvWaNSoUYqPj5dlWVq2bFmt9YWFhZo0aZLat2+v0NBQdevWTXPmzHG6HsEEfF5WVpYsy9LRo0ftbgoAuAX9HABfRz8HNK2ioiL16tVLs2fPrnP973//e61cuVJvvvmmtm/frt///veaPHmyli9f7lQ9yxhjGtNgwNMkJyfroosu0qxZsyRJ5eXl+umnn9SuXTtZlmVv4wDABejnAPg6+jn4gtLSUuXl5SkxMVEhISEyxqiiosKWtgQGBjr9u2NZljIzMzV69GjHsqSkJN1888165JFHHMv69Omjq6++Wk8++eRpxzj1tThVgFMtA7xIUFCQYmNj7W4GALgN/RwAX0c/B19QUVGhGTNm2FJ72rRpCgoKctnxBg4cqBUrVmjixImKj49XVlaWvv76a73wwgtOHY+pHPAp48ePV3Z2tl544QVZliXLsrRgwYJaQ/8WLFigVq1a6d1331WXLl0UFhamG264QUVFRVq4cKE6deqk1q1ba/LkyaqqqnIcu7y8XPfff7/OO+88hYeHq3///srKyrLnRAE0W/RzAHwd/Rzg+V588UV1795d7du3V1BQkIYPH66XX35ZAwcOdOp4jJiAT3nhhRf09ddfKykpSU888YQk6csvvzxtu+LiYr344otasmSJjh8/rjFjxmjMmDFq1aqV3nvvPX377be6/vrrNXDgQN18882SpAkTJmj37t1asmSJ4uPjlZmZqeHDh2vr1q3q3Llzk54ngOaLfg6Ar6Ofg68KDAzUtGnTbKvtSi+++KLWr1+vFStWqGPHjlqzZo3uuecexcXFaejQoQ0+HsEEfEpkZKSCgoIUFhbmGO731VdfnbZdRUWF5syZo/PPP1+SdMMNN+iNN97Qjz/+qBYtWqh79+4aMmSIPv74Y91888365ptvtHjxYn3//feKj4+XJP3hD3/QypUrNX/+fNuGZAFofujnAPg6+jn4KsuyXDqdwi4lJSWaNm2aMjMzNWLECEnShRdeqNzcXP3v//4vwQRQX2FhYY6LmCS1a9dOnTp1UosWLWoty8/PlyRt3rxZxhhdcMEFtY5TVlam6Ojopmk0ADQA/RwAX0c/B9ijoqJCFRUV8vOrfWcIf39/VVdXO3VMggk0S6cOZbIsq85lNb9Y1dXV8vf3V05Ojvz9/Wttd/LFDwA8Bf0cAF9HPwe4T2FhoXbt2uV4npeXp9zcXEVFRSkhIUGDBw/Wfffdp9DQUHXs2FHZ2dl6/fXX9fzzzztVj2ACPicoKKjWTY5coXfv3qqqqlJ+fr6uuOIKlx4bABqKfg6Ar6OfA+y1adMmDRkyxPE8IyNDkpSWlqYFCxZoyZIleuihhzR27Fj99NNP6tixo55++mndfffdTtUjmIDP6dSpkzZs2KDdu3erRYsWTg8nOtkFF1ygsWPH6vbbb9dzzz2n3r1769ChQ/roo4/Us2dPXX311S5oOQDUD/0cAF9HPwfYKzk5WcaYM66PjY3V/PnzXVaPrwuFz/nDH/4gf39/de/eXW3bttWePXtcctz58+fr9ttv17333qsuXbrommuu0YYNG9ShQweXHB8A6ot+DoCvo58DmhfLnC0GAQAAAAAADVZaWqq8vDwlJiYqJCTE7ubY6lyvBSMmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAw5o1azRq1CjFx8fLsiwtW7as1voff/xR48ePV3x8vMLCwjR8+HDt3LnT6XoEEwAAAAAAwKGoqEi9evXS7NmzT1tnjNHo0aP17bffavny5fr888/VsWNHDR06VEVFRU7VC2hsgwEAAAAAwNkZY1RdXWJLbT+/UFmWVe/tU1NTlZqaWue6nTt3av369friiy/Uo0cPSdLLL7+smJgYLV68WHfeeWeD20cwAQAAAACAm1VXlygru6cttZMHb5W/f5hLjlVWViZJCgkJcSzz9/dXUFCQ1q5d61QwwVQOAAAAAABQL127dlXHjh310EMP6ciRIyovL9czzzyjAwcOaP/+/U4dkxETAAAAAAC4mZ9fqJIHb7WttqsEBgbq7bff1h133KGoqCj5+/tr6NChZ5z6UR8EEwAAAAAAuJllWS6bTmG3Pn36KDc3VwUFBSovL1fbtm3Vv39/9e3b16njMZUDAAAAAAA0WGRkpNq2baudO3dq06ZNuvbaa506DiMmAAAAAACAQ2FhoXbt2uV4npeXp9zcXEVFRSkhIUF/+9vf1LZtWyUkJGjr1q363e9+p9GjRyslJcWpegQTAAAAAADAYdOmTRoyZIjjeUZGhiQpLS1NCxYs0P79+5WRkaEff/xRcXFxuv322/XII484Xc8yxphGtxoAAAAAADiUlpYqLy9PiYmJtb5aszk612vBPSYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIAkaebMmerXr58iIiIUExOj0aNHa8eOHbW2McZo+vTpio+PV2hoqJKTk/Xll186XZNgAgAAAAAASJKys7OVnp6u9evXa/Xq1aqsrFRKSoqKiooc2/zpT3/S888/r9mzZ2vjxo2KjY3Vr371Kx0/ftypmpYxxrjqBAAAAAAAgFRaWqq8vDwlJiYqJCTE7uY47eDBg4qJiVF2drYGDRokY4zi4+M1depUPfDAA5KksrIytWvXTs8++6zuuuuu045xrtciwO1nAQAAAABAM2eMUXF1tS21w/z8ZFmWU/sWFBRIkqKioiRJeXl5OnDggFJSUhzbBAcHa/DgwVq3bl2dwcS5EEwAAAAAAOBmxdXVOn/NVltqfzOop8L9/Ru8nzFGGRkZGjhwoJKSkiRJBw4ckCS1a9eu1rbt2rXTd99951T7CCYAAAAAAMBpJk2apC1btmjt2rWnrTt1BIYxxulRGQQTAAAAAAC4WZifn74Z1NO22g01efJkrVixQmvWrFH79u0dy2NjYyWdGDkRFxfnWJ6fn3/aKIr6IpgAAAAAAMDNLMtyajpFUzPGaPLkycrMzFRWVpYSExNrrU9MTFRsbKxWr16t3r17S5LKy8uVnZ2tZ5991qmaBBMAAAAAAECSlJ6erkWLFmn58uWKiIhw3FMiMjJSoaGhsixLU6dO1YwZM9S5c2d17txZM2bMUFhYmG677TanahJMAAAAAAAASdKcOXMkScnJybWWz58/X+PHj5ck3X///SopKdE999yjI0eOqH///vrggw8UERHhVE3LGGMa02gAAAAAAFBbaWmp8vLylJiYqJCQELubY6tzvRYNvwMGAAAAAACAixBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAASdLMmTPVr18/RUREKCYmRqNHj9aOHTtqbfPOO+9o2LBhatOmjSzLUm5ubqNqEkwAAAAAAABJUnZ2ttLT07V+/XqtXr1alZWVSklJUVFRkWOboqIiXX755XrmmWdcUjPAJUcBAAAAAABeb+XKlbWez58/XzExMcrJydGgQYMkSePGjZMk7d692yU1CSYAAAAAAHAzY4xKKqpsqR0a6C/Lspzat6CgQJIUFRXlyibVQjABAAAAAICblVRUqfujq2ypve2JYQoLavjbf2OMMjIyNHDgQCUlJbmhZScQTAAAAAAAgNNMmjRJW7Zs0dq1a91ah2ACAAAAAAA3Cw3017YnhtlWu6EmT56sFStWaM2aNWrfvr0bWvUzggkAAAAAANzMsiynplM0NWOMJk+erMzMTGVlZSkxMdHtNT3/VQEAAAAAAE0iPT1dixYt0vLlyxUREaEDBw5IkiIjIxUaGipJ+umnn7Rnzx7t27dPkrRjxw5JUmxsrGJjYxtc089FbQcAAAAAAF5uzpw5KigoUHJysuLi4hyPpUuXOrZZsWKFevfurREjRkiSbrnlFvXu3VuvvPKKUzUtY4xxSesBAAAAAIAkqbS0VHl5eUpMTFRISIjdzbHVuV4LRkwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAABJ0syZM9WvXz9FREQoJiZGo0eP1o4dOxzrKyoq9MADD6hnz54KDw9XfHy8br/9du3bt8/pmgQTAAAAAABAkpSdna309HStX79eq1evVmVlpVJSUlRUVCRJKi4u1ubNm/XII49o8+bNeuedd/T111/rmmuucbqmZYwxrjoBAAAAAAAglZaWKi8vT4mJiQoJCbG7OU47ePCgYmJilJ2drUGDBtW5zcaNG3XJJZfou+++U0JCwmnrz/VaBLi81QAAAAAAoDZjpIpie2oHhkmW5dSuBQUFkqSoqKizbmNZllq1auVUDYIJAAAAAADcraJYmhFvT+1p+6Sg8AbvZoxRRkaGBg4cqKSkpDq3KS0t1YMPPqjbbrtNLVu2dKp5BBMAAAAAAOA0kyZN0pYtW7R27do611dUVOiWW25RdXW1Xn75ZafrEEwAAAAAAOBugWEnRi7YVbuBJk+erBUrVmjNmjVq3779aesrKip00003KS8vTx999JHToyUkggkAAAAAANzPspyaTtHUjDGaPHmyMjMzlZWVpcTExNO2qQkldu7cqY8//ljR0dGNqkkwAQAAAAAAJEnp6elatGiRli9froiICB04cECSFBkZqdDQUFVWVuqGG27Q5s2b9e6776qqqsqxTVRUlIKCghpck68LBQAAAADAxbz160KtM3x7x/z58zV+/Hjt3r27zlEUkvTxxx8rOTn5tOV8XSgAAAAAAKiXc41d6NSp0zm3aSg/lx4NAAAAAACgAQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAJGnmzJnq16+fIiIiFBMTo9GjR2vHjh21tpk+fbq6du2q8PBwtW7dWkOHDtWGDRucrkkwAQAAAAAAJEnZ2dlKT0/X+vXrtXr1alVWViolJUVFRUWObS644ALNnj1bW7du1dq1a9WpUyelpKTo4MGDTtW0jDHGVScAAAAAAACk0tJS5eXlKTExUSEhIXY3x2kHDx5UTEyMsrOzNWjQoDq3OXbsmCIjI/Xhhx/qqquuOm39uV6LAJe3GgAAAAAA1GKMUUlliS21QwNCZVmWU/sWFBRIkqKioupcX15errlz5yoyMlK9evVyqgbBBAAAAAAAblZSWaL+i/rbUnvDbRsUFhjW4P2MMcrIyNDAgQOVlJRUa927776rW265RcXFxYqLi9Pq1avVpk0bp9rHPSYAAAAAAMBpJk2apC1btmjx4sWnrRsyZIhyc3O1bt06DR8+XDfddJPy8/OdqsM9JgAAAAAAcLFT76vgbVM5Jk+erGXLlmnNmjVKTEw85/adO3fWxIkT9dBDD522jntMAAAAAABgM8uynJpO0dSMMZo8ebIyMzOVlZVVr1CiZr+ysjKnahJMAAAAAAAASVJ6eroWLVqk5cuXKyIiQgcOHJAkRUZGKjQ0VEVFRXr66ad1zTXXKC4uTocPH9bLL7+s77//XjfeeKNTNQkmAAAAAACAJGnOnDmSpOTk5FrL58+fr/Hjx8vf319fffWVFi5cqEOHDik6Olr9+vXTJ598oh49ejhVk2ACAAAAAABIOjEl42xCQkL0zjvvuLQm38oBAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAAAkSTNnzlS/fv0UERGhmJgYjR49Wjt27Djj9nfddZcsy9KsWbOcrkkwAQAAAAAAJEnZ2dlKT0/X+vXrtXr1alVWViolJUVFRUWnbbts2TJt2LBB8fHxjaoZ0Ki9AQAAAACAz1i5cmWt5/Pnz1dMTIxycnI0aNAgx/IffvhBkyZN0qpVqzRixIhG1SSYAAAAAADAzYwxMiUlttS2QkNlWZZT+xYUFEiSoqKiHMuqq6s1btw43XffferRo0ej20cwAQAAAACAm5mSEu24uI8ttbtszpEVFtbg/YwxysjI0MCBA5WUlORY/uyzzyogIEBTpkxxSfsIJgAAAAAAwGkmTZqkLVu2aO3atY5lOTk5euGFF7R582anR2GcimACAAAAAAA3s0JD1WVzjm21G2ry5MlasWKF1qxZo/bt2zuWf/LJJ8rPz1dCQoJjWVVVle69917NmjVLu3fvbnAtggkAAAAAANzMsiynplM0NWOMJk+erMzMTGVlZSkxMbHW+nHjxmno0KG1lg0bNkzjxo3ThAkTnKpJMAEAAAAAACRJ6enpWrRokZYvX66IiAgdOHBAkhQZGanQ0FBFR0crOjq61j6BgYGKjY1Vly5dnKrp1+hWAwAAAAAAnzBnzhwVFBQoOTlZcXFxjsfSpUvdVpMREwAAAAAAQNKJqRwN5cx9JU7GiAkAAAAAAGAbggkAAAAAAGAbggkAAAAAAGAbggkAAAAAAGAbggkAAAAAANzEmZtJ+ppzvQYEEwAAAAAAuJi/v78kqby83OaW2K+4uFiSFBgYWOd6vi4UAAAAAAAXCwgIUFhYmA4ePKjAwED5+TW/cQHGGBUXFys/P1+tWrVyhDWnsgzjSgAAAAAAcLny8nLl5eWpurra7qbYqlWrVoqNjZVlWXWuJ5gAAAAAAMBNqqurm/V0jsDAwDOOlKhBMAEAAAAAAGzT/Ca5AAAAAAAAj0EwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbPP/A1Brg5HM4mxiAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff['rh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAIjCAYAAABCuxM+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABezklEQVR4nO3deXhU5d3/8c/JnkAIBMgyZYsLiICIYBFFCJUGI6AI7j4YcH9kKcYVrIpWQWv1QqWg9KEsVcD2gSBVi8ZKEnkEJEAqFUTQIAiJEZRAQvbcvz/8ZUpMyDKcmclM3q/ryqVztu99JuSezCffc8YyxhgBAAAAAADYJMDbAwAAAAAAAP6FsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANgqyNsDAAAAAADA11RVVamiosLbw/Ca4OBgBQYGnnY9YQMAAAAAAE1kjFF+fr6OHTvm7aF4Xfv27RUXFyfLsuqsI2wAAAAAAKCJaoKGmJgYRURE1PtG298ZY3Ty5EkVFBRIkuLj4+tsQ9gAAAAAAEATVFVVOYOGjh07ens4XhUeHi5JKigoUExMTJ1LKrhBJAAAAAAATVBzj4aIiAgvj6RlqHke6rt3BWEDAAAAAADN0BovnahPQ88DYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAAC1YYmKiZsyYcdr1PXr00Lx58zw2nqbg0ygAAAAAAGjB1qxZo+DgYG8Po1kIGwAAAAAAaMGio6O9PYRm4zIKAAAAAABasFMvoygoKNDYsWMVHh6uhIQEvfnmm94d3GnQ2QAAAAAAgI+YNGmSDh48qI8++kghISGaPn26CgoKvD2sOggbAAAAAADwAV9++aX+8Y9/aPPmzRo8eLAkafHixerdu7eXR1YXl1EAAAAAAOADdu/eraCgIA0aNMi57LzzzlP79u29N6jTIGwAAAAAAMAHGGMkSZZleXkkjSNsAAAAAADAB/Tu3VuVlZXKzs52LtuzZ4+OHTvmvUGdBmEDAAAAAAA+oFevXrryyit11113acuWLdq2bZvuvPNOhYeHe3todRA2AAAAAADgI5YsWaKuXbtq+PDhGj9+vO6++27FxMR4e1h1WKbmog8AAAAAAHBapaWlys3NVUJCgsLCwrw9HK9r6PmgswEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAAD+XlZWlsWPHyuFwyLIsrV271q31CBsAAAAAAPBzxcXF6t+/v+bPn++RekEeqQIAAAAAgB8yxqikosrjdcODA2VZVpO3T05OVnJyshtHVBthAwAAAAAALiqpqNL5T7zv8bq7nh6liJCW+5aeyygAAAAAAICtWm4MAgAAAABACxceHKhdT4/ySt2WjLABAAAAAAAXWZbVoi9n8BYuowAAAAAAALYifgEAAAAAwM8VFRVp3759zse5ubnKyclRdHS0unXrZns9wgYAAAAAAPxcdna2RowY4XycmpoqSUpJSdHSpUttr0fYAAAAAACAn0tMTJQxxmP1uGcDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAAB+bu7cubr44osVGRmpmJgYjRs3Tnv27HFbPcIGAAAAAAD8XGZmpqZMmaLNmzcrPT1dlZWVSkpKUnFxsVvqBbnlqAAAAAAAoMVYv359rcdLlixRTEyMtm3bpmHDhtlej7ABAAAAAABXGSNVnPR83eAIybJc3r2wsFCSFB0dbdeIaiFsAAAAAADAVRUnpTkOz9eddVgKaePSrsYYpaamaujQoerbt6/NA/sJYQMAAAAAAK3I1KlT9dlnn2njxo1uq0HYAAAAAACAq4Ijfuoy8EZdF0ybNk3r1q1TVlaWunTpYvOg/oOwAQAAAAAAV1mWy5czeJIxRtOmTVNaWpoyMjKUkJDg1nqEDQAAAAAA+LkpU6ZoxYoVevvttxUZGan8/HxJUlRUlMLDw22vF2D7EQEAAAAAQIuycOFCFRYWKjExUfHx8c6vt956yy316GwAAAAAAMDPGWM8Wo/OBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAA/NzChQt1wQUXqF27dmrXrp2GDBmif/zjH26rR9gAAAAAAICf69Kli5577jllZ2crOztbv/rVr3TNNdfo888/d0s9yxhj3HJkAAAAAAD8SGlpqXJzc5WQkKCwsDBvD+eMRUdH64UXXtAdd9zh0v4NPR9BdgwQAAAAAIDWyBijksoSj9cNDwqXZVku7VtVVaW//e1vKi4u1pAhQ2we2U8IGwAAAAAAcFFJZYkGrxjs8bpbbtmiiOCIZu2zc+dODRkyRKWlpWrbtq3S0tJ0/vnnu2V83LMBAAAAAIBWoFevXsrJydHmzZv13//930pJSdGuXbvcUot7NgAAAAAA0AT13aPAFy+jqDFy5EidffbZev31113an3s2AAAAAADgBpZlNftyhpbCGKOysjK3HJuwAQAAAAAAPzdr1iwlJyera9euOnHihFatWqWMjAytX7/eLfUIGwAAAAAA8HPfffedJk6cqLy8PEVFRemCCy7Q+vXr9etf/9ot9QgbAAAAAADwc4sXL/ZoPT6NAgAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAACAVmTu3LmyLEszZsxwWw3CBgAAAAAAWomtW7dq0aJFuuCCC9xah7ABAAAAAIBWoKioSLfeeqv+9Kc/qUOHDm6tFeTWowMAAAAA4MeMMTIlJR6va4WHy7KsZu0zZcoUjR49WiNHjtQzzzzjppH9hLABAAAAAAAXmZIS7blooMfr9tq+TVZERJO3X7VqlbZv366tW7e6cVT/QdgAAAAAAIAfO3jwoH7zm9/ogw8+UFhYmEdqWsYY45FKAAAAAAD4sNLSUuXm5iohIcH5pt0XLqNYu3atrr32WgUGBjqXVVVVybIsBQQEqKysrNa6pqrv+ahBZwMAAAAAAC6yLKtZlzN4wxVXXKGdO3fWWjZ58mSdd955euSRR1wKGhpD2AAAAAAAgB+LjIxU3759ay1r06aNOnbsWGe5XfjoSwAAAAAAYCs6GwAAAAAAaGUyMjLcenw6GwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAA8HOzZ8+WZVm1vuLi4txWL8htRwYAAAAAAC1Gnz599OGHHzofBwYGuq0WYQMAAAAAAK1AUFCQW7sZatXySBUAAAAAAPyQMUaV5dUerxsUEiDLspq1z969e+VwOBQaGqrBgwdrzpw5Ouuss9wzPrccFQAAAACAVqCyvFqLfpPp8bp3vzxcwaFNvwxi8ODBWr58uXr27KnvvvtOzzzzjC699FJ9/vnn6tixo+3jI2wAAAAAAMDPJScnO/+/X79+GjJkiM4++2wtW7ZMqampttcjbAAAAAAAwEVBIQG6++XhXql7Jtq0aaN+/fpp7969No2oNsIGAAAAAABcZFlWsy5naCnKysq0e/duXX755W45/plFIQAAAAAAoMV78MEHlZmZqdzcXG3ZskXXXXedjh8/rpSUFLfUo7MBAAAAAAA/9+233+rmm2/WkSNH1LlzZ11yySXavHmzunfv7pZ6hA0AAAAAAPi5VatWebQel1EAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETbAZ/zv//6v+vXrp/DwcHXs2FEjR45UcXGxJGnSpEkaN26cnnrqKcXExKhdu3a65557VF5e7tx//fr1Gjp0qNq3b6+OHTtqzJgx+uqrr2rV+Pbbb3XTTTcpOjpabdq00aBBg7Rlyxbn+r///e8aOHCgwsLCdNZZZ+mpp55SZWWlW87XGKORI0fqyiuvlDFGknTs2DF169ZNjz32mFtqAnCv1jaP7d+/XwEBAcrOzq61/NVXX1X37t2dcxsA39Ha5jFJ6tGjhyzLqvMF+KJDhw7pv/7rv9SxY0dFRETowgsv1LZt29xSi7ABPiEvL08333yzbr/9du3evVsZGRkaP358rV9U//nPf2r37t3asGGDVq5cqbS0ND311FPO9cXFxUpNTdXWrVv1z3/+UwEBAbr22mtVXV0tSSoqKtLw4cN1+PBhrVu3Tv/617/08MMPO9e///77+q//+i9Nnz5du3bt0uuvv66lS5fq2WefPe2433zzTbVt27bBrzfffLPefS3L0rJly/Tpp5/qlVdekSTde++9io2N1ezZs8/0KQXgYa1xHuvRo4dGjhypJUuW1Fq+ZMkSTZo0iV/WAR/TGucxSdq6davy8vKUl5enb7/9Vpdccokuv/zyM306AY/78ccfddlllyk4OFj/+Mc/tGvXLr344otq3769ewqaViwzM9OMGTPGxMfHG0kmLS3N6/VOnDhhpkyZYn7xi1+YsLAwc95555kFCxa4dVy+YNu2bUaS2b9/f73rU1JSTHR0tCkuLnYuW7hwoWnbtq2pqqqqd5+CggIjyezcudMYY8zrr79uIiMjzdGjR+vd/vLLLzdz5syptewvf/mLiY+PP+24jx8/bvbu3dvg1/Hjxxs897/+9a8mNDTUzJw500RERJg9e/Y0uD2Alqm1zmNvvfWW6dChgyktLTXGGJOTk2MsyzK5ubmn3QdAy9Ra57FTTZ8+3XTv3t0UFBQ0aXv4n5KSErNr1y5TUlLi7aE02yOPPGKGDh1q6zEbej6C3BNh+Ibi4mL1799fkydP1oQJE1pEvfvvv18bNmzQG2+8oR49euiDDz7QfffdJ4fDoWuuucbtY2yp+vfvryuuuEL9+vXTqFGjlJSUpOuuu04dOnSotU1ERITz8ZAhQ1RUVKSDBw+qe/fu+uqrr/T4449r8+bNOnLkiDMhP3DggPr27aucnBwNGDBA0dHR9Y5h27Zt2rp1a63kvKqqSqWlpTp58mSt2jUiIyMVGRl5Rud+/fXXKy0tTXPnztXChQvVs2fPMzoeAO9orfPYuHHjNHXqVKWlpemmm27Sn//8Z40YMUI9evRw+ZgAvKO1zmM1Fi1apMWLF+v//u//1Llz5zM+HvyHMUaVZWUerxsUGtqsLsF169Zp1KhRuv7665WZmalf/OIXuu+++3TXXXe5Z3xuOaqPSE5OVnJy8mnXl5eX67e//a3efPNNHTt2TH379tXzzz+vxMREt9STpE2bNiklJcVZ4+6779brr7+u7OzsVh02BAYGKj09XZ988ok++OADvfrqq3rssce0ZcsWJSQkNLhvzQ/g2LFj1bVrV/3pT3+Sw+FQdXW1+vbt67yOMDw8vMHjVFdX66mnntL48ePrrAsLC6t3nzfffFP33HNPg8d9/fXXdeutt552/cmTJ7Vt2zYFBgZq7969DR4LQMvVWuexkJAQTZw4UUuWLNH48eO1YsUKzZs3r8HjAWiZWus8JkkZGRmaNm2aVq5cqf79+zd4LLQ+lWVleiXlOo/Xnb7sfxV8mn/39fn666+1cOFCpaamatasWfr00081ffp0hYaG6rbbbrN9fK06bGjM5MmTtX//fq1atUoOh0NpaWm68sortXPnTp177rluqTl06FCtW7dOt99+uxwOhzIyMvTll1/q5Zdfdks9X2JZli677DJddtlleuKJJ9S9e3elpaUpNTVVkvSvf/1LJSUlzhepzZs3q23bturSpYuOHj2q3bt36/XXX3deY7dx48Zax7/gggv0P//zP/rhhx/qTdMvuugi7dmzR+ecc06Tx3z11Vdr8ODBDW4TGxvb4PoHHnhAAQEB+sc//qGrrrpKo0eP1q9+9asmjwFAy9Fa57E777xTffv21YIFC1RRUVHvmwQAvqE1zmP79u3ThAkTNGvWLOYv+LTq6moNGjRIc+bMkSQNGDBAn3/+uRYuXEjY4ElfffWVVq5cqW+//VYOh0OS9OCDD2r9+vVasmSJ8xtkt1deeUV33XWXunTpoqCgIAUEBOh//ud/NHToULfU8xVbtmzRP//5TyUlJSkmJkZbtmzR999/r969ezu3KS8v1x133KHf/va3+uabb/Tkk09q6tSpCggIUIcOHdSxY0ctWrRI8fHxOnDggB599NFaNW6++WbNmTNH48aN09y5cxUfH68dO3bI4XBoyJAheuKJJzRmzBh17dpV119/vQICAvTZZ59p586deuaZZ+od95m27b377rv685//rE2bNumiiy7So48+qpSUFH322We1WhYBtHytdR6TpN69e+uSSy7RI488ottvv73Rv1wCaJla4zxWUlKisWPH6sILL9Tdd9+t/Px857q4uDiXjgn/ExQaqunL/tcrdZsjPj5e559/fq1lvXv31urVq+0c1n/YencIH6af3bDxr3/9q5Fk2rRpU+srKCjI3HDDDcYYY3Jzc42kBr+mTJnSpHo1XnjhBdOzZ0+zbt06869//cu8+uqrpm3btiY9Pd0dp+0zdu3aZUaNGmU6d+5sQkNDTc+ePc2rr77qXJ+SkmKuueYa88QTT5iOHTuatm3bmjvvvNN5QzJjjElPTze9e/c2oaGh5oILLjAZGRl1vg/79+83EyZMMO3atTMRERFm0KBBZsuWLc7169evN5deeqkJDw837dq1M7/85S/NokWL3HLOBQUFJjY2ttZNkCoqKswvf/lL579BAL6jNc5jp1q8eLGRZD799FO31wLgHq1xHmvo9320Tr58g8ibb765zg0iZ8yYYYYMGeLyMRt6Pixj+JBr6aeWsLS0NI0bN06S9NZbb+nWW2/V559/rsDAwFrbtm3bVnFxcaqoqKjzucA/16FDh3rbsn5eT/opOY2KilJaWppGjx7tXH7nnXfq22+/1fr1610/QT83adIkHTt2TGvXrvX2UADAJf4+jz377LNatWqVdu7c6e2hAHATf5/HAEkqLS1Vbm6uEhISTnufkJZq69atuvTSS/XUU0/phhtu0Keffqq77rpLixYtavCeJQ1p6PngMorTGDBggKqqqlRQUHDaz9ENDg7WeeedZ1vNiooKVVRUKCAgoNbywMBA5516AQDwJUVFRdq9e7deffVV/e53v/P2cAAAaLUuvvhipaWlaebMmXr66aeVkJCgefPmuRw0NKZVhw1FRUXat2+f83Fubq5ycnIUHR2tnj176tZbb9Vtt92mF198UQMGDNCRI0f00UcfqV+/frrqqqtsrdetWze1a9dOw4cP10MPPaTw8HB1795dmZmZWr58uV566SVbzhkAAE+aOnWqVq5cqXHjxun222/39nAAAGjVxowZozFjxnikVqu+jCIjI0MjRoyoszwlJUVLly5VRUWFnnnmGS1fvlyHDh1Sx44dNWTIED311FPq16+f7fUkKT8/XzNnztQHH3ygH374Qd27d9fdd9+t+++/v1mfoQoAAAAAsJcvX0bhDg09H606bAAAAAAAoKkIG2pr6PkIOM0+AAAAAAAALml192yorq7W4cOHFRkZyWUJADzGGKMTJ07I4XDUuQlsczGPAfAG5jEAvs7OeQyNa3Vhw+HDh9W1a1dvDwNAK3Xw4EF16dLljI7BPAbAm5jHAPg6O+YxNK7VhQ2RkZGSfvoH1q5dOy+PBkBrcfz4cXXt2tU5B50J5jEA3sA8BsDX2TmPoXGtLmyoadVr164dL24APM6OdmHmMQDexDwGwNdx+ZZncKEKAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAAB+rkePHrIsq87XlClT3FKv1d0gEgAAAACA1mbr1q2qqqpyPv73v/+tX//617r++uvdUo+wAQAAwE9VVlbq6NGj9a4LCQlRhw4dPDwiAIC3dO7cudbj5557TmeffbaGDx/ulnqEDQAAAH5q+fLlOnDgQL3rEhISlJKS4uERAYD/McbIVFR7vK4VHODyx3iWl5frjTfeUGpqqts+CpSwAQAAwE/l5+dLksLDw+v8MhkWFuaNIQGA3zEV1Tr8xCcer+t4+lJZIYEu7bt27VodO3ZMkyZNsndQpyBsAAAA8FPGGEnS3XffzSUTAACnxYsXKzk5WQ6Hw201CBsAAAD8VHX1T2297mqRBQD8dDmD4+lLvVLXFd98840+/PBDrVmzxuYR1UbYAAAA4KdqOhsCAvi0cwBwF8uyXL6cwRuWLFmimJgYjR492q11eOUBAADwU3Q2AABOVV1drSVLliglJUVBQe7tPSBsAAAA8EM1XQ0SnQ0AgJ98+OGHOnDggG6//Xa31+IyCgAAAD9U09Ug0dkAAPhJUlJSrTDanYi5AQAA/BCdDQAAb+KVBwAAwA/R2QAA8CbCBgAAAD9EZwMAwJt45QEAAPBDdDYAALyJsAEAAMAPnRo20NkAAPA0XnkAAAD80KmXUdDZAADwNMIGAAAAP1TT2WBZFmEDAMDjCBsAAAD8UE1nA5dQAAC8gVcfAAAAP3RqZwMAAJ5G2AAAAOCH6GwAAHgTrz4AAAB+iM4GAMCpKisr9dvf/lYJCQkKDw/XWWedpaeffrrWpxfZKcgtRwUAAIBX0dkAADjV888/r9dee03Lli1Tnz59lJ2drcmTJysqKkq/+c1vbK9H2AAAAOCH6GwAAM8wxqiiosLjdYODg5s1x2/atEnXXHONRo8eLUnq0aOHVq5cqezsbLeMj7ABAADAD9HZAACeUVFRoTlz5ni87qxZsxQSEtLk7YcOHarXXntNX375pXr27Kl//etf2rhxo+bNm+eW8Xn11ScrK0tjx46Vw+GQZVlau3Ztg9tnZGQ4Pyv61K8vvvjCMwMGAADwEXQ2AABO9cgjj+jmm2/Weeedp+DgYA0YMEAzZszQzTff7JZ6Xu1sKC4uVv/+/TV58mRNmDChyfvt2bNH7dq1cz7u3LmzO4YHAADgs+hsAADPCA4O1qxZs7xStzneeustvfHGG1qxYoX69OmjnJwczZgxQw6HQykpKbaPz6thQ3JyspKTk5u9X0xMjNq3b2//gAAAAPwEnQ0A4BmWZTXrcgZveeihh/Too4/qpptukiT169dP33zzjebOnet/YYOrBgwYoNLSUp1//vn67W9/qxEjRpx227KyMpWVlTkfHz9+3BNDBADbMI8BcEVL6mxgHgMA7zt58mSd14TAwEC3ffSl9199miE+Pl6LFi3S6tWrtWbNGvXq1UtXXHGFsrKyTrvP3LlzFRUV5fzq2rWrB0cMAGeOeQyAK1pSZwPzGAB439ixY/Xss8/q3Xff1f79+5WWlqaXXnpJ1157rVvqWaYm9vYyy7KUlpamcePGNWu/sWPHyrIsrVu3rt719SXpXbt2VWFhYa37PgCAOx0/flxRUVEuzT3MYwBckZubq2XLlqlz586aMmXKGR+PeQyArzuTeaxGaWmpcnNzlZCQoLCwMJtH6F4nTpzQ448/rrS0NBUUFMjhcOjmm2/WE0884fJlIA09Hz55GcWpLrnkEr3xxhunXR8aGqrQ0FAPjggA7MU8BsAVLamzgXkMALwvMjJS8+bNc9tHXf6cT11GUZ8dO3YoPj7e28MAAABoUWrChpZwzwYAQOvj1c6GoqIi7du3z/k4NzdXOTk5io6OVrdu3TRz5kwdOnRIy5cvlyTNmzdPPXr0UJ8+fVReXq433nhDq1ev1urVq711CgAAAC1SzZWyLaGzAQDQ+ng1bMjOzq71SRKpqamSpJSUFC1dulR5eXk6cOCAc315ebkefPBBHTp0SOHh4erTp4/effddXXXVVR4fOwAAQEtGZwMAwJu8GjYkJiaqoftTLl26tNbjhx9+WA8//LCbRwUAAOD7WtJHXwIAWh9efQAAAPxQS7pBJACg9SFsAAAA8EN0NgAAvIlXHwAAAD9EZwMAwJsIGwAAAPwQnQ0AAG/i1QcAAMAP0dkAAPAmwgYAAAA/RGcDAMCbePUBAADwQ3Q2AABOdeLECc2YMUPdu3dXeHi4Lr30Um3dutVt9QgbAAAA/BCdDQCAU915551KT0/XX/7yF+3cuVNJSUkaOXKkDh065JZ6QW45KgAAALyKzgYA8AxjjKqrSzxeNyAgvMlzfElJiVavXq23335bw4YNkyTNnj1ba9eu1cKFC/XMM8/YPj7CBgAAAD9EZwMAeEZ1dYkyMvt5vG7i8J0KDIxo0raVlZWqqqpSWFhYreXh4eHauHGjO4bHZRQAAAD+iM4GAECNyMhIDRkyRL/73e90+PBhVVVV6Y033tCWLVuUl5fnlpp0NgAAAPghOhsAwDMCAsKVOHynV+o2x1/+8hfdfvvt+sUvfqHAwEBddNFFuuWWW7R9+3a3jI+wAQAAwA/R2QAAnmFZVpMvZ/Cms88+W5mZmSouLtbx48cVHx+vG2+8UQkJCW6pR9QNAADgh2rCBjobAACnatOmjeLj4/Xjjz/q/fff1zXXXOOWOnQ2AAAA+KGayyjobAAASNL7778vY4x69eqlffv26aGHHlKvXr00efJkt9Qj6gYAAPBDdDYAAE5VWFioKVOm6LzzztNtt92moUOH6oMPPlBwcLBb6tHZAAAA4Ie4QSQA4FQ33HCDbrjhBo/V49UHAADAD3GDSACANxE2AAAA+CE6GwAA3sSrDwAAgB+iswEA4E2EDQAAAH6IzgYAgDfx6gMAAOCH6GwAAHgTYQMAAIAforMBAOBNvPoAAAD4ITobAADeRNgAAADgh+hsAAB4E68+AAAAfojOBgCANxE2AAAA+CE6GwAA3sSrDwAAgB+iswEAcKqsrCyNHTtWDodDlmVp7dq1tdYbYzR79mw5HA6Fh4crMTFRn3/+ucv1CBsAAAD8EJ0NAIBTFRcXq3///po/f36963//+9/rpZde0vz587V161bFxcXp17/+tU6cOOFSvaAzGSwAAABaJjobAMAzjDE6+f/nXE+KCAho1hyfnJys5OTketcZYzRv3jw99thjGj9+vCRp2bJlio2N1YoVK3TPPfc0e3yEDQAAAH6oJmygswEA3OtkdbXOztrp8bpfDeunNoGBthwrNzdX+fn5SkpKci4LDQ3V8OHD9cknn7gUNvDqAwAA4IdqLqOgswEA0Jj8/HxJUmxsbK3lsbGxznXNRWcDAACAH6KzAQA8IyIgQF8N6+eVunb7eUBtjHE5tCZsAAAA8EPcIBIAPMOyLNsuZ/CWuLg4ST91OMTHxzuXFxQU1Ol2aCpefQAAAPwQN4gEADRVQkKC4uLilJ6e7lxWXl6uzMxMXXrppS4dk84GAAAAP0RnAwDgVEVFRdq3b5/zcW5urnJychQdHa1u3bppxowZmjNnjs4991yde+65mjNnjiIiInTLLbe4VI+wAQAAwA/R2QAAOFV2drZGjBjhfJyamipJSklJ0dKlS/Xwww+rpKRE9913n3788UcNHjxYH3zwgSIjI12qR9gAAADgh+hsAACcKjEx0fnaUB/LsjR79mzNnj3blnq8+gAAAPghOhsAAN5E2AAAAOCH6GwAAHgTrz4AAAB+iM4GAIA3ETYAAAD4ITobAADexKsPAACAH6KzAQDgTV4NG7KysjR27Fg5HA5ZlqW1a9c2uk9mZqYGDhyosLAwnXXWWXrttdfcP1AAAAAfQ2cDAMCbvPrqU1xcrP79+2v+/PlN2j43N1dXXXWVLr/8cu3YsUOzZs3S9OnTtXr1ajePFAAAwLfQ2QAA8KYgbxZPTk5WcnJyk7d/7bXX1K1bN82bN0+S1Lt3b2VnZ+sPf/iDJkyY4KZRAgAA+B46GwAA3uTVsKG5Nm3apKSkpFrLRo0apcWLF6uiokLBwcF19ikrK1NZWZnz8fHjx90+TgCwE/MYAFe0pM4G5jEAaH18KurOz89XbGxsrWWxsbGqrKzUkSNH6t1n7ty5ioqKcn517drVE0MFANswjwFwRU3Y0BI6G5jHAKD18f6rTzP9PJ2vaRE8XWo/c+ZMFRYWOr8OHjzo9jECgJ2YxwC4orHfkTyJeQwAvK+xD2hYs2aNRo0apU6dOsmyLOXk5JxRPZ+6jCIuLk75+fm1lhUUFCgoKEgdO3asd5/Q0FCFhoZ6YngA4BbMYwBc0ZI6G5jHAMD7aj6gYfLkyfXe87C4uFiXXXaZrr/+et11111nXM+nwoYhQ4bo73//e61lH3zwgQYNGlTv/RoAAABaK24QCQCeYYxRSUWVx+uGBwc2q3utsQ9omDhxoiRp//79Zzo0SV4OG4qKirRv3z7n49zcXOXk5Cg6OlrdunXTzJkzdejQIS1fvlySdO+992r+/PlKTU3VXXfdpU2bNmnx4sVauXKlt04BAACgRWpJN4gEAH9WUlGl85943+N1dz09ShEhLbd/wKsjy87O1ogRI5yPU1NTJUkpKSlaunSp8vLydODAAef6hIQEvffee7r//vv1xz/+UQ6HQ6+88gofewkAAPAzdDYAALzJq2FDYmKi84WwPkuXLq2zbPjw4dq+fbsbRwUAAOD76GwAAM8IDw7UrqdHeaVuS9Zyey4AAADgMjobAMAzLMtq0ZczeAuvPgAAAH6IzgYAgDcRvwAAAPghOhsAAKdq7AMafvjhBx04cECHDx+WJO3Zs0eSFBcXp7i4uGbXa3bYUFxcrOeee07//Oc/VVBQ4EzNa3z99dfNHgQAAADsc+rvZ3Q2AACkxj+gYd26dZo8ebJz/U033SRJevLJJzV79uxm12t22HDnnXcqMzNTEydOVHx8PC9gAAAALcypN+CmswEAIDX+AQ2TJk3SpEmTbKvX7LDhH//4h959911ddtlltg0CAAAA9qGzAQDgbc0OGzp06KDo6Gh3jAVAC1ZdXa2srCz98MMP3h5KixAdHa3ExERvDwNAC/HFF19o165d3h6G06lhA50NAABvaHbY8Lvf/U5PPPGEli1bpoiICHeMCUALlJ+fr4yMDG8Po8Xo0qULYQMAp3feeUdFRUXeHkYdISEhCgxs2Z/DDgDwT00KGwYMGFCrBW/fvn2KjY1Vjx49FBwcXGvb7du32ztCAC1CRUWFJCkiIkJDhw718mi8r23btt4eAoAW5OTJk5KkYcOGKSwszMuj+Y+uXbsSNgAAvKJJYcO4cePcPAwALV1NS25ERIQuvfRSL48GAFqOyspK5xw5ZMgQhYeHe3lEAAB4X5PChieffNL5/5MnT9att96qK664ghsOAa0In9cOAPUrLy93/n9ISIgXRwIAQMvR7HcNR48e1ZgxY9SlSxc9+OCDysnJccOwALQ0NX+1I2QEgNpqwobAwEAuWQAA4P9rdtiwbt065efn68knn1R2drYGDhyo888/X3PmzNH+/fvdMEQALUFN2EBnAwDUVhM20NUAAMB/uPSuoX379rr77ruVkZGhb775RpMnT9Zf/vIXnXPOOXaPD0ALUXMZBZ0NAFAbYQMAAHWd0Z8oKyoqlJ2drS1btmj//v2KjY21a1wAWhg6GwCgfoQNAADU5dK7hg0bNuiuu+5SbGysUlJSFBkZqb///e86ePCg3eMD0EJwg0gAqF9N2BAaGurlkQAAcHpZWVkaO3asHA6HLMvS2rVrnesqKir0yCOPqF+/fmrTpo0cDoduu+02HT582OV6zX7X0KVLF1111VX6/vvv9frrr+u7777TkiVLNHLkSN6EAH6MG0QCQP3obAAA+ILi4mL1799f8+fPr7Pu5MmT2r59ux5//HFt375da9as0Zdffqmrr77a5XpN+ujLUz3xxBO6/vrr1aFDB5eLAvA9dDYAQP3KysokETYAQKtljFRx0vN1gyOkZvwhMDk5WcnJyfWui4qKUnp6eq1lr776qn75y1/qwIED6tatW7OH1+yw4e677252EQC+j84GAKgfnQ0A0MpVnJTmODxfd9ZhKaSN2w5fWFgoy7LUvn17l/bnT5QAmoTOBgCoH2EDAMDflJaW6tFHH9Utt9yidu3auXSMZnc2AGid6GwAgPoRNgBAKxcc8VOXgTfqukFFRYVuuukmVVdXa8GCBS4fh7ABQJPQ2QAA9SNsAIBWzrLcejmDJ1VUVOiGG25Qbm6uPvroI5e7GiTCBgBNRGcDANSPsAEA4A9qgoa9e/dqw4YN6tix4xkdj7ABQJPQ2QAA9asJG0JDQ708EgAATq+oqEj79u1zPs7NzVVOTo6io6PlcDh03XXXafv27XrnnXdUVVWl/Px8SVJ0dLRLgTphA4AmobMBAOpHZwMAwBdkZ2drxIgRzsepqamSpJSUFM2ePVvr1q2TJF144YW19tuwYYMSExObXY+wAUCT0NkAAPUrKyuTRNgAAGjZEhMTnb/T16ehda7gXQOAJqGzAQDqR2cDAAB1ETYAaBI6GwCgfoQNAADUxbsGAE1CZwMA1I+wAQCAurhnA4AmqQkb6GwA4K+MMfruu+9UUlIiy7LkcDhqBQjV1dU6dOiQKisra+1H2AAAQF2EDQCapOYyCjobAPir3bt3669//avzcffu3TV58mTn48zMTGVmZp52fz76EgCA/yBsANAkdDYA8HdHjhyRJAUFBamystL5+Ofr27Ztq7CwsFrrzjrrrDrLAABozQgbADQJN4gE4O9qLoc455xz9MUXXzgf/3z9FVdcoQEDBnh8fAAA+BLeNQBoEm4QCcDf1YQJbdu2lSRVVFQ4575T13NvBgAAGkfYAKBJ6GwA4O/Kysok/SdskH4KHH6+nrABAIDG8a4BQJPQ2QDA39V0LkRERDjnupqA4dT1hA0AADSOsAFAk9DZAMDfnRom1AQKp963gbABAODLsrKyNHbsWDkcDlmWpbVr19ZaP3v2bJ133nlq06aNOnTooJEjR2rLli0u1+NdA4AmobMBgL8jbAAA+LPi4mL1799f8+fPr3d9z549NX/+fO3cuVMbN25Ujx49lJSUpO+//96lenwaBYAmobMBgL+rCRNCQ0PrhA3GmFrrAQCoYYxRSWWJx+uGB4U36w+BycnJSk5OPu36W265pdbjl156SYsXL9Znn32mK664otnjI2wA0CR0NgDwdw11Npx6o0g6GwAApyqpLNHgFYM9XnfLLVsUERzhlmOXl5dr0aJFioqKUv/+/V06BmEDgCahswGAv2sobDj1coqgIH59AgD4p3feeUc33XSTTp48qfj4eKWnp6tTp04uHYtXSwBNQmcDAH93athQc6nEz8OGkJAQQlcAQC3hQeHacovrN1I8k7p2GzFihHJycnTkyBH96U9/0g033KAtW7YoJiam2ccibADQJHQ2APBn1dXVzkslGups4BIKAMDPWZbltssZPK1NmzY655xzdM455+iSSy7Rueeeq8WLF2vmzJnNPpbX3zUsWLBACQkJCgsL08CBA/Xxxx+fdtuMjAxZllXn64svvvDgiIHWic4GAP7s1Msk6gsbysrKnOsAAGgtjDHO18Dm8mpnw1tvvaUZM2ZowYIFuuyyy/T6668rOTlZu3btUrdu3U673549e9SuXTvn486dO3tiuECrRmcDAH9WEypYlqWgoCA6GwAAfqeoqEj79u1zPs7NzVVOTo6io6PVsWNHPfvss7r66qsVHx+vo0ePasGCBfr22291/fXXu1TPq+8aXnrpJd1xxx2688471bt3b82bN09du3bVwoULG9wvJiZGcXFxzq/AwEAPjRhovehsAODPTg0TLMtyhgo1f80hbAAA+Lrs7GwNGDBAAwYMkCSlpqZqwIABeuKJJxQYGKgvvvhCEyZMUM+ePTVmzBh9//33+vjjj9WnTx+X6nmts6G8vFzbtm3To48+Wmt5UlKSPvnkkwb3HTBggEpLS3X++efrt7/9rUaMGHHabcvKymq1fRw/fvzMBg60UjVhA50Nnsc8Brjfz8MEOhvsxTwGAN6XmJjo7Fauz5o1a2yt57V3DUeOHFFVVZViY2NrLY+NjVV+fn69+8THx2vRokVavXq11qxZo169eumKK65QVlbWaevMnTtXUVFRzq+uXbvaeh5Aa1EzMdHZ4HnMY4D7ETa4F/MYALQ+Xv8T5c/fuBhjTvtmplevXrrrrrt00UUXaciQIVqwYIFGjx6tP/zhD6c9/syZM1VYWOj8OnjwoK3jB1oLOhu8h3kMcL+aMKHmIy9PFzbUrEfzMI8BQOvjtcsoOnXqpMDAwDpdDAUFBXW6HRpyySWX6I033jjt+tDQUH4xAGzADSK9h3kMcL+fdy7U/MzR2WAP5jEAaH289q4hJCREAwcOVHp6eq3l6enpuvTSS5t8nB07dig+Pt7u4QH4GW4QCcCfcRkFAAD28upHX6ampmrixIkaNGiQhgwZokWLFunAgQO69957Jf3Ucnfo0CEtX75ckjRv3jz16NFDffr0UXl5ud544w2tXr1aq1ev9uZpAK0CnQ0A/FnNzQtPFzb8fD0AAGiYV8OGG2+8UUePHtXTTz+tvLw89e3bV++99566d+8uScrLy9OBAwec25eXl+vBBx/UoUOHFB4erj59+ujdd9/VVVdd5a1TAFoNOhsA+DM6GwAAsJdXwwZJuu+++3TffffVu27p0qW1Hj/88MN6+OGHPTAqAD9HZwMAX1dWVqaVK1eqsLCwzrqSkhJJdcOG4uJivfzyyzpx4kSt5QAAoGFeDxsA+AY6GwD4uoMHD2r//v0NbhMTEyNJatu2rcLDw1VSUqIff/yxznoAANAwwgYATUJnAwBfV3Pfhbi4OI0ePbrO+tDQUHXu3FmSFBwcrGnTpuno0aPO9W3btlWHDh08M1gAAHwcYQOAJqGzAYCvq7nvQmRkpLp27dro9hEREYqIiHD3sAAA8Ev8iRJAk9DZAMDXcZNHAEBrlpWVpbFjx8rhcMiyLK1du/a0295zzz2yLEvz5s1zuR7vGgA0CZ0NAHwdYQMAoDUrLi5W//79NX/+/Aa3W7t2rbZs2SKHw3FG9biMAkCT0NkAwNcRNgAA3MEYI/P/P9XIk6zw8Gb9ITA5OVnJyckNbnPo0CFNnTpV77//fr33N2oOwgYATUJnAwBfV3ODSMIGAICdTEmJ9lw00ON1e23fJsvGewtVV1dr4sSJeuihh9SnT58zPh5/ogTQJHQ2APB1dDYAAHB6zz//vIKCgjR9+nRbjkdnA4AmobMBgK8jbAAAuIMVHq5e27d5pa5dtm3bppdfflnbt2+37fd9wgYATVITNtDZAMBX1YQNoaGhXh4JAMCfWJZl6+UM3vDxxx+roKBA3bp1cy6rqqrSAw88oHnz5mn//v3NPiZhA4Am4TIKAL6OzgYAAOo3ceJEjRw5stayUaNGaeLEiZo8ebJLxyRsANAkXEYBwNcRNgAAWrOioiLt27fP+Tg3N1c5OTmKjo5Wt27d1LFjx1rbBwcHKy4uTr169XKpHmEDgCahswGAryNsAAC0ZtnZ2RoxYoTzcWpqqiQpJSVFS5cutb0eYQOAJqGzAYCvI2wAALRmiYmJzj8gNoUr92k4FX+iBNAkdDYA8HWEDQAAeA7vGgA0CZ0NAHxZdXU1YQMAAB5E2ACgSehsAODLKioqnP9P2AAAgPvxrgFAk9DZAMCX1XQ1WJal4OBgL48GAAD/R9gAoFGn3kiGzgYAvujUSygITQEAcD/eNQBoVE1Xg0RnAwDfxP0aAADwLMIGAI2iswGAryNsAADAs3jXAKBRdDYA8HWEDQAAeFaQtwcAoOWjswFAS5Cbm6uDBw+6tG9BQYEkwgYAADyFsAFAo+hsAOBtlZWVWrFiRa2PsHRFRESETSMCAAANIWwA0Cg6GwB4W1FRkSoqKhQQEKALL7zQpWMEBgZq0KBB9g4MAAAfkZWVpRdeeEHbtm1TXl6e0tLSNG7cOOf6SZMmadmyZbX2GTx4sDZv3uxSPcIGAI2iswGAt504cUKSFBkZqauvvtrLowEAwPcUFxerf//+mjx5siZMmFDvNldeeaWWLFnifHwmlx8SNgBoVE3YYFkWYQMArygqKpIktW3b1ssjAQCgNmOMKsurG9/QZkEhAc363Tw5OVnJyckNbhMaGqq4uLgzHZokwgYATVBzGQWXUADwFsIGAEBLVVlerUW/yfR43btfHq7g0EBbj5mRkaGYmBi1b99ew4cP17PPPquYmBiXjsU7BwCNOrWzAQC8oSZsiIyM9PJIAADwT8nJyXrzzTf10Ucf6cUXX9TWrVv1q1/9SmVlZS4dj84GAI2iswGAt9HZAABoqYJCAnT3y8O9UtdON954o/P/+/btq0GDBql79+569913NX78+GYfj7ABQKPobADgbTU3iCRsAAC0NJZl2X45Q0sQHx+v7t27a+/evS7tz58pATSKzgYA3kZnAwAAnnX06FEdPHhQ8fHxLu1PZwOARtHZAMDbuGcDAABnpqioSPv27XM+zs3NVU5OjqKjoxUdHa3Zs2drwoQJio+P1/79+zVr1ix16tRJ1157rUv1CBsANIrOBgDeVF1dTWcDAABnKDs7WyNGjHA+Tk1NlSSlpKRo4cKF2rlzp5YvX65jx44pPj5eI0aM0FtvveVy0E/YAKBRdDYAcLePP/5Yu3fvrnedMcY5D7Vp08aTwwIAwG8kJiY6/4hYn/fff9/WeoQNABpFZwMAdyovL9dHH33U4C9AktSpUycFBfGrCwAAvoBXbACNorMBgDt99913MsYoIiJC48aNO+12DofDc4MCAABnhLABQKPobADgTnl5eZJ+ChN69uzp5dEAAAA78M4BQKPobADgTvn5+ZLk8kdrAQCAloewAUCj6GwA4E41nQ1xcXFeHgkAALALl1EAaBSdDQDsdPjwYa1atUplZWWS5PwvYQMAAP6DsAFAo2rCBjobANjh//7v/3T8+PFayzp16qQOHTp4aUQAAMBuhA0AGlVzGQWdDQCaqrq6ut6PsiwpKdEXX3whSbr11lsVHR0tSYqKiiLQBADAj3g9bFiwYIFeeOEF5eXlqU+fPpo3b54uv/zy026fmZmp1NRUff7553I4HHr44Yd17733enDEQOtDZwOA5igsLNSf/vQnFRUVnXabuLg4nXvuuR4cFQAA8CSvvnN46623NGPGDD322GPasWOHLr/8ciUnJ+vAgQP1bp+bm6urrrpKl19+uXbs2KFZs2Zp+vTpWr16tYdHDrQu3CASQHNs27atwaDBsixdeumlHhwRAADwNK92Nrz00ku64447dOedd0qS5s2bp/fff18LFy7U3Llz62z/2muvqVu3bpo3b54kqXfv3srOztYf/vAHTZgwwS1jPPLdd8p4/2/Ov+zCcwIsSwqU5KOd+20VKCskQrLqf4MepGCp5rKEkOA65xkaIJnAMIUGWrIsS9WBp2zvYXnff69qVam4qFCfbf7EK2NoilA1/SkKDJKCQkLO6NKQ2HP6KDQszOX9PYV5zHta6zz26b9yVF1VrgHn9VRMjEOhQT/NYyYwRLIsBVoB0smiFj2feAvzWP2O/fCD1n30riI/SlePz3bJqvSj+awFzA91L3hq/hYtlSVLh5N6qbJdeJ11gQqQIyDyP9u2j5AVGFhrm4AASQHBCgs0sizpROxgmaC6x8J/hMiz89gvel6gsHC+J02RlZWlF154Qdu2bVNeXp7S0tI0bty4Wtvs3r1bjzzyiDIzM1VdXa0+ffror3/9q7p169bsel4LG8rLy7Vt2zY9+uijtZYnJSXpk0/q/+Vj06ZNSkpKqrVs1KhRWrx4sSoqKhQcHFxnn7KyMuddriXVuSFVYzb+c7X6zPljs/YBavS6Lk8BQa6/QPcu/bNS9v9FwabSxlG5JkpShaT0Tz7y9lBs0+/2LxQYfCa/QP1V3foOtG08p8M8Bm9yZR67QvopZNkr9d7ZcuYxf9Ra5rFN/7deYX+aq/O+8d03vfCeK068r1CrqvENC+tfXPv3sU9tHRtsmMe+XKGz+w+2b0B+rLi4WP3799fkyZPr/WP9V199paFDh+qOO+7QU089paioKO3evVthLobSXgsbjhw5oqqqKsXGxtZaHhsbq/z8/Hr3yc/Pr3f7yspKHTlyRPHx8XX2mTt3rp566in7Bg4AHsY8BsDXnek8FhAYrP2x0rmHpb9eHqDc2Mb3ASTpj/nfKzjCjzph0CIZY1R5SqDqKUGhoc3qCklOTlZycvJp1z/22GO66qqr9Pvf/9657KyzznJ9fC7vaZOfPznGmAafsPq2r295jZkzZyo1NdX5+Pjx4+ratWuTxzf0igna2OStYafAAN9uP45QoPJC2so6TftxqBX8n1MLDqrTbxYcaGl5YJhCg6b+1H4cEOa1yyh8hTfajz2Becx3MY8xjzUX81j9Bl9yhaqPFejDc7/V4fadFMy/I4/pddY5PjuHSdIOq1qBAc17Q3aq0EBpcUCYQoPu51O5msgbl1F4W2VZmV5Juc7jdacv+18F23QpXHV1td599109/PDDGjVqlHbs2KGEhATNnDmzzqUWTeW1sKFTp04KDAys08VQUFBQp3uhRlxcXL3bBwUFqWPHjvXuExoaqtDQUNfHGRurcbfc5/L+AHCmmMcA+LozncfaR0cr+ZYp+lVZmXbu3Kny8nIbR4eGcDNXoHUoKChQUVGRnnvuOT3zzDN6/vnntX79eo0fP14bNmzQ8OHDm31Mr4UNISEhGjhwoNLT03Xttdc6l6enp+uaa66pd58hQ4bo73//e61lH3zwgQYNGlTv/RoAAADgP0JDQzVo0CBvDwMAagkKDdX0Zf/rlbp2qbmR+DXXXKP7779fknThhRfqk08+0WuvveZbYYMkpaamauLEiRo0aJCGDBmiRYsW6cCBA7r33nsl/dRyd+jQIS1fvlySdO+992r+/PlKTU3VXXfdpU2bNmnx4sVauXKlN08DAAAAANBKWZZl2+UM3tKpUycFBQXp/PPPr7W8d+/e2rjRtQtyvRo23HjjjTp69Kiefvpp5eXlqW/fvnrvvffUvXt3SVJeXp4OHDjg3D4hIUHvvfee7r//fv3xj3+Uw+HQK6+84raPvQQAAAAAwN+FhITo4osv1p49e2ot//LLL53vz5vL6zeIvO+++3TfffVfS7x06dI6y4YPH67t27e7eVQAAAAAAPiPoqIi7du3z/k4NzdXOTk5io6OVrdu3fTQQw/pxhtv1LBhwzRixAitX79ef//735WRkeFSPa+HDQAAAAAAwL2ys7M1YsQI5+OaTwlKSUnR0qVLde211+q1117T3LlzNX36dPXq1UurV6/W0KFDXapH2AAAAAAAgJ9LTEyUMabBbW6//XbdfvvtttSr/4OzAQAAAAAAXNTqOhtqkpzjx497eSQAWpOaOaexNLkpmMcAeAPzGABfZ+c8hsa1urDhxIkTkqSuXbt6eSQAWqMTJ04oKirqjI8hMY8B8A7mMQC+zo55DI1rdWGDw+HQwYMHFRkZKcuymrTP8ePH1bVrVx08eFDt2rVz8wg9j/PzbZyfbzDG6MSJE3I4HGd8LOaxujg/38b5+QbmMffi/Hwb5+cb7JzH0LhWFzYEBASoS5cuLu3brl07n/7hagzn59s4v5bPrgSdeez0OD/fxvm1fMxj7sf5+TbOr+Wjo8FzuEEkAAAAAACwFWEDAAAAAACwFWFDE4SGhurJJ59UaGiot4fiFpyfb+P80BT+/jxyfr6N80NT+PvzyPn5Ns4PqMsyfO4HAAAAAACNKi0tVW5urhISEhQWFubt4XhdQ88HnQ0AAAAAAMBWhA0AAAAAAPi5rKwsjR07Vg6HQ5Zlae3atbXWW5ZV79cLL7zgUj3CBgAAAAAA/FxxcbH69++v+fPn17s+Ly+v1tef//xnWZalCRMmuFQv6EwGCwAAAABAa2aMkamo9nhdKzhAlmU1efvk5GQlJyefdn1cXFytx2+//bZGjBihs846y6Xx0dnQBAsWLHDe8GLgwIH6+OOPvT2kOmbPnl2n3eXUfyzGGM2ePVsOh0Ph4eFKTEzU559/XusYZWVlmjZtmjp16qQ2bdro6quv1rfffltrmx9//FETJ05UVFSUoqKiNHHiRB07dsz282msxceT53PgwAGNHTtWbdq0UadOnTR9+nSVl5e77dwmTZpU53t5ySWX+MS5SdLcuXN18cUXKzIyUjExMRo3bpz27NlTaxtf/v75Kuax//DEPObPc1hTzo95rOWfoy9iHvsP5jHmscYwj3mWqajW4Sc+8fiXOwOO7777Tu+++67uuOMOl49B2NCIt956SzNmzNBjjz2mHTt26PLLL1dycrIOHDjg7aHV0adPn1ptLzt37nSu+/3vf6+XXnpJ8+fP19atWxUXF6df//rXOnHihHObGTNmKC0tTatWrdLGjRtVVFSkMWPGqKqqyrnNLbfcopycHK1fv17r169XTk6OJk6caPu5NNbi46nzqaqq0ujRo1VcXKyNGzdq1apVWr16tR544AG3nZskXXnllbW+l++9916t9S313CQpMzNTU6ZM0ebNm5Wenq7KykolJSWpuLjYuY0vf/98EfOY5+cxf57DmnJ+EvNYSz9HX8M8xjzGPNY8zGM4U8uWLVNkZKTGjx/v+kEMGvTLX/7S3HvvvbWWnXfeeebRRx/10ojq9+STT5r+/fvXu666utrExcWZ5557zrmstLTUREVFmddee80YY8yxY8dMcHCwWbVqlXObQ4cOmYCAALN+/XpjjDG7du0ykszmzZud22zatMlIMl988YUbzuonkkxaWppXzue9994zAQEB5tChQ85tVq5caUJDQ01hYaHt52aMMSkpKeaaa6457T6+cm41CgoKjCSTmZlpjPGv75+vYB7z7jzmz3NYfednDPOYL55jS8c8xjzGPHZmmMfsU1JSYnbt2mVKSkqcy6qrq01VWaXHv6qrq10+j/r+3Z+qV69eZurUqS49HzXobGhAeXm5tm3bpqSkpFrLk5KS9Mknn3hpVKe3d+9eORwOJSQk6KabbtLXX38tScrNzVV+fn6t8wgNDdXw4cOd57Ft2zZVVFTU2sbhcKhv377ObTZt2qSoqCgNHjzYuc0ll1yiqKgojz4fnjyfTZs2qW/fvnI4HM5tRo0apbKyMm3bts1t55iRkaGYmBj17NlTd911lwoKCpzrfO3cCgsLJUnR0dGSWsf3ryVhHmt581hr+RlgHvOtc2zJmMeYx5jHzhzzmHtZlqWAkECPfzXnfg3N8fHHH2vPnj268847z+g4hA0NOHLkiKqqqhQbG1treWxsrPLz8700qvoNHjxYy5cv1/vvv68//elPys/P16WXXqqjR486x9rQeeTn5yskJEQdOnRocJuYmJg6tWNiYjz6fHjyfPLz8+vU6dChg0JCQtx2zsnJyXrzzTf10Ucf6cUXX9TWrVv1q1/9SmVlZc4x+cq5GWOUmpqqoUOHqm/fvs66NeNtaPy+co4tHfNYy5vHWsPPAPOYb51jS8c8xjzGPHZmmMfQXIsXL9bAgQPVv3//MzoOn0bRBD9PjIwxbkuRXHXqXUX79eunIUOG6Oyzz9ayZcucN7Nx5Tx+vk1923vr+fDU+Xj6nG+88Ubn//ft21eDBg1S9+7d9e677zZ4zVRLPLepU6fqs88+08aNG+us89fvX0vFPNby/k34888A89jptcRz9BXMYy3v34Q//wwwj51eSzxHNE1RUZH27dvnfJybm6ucnBxFR0erW7dukqTjx4/rb3/7m1588cUzrkdnQwM6deqkwMDAOolbQUFBnXSupWnTpo369eunvXv3Ou+C3NB5xMXFqby8XD/++GOD23z33Xd1an3//fcefT48eT5xcXF16vz444+qqKjw2DnHx8ere/fu2rt3r3NMvnBu06ZN07p167RhwwZ16dLFuby1ff+8jXms5c1jrfFngHns9Nt4+xx9AfMY81hL+BlgHjv9Nt4+RzRddna2BgwYoAEDBkiSUlNTNWDAAD3xxBPObVatWiVjjG6++eYzrkfY0ICQkBANHDhQ6enptZanp6fr0ksv9dKomqasrEy7d+9WfHy8EhISFBcXV+s8ysvLlZmZ6TyPgQMHKjg4uNY2eXl5+ve//+3cZsiQISosLNSnn37q3GbLli0qLCz06PPhyfMZMmSI/v3vfysvL8+5zQcffKDQ0FANHDjQredZ4+jRozp48KDi4+MltfxzM8Zo6tSpWrNmjT766CMlJCTUWt/avn/exjzW8uax1vgzwDzW8s7RlzCPMY+1hJ8B5rGWd45ovsTERBlj6nwtXbrUuc3dd9+tkydPKioq6swLNnp7yVZu1apVJjg42CxevNjs2rXLzJgxw7Rp08bs37/f20Or5YEHHjAZGRnm66+/Nps3bzZjxowxkZGRznE+99xzJioqyqxZs8bs3LnT3HzzzSY+Pt4cP37ceYx7773XdOnSxXz44Ydm+/bt5le/+pXp37+/qaysdG5z5ZVXmgsuuMBs2rTJbNq0yfTr18+MGTPG9vM5ceKE2bFjh9mxY4eRZF566SWzY8cO880333j0fCorK03fvn3NFVdcYbZv324+/PBD06VLlybdmdWVcztx4oR54IEHzCeffGJyc3PNhg0bzJAhQ8wvfvELnzg3Y4z57//+bxMVFWUyMjJMXl6e8+vkyZPObXz5++eLmMc8P4/58xzW2Pkxj/nGOfoa5jHmMeax5mEec5+GPn2hNWro+SBsaII//vGPpnv37iYkJMRcdNFFzo+MaUluvPFGEx8fb4KDg43D4TDjx483n3/+uXN9dXW1efLJJ01cXJwJDQ01w4YNMzt37qx1jJKSEjN16lQTHR1twsPDzZgxY8yBAwdqbXP06FFz6623msjISBMZGWluvfVW8+OPP9p+Phs2bDCS6nylpKR4/Hy++eYbM3r0aBMeHm6io6PN1KlTTWlpqVvO7eTJkyYpKcl07tzZBAcHm27dupmUlJQ6426p52aMqffcJJklS5Y4t/Hl75+vYh77D0/MY/48hzV2fsxjvnGOvoh57D+Yx5jHGsM85j6EDbU19HxYxhhz5v0RAAAAAAD4t9LSUuXm5iohIUFhYWHeHo7XNfR8cM8GAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAAD8XFZWlsaOHSuHwyHLsrR27dpa64uKijR16lR16dJF4eHh6t27txYuXOhyPcIGQFJGRoYsy9KxY8e8PRQAcAnzGABfxzwGuFdxcbH69++v+fPn17v+/vvv1/r16/XGG29o9+7duv/++zVt2jS9/fbbLtWzjDHmTAYM+KLExERdeOGFmjdvniSpvLxcP/zwg2JjY2VZlncHBwBNwDwGwNcxj8EXlZaWKjc3VwkJCQoLC5MkGWNUUVHh8bEEBwe7/LNiWZbS0tI0btw457K+ffvqxhtv1OOPP+5cNnDgQF111VX63e9+V+9x6ns+agS5NDLAz4SEhCguLs7bwwAAlzGPAfB1zGPwVRUVFZozZ47H686aNUshISG2HW/o0KFat26dbr/9djkcDmVkZOjLL7/Uyy+/7NLxuIwCrc6kSZOUmZmpl19+WZZlybIsLV26tFbb3tKlS9W+fXu988476tWrlyIiInTdddepuLhYy5YtU48ePdShQwdNmzZNVVVVzmOXl5fr4Ycf1i9+8Qu1adNGgwcPVkZGhndOFIDfYh4D4OuYx4CW55VXXtH555+vLl26KCQkRFdeeaUWLFigoUOHunQ8OhvQ6rz88sv68ssv1bdvXz399NOSpM8//7zOdidPntQrr7yiVatW6cSJExo/frzGjx+v9u3b67333tPXX3+tCRMmaOjQobrxxhslSZMnT9b+/fu1atUqORwOpaWl6corr9TOnTt17rnnevQ8Afgv5jEAvo55DP4kODhYs2bN8kpdO73yyivavHmz1q1bp+7duysrK0v33Xef4uPjNXLkyGYfj7ABrU5UVJRCQkIUERHhbNX74osv6mxXUVGhhQsX6uyzz5YkXXfddfrLX/6i7777Tm3bttX555+vESNGaMOGDbrxxhv11VdfaeXKlfr222/lcDgkSQ8++KDWr1+vJUuWeKW1CoB/Yh4D4OuYx+BPLMuy9XIGbygpKdGsWbOUlpam0aNHS5IuuOAC5eTk6A9/+ANhA2CniIgI5wubJMXGxqpHjx5q27ZtrWUFBQWSpO3bt8sYo549e9Y6TllZmTp27OiZQQPAKZjHAPg65jHAMyoqKlRRUaGAgNp3WggMDFR1dbVLxyRsAE7j521JlmXVu6zmh6+6ulqBgYHatm2bAgMDa2136gsiAHgK8xgAX8c8BtinqKhI+/btcz7Ozc1VTk6OoqOj1a1bNw0fPlwPPfSQwsPD1b17d2VmZmr58uV66aWXXKpH2IBWKSQkpNaNhOwwYMAAVVVVqaCgQJdffrmtxwaAn2MeA+DrmMcAz8rOztaIESOcj1NTUyVJKSkpWrp0qVatWqWZM2fq1ltv1Q8//KDu3bvr2Wef1b333utSPcIGtEo9evTQli1btH//frVt29bl1qBT9ezZU7feeqtuu+02vfjiixowYICOHDmijz76SP369dNVV11lw8gB4CfMYwB8HfMY4FmJiYkyxpx2fVxcnJYsWWJbPT76Eq3Sgw8+qMDAQJ1//vnq3LmzDhw4YMtxlyxZottuu00PPPCAevXqpauvvlpbtmxR165dbTk+ANRgHgPg65jHAP9mmYaiDQAAAAAAIEkqLS1Vbm6uEhISFBYW5u3heF1DzwedDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAA+LmsrCyNHTtWDodDlmVp7dq1tdZ/9913mjRpkhwOhyIiInTllVdq7969LtcjbAAAAAAAwM8VFxerf//+mj9/fp11xhiNGzdOX3/9td5++23t2LFD3bt318iRI1VcXOxSvaAzHTAAAAAAAK2VMUbV1SUerxsQEC7Lspq8fXJyspKTk+tdt3fvXm3evFn//ve/1adPH0nSggULFBMTo5UrV+rOO+9s9vgIGwAAAAAAcFF1dYkyMvt5vG7i8J0KDIyw5VhlZWWSpLCwMOeywMBAhYSEaOPGjS6FDVxGAQAAAABAK3beeeepe/fumjlzpn788UeVl5frueeeU35+vvLy8lw6Jp0NAAAAAAC4KCAgXInDd3qlrl2Cg4O1evVq3XHHHYqOjlZgYKBGjhx52ssumoKwAQAAAAAAF1mWZdvlDN40cOBA5eTkqLCwUOXl5ercubMGDx6sQYMGuXQ8LqMAAAAAAACSpKioKHXu3Fl79+5Vdna2rrnmGpeOQ2cDAAAAAAB+rqioSPv27XM+zs3NVU5OjqKjo9WtWzf97W9/U+fOndWtWzft3LlTv/nNbzRu3DglJSW5VI+wAQAAAAAAP5edna0RI0Y4H6empkqSUlJStHTpUuXl5Sk1NVXfffed4uPjddttt+nxxx93uZ5ljDFnPGoAAAAAAPxcaWmpcnNzlZCQUOtjIlurhp4P7tkAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAICfmzt3ri6++GJFRkYqJiZG48aN0549e2ptY4zR7Nmz5XA4FB4ersTERH3++ecu1SNsAAAAAADAz2VmZmrKlCnavHmz0tPTVVlZqaSkJBUXFzu3+f3vf6+XXnpJ8+fP19atWxUXF6df//rXOnHiRLPrWcYYY+cJAAAAAADgj0pLS5Wbm6uEhASFhYV5ezhn5Pvvv1dMTIwyMzM1bNgwGWPkcDg0Y8YMPfLII5KksrIyxcbG6vnnn9c999xT5xgNPR9BHjkLAAAAAAD8kDFGJ6urPV43IiBAlmW5vH9hYaEkKTo6WpKUm5ur/Px8JSUlObcJDQ3V8OHD9cknn9QbNjSEsAEAAAAAABedrK7W2Vk7PV73q2H91CYw0KV9jTFKTU3V0KFD1bdvX0lSfn6+JCk2NrbWtrGxsfrmm2+aXYOwAQAAAACAVmTq1Kn67LPPtHHjxjrrft4tYYxxqYOCsAEAAAAAABdFBAToq2H9vFLXFdOmTdO6deuUlZWlLl26OJfHxcVJ+qnDIT4+3rm8oKCgTrdDUxA2AAAAAADgIsuyXL6cwZOMMZo2bZrS0tKUkZGhhISEWusTEhIUFxen9PR0DRgwQJJUXl6uzMxMPf/8882uR9gAAAAAAICfmzJlilasWKG3335bkZGRzns0REVFKTw8XJZlacaMGZozZ47OPfdcnXvuuZozZ44iIiJ0yy23NLseYQMAAAAAAH5u4cKFkqTExMRay5csWaJJkyZJkh5++GGVlJTovvvu048//qjBgwfrgw8+UGRkZLPrWcYYc6aDBgAAAADA35WWlio3N1cJCQkKCwvz9nC8rqHnw7U7SgAAAAAAAJwGYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAAH5u7ty5uvjiixUZGamYmBiNGzdOe/bsqbXNmjVrNGrUKHXq1EmWZSknJ8fleoQNAAAAAAD4uczMTE2ZMkWbN29Wenq6KisrlZSUpOLiYuc2xcXFuuyyy/Tcc8+dcb2gMz4CAAAAAABo0davX1/r8ZIlSxQTE6Nt27Zp2LBhkqSJEydKkvbv33/G9QgbAAAAAABwkTFGJRVVHq8bHhwoy7Jc3r+wsFCSFB0dbdeQaiFsAAAAAADARSUVVTr/ifc9XnfX06MUEeLaW3pjjFJTUzV06FD17dvX5pH9hLABAAAAAIBWZOrUqfrss8+0ceNGt9UgbAAAAAAAwEXhwYHa9fQor9R1xbRp07Ru3TplZWWpS5cuNo/qPwgbAAAAAABwkWVZLl/O4EnGGE2bNk1paWnKyMhQQkKCW+u1/GcEAAAAAACckSlTpmjFihV6++23FRkZqfz8fElSVFSUwsPDJUk//PCDDhw4oMOHD0uS9uzZI0mKi4tTXFxcs+oF2Dh2AAAAAADQAi1cuFCFhYVKTExUfHy88+utt95ybrNu3ToNGDBAo0ePliTddNNNGjBggF577bVm17OMMca20QMAAAAA4KdKS0uVm5urhIQEhYWFeXs4XtfQ80FnAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAfm7u3Lm6+OKLFRkZqZiYGI0bN0579uxxrq+oqNAjjzyifv36qU2bNnI4HLrtttt0+PBhl+oRNgAAAAAA4OcyMzM1ZcoUbd68Wenp6aqsrFRSUpKKi4slSSdPntT27dv1+OOPa/v27VqzZo2+/PJLXX311S7Vs4wxxs4TAAAAAADAH5WWlio3N1cJCQkKCwvz9nDOyPfff6+YmBhlZmZq2LBh9W6zdetW/fKXv9Q333yjbt261Vnf0PMR5JZRAwAAAADQGhgjVZz0fN3gCMmyXN69sLBQkhQdHd3gNpZlqX379s0+PmEDAAAAAACuqjgpzXF4vu6sw1JIG5d2NcYoNTVVQ4cOVd++fevdprS0VI8++qhuueUWtWvXrtk1CBsAAAAAAGhFpk6dqs8++0wbN26sd31FRYVuuukmVVdXa8GCBS7VIGwAAAAAAMBVwRE/dRl4o64Lpk2bpnXr1ikrK0tdunSps76iokI33HCDcnNz9dFHH7nU1SARNgAAAAAA4DrLcvlyBk8yxmjatGlKS0tTRkaGEhIS6mxTEzTs3btXGzZsUMeOHV2uR9gAAAAAAICfmzJlilasWKG3335bkZGRys/PlyRFRUUpPDxclZWVuu6667R9+3a98847qqqqcm4THR2tkJCQZtXjoy8BAAAAAGgCX/7oS+s0n1yxZMkSTZo0Sfv376+320GSNmzYoMTExDrL+ehLAAAAAABascb6DHr06NHoNs0RYNuRAAAAAAAARNgAAAAAAABsRtgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAICfmzt3ri6++GJFRkYqJiZG48aN0549e2ptM3v2bJ133nlq06aNOnTooJEjR2rLli0u1SNsAAAAAADAz2VmZmrKlCnavHmz0tPTVVlZqaSkJBUXFzu36dmzp+bPn6+dO3dq48aN6tGjh5KSkvT99983u55ljDF2ngAAAAAAAP6otLRUubm5SkhIUFhYmLeHc0a+//57xcTEKDMzU8OGDat3m+PHjysqKkoffvihrrjiijrrG3o+gtwyagAAAAAAWgFjjEoqSzxeNzwoXJZlubx/YWGhJCk6Orre9eXl5Vq0aJGioqLUv3//Zh+fsAEAAAAAABeVVJZo8IrBHq+75ZYtigiOcGlfY4xSU1M1dOhQ9e3bt9a6d955RzfddJNOnjyp+Ph4paenq1OnTs2uwT0bAAAAAABoRaZOnarPPvtMK1eurLNuxIgRysnJ0SeffKIrr7xSN9xwgwoKCppdg3s2AAAAAADQBPXdo8DXLqOYNm2a1q5dq6ysLCUkJDS6/bnnnqvbb79dM2fOrLOOezYAAAAAAOAGlmW5fDmDJxljNG3aNKWlpSkjI6NJQUPNfmVlZc2uR9gAAAAAAICfmzJlilasWKG3335bkZGRys/PlyRFRUUpPDxcxcXFevbZZ3X11VcrPj5eR48e1YIFC/Ttt9/q+uuvb3Y9wgYAAAAAAPzcwoULJUmJiYm1li9ZskSTJk1SYGCgvvjiCy1btkxHjhxRx44ddfHFF+vjjz9Wnz59ml2PsAEAAAAAAD/X2O0aw8LCtGbNGtvq8WkUAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAAD4ublz5+riiy9WZGSkYmJiNG7cOO3Zs+e0299zzz2yLEvz5s1zqR5hAwAAAAAAfi4zM1NTpkzR5s2blZ6ersrKSiUlJam4uLjOtmvXrtWWLVvkcDhcrhd0JoMFAAAAAAAt3/r162s9XrJkiWJiYrRt2zYNGzbMufzQoUOaOnWq3n//fY0ePdrleoQNAAAAAAC4yBgjU1Li8bpWeLgsy3J5/8LCQklSdHS0c1l1dbUmTpyohx56SH369Dmj8RE2AAAAAADgIlNSoj0XDfR43V7bt8mKiHBpX2OMUlNTNXToUPXt29e5/Pnnn1dQUJCmT59+xuMjbAAAAAAAoBWZOnWqPvvsM23cuNG5bNu2bXr55Ze1ffv2M+qYqEHYAAAAAACAi6zwcPXavs0rdV0xbdo0rVu3TllZWerSpYtz+ccff6yCggJ169bNuayqqkoPPPCA5s2bp/379zerDmEDAAAAAAAusizL5csZPMkYo2nTpiktLU0ZGRlKSEiotX7ixIkaOXJkrWWjRo3SxIkTNXny5GbXI2wAAAAAAMDPTZkyRStWrNDbb7+tyMhI5efnS5KioqIUHh6ujh07qmPHjrX2CQ4OVlxcnHr16tXsegG2jBoAAAAAALRYCxcuVGFhoRITExUfH+/8euutt9xSj84GAAAAAAD8nDGm2fs09z4Np6KzAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAACAZnDlZov+qKHngbABAAAAAIAmCA4OliSdPHnSyyNpGWqeh5rn5VR89CUAAAAAAE0QGBio9u3bq6CgQJIUEREhy7K8PCrPM8bo5MmTKigoUPv27RUYGFhnG8vQ/wEAAAAAQJMYY5Sfn69jx455eyhe1759e8XFxdUbuBA2AAAAAADQTFVVVaqoqPD2MLwmODi43o6GGoQNAAAAAADAVtwgEgAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2Or/AdWd9EgAauThAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff['vh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (My debug_env Kernel)", + "language": "python", + "name": "debug_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.py b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.py new file mode 100644 index 000000000..dce7c1358 --- /dev/null +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.py @@ -0,0 +1,9 @@ +import swiftest +import numpy as np + +swiftersim = swiftest.Simulation(simdir="swifter_sim", read_data=True, codename="Swifter") +swiftestsim = swiftest.Simulation(simdir="swiftest_sim",read_data=True) +swiftestsim.data = swiftestsim.data.swap_dims({"name" : "id"}) +swiftdiff = swiftestsim.data - swiftersim.data +swiftdiff['rh'].plot(x="time",hue="id",col="space") +swiftdiff['vh'].plot(x="time",hue="id",col="space") \ No newline at end of file From 027a2f1abed5daed5bca69387d1c30384c1ba5a3 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 16 May 2023 16:31:53 -0400 Subject: [PATCH 066/149] Changed behavior so lrhill_present is only set by the parameter input file to give the user the option to set a fixed value as in the previous versions. --- src/swiftest/swiftest_io.f90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 8cb720ec0..96f974362 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -983,7 +983,6 @@ module subroutine swiftest_io_netcdf_open(self, param, readonly) ! Optional variables The User Doesn't Need to Know About status = nf90_inq_varid(nc%id, nc%mass_varname, nc%mass_varid) status = nf90_inq_varid(nc%id, nc%rhill_varname, nc%rhill_varid) - param%lrhill_present = (status == NF90_NOERR) status = nf90_inq_varid(nc%id, nc%npl_varname, nc%npl_varid) status = nf90_inq_varid(nc%id, nc%status_varname, nc%status_varid) status = nf90_inq_varid(nc%id, nc%ntp_varname, nc%ntp_varid) @@ -1232,7 +1231,6 @@ module function swiftest_io_netcdf_read_frame_system(self, nc, param) result(ier status = nf90_get_var(nc%id, nc%rhill_varid, rtemp, start=[1, tslot], count=[idmax,1]) if (status == NF90_NOERR) then pl%rhill(:) = pack(rtemp, plmask) - param%lrhill_present = .true. end if end if From 45c298e9414177ef6147ec498c486fe348ef6120 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Tue, 16 May 2023 16:59:17 -0400 Subject: [PATCH 067/149] Switched a bunch of subroutines to the strict math compiler flag category to tamp down on floating point nonsense. --- src/CMakeLists.txt | 50 +++++++++++++++++----------------- src/swiftest/swiftest_util.f90 | 4 +-- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d2a04991c..6212ee56b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,17 +13,36 @@ # Add the source files SET(STRICT_MATH_FILES - ${SRC}/swiftest/swiftest_kick.f90 - ${SRC}/helio/helio_kick.f90 - ${SRC}/rmvs/rmvs_kick.f90 - ${SRC}/symba/symba_kick.f90 - ${SRC}/whm/whm_kick.f90 - ${SRC}/swiftest/swiftest_user.f90 ${SRC}/fraggle/fraggle_generate.f90 ${SRC}/fraggle/fraggle_util.f90 ${SRC}/fraggle/fraggle_module.f90 + ${SRC}/helio/helio_drift.f90 + ${SRC}/helio/helio_gr.f90 + ${SRC}/helio/helio_kick.f90 + ${SRC}/helio/helio_step.f90 ${SRC}/misc/lambda_function_module.f90 ${SRC}/misc/solver_module.f90 + ${SRC}/operator/operator_module.f90 + ${SRC}/operator/operator_cross.f90 + ${SRC}/operator/operator_mag.f90 + ${SRC}/operator/operator_unit.f90 + ${SRC}/rmvs/rmvs_kick.f90 + ${SRC}/rmvs/rmvs_step.f90 + ${SRC}/swiftest/swiftest_drift.f90 + ${SRC}/swiftest/swiftest_gr.f90 + ${SRC}/swiftest/swiftest_kick.f90 + ${SRC}/swiftest/swiftest_user.f90 + ${SRC}/swiftest/swiftest_obl.f90 + ${SRC}/swiftest/swiftest_orbel.f90 + ${SRC}/symba/symba_drift.f90 + ${SRC}/symba/symba_gr.f90 + ${SRC}/symba/symba_kick.f90 + ${SRC}/symba/symba_step.f90 + ${SRC}/whm/whm_coord.f90 + ${SRC}/whm/whm_drift.f90 + ${SRC}/whm/whm_gr.f90 + ${SRC}/whm/whm_kick.f90 + ${SRC}/whm/whm_step.f90 ) SET(FAST_MATH_FILES @@ -33,7 +52,6 @@ SET(FAST_MATH_FILES ${SRC}/misc/io_progress_bar_module.f90 ${SRC}/encounter/encounter_module.f90 ${SRC}/collision/collision_module.f90 - ${SRC}/operator/operator_module.f90 ${SRC}/walltime/walltime_module.f90 ${SRC}/swiftest/swiftest_module.f90 ${SRC}/whm/whm_module.f90 @@ -49,36 +67,18 @@ SET(FAST_MATH_FILES ${SRC}/encounter/encounter_check.f90 ${SRC}/encounter/encounter_io.f90 ${SRC}/encounter/encounter_util.f90 - ${SRC}/helio/helio_drift.f90 - ${SRC}/helio/helio_gr.f90 - ${SRC}/helio/helio_step.f90 ${SRC}/helio/helio_util.f90 ${SRC}/netcdf_io/netcdf_io_implementations.f90 - ${SRC}/operator/operator_cross.f90 - ${SRC}/operator/operator_mag.f90 - ${SRC}/operator/operator_unit.f90 ${SRC}/rmvs/rmvs_discard.f90 ${SRC}/rmvs/rmvs_encounter_check.f90 - ${SRC}/rmvs/rmvs_step.f90 ${SRC}/rmvs/rmvs_util.f90 ${SRC}/swiftest/swiftest_discard.f90 - ${SRC}/swiftest/swiftest_drift.f90 - ${SRC}/swiftest/swiftest_gr.f90 ${SRC}/swiftest/swiftest_io.f90 - ${SRC}/swiftest/swiftest_obl.f90 - ${SRC}/swiftest/swiftest_orbel.f90 ${SRC}/swiftest/swiftest_util.f90 ${SRC}/symba/symba_discard.f90 - ${SRC}/symba/symba_drift.f90 ${SRC}/symba/symba_encounter_check.f90 - ${SRC}/symba/symba_gr.f90 - ${SRC}/symba/symba_step.f90 ${SRC}/symba/symba_util.f90 ${SRC}/walltime/walltime_implementations.f90 - ${SRC}/whm/whm_coord.f90 - ${SRC}/whm/whm_drift.f90 - ${SRC}/whm/whm_gr.f90 - ${SRC}/whm/whm_step.f90 ${SRC}/whm/whm_util.f90 ${SRC}/swiftest/swiftest_driver.f90 ) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 3fc664af9..b620a7f9d 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -110,7 +110,6 @@ module subroutine swiftest_util_append_arr_kin(arr, source, nold, lsource_mask) end subroutine swiftest_util_append_arr_kin - module subroutine swiftest_util_append_body(self, source, lsource_mask) !! author: David A. Minton !! @@ -829,7 +828,6 @@ module subroutine swiftest_util_dealloc_tp(self) end subroutine swiftest_util_dealloc_tp - module subroutine swiftest_util_fill_arr_info(keeps, inserts, lfill_list) !! author: David A. Minton !! @@ -2825,6 +2823,7 @@ module subroutine swiftest_util_sort_tp(self, sortby, ascending) return end subroutine swiftest_util_sort_tp + module subroutine swiftest_util_sort_rearrange_body(self, ind) !! author: David A. Minton !! @@ -3200,7 +3199,6 @@ module subroutine swiftest_util_spill_tp(self, discards, lspill_list, ldestructi end subroutine swiftest_util_spill_tp - module subroutine swiftest_util_valid_id_system(self, param) !! author: David A. Minton !! From 4ed6bf4a91533a3476a7f2e454bc9b0c1942a20b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 17 May 2023 08:45:44 -0400 Subject: [PATCH 068/149] Removed compiler flag that was causing test particle orbits to go haywire (-assume contiguous_assumed_shape) and moved more subroutines to the strict math category. --- cmake/Modules/SetFortranFlags.cmake | 25 +++--- .../swiftest_vs_swifter.ipynb | 88 +++++++++++++++++-- src/CMakeLists.txt | 2 +- 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 403bc6ed0..76f23f5cf 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -158,7 +158,7 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" # Strict model for floating-point calculations (precise and except) SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fp-model strict" # Intel + Fortran "-fp-model=strict" # Intel ) # Enables floating-point invalid, divide-by-zero, and overflow exceptions @@ -273,11 +273,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-align all" # Intel ) -# Assume all objects are contiguous in memory -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-assume contiguous_assumed_shape" # Intel - ) - # Generate an extended set of vector functions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" Fortran "-vecabi=cmdtarget" # Intel @@ -289,17 +284,17 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" ) # Generate fused multiply-add instructions -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-fma" # Intel - ) + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" + Fortran "-fma" # Intel + ) # Generate fused multiply-add instructions -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-qmkl=cluster" # Intel - Fortran "-qmkl" # Intel - Fortran "-mkl" # Old Intel - ) - + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" + Fortran "-qmkl=cluster" # Intel + Fortran "-qmkl" # Intel + Fortran "-mkl" # Old Intel + ) + ##################### ### MATH FLAGS ### ##################### diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb index 920def237..322bbb1d2 100644 --- a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/swiftest_vs_swifter.ipynb @@ -71,20 +71,60 @@ "cell_type": "code", "execution_count": 5, "metadata": {}, + "outputs": [], + "source": [ + "plid = swiftdiff['id'].where(swiftdiff['id'] < 9, drop=True)\n", + "tpid = swiftdiff['id'].where(swiftdiff['id'] >= 9, drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABCYAAAIjCAYAAADfthq2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABeJklEQVR4nO3deXxU5d3///fJvhACCYQkQiC1yBZEBEQUISgNREARdykG0FZvA5TGumFV3EDbW3+ofEXxpoALS1sNUKsgVhOkCEIwBQURNAgKGEAIZN+u3x80I4EAyWQmZ2byej4e0zpn+1xnQq6Tec91nbGMMUYAAAAAAAA28LO7AQAAAAAAoPkimAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYhmAAAAAAAALYJsLsBAAAAAAD4qurqapWXl9vdDNsEBgbK39//rNsQTAAAAAAA4Abl5eXKy8tTdXW13U2xVatWrRQbGyvLsupcTzABAAAAAICLGWO0f/9++fv7q0OHDvLza353UjDGqLi4WPn5+ZKkuLi4OrcjmAAAAAAAwMUqKytVXFys+Ph4hYWF2d0c24SGhkqS8vPzFRMTU+e0juYX2QAAAAAA4GZVVVWSpKCgIJtbYr+aYKaioqLO9QQTAAAAAAC4yZnuq9CcnOs1IJgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAMAHJCcna+rUqWdc36lTJ82aNavJ2lNffCsHAAAAAAA+4J133lFgYKDdzWgwggkAAAAAAHxAVFSU3U1wClM5AAAAAADwASdP5cjPz9eoUaMUGhqqxMREvfXWW/Y27iwYMQEAAAAAgI8ZP3689u7dq48++khBQUGaMmWK8vPz7W5WnQgmAAAAAADwIV9//bXef/99rV+/Xv3795ckzZs3T926dbO5ZXVjKgcAAAAAAD5k+/btCggIUN++fR3LunbtqlatWtnXqLMgmAAAAAAAwIcYYyRJlmXZ3JL6IZgAAAAAAMCHdOvWTZWVldq0aZNj2Y4dO3T06FH7GnUWBBMAAAAAAPiQLl26aPjw4frNb36jDRs2KCcnR3feeadCQ0PtblqdCCYAAAAAAPAx8+fPV4cOHTR48GCNGTNGv/3tbxUTE2N3s+pkmZrJJwAAAAAAwCVKS0uVl5enxMREhYSE2N0cW53rtWDEBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAsA3BBAAAAAAAcFizZo1GjRql+Ph4WZalZcuWubUewQQAAAAAAHAoKipSr169NHv27CapF9AkVQAAAAAAaMaMMSqpqLKldmigvyzLqvf2qampSk1NdWOLaiOYAAAAAADAzUoqqtT90VW21N72xDCFBXnu23+mcgAAAAAAANt4bmQCAAAAAICPCA3017YnhtlW25MRTAAAAAAA4GaWZXn0dAo7MZUDAAAAAADYhrgGAAAAAAA4FBYWateuXY7neXl5ys3NVVRUlBISElxej2ACAAAAAAA4bNq0SUOGDHE8z8jIkCSlpaVpwYIFLq9HMAEAAAAAABySk5NljGmyetxjAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAA2IZgAgAAAAAASJJmzpypfv36KSIiQjExMRo9erR27Njh1poEEwAAAAAAQJKUnZ2t9PR0rV+/XqtXr1ZlZaVSUlJUVFTktpoBbjsyAAAAAADwKitXrqz1fP78+YqJiVFOTo4GDRrklpoEEwAAAAAAuJsxUkWxPbUDwyTLcmrXgoICSVJUVJQrW1QLwQQAAAAAAO5WUSzNiLen9rR9UlB4g3czxigjI0MDBw5UUlKSGxp2AsEEAAAAAAA4zaRJk7RlyxatXbvWrXUIJgAAAAAAcLfAsBMjF+yq3UCTJ0/WihUrtGbNGrVv394NjfoZwQQAAAAAAO5mWU5Np2hqxhhNnjxZmZmZysrKUmJiottrEkwAAAAAAABJUnp6uhYtWqTly5crIiJCBw4ckCRFRkYqNDTULTX93HJUAAAAAADgdebMmaOCggIlJycrLi7O8Vi6dKnbajJiAgAAAAAASDoxlaOpMWICAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAABIkubMmaMLL7xQLVu2VMuWLTVgwAC9//77bq1JMAEAAAAAACRJ7du31zPPPKNNmzZp06ZNuvLKK3Xttdfqyy+/dFtNyxhj3HZ0AAAAAACaodLSUuXl5SkxMVEhISF2N6dRoqKi9Oc//1l33HGHU/uf67UIaGwDAQAAAADA2RljVFJZYkvt0IBQWZbV4P2qqqr0t7/9TUVFRRowYIAbWnYCwQQAAAAAAG5WUlmi/ov621J7w20bFBYYVu/tt27dqgEDBqi0tFQtWrRQZmamunfv7rb2cY8JAAAAAADg0KVLF+Xm5mr9+vX6n//5H6WlpWnbtm1uq8c9JgAAAAAAcLFT76vgjVM5agwdOlTnn3++Xn31Vaf25x4TAAAAAADYzLKsBk2n8CTGGJWVlbnt+AQTAAAAAABAkjRt2jSlpqaqQ4cOOn78uJYsWaKsrCytXLnSbTUJJgAAAAAAgCTpxx9/1Lhx47R//35FRkbqwgsv1MqVK/WrX/3KbTUJJgAAAAAAgCRp3rx5TV6Tb+UAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAC2IZgAAAAAAAB1mjlzpizL0tSpU91Wg2ACAAAAAACcZuPGjZo7d64uvPBCt9YhmAAAAAAAALUUFhZq7Nixeu2119S6dWu31gpw69EBAAAAAICMMTIlJbbUtkJDZVlWg/ZJT0/XiBEjNHToUD311FNuatkJBBMAAAAAALiZKSnRjov72FK7y+YcWWFh9d5+yZIl2rx5szZu3OjGVv2MYAIAAAAAAEiS9u7dq9/97nf64IMPFBIS0iQ1LWOMaZJKAAAAAAA0E6WlpcrLy1NiYqJCQkK8ZirHsmXLdN1118nf39+xrKqqSpZlyc/PT2VlZbXW1cepr8WpGDEBAAAAAICbWZbVoOkUdrnqqqu0devWWssmTJigrl276oEHHmhwKFEfBBMAAAAAAECSFBERoaSkpFrLwsPDFR0dfdpyV+HrQgEAAAAAgG0YMQEAAAAAAM4oKyvLrcdnxAQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAALANwQQAAAAAAJAkTZ8+XZZl1XrExsa6tWaAW48OAAAAAAC8So8ePfThhx86nvv7+7u1HsEEAAAAAABwCAgIcPsoiVr1mqwSAAAAAADNlDFGleXVttQOCPKTZVn13n7nzp2Kj49XcHCw+vfvrxkzZugXv/iF+9rntiMDAAAAAABJUmV5teb+LtuW2r99YbACg+s3HaN///56/fXXdcEFF+jHH3/UU089pcsuu0xffvmloqOj3dI+ggkAAAAAACBJSk1Ndfx3z549NWDAAJ1//vlauHChMjIy3FKTYAIAAAAAADcLCPLTb18YbFttZ4WHh6tnz57auXOnC1tUG8EEAAAAAABuZllWvadTeJKysjJt375dV1xxhdtqOB+bAAAAAAAAn/KHP/xB2dnZysvL04YNG3TDDTfo2LFjSktLc1tNRkwAAAAAAABJ0vfff69bb71Vhw4dUtu2bXXppZdq/fr16tixo9tqEkwAAAAAAABJ0pIlS5q8JlM5AAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgm4BH+/ve/q2fPngoNDVV0dLSGDh2qoqIiSdL48eM1evRoPf7444qJiVHLli111113qby83LH/ypUrNXDgQLVq1UrR0dEaOXKkvvnmm1o1vv/+e91yyy2KiopSeHi4+vbtqw0bNjjW/+Mf/1CfPn0UEhKiX/ziF3r88cdVWVnplvM1xmjo0KEaPny4jDGSpKNHjyohIUEPP/ywW2oCsE9z6+N2794tPz8/bdq0qdbyl156SR07dnT0ewB8S3Pr6ySpU6dOsizrtAfg7X744Qf9+te/VnR0tMLCwnTRRRcpJyfHbfUIJmC7/fv369Zbb9XEiRO1fft2ZWVlacyYMbX+cP3Xv/6l7du36+OPP9bixYuVmZmpxx9/3LG+qKhIGRkZ2rhxo/71r3/Jz89P1113naqrqyVJhYWFGjx4sPbt26cVK1boP//5j+6//37H+lWrVunXv/61pkyZom3btunVV1/VggUL9PTTT5+x3W+99ZZatGhx1sdbb71V576WZWnhwoX67LPP9OKLL0qS7r77brVr107Tp09v7EsKwIM0xz6uU6dOGjp0qObPn19r+fz58zV+/Hj+aAd8UHPs6yRp48aN2r9/v/bv36/vv/9el156qa644orGvpyArY4cOaLLL79cgYGBev/997Vt2zY999xzatWqlfuKGh+RnZ1tRo4caeLi4owkk5mZaXu9tLQ0I6nWo3///m5tlzfKyckxkszu3bvrXJ+WlmaioqJMUVGRY9mcOXNMixYtTFVVVZ375OfnG0lm69atxhhjXn31VRMREWEOHz5c5/ZXXHGFmTFjRq1lb7zxhomLiztju48dO2Z27tx51sexY8fOeu5//etfTXBwsHnooYdMWFiY2bFjx1m3B+B9mmsft3TpUtO6dWtTWlpqjDEmNzfXWJZl8vLyzrgPAO/VXPu6k02ZMsV07NjR5Ofn12t7+L6SkhKzbds2U1JSYndTGuSBBx4wAwcOdOkxz/VaBLgv8mhaRUVF6tWrlyZMmKDrr7/eY+oNHz681idGQUFBbm+bt+nVq5euuuoq9ezZU8OGDVNKSopuuOEGtW7dutY2YWFhjucDBgxQYWGh9u7dq44dO+qbb77RI488ovXr1+vQoUOO5HzPnj1KSkpSbm6uevfuraioqDrbkJOTo40bN9ZK1KuqqlRaWqri4uJatWtEREQoIiKiUed+4403KjMzUzNnztScOXN0wQUXNOp4ADxPc+3jRo8erUmTJikzM1O33HKL/vKXv2jIkCHq1KmT08cE4Lmaa19XY+7cuZo3b57+/e9/q23bto0+HnyTMUaVZWW21A4IDq73iMUVK1Zo2LBhuvHGG5Wdna3zzjtP99xzj37zm9+4r31uO3ITS01NVWpq6hnXl5eX649//KPeeustHT16VElJSXr22WeVnJzslno1goODFRsb61SN5sLf31+rV6/WunXr9MEHH+ill17Sww8/rA0bNigxMfGs+9b8co0aNUodOnTQa6+9pvj4eFVXVyspKckxbzE0NPSsx6murtbjjz+uMWPGnLYuJCSkzn3eeust3XXXXWc97quvvqqxY8eecX1xcbFycnLk7++vnTt3nvVYALxTc+3jgoKCNG7cOM2fP19jxozRokWLNGvWrLMeD4D3aq59nSRlZWVp8uTJWrx4sXr16nXWY6F5qywr04tpN9hSe8rCvyvwDL8Hp/r22281Z84cZWRkaNq0afrss880ZcoUBQcH6/bbb3dL+3wmmDiXCRMmaPfu3VqyZIni4+OVmZmp4cOHa+vWrercubPb6mZlZSkmJkatWrXS4MGD9fTTTysmJsZt9byVZVm6/PLLdfnll+vRRx9Vx44dlZmZqYyMDEnSf/7zH5WUlDguSOvXr1eLFi3Uvn17HT58WNu3b9err77qmNO3du3aWse/8MIL9X//93/66aef6kzZL774Yu3YsUO//OUv693ma665Rv379z/rNu3atTvr+nvvvVd+fn56//33dfXVV2vEiBG68sor690GAN6hufZxd955p5KSkvTyyy+roqKizjcLAHxHc+zrdu3apeuvv17Tpk2jj4PPqK6uVt++fTVjxgxJUu/evfXll19qzpw5BBON8c0332jx4sX6/vvvFR8fL0n6wx/+oJUrV2r+/PmOF9zVUlNTdeONN6pjx47Ky8vTI488oiuvvFI5OTkKDg52S01vtGHDBv3rX/9SSkqKYmJitGHDBh08eFDdunVzbFNeXq477rhDf/zjH/Xdd9/pscce06RJk+Tn56fWrVsrOjpac+fOVVxcnPbs2aMHH3ywVo1bb71VM2bM0OjRozVz5kzFxcXp888/V3x8vAYMGKBHH31UI0eOVIcOHXTjjTfKz89PW7Zs0datW/XUU0/V2e7GDv375z//qb/85S/69NNPdfHFF+vBBx9UWlqatmzZUmvYIwDv1lz7OEnq1q2bLr30Uj3wwAOaOHHiOT/tBOC9mmNfV1JSolGjRumiiy7Sb3/7Wx04cMCxjhHTqEtAcLCmLPy7bbXrKy4uTt27d6+1rFu3bnr77bdd3ayfufSOFh5Cp9yM8q9//auRZMLDw2s9AgICzE033WSMMSYvL++0G1We+khPT69XvTPZt2+fCQwMNG+//bYrTtNnbNu2zQwbNsy0bdvWBAcHmwsuuMC89NJLjvVpaWnm2muvNY8++qiJjo42LVq0MHfeeafjhmrGGLN69WrTrVs3ExwcbC688EKTlZV12s9l9+7d5vrrrzctW7Y0YWFhpm/fvmbDhg2O9StXrjSXXXaZCQ0NNS1btjSXXHKJmTt3rlvOOT8/37Rr167WDZoqKirMJZdc4vg3CcA3NMc+7mTz5s0zksxnn33m9loA7NMc+7qzvX8AjPHem1/eeuutp938curUqWbAgAFOH/Ncr4VljO99mbhlWcrMzNTo0aMlSUuXLtXYsWP15Zdfyt/fv9a2LVq0UGxsrCoqKk77nuRTtW7dus6hXKfWO5vOnTvrzjvv1AMPPFDv82nuxo8fr6NHj2rZsmV2NwUAXM7X+7inn35aS5Ys0datW+1uCgAb+XpfB9SltLRUeXl5SkxMPON9TjzRxo0bddlll+nxxx/XTTfdpM8++0y/+c1vNHfu3LPeb+VszvVaNIupHL1791ZVVZXy8/PP+L3CgYGB6tq1q1vbcfjwYe3du1dxcXFurQMAgN0KCwu1fft2vfTSS3ryySftbg4AAKinfv36KTMzUw899JCeeOIJJSYmatasWU6HEvXhM8FEYWGhdu3a5Xiel5en3NxcRUVF6YILLtDYsWN1++2367nnnlPv3r116NAhffTRR+rZs6euvvpql9ZLSEhQYWGhpk+fruuvv15xcXHavXu3pk2bpjZt2ui6665zyTkDAOCpJk2apMWLF2v06NGaOHGi3c0BAAANMHLkSI0cObLJ6vnMVI6srCwNGTLktOVpaWlasGCBKioq9NRTT+n111/XDz/8oOjoaA0YMECPP/64evbs6fJ6JSUlGj16tD7//HMdPXpUcXFxGjJkiJ588kl16NDBqXMEAAAAAHgHb53K4Q7nei18JpgAAAAAAMBTEEz87FyvhZ8NbQIAAAAAAJDk5feYqK6u1r59+xQRESHLsuxuDgDIGKPjx48rPj5efn6Nz37p5wB4Gvo5AL7O1f0czs2rg4l9+/ZxvwYAHmnv3r1q3759o49DPwfAU9HPAfB1rurncG5eHUxERERIOvEPpmXLlja3BgCkY8eOqUOHDo7+qbHo5wB4Gvo5AL7O1f0czs2rg4ma4X4tW7bkQgbAo7hqODL9HABPRT8HwNcxvazpMGEGAAAAAADYhmACAAAAAADYhmACAAAAAADYhmACAAAAAABIkjp16iTLsk57pKenu62mV9/8EgAAAAAAuM7GjRtVVVXleP7FF1/oV7/6lW688Ua31SSYAAAA8EHHjh1TSUnJObfz9/dXmzZtmqBFAABv0LZt21rPn3nmGZ1//vkaPHiw22oSTAAAAPiY7777TvPnz6/Xtq1atdLUqVPd2yAAgIwxMhXVttS2Av2c+vrT8vJyvfnmm8rIyHDr16cSTAAAAPiY/Px8SSdGQwQHB59127CwsKZoEgA0e6aiWvseXWdL7fgnLpMV5N/g/ZYtW6ajR49q/Pjxrm/USQgmAAAAfEx19YlP5Lp06aKbbrrJ5tYAALzVvHnzlJqaqvj4eLfWIZgAAADwMTXBhJ8fX8AGAJ7CCvRT/BOX2Va7ob777jt9+OGHeuedd9zQotoIJgAAAHyMMUYSwQQAeBLLspyaTmGX+fPnKyYmRiNGjHB7La5WAAAAPqZmxIQ7b1QGAPBd1dXVmj9/vtLS0hQQ4P7xDAQTAAAAPoapHACAxvjwww+1Z88eTZw4sUnqMZUDAADAxzCVAwDQGCkpKY5rSVPgagUAAOBjmMoBAPAmBBMAAAA+hqkcAABvwtUKAADAxzCVAwDgTbhaAQAA+BimcgAAvAnBBAAAgI9hxAQAwJtwtQIAAPAx3GMCAOBNuFoBAAD4GKZyAAC8CcEEAACAj2EqBwDAm3C1AgAA8DFM5QAAeBOuVgAAAD6GqRwAAG9CMAEAAOBjmMoBAHBWZWWl/vjHPyoxMVGhoaH6xS9+oSeeeMIRertDgNuODAAAAFswlQMA4Kxnn31Wr7zyihYuXKgePXpo06ZNmjBhgiIjI/W73/3OLTUJJgAAAHwMUzkAwPMYY1RRUWFL7cDAwHpfEz799FNde+21GjFihCSpU6dOWrx4sTZt2uS29hFMAAAA+BimcgCA56moqNCMGTNsqT1t2jQFBQXVa9uBAwfqlVde0ddff60LLrhA//nPf7R27VrNmjXLbe0jmAAAAPAxjJgAADjrgQceUEFBgbp27Sp/f39VVVXp6aef1q233uq2mgQTAAAAPoZ7TACA5wkMDNS0adNsq11fS5cu1ZtvvqlFixapR48eys3N1dSpUxUfH6+0tDS3tI9gAgAAwMcwlQMAPI9lWfWeTmGn++67Tw8++KBuueUWSVLPnj313XffaebMmW4LJmy/Wv3www/69a9/rejoaIWFhemiiy5STk6O3c0CAADwWkzlAAA4q7i4+LRg29/f33e/LvTIkSO6/PLLNWTIEL3//vuKiYnRN998o1atWtnZLAAAAK/GVA4AgLNGjRqlp59+WgkJCerRo4c+//xzPf/885o4caLbatoaTDz77LPq0KGD5s+f71jWqVOnM25fVlamsrIyx/Njx465s3kA0OTo5wC4gidP5aCfAwDP9tJLL+mRRx7RPffco/z8fMXHx+uuu+7So48+6raatl6tVqxYob59++rGG29UTEyMevfurddee+2M28+cOVORkZGOR4cOHZqwtQDgfvRzAFzBk6dy0M8BgGeLiIjQrFmz9N1336mkpETffPONnnrqKbfeH8PWYOLbb7/VnDlz1LlzZ61atUp33323pkyZotdff73O7R966CEVFBQ4Hnv37m3iFgOAe9HPAXAFT57KQT8HADiVrVM5qqur1bdvX82YMUOS1Lt3b3355ZeaM2eObr/99tO2Dw4OVnBwcFM3EwCaDP0cAFfw5Kkc9HMAgFPZerWKi4tT9+7day3r1q2b9uzZY1OLAAAAvJ8nT+UAAOBUtgYTl19+uXbs2FFr2ddff62OHTva1CIAAADv58lTOQAAOJWtV6vf//73Wr9+vWbMmKFdu3Zp0aJFmjt3rtLT0+1sFgAAgFfz5KkcAACcytarVb9+/ZSZmanFixcrKSlJTz75pGbNmqWxY8fa2SwAAACvxlQOAIA3sfXml5I0cuRIjRw50u5mAAAA+AymcgAAvAlXKwAAAB/DVA4AgDfhagUAAOBjmMoBAPAmBBMAAAA+hqkcAABvwtUKAADAxzCVAwDQGMePH9fUqVPVsWNHhYaG6rLLLtPGjRvdVo+rFQAAgI9hKgcAoDHuvPNOrV69Wm+88Ya2bt2qlJQUDR06VD/88INb6tn+rRwAAABwLaZyAIDnMcaourrEltp+fqH1DqtLSkr09ttva/ny5Ro0aJAkafr06Vq2bJnmzJmjp556yuXtI5gAAADwMUzlAADPU11doqzsnrbUTh68Vf7+YfXatrKyUlVVVQoJCam1PDQ0VGvXrnVH85jKAQAA4GuYygEAcFZERIQGDBigJ598Uvv27VNVVZXefPNNbdiwQfv373dLTUZMAAAA+BimcgCA5/HzC1Xy4K221W6IN954QxMnTtR5550nf39/XXzxxbrtttu0efNmt7SPYAIAAMDHMJUDADyPZVn1nk5ht/PPP1/Z2dkqKirSsWPHFBcXp5tvvlmJiYluqcfVCgAAwMcwlQMA4Arh4eGKi4vTkSNHtGrVKl177bVuqcOICQAAAB9SE0pIjJgAADhn1apVMsaoS5cu2rVrl+677z516dJFEyZMcEs9rlYAAAA+pGYah0QwAQBwTkFBgdLT09W1a1fdfvvtGjhwoD744AMFBga6pR4jJgAAAHzIySMmmMoBAHDGTTfdpJtuuqnJ6hGjAwAA+BCmcgAAvA1XKwAAAB/CVA4AgLfhagUAAOBDmMoBAPA2BBMAAAA+hKkcAABvw9UKAADAh9RM5bAsixETAACvQDABAADgQ2pGTBBKAAC8BcEEAACAD6kZMcE0DgCAt+CKBQAA4ENqRkwQTAAAvAVXLAAAAB/CVA4AgLchmAAAAPAhTOUAAHgbrlgAAAA+hKkcAIDGWrNmjUaNGqX4+HhZlqVly5bVWm+M0fTp0xUfH6/Q0FAlJyfryy+/dLoeVywAAAAfwlQOAEBjFRUVqVevXpo9e3ad6//0pz/p+eef1+zZs7Vx40bFxsbqV7/6lY4fP+5UvYDGNBYAAACehakcAOCZjDEq/m943NTC/PwaFFinpqYqNTW1znXGGM2aNUsPP/ywxowZI0lauHCh2rVrp0WLFumuu+5qcPsIJgAAAHwIUzkAwDMVV1fr/DVbban9zaCeCvf3d8mx8vLydODAAaWkpDiWBQcHa/DgwVq3bp1TwQRXLAAAAB/CVA4AgDsdOHBAktSuXbtay9u1a+dY11CMmAAAAPAhTOUAAM8U5uenbwb1tK22q50agBtjnA7FCSYAAAB8CCMmAMAzWZblsukUdoqNjZV0YuREXFycY3l+fv5poyjqiygdAADAh3CPCQCAOyUmJio2NlarV692LCsvL1d2drYuu+wyp47JiAkAAAAfwlQOAEBjFRYWateuXY7neXl5ys3NVVRUlBISEjR16lTNmDFDnTt3VufOnTVjxgyFhYXptttuc6oewQQAAIAPYSoHAKCxNm3apCFDhjieZ2RkSJLS0tK0YMEC3X///SopKdE999yjI0eOqH///vrggw8UERHhVD2CCQAAAB/CVA4AQGMlJyc7RuDVxbIsTZ8+XdOnT3dJPa5YAAAAPoSpHAAAb8MVCwAAwIcwlQMA4G0IJgAAAHwIUzkAAN6GKxYAAIAPYSoHAMDbeMwVa+bMmbIsS1OnTrW7KQAAAF6LqRwAAG/jEcHExo0bNXfuXF144YV2NwUAAMCrMZUDAOBtbL9iFRYWauzYsXrttdfUunXrs25bVlamY8eO1XoAgC+hnwPQWJ4+lYN+DgBwKtuvWOnp6RoxYoSGDh16zm1nzpypyMhIx6NDhw5N0EIAaDr0cwAay9OnctDPAQBOZWswsWTJEm3evFkzZ86s1/YPPfSQCgoKHI+9e/e6uYUA0LTo5wA0lqdP5aCfAwCcKsCuwnv37tXvfvc7ffDBBwoJCanXPsHBwQoODnZzywDAPvRzABrL06dy0M8BAE5l2xUrJydH+fn56tOnjwICAhQQEKDs7Gy9+OKLCggIUFVVlV1NAwAA8FqePpUDAOD51qxZo1GjRik+Pl6WZWnZsmW11r/zzjsaNmyY2rRpI8uylJub26h6tgUTV111lbZu3arc3FzHo2/fvho7dqxyc3Pl7+9vV9MAAAC8lqdP5QAAeL6ioiL16tVLs2fPPuP6yy+/XM8884xL6tk2lSMiIkJJSUm1loWHhys6Ovq05QAAAKgfT5/KAQDNlTFGJRX2zAwIDfRv0Ei61NRUpaamnnH9uHHjJEm7d+9ubNMk2RhMAAAAwPWYygEAnqmkokrdH11lS+1tTwxTWJDnvv33qJZlZWXZ3QQAAACvxlQOAIC38ahgAgAAAI3DVA4A8Eyhgf7a9sQw22p7MoIJAAAAH8JUDgDwTJZlefR0CjvxqgBoclVVVfriiy9UVFRkd1Pq5dJLL+WTR8BL7Nu3z2U34vJW33//vSRGTAAAvAfBBIAmt2PHDmVmZtrdjHrr37+/3U0AUE9vvfWW14Se7hYYGGh3EwAAXqqwsFC7du1yPM/Ly1Nubq6ioqKUkJCgn376SXv27NG+ffsknfj7XpJiY2MVGxvb4HoEEwCaXHFxsSSpZcuW6tSpk72NAeBTakKJbt26Nes35kFBQerTp4/dzQAAeKlNmzZpyJAhjucZGRmSpLS0NC1YsEArVqzQhAkTHOtvueUWSdJjjz2m6dOnN7gewQSAJlcz//m8887TmDFjbG4NAF9Rc9NHSRo5cqTCw8NtbA0AAN4rOTm51nX1VOPHj9f48eNdVq9RwUR5ebny8/MdbzJqJCQkNKpRAHwbX2UHwB1O/nuEGz8CAOA9nAomdu7cqYkTJ2rdunW1lhtjZFmWqqqqXNI4AL6Jr7ID4A4nBxP0LwAAeA+ngonx48crICBA7777ruLi4vhUAkCD8FV2ANzh5CGnBBMAAHgPp4KJ3Nxc5eTkqGvXrq5uD4BmgBETANyBqRwAAHgnp94VdO/eXYcOHXJ1WwA0E9xjAoA7MJUDAADvVO+r9rFjxxyPZ599Vvfff7+ysrJ0+PDhWuuOHTvmzvYC8AFM5QDgDkzlAADAO9V7KkerVq1qvYkwxuiqq66qtQ03vwRQH0zlAOAOTOUAAMA71TuY+PjjjyVJZWVlmjFjhm699VbuMQHAKUzlAOAO9C0AAHinegcTgwcPdvz32LFjdeWVV6pz585uaRQA38ZUDgDuwGgsAAC8k1NX7ttvv13z5s1zdVsANBO8eQDgDoSeAAB4J6e+LrS8vFz/93//p9WrV6tv374KDw+vtf755593SeMA+CaGWwNwB/oWAABcY82aNfrzn/+snJwc7d+/X5mZmRo9erQkqaKiQn/84x/13nvv6dtvv1VkZKSGDh2qZ555RvHx8U7VcyqY+OKLL3TxxRdLkr7++uta6/iUAsC58KkmAHdgNBYAAK5RVFSkXr16acKECbr++utrrSsuLtbmzZv1yCOPqFevXjpy5IimTp2qa665Rps2bXKqnlPBRM2NMAHAGbx5AOAOhJ4AAI9mjFRRbE/twDCpAdfH1NRUpaam1rkuMjJSq1evrrXspZde0iWXXKI9e/YoISGhwc1zKpgAgMbgzQMAdyD0BAB4tIpiaYZzUx0abdo+KSj83Ns5qaCgQJZlqVWrVk7tz5UbQJNjHjgAd6BvAQCg6ZWWlurBBx/UbbfdppYtWzp1DEZMAGhyfKoJwB0YjQUA8GiBYSdGLthV2w0qKip0yy23qLq6Wi+//LLTxyGYANDkePMAwB0IPQEAHs2y3DqdoqlVVFTopptuUl5enj766COnR0tIBBMAbMBwawDuQN8CAEDTqAkldu7cqY8//ljR0dGNOh7BBIAmx6eaANyB0VgAALhGYWGhdu3a5Xiel5en3NxcRUVFKT4+XjfccIM2b96sd999V1VVVTpw4IAkKSoqSkFBQQ2uRzABoMnx5gGAOxB6AgDgGps2bdKQIUMczzMyMiRJaWlpmj59ulasWCFJuuiii2rt9/HHHys5ObnB9QgmADQ5hlsDcAdCTwAAXCM5OdkR+NflbOucwbsCAE2OTzUBuAOhJwAA3okrN4Amx6eaANyB0BMAAO/ElRtAk+NTTQDuQOgJAIB34l0BgCbHp5oA3IHQEwAA78SVG0CT41NNAO5A6AkAgHfiyg2gyfGpJgB3IPQEAMA78a4AQJPjU00A7kDoCQCAd+LKDaDJ8akmAHcg9AQAwDtx5QbQ5PhUE4A7EHoCAOCdeFcAoMnxqSYAdyD0BADAO3HlBtDk+FQTgDsQegIA4Bpr1qzRqFGjFB8fL8uytGzZslrrp0+frq5duyo8PFytW7fW0KFDtWHDBqfrceUG0OT4VBOAOxB6AgDgGkVFRerVq5dmz55d5/oLLrhAs2fP1tatW7V27Vp16tRJKSkpOnjwoFP1AhrT2MaaOXOm3nnnHX311VcKDQ3VZZddpmeffVZdunSxs1kA3IxPNQG4A6EnAMCTGWNUUlliS+3QgNAGBfepqalKTU094/rbbrut1vPnn39e8+bN05YtW3TVVVc1uH22BhPZ2dlKT09Xv379VFlZqYcfflgpKSnatm2bwsPD7WwaADfiU00A7kDoCQDwZCWVJeq/qL8ttTfctkFhgWFuOXZ5ebnmzp2ryMhI9erVy6lj2BpMrFy5stbz+fPnKyYmRjk5ORo0aNBp25eVlamsrMzx/NixY25vIwDX483DmdHPAc4j9PQO9HMA4Bveffdd3XLLLSouLlZcXJxWr16tNm3aOHUsW4OJUxUUFEiSoqKi6lw/c+ZMPf74403ZJABuwHDrM6OfA5xH3+Id6OcANFehAaHacJvzN4hsbG1XGzJkiHJzc3Xo0CG99tpruummm7RhwwbFxMQ0+Fgec+U2xigjI0MDBw5UUlJSnds89NBDKigocDz27t3bxK0E4Ap8qnlm9HOA8xiN5R3o5wA0V5ZlKSwwzJaHO/7uDg8P1y9/+UtdeumlmjdvngICAjRv3jynjuUxIyYmTZqkLVu2aO3atWfcJjg4WMHBwU3YKgDuwJuHM6OfA5xH6Okd6OcAwDcZY2pN1WsIjwgmJk+erBUrVmjNmjVq37693c0B4GYMtwbgDoSeAAC4RmFhoXbt2uV4npeXp9zcXEVFRSk6OlpPP/20rrnmGsXFxenw4cN6+eWX9f333+vGG290qp6twYQxRpMnT1ZmZqaysrKUmJhoZ3MANEBZWZkqKioavF9QUBCfagJwC0JPAABcY9OmTRoyZIjjeUZGhiQpLS1Nr7zyir766istXLhQhw4dUnR0tPr166dPPvlEPXr0cKqercFEenq6Fi1apOXLlysiIkIHDhyQJEVGRio01PU35wDgGnl5eXrzzTdVVVXV4H0DAgJUWVkpiTcPAFyL0BMAANdITk52jESsyzvvvOPSera+K5gzZ44KCgqUnJysuLg4x2Pp0qV2NgvAOezdu9epUEKSI5SQCCYAuBZTOQAA8E62T+UA4H1qQom+fftq5MiR9d7v7bff1tatWx3P+VQTgCsxlQMAAO/ElRtAg9UEE/7+/g3a79TtefMAwJWYygEAgHfiXQGABqsJJgICGjboimACgDsxlQMAAO/ElRtAg9XcJ6KxIyb4VBOAKzFiAgAA70QwAaDBnJ3KceoICz7VBOBK3GMCAADvxJUbQIMxlQOAJ2IqBwAA3okrN4AGYyoHAE/EVA4AALwTwQSABmMqBwBPxFQOAAC8E1duAA3mqqkcfKoJwJWYygEAgHfiyg2gwVwxlcOyLIIJAC7FVA4AAFxjzZo1GjVqlOLj42VZlpYtW3bGbe+66y5ZlqVZs2Y5XY9gAkCDuWIqB59oAnA1pnIAAOAaRUVF6tWrl2bPnn3W7ZYtW6YNGzYoPj6+UfUaNg4bAOSaqRy8cQDgakzlAAB4MmOMTEmJLbWt0NAGjShMTU1VamrqWbf54YcfNGnSJK1atUojRoxoVPsIJgA0mKumcgCAKzGVAwDgyUxJiXZc3MeW2l0258gKC3PZ8aqrqzVu3Djdd9996tGjR6OPx0cKABqMqRwAPBFTOQAAaBrPPvusAgICNGXKFJccjxETABrMFVM5+EQTgKsxlQMA4Mms0FB12ZxjW21XycnJ0QsvvKDNmze77G96ggkADeaKqRy8cQDgakzlAAB4MsuyXDqdwi6ffPKJ8vPzlZCQ4FhWVVWle++9V7NmzdLu3bsbfEyCCQAN5uxUDoIJAO7EVA4AANxv3LhxGjp0aK1lw4YN07hx4zRhwgSnjkkwAaDBXHGPCT7RBOBqTOUAAMA1CgsLtWvXLsfzvLw85ebmKioqSgkJCYqOjq61fWBgoGJjY9WlSxen6hFMAGgwvi4UgCdiKgcAAK6xadMmDRkyxPE8IyNDkpSWlqYFCxa4vB7BBIAGMcZwjwkAHompHAAAuEZycrJjJGJ9OHNfiZMRTACSKioq9MYbb+jw4cN2N8UlwsPDddttt6lVq1YuP3bNH/4SUzkAeBamcgAA4J0IJgBJ+/bt0549e+xuhssUFRXpm2++UZ8+fVx+7JppHBJTOQB4FqZyAADgnQgmAEllZWWSpLZt2+qGG26wuTWN8+GHH2rnzp2Oc3K1mmkcUuOmcvDGAYCrMWICAADvRDAB6OdgIjw8XO3atbO5NY0TGRkpSSovL3fL8U8eMdHQP/5PHmFx8pQQAHAF7jEBAIB34soN6Oc38UFBQTa3pPFqzsFdIyZO/kaOho56OHnERENupgMA9cFUDgAAvBPBBKCf38QHBwfb3JLGqzkHd42YcPYbOU7dhxETAFyNqRwAAHgnrtyAfg4mGDFxbjUjJpwJJk7+FJNgAoCrMZUDAADvxJUb0M+jC3xpxERTTOVoDKZyAHA1pnIAAOCdCCYAMZWjIRozleNkjJgA4GpM5QAAwDtx5QbEzS8bojFTOU5GMAHA1RgxAQCAdyKYAOSbIyaYygGgueEeEwAAuMaaNWs0atQoxcfHy7IsLVu2rNb68ePHy7KsWo9LL73U6XpcuQH5VjBRM2LCXVM5GDEBwFMxlQMAANcoKipSr169NHv27DNuM3z4cO3fv9/xeO+995yu17iPPAEf4UtTOdw9YoJ7TADwVEzlAAB4MmOMKsvt+Rs4IMivQdfH1NRUpaamnnWb4OBgxcbGNrZpkggmAEm+NWKi5hwqKytVVVXV6ADhVEzlAOCJjDGMmAAAeLTK8mrN/V22LbV/+8JgBQa79n1BVlaWYmJi1KpVKw0ePFhPP/20YmJinDoWV25AvhVMnDzqwx3TOZjKAcATnRx2EkwAAOBeqampeuutt/TRRx/pueee08aNG3XllVc6PWqbEROAfGsqR0BAgPz9/VVVVaWysjKFhoa69PhM5QDgiU7uU5jKAQDwRAFBfvrtC4Ntq+1KN998s+O/k5KS1LdvX3Xs2FH//Oc/NWbMmAYfj2ACzV5VVZXjzbYvjJiQTgQsJSUlbh0x0dipHADgSicHE4yYAAB4IsuyXD6dwlPExcWpY8eO2rlzp1P7884Czd7Jw418YcSEdCJgKSkpccsNMF01lQMAXImpHAAkqeLHH3Vk8WJVHzt+zm39WkYoZupU9zcKaAYOHz6svXv3Ki4uzqn9CSbQ7NWMKvD39/eZUQDu/GYOV03lAABXYioH4PmOvfeeDr40W9WlpW6rUfXTTzL1/PsnoF07ggngDAoLC7Vr1y7H87y8POXm5ioqKkpRUVGaPn26rr/+esXFxWn37t2aNm2a2rRpo+uuu86per7xLgxoBF+68WWNmpEfTOUA0FwwlaP5qfjhBxX84x8ylVV2NwX1YMrKdHj+fOm/H3C4U0ivC9Xi8svPuZ1feAu3twXwVps2bdKQIUMczzMyMiRJaWlpmjNnjrZu3arXX39dR48eVVxcnIYMGaKlS5cqIiLCqXq2v7N4+eWX9ec//1n79+9Xjx49NGvWLF1xxRV2NwvNiC/d+LKGO0dMMJUDgCc6eSoHIyaah33THlbxhg12NwMN1PLqVEVNvMNtx/cLDlLQ+efLIqAEGiU5ObnWtfVUq1atcmk9W4OJpUuXaurUqXr55Zd1+eWX69VXX1Vqaqq2bdumhIQEt9Q89OOPWvuvt91ybNSPv58l+UvykL8bjx4tVGXlQRUfP65//H1OvfYJk7+soBayrLovesFW4M+nFxggnfJHcqC/JeMfouAAy/EHtPELOW07ZxUXFqhaVfp2xzb5l5W45Jg19v+wV9Wq0qED+7Vl/boG71+tnz/Zcmb/YDXsZfIPkAKCghr1RqXdL3soOCTE6f2b0uH8fH3y0d/tbkaz5u9nnejfvPxv4jAFyAo+Sz+ngJ9/r4Lq2c/5u66fO1VxWcmJ/sWytDVnvVtqNJVgq+n7udhOPRTkJSMHC44c0X+WLVHbDRtkLEvFl/aV/Dzkj4omYaT/vlewdKY3DXUtN2dZfY7tT35es9g6+f/OsO0pz6tbtdShyxKU981HqjZG5r+nYlQtY4ys/74JMv9deuJN0X8ftdbV/JeRdVI1Y6pPbLe5Zm3t+ka1WxYkf1lWoE7/o9TIT5Yi/cJPe1VOFugnWVXlalH5kyxLKo7uoYqIjmfdByc05O85/4ATgbN/YGCz+XuuObI1mHj++ed1xx136M4775QkzZo1S6tWrdKcOXM0c+bM07YvKyur9QnwsWPHGlxz+dv/p8tefNP5RsMn9XViny437JdfwJlTxHPpVvoXpe1+Q4HGPUMaIyXt3v65drvp2Ae2f64D7zm3b43V2z93VZPOqufEr+Qf6PzPSvqrEpL6uKw9Z9PYfu6Tj/6uLtNfcnWz0Ex5ej93qpr+ZfW2zU1Sz5M0vp/7mxK6XOyy9pxNY/u5nPUfKffDl5QqaWNn6X+Tm+ZaAhcp+szuFjhs2L1XYWf5RLg+TvRza/7bz+W6pF2omzf9PYeGs+3znPLycuXk5CglJaXW8pSUFK1bV/enqDNnzlRkZKTj0aFDh4bXLW2aP44AwBmu6OcAwJM1up+rqNCgL068OVl1cXMaKQEAvssyZ5s44kb79u3Teeedp3//+9+67LLLHMtnzJihhQsXaseOHaftU1fC3qFDBxUUFKhly5b1qrt9yxat+/e7jT8BOM+yTkRiHvS3hCVLkS0iFBRYv6zO06dyVFVX6cChQ6qock8QFxwQqHZt2sjvDOd/NlXVVTp8pEDRrSPl79fw+1R4+lSOY8eOKTIyskH90ska288dzs/X+0sXqKzM9Tc+Rf1ZlmQsNayfK62WCk66JAdKirLvXi5hxk/yD5XO8HvurwBZNYO4AwJ06skGWVK1X5CC/CzH72yVdXp/6GptwqLUwkumJJxJsBr2T8ffTwoIbEQ/F2ApdlDvek/lsLufKzhyRC/+f4+p/bf7tPHibrX+TVl1vXL1fFka9OpZfqqWv8wpr/nJ0wp+XtjQn4tVx3+e63prqVz+KpOfglStln6nXv/r6pD+O+3sXGdunemJOcOulqPJNT+Pn//3xJKaf6snL699hLpeg9pPigMCtSaqgwoDnLtHWGJRge75/itZCjqt/s/lfl5e3jpSCqxdK9Dvv/2cv+Xurs02P+Yf1N79+11+XP/qqv/OwDrxt7BlSYH+gSqvrJBkFBocosrKKpVVlEtW9X//mfg5fQ05r107/Trtt03291yN0tJS5eXlKTExUSHNfBrJuV4L229+eepF1BhzxgtrcHBwo785oduFF6rbhRc26hgA4C6N7eeiY2L068n3u7BFaCqlu47q0P9tdTwP6RytNrd3t7FFgHs0tp+LbN1ajzzxoiRpgqsa5QMOHz6sl146MZUvMDDQJce0LEsDBgyodWd+T1JljEqqqs+9YR38LEth/s59rWFzU1FRUeubj05VUlKi7777ThUVFY5lLVq0UEJCggICAv4bPJz+qEtNnZpvVyovLz/rDRjry7Isn7rRvS+yLZho06aN/P39deDAgVrL8/Pz1a5dO5taBQCAPfwjg876HADOJjo6Wt27d9e2bdtqvUFsrLVr1+qSSy5RePjZbwRpB3/LUosAviXM3c4VdAUHB6tVq1YuqXXq1z0TJjQftgUTQUFB6tOnj1avXq3rrvs5rVy9erWuvfZau5oFAIAt/FsGnfU5AJzLDTfcoGPHjrnkE2ZJ+utf/6r9+/dr3bp16tmzp2N5y5YtFRYW5pIaACDZPJUjIyND48aNU9++fTVgwADNnTtXe/bs0d13321nswAAaHJ+wQGygv1lyk58pa5/S+++TwKApufn5+eyT64l6ZJLLtHy5cv173//W//+978dy6+55hpdfHHTfIsLgObB1mDi5ptv1uHDh/XEE09o//79SkpK0nvvvaeOHfn+XwBA8+PfMkiVB0sc/w0AdkpKStKWLVt08ODBWssDAmy/TR0AH2N7r3LPPffonnvusbsZAADYjmACgCcJDAxUWlqa3c0A0Aw0/Lv+AACAW5w8fYOpHAAAwC5r1qzRqFGjFB8fL8uytGzZstO22b59u6655hpFRkYqIiJCl156qfbs2eNUPYIJAAA8RM0oCSvQT1YId5oHAAD2KCoqUq9evTR79uw613/zzTcaOHCgunbtqqysLP3nP//RI488opCQEKfq2T6VAwAAnOD332DCv2XQGb/jHQAAeCdjjCrLymypHRAc3KC/LVJTU5WamnrG9Q8//LCuvvpq/elPf3Is+8UvfuF8+5zeEwAAuFRg2xNfvxfQJtTmlgAAAFerLCvTi2k32FJ7ysK/K9DJ0Qynqq6u1j//+U/df//9GjZsmD7//HMlJibqoYce0ujRo506JlM5AADwEMG/bKWo27qq1XW/tLspAAAAdcrPz1dhYaGeeeYZDR8+XB988IGuu+46jRkzRtnZ2U4dkxETAAB4CMvPUtiFbe1uBgAAcIOA4GBNWfh322q7SnV1tSTp2muv1e9//3tJ0kUXXaR169bplVde0eDBgxvePpe1DgAAAAAA1MmyLJdNp7BTmzZtFBAQoO7du9da3q1bN61du9apYzKVAwAAAAAA1EtQUJD69eunHTt21Fr+9ddfq2PHjk4dkxETAAAAAADAobCwULt27XI8z8vLU25urqKiopSQkKD77rtPN998swYNGqQhQ4Zo5cqV+sc//qGsrCyn6hFMAAAAAAAAh02bNmnIkCGO5xkZGZKktLQ0LViwQNddd51eeeUVzZw5U1OmTFGXLl309ttva+DAgU7VI5gAAAAAAAAOycnJMsacdZuJEydq4sSJLqnHPSYAAAAAAIBtvHrERE2Cc+zYMZtbAgAn1PRH50qY64t+DoCnoZ8D4Otc3c/h3Lw6mDh+/LgkqUOHDja3BABqO378uCIjI11yHIl+DoDnoZ8D4Otc1c/h3Lw6mIiPj9fevXsVEREhy7Lqvd+xY8fUoUMH7d27Vy1btnRjC+3DOfqO5nCevnSOxhgdP35c8fHxLjmeM/2cL72eZ9MczrM5nKPUPM7Tl86Rfq7pNIfz5Bx9hy+dp6v7OZybVwcTfn5+at++vdP7t2zZ0ut/ac6Fc/QdzeE8feUcXZmsN6af85XX81yaw3k2h3OUmsd5+so50s81reZwnpyj7/CV82SkRNPi5pcAAAAAAMA2BBMAAAAAAMA2zTKYCA4O1mOPPabg4GC7m+I2nKPvaA7n2RzOsSk1l9ezOZxnczhHqXmcZ3M4x6bUXF7P5nCenKPvaC7nCfewDN+BAgAAAACAS5WWliovL0+JiYkKCQmxuzm2Otdr0SxHTAAAAAAAAM9AMAEAAAAAABzWrFmjUaNGKT4+XpZladmyZbXWW5ZV5+PPf/6zU/UIJgAAAAAAgENRUZF69eql2bNn17l+//79tR5/+ctfZFmWrr/+eqfqBTSmsQAAAAAA4NyMMTIV1bbUtgL9ZFlWvbdPTU1VamrqGdfHxsbWer58+XINGTJEv/jFL5xqX7MbMfHyyy87brjRp08fffLJJ3Y3qU7Tp08/bVjMyT98Y4ymT5+u+Ph4hYaGKjk5WV9++WWtY5SVlWny5Mlq06aNwsPDdc011+j777+vtc2RI0c0btw4RUZGKjIyUuPGjdPRo0fddl7nGhLUlOe1Z88ejRo1SuHh4WrTpo2mTJmi8vJyt5/j+PHjT/vZXnrppV51jjNnzlS/fv0UERGhmJgYjR49Wjt27Ki1jS/8LL0V/Rz9XA36OefRz3k2+jn7+rnm0MfV5zzp57znPD2JqajWvkfX2fJwZyDy448/6p///KfuuOMOp4/RrIKJpUuXaurUqXr44Yf1+eef64orrlBqaqr27Nljd9Pq1KNHj1rDY7Zu3epY96c//UnPP/+8Zs+erY0bNyo2Nla/+tWvdPz4ccc2U6dOVWZmppYsWaK1a9eqsLBQI0eOVFVVlWOb2267Tbm5uVq5cqVWrlyp3NxcjRs3zm3ndK4hQU11XlVVVRoxYoSKioq0du1aLVmyRG+//bbuvfdet5+jJA0fPrzWz/a9996rtd7TzzE7O1vp6elav369Vq9ercrKSqWkpKioqMixjS/8LL0R/Rz9XA36ucahn/Nc9HP29nPNoY+rz3lK9HPecp5wv4ULFyoiIkJjxoxx/iCmGbnkkkvM3XffXWtZ165dzYMPPmhTi87sscceM7169apzXXV1tYmNjTXPPPOMY1lpaamJjIw0r7zyijHGmKNHj5rAwECzZMkSxzY//PCD8fPzMytXrjTGGLNt2zYjyaxfv96xzaeffmokma+++soNZ1WbJJOZmel43pTn9d577xk/Pz/zww8/OLZZvHixCQ4ONgUFBW47R2OMSUtLM9dee+0Z9/G2czTGmPz8fCPJZGdnG2N882fpLejn6Ofo5+jnfB39nOf0c82hj6vrPI2hn/Pmn2dTKikpMdu2bTMlJSXGmBOvaVVZpS2P6upqp8+jrt+Bk3Xp0sVMmjSpQa/FqZrNiIny8nLl5OQoJSWl1vKUlBStW7fOplad3c6dOxUfH6/ExETdcsst+vbbbyVJeXl5OnDgQK1zCQ4O1uDBgx3nkpOTo4qKilrbxMfHKykpybHNp59+qsjISPXv39+xzaWXXqrIyEhbXpOmPK9PP/1USUlJio+Pd2wzbNgwlZWVKScnx63nKUlZWVmKiYnRBRdcoN/85jfKz893rPPGcywoKJAkRUVFSWpeP0tPQj9HP+dJvxv0c955np6Ofs6z+7nm9ntBP+ed52kny7LkF+Rvy6Mh95doiE8++UQ7duzQnXfe2ajjNJtg4tChQ6qqqlK7du1qLW/Xrp0OHDhgU6vOrH///nr99de1atUqvfbaazpw4IAuu+wyHT582NHes53LgQMHFBQUpNatW591m5iYmNNqx8TE2PKaNOV5HThw4LQ6rVu3VlBQkNvPPTU1VW+99ZY++ugjPffcc9q4caOuvPJKlZWVOdrmTedojFFGRoYGDhyopKQkR+2aNp/tHLzpPL0B/Vzd29DP/Yx+zjn0c56Dfq7ubTyln2tOvxf0c77184Tz5s2bpz59+qhXr16NOk6z+1aOU5MiY4zb0qPGOPkOqD179tSAAQN0/vnna+HChY4b6zhzLqduU9f2dr8mTXVedp37zTff7PjvpKQk9e3bVx07dtQ///nPs87L8tRznDRpkrZs2aK1a9eets7Xf5aein7O8/99+PrvBv2c7/wsPRX9nGf/+2gOvxf0c77188TpCgsLtWvXLsfzvLw85ebmKioqSgkJCZKkY8eO6W9/+5uee+65RtdrNiMm2rRpI39//9MSt/z8/NPSOU8UHh6unj17aufOnY67OZ/tXGJjY1VeXq4jR46cdZsff/zxtFoHDx605TVpyvOKjY09rc6RI0dUUVHR5OceFxenjh07aufOnY62ecs5Tp48WStWrNDHH3+s9u3bO5Y315+l3ejn6t6Gfu5n9HMNRz/nWejn6t7GU/q55vx7QT/n+eeJhtm0aZN69+6t3r17S5IyMjLUu3dvPfroo45tlixZImOMbr311kbXazbBRFBQkPr06aPVq1fXWr569WpddtllNrWq/srKyrR9+3bFxcUpMTFRsbGxtc6lvLxc2dnZjnPp06ePAgMDa22zf/9+ffHFF45tBgwYoIKCAn322WeObTZs2KCCggJbXpOmPK8BAwboiy++0P79+x3bfPDBBwoODlafPn3cep6nOnz4sPbu3au4uDhJ3nGOxhhNmjRJ77zzjj766CMlJibWWt9cf5Z2o5+jn/PU3w36Oc89T29DP+fZ/Vxz/r2gn/Pc84RzkpOTZYw57bFgwQLHNr/97W9VXFysyMjIxhc8660zfcySJUtMYGCgmTdvntm2bZuZOnWqCQ8PN7t377a7aae59957TVZWlvn222/N+vXrzciRI01ERISjrc8884yJjIw077zzjtm6dau59dZbTVxcnDl27JjjGHfffbdp3769+fDDD83mzZvNlVdeaXr16mUqKysd2wwfPtxceOGF5tNPPzWffvqp6dmzpxk5cqTbzuv48ePm888/N59//rmRZJ5//nnz+eefm++++65Jz6uystIkJSWZq666ymzevNl8+OGHpn379ue8m2xjz/H48ePm3nvvNevWrTN5eXnm448/NgMGDDDnnXeeV53j//zP/5jIyEiTlZVl9u/f73gUFxc7tvGFn6U3op+jn6tBP9c49HOei37O3n6uOfRx5zpP+jnv+3na5VzfRNGcnOu1aFbBhDHG/L//9/9Mx44dTVBQkLn44osdX4fjaW6++WYTFxdnAgMDTXx8vBkzZoz58ssvHeurq6vNY489ZmJjY01wcLAZNGiQ2bp1a61jlJSUmEmTJpmoqCgTGhpqRo4cafbs2VNrm8OHD5uxY8eaiIgIExERYcaOHWuOHDnitvP6+OOPjaTTHmlpaU1+Xt99950ZMWKECQ0NNVFRUWbSpEmmtLTUredYXFxsUlJSTNu2bU1gYKBJSEgwaWlpp7Xf08+xrvOTZObPn+/Yxhd+lt6Kfo5+rgb9nPPo5zwb/Zx9/Vxz6OPOdZ70c97387QLwcTPzvVaWMYY0/hxFwAAAAAAoEZpaany8vKUmJiokJAQu5tjq3O9Fs3mHhMAAAAAAMDzEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAACHNWvWaNSoUYqPj5dlWVq2bFmt9YWFhZo0aZLat2+v0NBQdevWTXPmzHG6HsEEfF5WVpYsy9LRo0ftbgoAuAX9HABfRz8HNK2ioiL16tVLs2fPrnP973//e61cuVJvvvmmtm/frt///veaPHmyli9f7lQ9yxhjGtNgwNMkJyfroosu0qxZsyRJ5eXl+umnn9SuXTtZlmVv4wDABejnAPg6+jn4gtLSUuXl5SkxMVEhISEyxqiiosKWtgQGBjr9u2NZljIzMzV69GjHsqSkJN1888165JFHHMv69Omjq6++Wk8++eRpxzj1tThVgFMtA7xIUFCQYmNj7W4GALgN/RwAX0c/B19QUVGhGTNm2FJ72rRpCgoKctnxBg4cqBUrVmjixImKj49XVlaWvv76a73wwgtOHY+pHPAp48ePV3Z2tl544QVZliXLsrRgwYJaQ/8WLFigVq1a6d1331WXLl0UFhamG264QUVFRVq4cKE6deqk1q1ba/LkyaqqqnIcu7y8XPfff7/OO+88hYeHq3///srKyrLnRAE0W/RzAHwd/Rzg+V588UV1795d7du3V1BQkIYPH66XX35ZAwcOdOp4jJiAT3nhhRf09ddfKykpSU888YQk6csvvzxtu+LiYr344otasmSJjh8/rjFjxmjMmDFq1aqV3nvvPX377be6/vrrNXDgQN18882SpAkTJmj37t1asmSJ4uPjlZmZqeHDh2vr1q3q3Llzk54ngOaLfg6Ar6Ofg68KDAzUtGnTbKvtSi+++KLWr1+vFStWqGPHjlqzZo3uuecexcXFaejQoQ0+HsEEfEpkZKSCgoIUFhbmGO731VdfnbZdRUWF5syZo/PPP1+SdMMNN+iNN97Qjz/+qBYtWqh79+4aMmSIPv74Y91888365ptvtHjxYn3//feKj4+XJP3hD3/QypUrNX/+fNuGZAFofujnAPg6+jn4KsuyXDqdwi4lJSWaNm2aMjMzNWLECEnShRdeqNzcXP3v//4vwQRQX2FhYY6LmCS1a9dOnTp1UosWLWoty8/PlyRt3rxZxhhdcMEFtY5TVlam6Ojopmk0ADQA/RwAX0c/B9ijoqJCFRUV8vOrfWcIf39/VVdXO3VMggk0S6cOZbIsq85lNb9Y1dXV8vf3V05Ojvz9/Wttd/LFDwA8Bf0cAF9HPwe4T2FhoXbt2uV4npeXp9zcXEVFRSkhIUGDBw/Wfffdp9DQUHXs2FHZ2dl6/fXX9fzzzztVj2ACPicoKKjWTY5coXfv3qqqqlJ+fr6uuOIKlx4bABqKfg6Ar6OfA+y1adMmDRkyxPE8IyNDkpSWlqYFCxZoyZIleuihhzR27Fj99NNP6tixo55++mndfffdTtUjmIDP6dSpkzZs2KDdu3erRYsWTg8nOtkFF1ygsWPH6vbbb9dzzz2n3r1769ChQ/roo4/Us2dPXX311S5oOQDUD/0cAF9HPwfYKzk5WcaYM66PjY3V/PnzXVaPrwuFz/nDH/4gf39/de/eXW3bttWePXtcctz58+fr9ttv17333qsuXbrommuu0YYNG9ShQweXHB8A6ot+DoCvo58DmhfLnC0GAQAAAAAADVZaWqq8vDwlJiYqJCTE7ubY6lyvBSMmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAw5o1azRq1CjFx8fLsiwtW7as1voff/xR48ePV3x8vMLCwjR8+HDt3LnT6XoEEwAAAAAAwKGoqEi9evXS7NmzT1tnjNHo0aP17bffavny5fr888/VsWNHDR06VEVFRU7VC2hsgwEAAAAAwNkZY1RdXWJLbT+/UFmWVe/tU1NTlZqaWue6nTt3av369friiy/Uo0cPSdLLL7+smJgYLV68WHfeeWeD20cwAQAAAACAm1VXlygru6cttZMHb5W/f5hLjlVWViZJCgkJcSzz9/dXUFCQ1q5d61QwwVQOAAAAAABQL127dlXHjh310EMP6ciRIyovL9czzzyjAwcOaP/+/U4dkxETAAAAAAC4mZ9fqJIHb7WttqsEBgbq7bff1h133KGoqCj5+/tr6NChZ5z6UR8EEwAAAAAAuJllWS6bTmG3Pn36KDc3VwUFBSovL1fbtm3Vv39/9e3b16njMZUDAAAAAAA0WGRkpNq2baudO3dq06ZNuvbaa506DiMmAAAAAACAQ2FhoXbt2uV4npeXp9zcXEVFRSkhIUF/+9vf1LZtWyUkJGjr1q363e9+p9GjRyslJcWpegQTAAAAAADAYdOmTRoyZIjjeUZGhiQpLS1NCxYs0P79+5WRkaEff/xRcXFxuv322/XII484Xc8yxphGtxoAAAAAADiUlpYqLy9PiYmJtb5aszk612vBPSYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIBtCCYAAAAAAIAkaebMmerXr58iIiIUExOj0aNHa8eOHbW2McZo+vTpio+PV2hoqJKTk/Xll186XZNgAgAAAAAASJKys7OVnp6u9evXa/Xq1aqsrFRKSoqKiooc2/zpT3/S888/r9mzZ2vjxo2KjY3Vr371Kx0/ftypmpYxxrjqBAAAAAAAgFRaWqq8vDwlJiYqJCTE7uY47eDBg4qJiVF2drYGDRokY4zi4+M1depUPfDAA5KksrIytWvXTs8++6zuuuuu045xrtciwO1nAQAAAABAM2eMUXF1tS21w/z8ZFmWU/sWFBRIkqKioiRJeXl5OnDggFJSUhzbBAcHa/DgwVq3bl2dwcS5EEwAAAAAAOBmxdXVOn/NVltqfzOop8L9/Ru8nzFGGRkZGjhwoJKSkiRJBw4ckCS1a9eu1rbt2rXTd99951T7CCYAAAAAAMBpJk2apC1btmjt2rWnrTt1BIYxxulRGQQTAAAAAAC4WZifn74Z1NO22g01efJkrVixQmvWrFH79u0dy2NjYyWdGDkRFxfnWJ6fn3/aKIr6IpgAAAAAAMDNLMtyajpFUzPGaPLkycrMzFRWVpYSExNrrU9MTFRsbKxWr16t3r17S5LKy8uVnZ2tZ5991qmaBBMAAAAAAECSlJ6erkWLFmn58uWKiIhw3FMiMjJSoaGhsixLU6dO1YwZM9S5c2d17txZM2bMUFhYmG677TanahJMAAAAAAAASdKcOXMkScnJybWWz58/X+PHj5ck3X///SopKdE999yjI0eOqH///vrggw8UERHhVE3LGGMa02gAAAAAAFBbaWmp8vLylJiYqJCQELubY6tzvRYNvwMGAAAAAACAixBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAA2xBMAAAAAAAASdLMmTPVr18/RUREKCYmRqNHj9aOHTtqbfPOO+9o2LBhatOmjSzLUm5ubqNqEkwAAAAAAABJUnZ2ttLT07V+/XqtXr1alZWVSklJUVFRkWOboqIiXX755XrmmWdcUjPAJUcBAAAAAABeb+XKlbWez58/XzExMcrJydGgQYMkSePGjZMk7d692yU1CSYAAAAAAHAzY4xKKqpsqR0a6C/Lspzat6CgQJIUFRXlyibVQjABAAAAAICblVRUqfujq2ypve2JYQoLavjbf2OMMjIyNHDgQCUlJbmhZScQTAAAAAAAgNNMmjRJW7Zs0dq1a91ah2ACAAAAAAA3Cw3017YnhtlWu6EmT56sFStWaM2aNWrfvr0bWvUzggkAAAAAANzMsiynplM0NWOMJk+erMzMTGVlZSkxMdHtNT3/VQEAAAAAAE0iPT1dixYt0vLlyxUREaEDBw5IkiIjIxUaGipJ+umnn7Rnzx7t27dPkrRjxw5JUmxsrGJjYxtc089FbQcAAAAAAF5uzpw5KigoUHJysuLi4hyPpUuXOrZZsWKFevfurREjRkiSbrnlFvXu3VuvvPKKUzUtY4xxSesBAAAAAIAkqbS0VHl5eUpMTFRISIjdzbHVuV4LRkwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAADbEEwAAAAAAABJ0syZM9WvXz9FREQoJiZGo0eP1o4dOxzrKyoq9MADD6hnz54KDw9XfHy8br/9du3bt8/pmgQTAAAAAABAkpSdna309HStX79eq1evVmVlpVJSUlRUVCRJKi4u1ubNm/XII49o8+bNeuedd/T111/rmmuucbqmZYwxrjoBAAAAAAAglZaWKi8vT4mJiQoJCbG7OU47ePCgYmJilJ2drUGDBtW5zcaNG3XJJZfou+++U0JCwmnrz/VaBLi81QAAAAAAoDZjpIpie2oHhkmW5dSuBQUFkqSoqKizbmNZllq1auVUDYIJAAAAAADcraJYmhFvT+1p+6Sg8AbvZoxRRkaGBg4cqKSkpDq3KS0t1YMPPqjbbrtNLVu2dKp5BBMAAAAAAOA0kyZN0pYtW7R27do611dUVOiWW25RdXW1Xn75ZafrEEwAAAAAAOBugWEnRi7YVbuBJk+erBUrVmjNmjVq3779aesrKip00003KS8vTx999JHToyUkggkAAAAAANzPspyaTtHUjDGaPHmyMjMzlZWVpcTExNO2qQkldu7cqY8//ljR0dGNqkkwAQAAAAAAJEnp6elatGiRli9froiICB04cECSFBkZqdDQUFVWVuqGG27Q5s2b9e6776qqqsqxTVRUlIKCghpck68LBQAAAADAxbz160KtM3x7x/z58zV+/Hjt3r27zlEUkvTxxx8rOTn5tOV8XSgAAAAAAKiXc41d6NSp0zm3aSg/lx4NAAAAAACgAQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAbQgmAAAAAACAJGnmzJnq16+fIiIiFBMTo9GjR2vHjh21tpk+fbq6du2q8PBwtW7dWkOHDtWGDRucrkkwAQAAAAAAJEnZ2dlKT0/X+vXrtXr1alVWViolJUVFRUWObS644ALNnj1bW7du1dq1a9WpUyelpKTo4MGDTtW0jDHGVScAAAAAAACk0tJS5eXlKTExUSEhIXY3x2kHDx5UTEyMsrOzNWjQoDq3OXbsmCIjI/Xhhx/qqquuOm39uV6LAJe3GgAAAAAA1GKMUUlliS21QwNCZVmWU/sWFBRIkqKioupcX15errlz5yoyMlK9evVyqgbBBAAAAAAAblZSWaL+i/rbUnvDbRsUFhjW4P2MMcrIyNDAgQOVlJRUa927776rW265RcXFxYqLi9Pq1avVpk0bp9rHPSYAAAAAAMBpJk2apC1btmjx4sWnrRsyZIhyc3O1bt06DR8+XDfddJPy8/OdqsM9JgAAAAAAcLFT76vgbVM5Jk+erGXLlmnNmjVKTEw85/adO3fWxIkT9dBDD522jntMAAAAAABgM8uynJpO0dSMMZo8ebIyMzOVlZVVr1CiZr+ysjKnahJMAAAAAAAASVJ6eroWLVqk5cuXKyIiQgcOHJAkRUZGKjQ0VEVFRXr66ad1zTXXKC4uTocPH9bLL7+s77//XjfeeKNTNQkmAAAAAACAJGnOnDmSpOTk5FrL58+fr/Hjx8vf319fffWVFi5cqEOHDik6Olr9+vXTJ598oh49ejhVk2ACAAAAAABIOjEl42xCQkL0zjvvuLQm38oBAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAABsQzABAAAAAAAkSTNnzlS/fv0UERGhmJgYjR49Wjt27Djj9nfddZcsy9KsWbOcrkkwAQAAAAAAJEnZ2dlKT0/X+vXrtXr1alVWViolJUVFRUWnbbts2TJt2LBB8fHxjaoZ0Ki9AQAAAACAz1i5cmWt5/Pnz1dMTIxycnI0aNAgx/IffvhBkyZN0qpVqzRixIhG1SSYAAAAAADAzYwxMiUlttS2QkNlWZZT+xYUFEiSoqKiHMuqq6s1btw43XffferRo0ej20cwAQAAAACAm5mSEu24uI8ttbtszpEVFtbg/YwxysjI0MCBA5WUlORY/uyzzyogIEBTpkxxSfsIJgAAAAAAwGkmTZqkLVu2aO3atY5lOTk5euGFF7R582anR2GcimACAAAAAAA3s0JD1WVzjm21G2ry5MlasWKF1qxZo/bt2zuWf/LJJ8rPz1dCQoJjWVVVle69917NmjVLu3fvbnAtggkAAAAAANzMsiynplM0NWOMJk+erMzMTGVlZSkxMbHW+nHjxmno0KG1lg0bNkzjxo3ThAkTnKpJMAEAAAAAACRJ6enpWrRokZYvX66IiAgdOHBAkhQZGanQ0FBFR0crOjq61j6BgYGKjY1Vly5dnKrp1+hWAwAAAAAAnzBnzhwVFBQoOTlZcXFxjsfSpUvdVpMREwAAAAAAQNKJqRwN5cx9JU7GiAkAAAAAAGAbggkAAAAAAGAbggkAAAAAAGAbggkAAAAAAGAbggkAAAAAANzEmZtJ+ppzvQYEEwAAAAAAuJi/v78kqby83OaW2K+4uFiSFBgYWOd6vi4UAAAAAAAXCwgIUFhYmA4ePKjAwED5+TW/cQHGGBUXFys/P1+tWrVyhDWnsgzjSgAAAAAAcLny8nLl5eWpurra7qbYqlWrVoqNjZVlWXWuJ5gAAAAAAMBNqqurm/V0jsDAwDOOlKhBMAEAAAAAAGzT/Ca5AAAAAAAAj0EwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbEMwAQAAAAAAbPP/A1Brg5HM4mxiAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABB0AAAEiCAYAAAC1EapBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3rElEQVR4nO3de3xU1bn/8e9kkglJCIFEcoNwqQURAqhgEVABoVwKVEQ94kEFPfqrlYsWe7xgq6JC9I/6Qu0RxXIitgL2VGM5HsBiJVCKwRigIlBucicYQUi45rp+f9iMjlxMJrNn79nzeb9e0yYze9Z+VkaelTyznj0eY4wRAAAAAABAiMXYHQAAAAAAAHAnig4AAAAAAMASFB0AAAAAAIAlKDoAAAAAAABLUHQAAAAAAACWoOgAAAAAAAAsQdEBAAAAAABYgqIDAAAAAACwBEUHAAAAAABgCYoOAAAAAADAEhQdAAAAAACAJSg6AAAAAAAAS1B0gCP86U9/Uvfu3ZWQkKC0tDQNGTJEJ0+elCRNnDhRY8aM0YwZM5Senq4WLVroZz/7maqqqvzPX7Zsma6++mq1bNlSaWlpGjVqlHbu3Blwjv3792vcuHFKTU1VUlKSevfurbVr1/of/9///V/16tVLzZo10w9+8APNmDFDNTU1lszXGKMhQ4Zo+PDhMsZIko4dO6Z27drpscces+ScAOwVbXlu9+7diomJ0SeffBJw/0svvaT27dv7cx8A94i2PCdJHTp0kMfjOesG4BsUHWC70tJS3Xrrrbrrrru0ZcsWFRYWauzYsQG/kP71r3/Vli1btGLFCi1cuFAFBQWaMWOG//GTJ09q2rRpKi4u1l//+lfFxMTohhtuUF1dnSTpxIkTGjBggA4ePKjFixfrH//4hx566CH/4++//75uu+02TZ06VZs3b9arr76q119/XTNnzjxv3G+++aaaN29+wdubb755zud6PB7Nnz9fH3/8sV588UVJ0r333quMjAw9+eSTTf2RAnCYaMxzHTp00JAhQ5Sfnx9wf35+viZOnMgv5YDLRGOek6Ti4mKVlpaqtLRU+/fv11VXXaVrrrmmqT9OwF2MS6xcudKMGjXKZGVlGUmmoKDA9vNNmDDBSAq49enTx9K4IlFJSYmRZHbv3n3OxydMmGBSU1PNyZMn/ffNmTPHNG/e3NTW1p7zOWVlZUaS2bhxozHGmFdffdUkJyebI0eOnPP4a665xsyaNSvgvt///vcmKyvrvHFXVFSY7du3X/BWUVFxwbn/8Y9/NPHx8ebRRx81iYmJZuvWrRc8HkBkitY899Zbb5lWrVqZM2fOGGOM2bBhg/F4PGbXrl3nfQ6AyBStee7bpk6datq3b2/KysoadDwQLWLtKXWE3smTJ9WzZ0/deeeduvHGGx1zvuHDhwe8y+Pz+SyPLdL07NlTgwcPVvfu3TVs2DANHTpUN910k1q1ahVwTGJiov/7vn376sSJE9q3b5/at2+vnTt36te//rWKiop0+PBhf8V77969ys3N1YYNG3T55ZcrNTX1nDGUlJSouLg4oBJeW1urM2fO6NSpUwHnrpecnKzk5OQmzf3mm29WQUGB8vLyNGfOHHXu3LlJ4wFwpmjNc2PGjNHkyZNVUFCgcePG6b//+781aNAgdejQIegxAThTtOa5enPnztW8efP097//Xa1bt27yeICbuKa9YsSIEXrmmWc0duzYcz5eVVWlhx56SG3atFFSUpL69OmjwsJCy85XLz4+XpmZmf7b+ZJkNPN6vVq+fLmWLl2qrl276qWXXtIll1yiXbt2fe9z67fnjh49WkeOHNFrr72mtWvX+nv76vsEExISLjhOXV2dZsyYoQ0bNvhvGzdu1Pbt29WsWbNzPqep2/Ek6dSpUyopKZHX69X27du/d74AIlO05jmfz6fbb79d+fn5qqqq0oIFC3TXXXd975wBRJ5ozXOSVFhYqClTpuiNN95Qz549v3e+QLRxzU6H73PnnXdq9+7dWrRokbKzs1VQUKDhw4dr48aN6tSpk2XnLSwsVHp6ulq2bKkBAwZo5syZSk9Pt+x8kcrj8ah///7q37+/Hn/8cbVv314FBQWaNm2aJOkf//iHTp8+7V9sioqK1Lx5c7Vt21ZHjhzRli1b9Oqrr/p76FavXh0wfo8ePfS73/1OX3311TkLP1dccYW2bt2qH/7whw2O+ac//an69OlzwWMyMjIu+PiDDz6omJgYLV26VD/5yU80cuRIXXfddQ2OAUDkiNY8d/fddys3N1cvv/yyqqurv7dYDyByRWOe27Fjh2688UZNnz6d/Aacj939HVbQd66xsGPHDuPxeMyBAwcCjhs8eLB59NFHQ36+eosWLTLvvfee2bhxo1m8eLHp2bOn6datm7+3FV8rKioyM2fONMXFxWbPnj3mj3/8o/H5fGbJkiXGmK97AJs3b25uvfVWs2nTJrNkyRKTkZFhHnnkEWOMMbW1tSYtLc3cdtttZvv27eavf/2rufLKKwNel8rKStO5c2dzzTXXmNWrV5udO3eaP/3pT2bNmjXGGGOWLVtmYmNjzRNPPGE+++wzs3nzZrNo0SLz2GOPWTbv9957z/h8PlNSUmKMMeZXv/qVadu2rfnqq68sOycAe0RrnqvXr18/4/P5zL333mv5uQDYIxrz3KlTp0yXLl3MddddZw4ePGhKS0v9NwDfiIqiwx//+EcjySQlJQXcYmNjzb/9278ZY4zZtWvXWRd9/O5t0qRJDTrf+Rw8eNDExcWZt99+OxTTdI3NmzebYcOGmdatW5v4+HjTuXNn89JLL/kfnzBhgrn++uvN448/btLS0kzz5s3N3XffHVC8Wb58ubn00ktNfHy86dGjhyksLDzrddm9e7e58cYbTYsWLUxiYqLp3bu3Wbt2rf/xZcuWmX79+pmEhATTokUL86Mf/cjMnTvXkjmXlZWZjIyMgIsdVVdXmx/96Ef+/yYBuEc05rlvmzdvnpFkPv74Y8vPBcAe0ZjnLvT3A4BveIxx3wdlezweFRQUaMyYMZKkt956S+PHj9emTZvk9XoDjm3evLkyMzNVXV191ucAf1erVq3Oub3qu+e7kE6dOunuu+/Www8/3OD5RLuJEyfq2LFjevfdd+0OBQAs4fY8N3PmTC1atEgbN260OxQANnF7ngNwflFxTYfLL79ctbW1KisrO+/n5sbFxalLly6WxnHkyBHt27dPWVlZlp4HAAAnOHHihLZs2aKXXnpJTz/9tN3hAAAAG7im6HDixAnt2LHD//2uXbu0YcMGpaamqnPnzho/frzuuOMO/eY3v9Hll1+uw4cP68MPP1T37t31k5/8JKTna9eunU6cOKEnn3xSN954o7KysrR7925Nnz5dF110kW644YaQzBkAACebPHmyFi5cqDFjxvCpFQAARCnXtFcUFhZq0KBBZ90/YcIEvf7666qurtYzzzyjN954QwcOHFBaWpr69u2rGTNmqHv37iE/3+nTpzVmzBitX79ex44dU1ZWlgYNGqSnn35aOTk5Qc0RAAAAAIBI4pqiAwAAAAAAcJYYuwMAAAAAAADuRNEBAAAAAABYIqIvJFlXV6eDBw8qOTlZHo/H7nAAQJJkjNHx48eVnZ2tmJim1XbJcwCciDwHwO1CmeeiXUQXHQ4ePMhFGQE41r59+9S2bdsmjUGeA+Bk5DkAbheKPBftIrrokJycLOnr/xBatGhhczQA8LWKigrl5OT4c1RTkOcAOBF5DoDbhTLPRbuILjrUb8Fr0aIFixQAxwnFNmHyHAAnI88BcDvavpqO5hQAAAAAAGAJig4AAAAAAMASFB0AAAAAAIAlKDoAAAAAAABLUHQAAAAAAACWiOhPrwAAAMC5nThxQqWlpQ06Njs7W0lJSRZHBACIRhQdAAAAXOh3v/udjh071qBjx48fr06dOlkbEAAgKlF0AAAAcBljjL/gkJ6eLq/Xe8Hj4+PjwxAVACAaUXQAAABwmbq6Ov/XEydOVGJioo3RAACiGReSBAAAcBljjP/rmBh+3QMA2IdVCAAAwGW+vdPB4/HYGAkAINpRdAAAAHCZbxcd2OkAALATqxAAAIDL0F4BAHAKViEAAACXob0CAOAUFB0AAABchp0OAACnYBUCAABwmfqdDhQcAAB2i7U7AAAAAIRWfdGB1goAsJ8xRjU1NaqtrbU7FEt4vV7Fxsaed82h6AAAAOAy9e0V7HQAAHtVVVWptLRUp06dsjsUSyUmJiorK0s+n++sxyg6AAAAuAztFQBgv7q6Ou3atUter1fZ2dny+Xyu24FmjFFVVZW+/PJL7dq1S506dTpr7aHoAAAA4DK0VwCA/aqqqlRXV6ecnBwlJibaHY5lEhISFBcXpz179qiqqkrNmjULeJzyNwAAgMvQXgEAzhENufhCc3T/7AEAAKIM7RUAAKdgJQIAAHAZ2isAIDoMHDhQDzzwwHkf79Chg2bPnh22eM6FazoAAAC4DO0VABAd3nnnHcXFxdkdxgVRdAAAAHAZ2isAIDqkpqbaHcL3YiUCAABwGdorACA6fLu9oqysTKNHj1ZCQoI6duyoN998097g/oWdDgAAAC5DewUARJ+JEydq3759+vDDD+Xz+TR16lSVlZXZHRZFBwAAALehvQIAosu2bdu0dOlSFRUVqU+fPpKkefPm6dJLL7U5MtorAAAAXIf2CgCILlu2bFFsbKx69+7tv69Lly5q2bKlfUH9C0UHAAAAl6G9AgCiS33ed2Kx2faV6MCBA7rtttuUlpamxMREXXbZZSopKbE7LAAAgIjFTgcAiC6XXnqpampq9Mknn/jv27p1q44dO2ZfUP9i6zUdjh49qv79+2vQoEFaunSp0tPTtXPnTkdsAQEAAIhUXNMBAKLLJZdcouHDh+uee+7R3LlzFRsbqwceeEAJCQl2h2Zv0eG5555TTk6O8vPz/fd16NDBvoAAAABcgPYKAIg++fn5uvvuuzVgwABlZGTomWee0a9//Wu7w7K36LB48WINGzZMN998s1auXKk2bdrovvvu0z333HPO4ysrK1VZWen/vqKiIlyhAkBYkOcAhIKT2yvIcwAQOoWFhf6vMzMz9d577wU8fvvtt4c5orPZWv7+/PPPNWfOHHXq1Envv/++7r33Xk2dOlVvvPHGOY/Py8tTSkqK/5aTkxPmiAHAWuQ5AKHg5PYK8hwARBdbV6K6ujpdccUVmjVrli6//HL97Gc/0z333KM5c+ac8/hHH31U5eXl/tu+ffvCHDEAWIs8ByAUnNxeQZ4DgOhia3tFVlaWunbtGnDfpZdeqrfffvucx8fHxys+Pj4coQGALchzAELBye0V5DkAiC62lr/79++vrVu3Bty3bds2tW/f3qaIAAAAIp+T2ysAANHF1pXoF7/4hYqKijRr1izt2LFDCxYs0Ny5czVp0iQ7wwIAAIhoTm6vAABEF1tXoiuvvFIFBQVauHChcnNz9fTTT2v27NkaP368nWEBAABENCe3VwAAoout13SQpFGjRmnUqFF2hwEAAOAa7HQAADgFKxEAAIDLcE0HAIBTsBIBAAC4DO0VAACnoOgAAADgMrRXAACcgpUIAADAZWivAAA0xapVqzR69GhlZ2fL4/Ho3XffDXosViIAAACXob0CANAUJ0+eVM+ePfXb3/62yWPZ/ukVAAAACC3aKwAATTFixAiNGDEiJGNRdAAAAHAZ2isAwJmMMTpdXRv28ybEeW3b/UbRAQAAwGVorwAAZzpdXauuj78f9vNufmqYEn32/PlP+RsAAMBlaK8AADgFOx0AAABchvYKAHCmhDivNj81zJbz2oWiAwAAgMvQXgEAzuTxeGxrc7BLdM0WAAAgCtBeAQBoihMnTmjHjh3+73ft2qUNGzYoNTVV7dq1a9RYFB0AAABchvYKAEBTfPLJJxo0aJD/+2nTpkmSJkyYoNdff71RY1F0AAAAcBnaKwAATTFw4ED/rrmmovwNAADgMrRXAACcgpUIAADAZWivAAA4BSsRAACAy9BeAQBwCooOAAAALkN7BQDAKViJAAAAXIb2CgCAU7ASAQAAuAztFQAAp6DoAAAA4DK0VwAAnIKVCAAAwGVorwAAOAUrEQAAgMvQXgEAcAqKDgAAAC5DewUAwClYiQAAAFyG9goAQLDy8vJ05ZVXKjk5Wenp6RozZoy2bt0a9HisRAAAAC5Tv9OB9goAQGOtXLlSkyZNUlFRkZYvX66amhoNHTpUJ0+eDGq82BDHBwAAAJux0wEAEKxly5YFfJ+fn6/09HSVlJTo2muvbfR4FB0AAABchqIDADiUMVL1qfCfNy5RCnL3W3l5uSQpNTU1qOdTdAAAAHAZ2isAwKGqT0mzssN/3ukHJV9So59mjNG0adN09dVXKzc3N6hTU3QAAABwGXY6AABCYfLkyfr000+1evXqoMeg6AAAAOAyFB0AwKHiEr/edWDHeRtpypQpWrx4sVatWqW2bdsGfWqKDgAAAC5DewUAOJTHE1SbQzgZYzRlyhQVFBSosLBQHTt2bNJ4FB0AAABchp0OAIBgTZo0SQsWLNCf//xnJScn69ChQ5KklJQUJSQkNHo8ViIAAACXqS86sNMBANBYc+bMUXl5uQYOHKisrCz/7a233gpqPHY6AAAAuEx9ewU7HQAAjVW/hoQKKxEAAIDL0F4BAHAKViIAAACXob0CAOAUFB0AAABchvYKAIBTsBIBAAC4DO0VAACncMxKlJeXJ4/HowceeMDuUAAAACIa7RUAAKdwRNGhuLhYc+fOVY8ePewOBQAAIOLRXgEAcArbV6ITJ05o/Pjxeu2119SqVSu7wwEAAIh4tFcAAJzC9pVo0qRJGjlypIYMGfK9x1ZWVqqioiLgBgBuQp4DEApObq8gzwFAdLG16LBo0SKtW7dOeXl5DTo+Ly9PKSkp/ltOTo7FEQJAeJHnAISCk9sryHMAEF1sW4n27dun+++/X3/4wx/UrFmzBj3n0UcfVXl5uf+2b98+i6MEgPAizwFoqvpdDpIziw7kOQCILratRCUlJSorK1OvXr0UGxur2NhYrVy5Ui+++KJiY2NVW1t71nPi4+PVokWLgBsAuAl5DkBTfbvo4MT2CvIcADjbnDlz1KNHD3+O7tu3r5YuXRr0eLEhjK1RBg8erI0bNwbcd+edd6pLly56+OGH5fV6bYoMAAAgctW3VkjO3OkAAHC2tm3b6tlnn9UPf/hDSdL8+fN1/fXXa/369erWrVujx7Ot6JCcnKzc3NyA+5KSkpSWlnbW/QAAAGgYp7dXAACcbfTo0QHfz5w5U3PmzFFRUVFkFR0AAAAQek5vrwCAaGaM0ema02E/b0JsQlBrQm1trf7nf/5HJ0+eVN++fYM6t6OKDoWFhXaHAAAAENForwAA5zpdc1p9FvQJ+3nX/vtaJcYlNvj4jRs3qm/fvjpz5oyaN2+ugoICde3aNahzsxIBAAC4CO0VAICmuuSSS7RhwwYVFRXp5z//uSZMmKDNmzcHNZajdjoAAACgaep3OtBaAQDOkxCboLX/vtaW8zaGz+fzX0iyd+/eKi4u1gsvvKBXX3210eem6AAAAOAi9Tsd2OUAAM7j8Xga1ebgFMYYVVZWBvVcig4Aws4Yo+LiYn311Vd2h9Igffr0UatWrewOA0ADlJeXq7i4WDU1NXaHYpv6XwopOgAAgjF9+nSNGDFCOTk5On78uBYtWqTCwkItW7YsqPEoOgAIu7KyMi1ZssTuMBqsW7duFB2ACLFmzRqtXRv+batO1KxZM7tDAABEoC+++EK33367SktLlZKSoh49emjZsmX68Y9/HNR4FB0AhN2ZM2ckff0Lce/evW2O5vslJyfbHQKABqrPLx06dFDbtm1tjsZenTp1sjsEAEAEmjdvXkjHa1LRoaqqSmVlZQFXSZakdu3aNSkoAO5WnzOaN2+uIUOG2BwNADepzy+dO3dWv379bI4GAAAEVXTYvn277rrrLq1ZsybgfmOMPB6PamtrQxIcAHeqv7I6/cYAQo38AgCAswRVdJg4caJiY2P13nvvKSsri49kAtAo9e9EkjsAhBr5BQAAZwmq6LBhwwaVlJSoS5cuoY4HQBTg49wAWIX8AgCAswS1Inft2lWHDx8OdSwAogTbnwFYhfwCAICzNHhFrqio8N+ee+45PfTQQyosLNSRI0cCHquoqLAyXgAuwPZnAFYhvwAA4CwNbq9o2bJlwAJujNHgwYMDjuFCkgAagnciAViF/AIAgLM0uOiwYsUKSVJlZaVmzZqlW2+9lWs6AAgKPdcArEJ+AQDAWRpcdBgwYID/6/Hjx+u6665Tp06dLAkKgLux/RmAVcgvAAA4S1BvA9xxxx2aN29eqGMBECXY/gzAKuQXAACcJaiPzKyqqtLvfvc7LV++XL1791ZSUlLA488//3xIggPgTmx/BmAV8gsAAKGVl5en6dOn6/7779fs2bMb/fygig6fffaZrrjiCknStm3bAh5jOyOA78P2ZwBWIb8AABA6xcXFmjt3rnr06BH0GEEVHeovKgkAwWD7MwCrkF8AAAiNEydOaPz48Xrttdf0zDPPBD1OUEUHAGgKtj8DsAr5BQDgZMYYmdOnw35eT0JCo3cBTpo0SSNHjtSQIUMoOgCILGx/BmAV8gsAwMnM6dPaekWvsJ/3knUl8iQmNvj4RYsWad26dSouLm7yuSk6AAg7tj8DsAr5BQCAptm3b5/uv/9+/eUvf1GzZs2aPB5FBwBhxzuRAKxCfgEAOJknIUGXrCux5bwNVVJSorKyMvXq9c2OjNraWq1atUq//e1vVVlZKa/X2+DxKDoACDt6rgFYhfwCAHAyj8fTqDYHOwwePFgbN24MuO/OO+9Uly5d9PDDDzeq4CBRdABgA7Y/A7AK+QUAgKZJTk5Wbm5uwH1JSUlKS0s76/6GYEUGEHZsfwZgFfILAADOwk4HAGHH9mcAViG/AAAQeoWFhUE/lxUZQNix/RmAVcgvAAA4CysygLBj+zMAq5BfAABwFooOAMKO7c8ArEJ+AQDAWViRAYQd258BWIX8AgCAs7AiAwg7tj8DsAr5BQAAZ6HoACDs2P4MwCrkFwAAnIUVGUDYsf0ZgFXILwAAOAsrMoCwY/szACsYY/xFB/ILAADOQNEBQNix/RmAFepzi0R+AQDAKViRAYQd258BWKE+t0jkFwAAnIIVGUDY0V4BwArf3ulAfgEAIDhPPvmkPB5PwC0zMzPo8WJDGBsANAjtFQCsQHsFAACh0a1bN33wwQf+771eb9Bj2boi5+Xl6corr1RycrLS09M1ZswYbd261c6QAIQB7RUArEB7BQAAoREbG6vMzEz/rXXr1kGPZeuKvHLlSk2aNElFRUVavny5ampqNHToUJ08edLOsABYjPYKAFagvQIA4HTGGFVX1ob99u3CfENs375d2dnZ6tixo8aNG6fPP/886Dnb2l6xbNmygO/z8/OVnp6ukpISXXvttTZFBcBqtFcAsALtFQAAp6upqtPc+1eG/bz/74UBiotvWItEnz599MYbb6hz58764osv9Mwzz6hfv37atGmT0tLSGn1uR13Toby8XJKUmpp6zscrKytVWVnp/76ioiIscQEILdorzo88BwSP3BIZyHMA4GwjRozwf929e3f17dtXF198sebPn69p06Y1ejzHFB2MMZo2bZquvvpq5ebmnvOYvLw8zZgxI8yRAQg12ivOjzwHBI/cEhnIcwCiWawvRv/vhQG2nDdYSUlJ6t69u7Zv3x7U8x3zVsDkyZP16aefauHChec95tFHH1V5ebn/tm/fvjBGCCBUeDfy/MhzQPBo3YoM5DkA0czj8Sgu3hv2W1MK8pWVldqyZYuysrKCer4jdjpMmTJFixcv1qpVq9S2bdvzHhcfH6/4+PgwRgbACvxhcH7kOSB4FDQjA3kOAJztl7/8pUaPHq127dqprKxMzzzzjCoqKjRhwoSgxrO16GCM0ZQpU1RQUKDCwkJ17NjRznAAhAlboAFYgdwCAEDT7d+/X7feeqsOHz6s1q1b66qrrlJRUZHat28f1Hi2Fh0mTZqkBQsW6M9//rOSk5N16NAhSVJKSooSEhLsDA3A96ioqNCJEyca/by4uDh2OgCwBLkFAICmW7RoUUjHs7XoMGfOHEnSwIEDA+7Pz8/XxIkTwx8QgAb58ssv9fLLLzf6836/iz8MAIQS7RUAADiP7e0VACLPl19+KWOMvF6vkpKSGvy8M2fOqKqqyv89W6ABhBLtFQAAOI8jLiQJILLU1tZKknJychq1K2nVqlX68MMP/d/zbiSAUKK9AgAA52FVBtBoNTU1kqTY2MbVLb97PH8YAAgl2isAAHAeVmUAjVa/08Hr9Tbqed89ni3QAEKJ9goAAJyHogOARgtV0YF3IwGEEu0VAAA4D6sygEarLzrQXgHASWivAADAeViVATRa/TUdaK8A4CS0VwAA4DwUHQA0Gu0VAJyInQ4AADgPqzKARqO9AoATcU0HAACch1UZQKPRXgHAiWivAAAgNA4cOKDbbrtNaWlpSkxM1GWXXaaSkpKgxmrc25QAINorADgT7RUAADTd0aNH1b9/fw0aNEhLly5Venq6du7cqZYtWwY1HkUHAI0WbHsFRQcAVqK9AgCApnvuueeUk5Oj/Px8/30dOnQIejyKDgAaLdj2iu8WKdgCDSCUaK8AADidMUY1lZVhP29sfHyD18fFixdr2LBhuvnmm7Vy5Uq1adNG9913n+65557gzh3UswBENdorADgR7RUAAKerqazUixNuCvt5p87/k+KaNWvQsZ9//rnmzJmjadOmafr06fr44481depUxcfH64477mj0uSk6AGg02isAOBHtFQAANF1dXZ169+6tWbNmSZIuv/xybdq0SXPmzKHoACA8aK8A4ES0VwAAnC42Pl5T5//JlvM2VFZWlrp27Rpw36WXXqq33347uHMH9SwAUY32CgBORHsFAMDpPB5Pg9sc7NK/f39t3bo14L5t27apffv2QY3Hqgyg0ULVXsG7kQBCiZ0OAAA03S9+8QsVFRVp1qxZ2rFjhxYsWKC5c+dq0qRJQY1H0QFAo4WqvYJ3IwGEEtd0AACg6a688koVFBRo4cKFys3N1dNPP63Zs2dr/PjxQY1HewWARqO9AoAT0V4BAEBojBo1SqNGjQrJWBQdgH8xxvjfJYt0Ho/H0l+6gy06fDcmtkADCCXaKwAAcB6KDoC+/iP6tdde06FDh+wOJSTi4uI0btw4XXzxxZaMH+w1Hb6LdyMBhBLtFQAAOA+rMiDp2LFjrik4SFJ1dbV27txp2fjBXtPhu/jDAEAo0V4BAIDzsNMBkFRVVSVJat68edBXZXWK1atX6+9//7t/TlYItr0CAKxEewUAAM5D0QGQVFlZKUmKj49XQkKCzdE0TVJSkqRv5mSFULVXAEAo0V4BAIDzsCoDCiw6RDqfzyfJ2qJDU9or+GMAgFVorwAAwHlYlQF9017hhqJD/Rysaq8wxjSpvYJtzwCsQnsFAADOQ9EB0De7Aup3CUSy+qKDVTsdvv2xosG0V/AOJACr0F4BAIDzsCoDctdOh/rCiVU7HepbKyTaKwA4C+0VAAA4D6syIHY6NEZ9a4VEewUAZ6G9AgAA56HoAMhdF5IMV9HB4/Gw0wGAo9BeAQBAaHTo0EEej+es26RJkxo9Fp93B8id7RXV1dWqq6sL+S/fTfnkCok/BgBYh/YKAABCo7i4OGCH82effaYf//jHuvnmmxs9FkUHQO5sr5C+LqY0a9YspOM35ZMrJLY9A7AO7RUAAIRG69atA75/9tlndfHFF2vAgAGNHouiAyB3tVfExsbK4/HIGKPKykrLig7BfHKFxDuQAKxDewUAwOmMMTLVdd9/YIh54mKCLspXVVXpD3/4g6ZNmxbUGBQdALmrvcLj8Sg+Pl5nzpyx5BMs2OkAwKlorwAAOJ2prtPBx9eE/bzZT/WTxxfc7+/vvvuujh07pokTJwb1fFZlQO5qr5CsvZgk13QA4FS0VwAAEHrz5s3TiBEjlJ2dHdTz2ekAyF3tFdI3xRMrig60VwBwKtorAABO54mLUfZT/Ww5bzD27NmjDz74QO+8807Q56boAMhd7RXSN/OgvQJANKG9AgDgdB6PJ+g2Bzvk5+crPT1dI0eODHoMVmVAtFc0Bu0VAJyK9goAAEKnrq5O+fn5mjBhQtC7nCWKDoBqamr87967ZadDffHEyp0OtFcAcBraKwAACJ0PPvhAe/fu1V133dWkcWxflV9++WV17NhRzZo1U69evfS3v/3N7pAQZb79hzk7Hb4f7RUAnIr2CgAAQmfo0KEyxqhz585NGsfWVfmtt97SAw88oMcee0zr16/XNddcoxEjRmjv3r12hoUoU/+HeWxsbNB/SDsN7RUAohHtFQAAOI+tF5J8/vnn9R//8R+6++67JUmzZ8/W+++/rzlz5igvL8+Sc1ZVVqps5w5LxkZk+upYuSQp1uvV/s2bbI4mNCorKiRJXx06FPI5Hdm/X5JUfepUUGNXnznj/zpSft7pF/9QvghqvamuqtIXe8hziD6nT56QJB0rK9X+7XE2RxNZMtr/UHERtNuP3+cANFak/T7nJrYVHaqqqlRSUqJHHnkk4P6hQ4dqzZo153xOZWVlwDu3Ff/6w6oxVixbqi0l2xr9PLhXjeqkGMl3OkZ64yu7wwmJOG+VFCft2L5bB7cdDOnYlZ4aySPF7K2Rdjb+5+Xx1XyzxypCft5ld+xQ267dwnKuUOS51YuWat32zaEMC2FUXff16x8Xwy9GjXUqpkrySKeXl6q85rTd4USWMdVqe23PsJwqFHnu41Ur9cnfNoQwKgDn09q0UG5NO8Xq/Ltc402cmimw2HtGVTrjqdbXjW9Gxv+/X///d7+v9RjVqFaJxqfTnmrVqlbxJk6JCs16WDWuWj/oeVlIxkLj2FZ0OHz4sGpra5WRkRFwf0ZGhg4dOnTO5+Tl5WnGjBlNOu+ZykodiznVpDHgTi1Nkt0hhEwr01ySVOWpUZWnxpJzpJjEoJ7XvaadDvmOqW1tWogjcodQ5LnKM1U67g19aw3C5F+/050Rr2GwMr3pSolpYXcYkaXGhO1UochzVdXV/D4HhMkxndJ277n/PvMzUiuTJO+/3lmqVZ2OxpwMQ3QNN/BgH4oONrG1vUI6u+/SGHPeXsxHH31U06ZN839fUVGhnJycRp2v74CByk7P+P4DEVU8Humi1FTJJdd0aKdUjS3PUGVl6D+9QpK83hhd1CpViml833R7perfTrRR88REKUKu75B+8Q/Ddq5Q5LkrftxfaZ+mhjo0WM0Yae13/ojqnSjFcn2CxkiMb6aUxOZ2hxFxWvfoFLZzhSLPXdbnKiUnuufNAsCpamprtHHrNn159AK7U41UXVOjo56ziwy+uDh5PB59/eedx/+1519f61tfezwexXq9Onn6tBKaxSsuNk6nz5zRmarQFOFbZWWFZBw0nm1Fh4suukher/esXQ1lZWVn7X6oFx8f3+SPNEzPyFD6ecYH3KSt3QEgKCHJcx2zld4xO0QRIZwObFgjc+brT4iR16M2N/SXJ4jiHuBkochzqWlpSh04MDQBAbigvoOHfO8x5eXlKisrC7gvMzNTycnJVoWFCGJb0cHn86lXr15avny5brjhBv/9y5cv1/XXX29XWAAA2MbbIl41Z77e7eBN9lFwAABEhJSUFKWkpNgdBhzK1vaKadOm6fbbb1fv3r3Vt29fzZ07V3v37tW9995rZ1gAANjC28KnmrJT/q8BAAAina1Fh1tuuUVHjhzRU089pdLSUuXm5mrJkiVq3769nWEBAGCLbxcaKDoAAAA3sP1Ckvfdd5/uu+8+u8MAAMB23hbx5/waAAAgUkXGpeMBAIgC3pRvdjfEsNMBAADYoKamRr/61a/UsWNHJSQk6Ac/+IGeeuop1dXVBTWe7TsdAADA17zJ32qvSGGnAwAACL/nnntOr7zyiubPn69u3brpk08+0Z133qmUlBTdf//9jR6PogMAAA7x7UID13QAAAB2+Oijj3T99ddr5MiRkqQOHTpo4cKF+uSTT4Iaj6IDAAAOEcOFJAEAcDVjjKqrq8N+3ri4OHk8Dfso7quvvlqvvPKKtm3bps6dO+sf//iHVq9erdmzZwd1booOAAA4hLe5TzFJsTI1RrEtaa8AAMBtqqurNWvWrLCfd/r06fL5GvaGxsMPP6zy8nJ16dJFXq9XtbW1mjlzpm699dagzk3RAQAAh/B4PUq/7zKZOiNPnNfucAAAQBR666239Ic//EELFixQt27dtGHDBj3wwAPKzs7WhAkTGj0eRQcAABwkNi3B7hAAAIBF4uLiNH36dFvO21D/+Z//qUceeUTjxo2TJHXv3l179uxRXl4eRQcAAAAAAJzK4/E0uM3BLqdOnVJMTEzAfV6vl4/MBAAAAAAATTN69GjNnDlT7dq1U7du3bR+/Xo9//zzuuuuu4Iaj6IDAAAAAACQJL300kv69a9/rfvuu09lZWXKzs7Wz372Mz3++ONBjUfRAQAAAAAASJKSk5M1e/bsoD8i87tivv8QAAAAAACAxqPoAAAAAAAALBHR7RXGGElSRUWFzZEAwDfqc1J9jmoK8hwAJyLPAXC7UOa5aBfRRYfjx49LknJycmyOBADOdvz4caWkpDR5DIk8B8CZyHMA3C4UeS7aRXTRITs7W/v27VNycrI8Hk+Dn1dRUaGcnBzt27dPLVq0sDBC+zBH94iGebptjsYYHT9+XNnZ2U0eizx3fszRPaJhnm6bI3kuPKJhjlJ0zJM5Rp5Q5rloF9FFh5iYGLVt2zbo57do0cIV/yAuhDm6RzTM001zDFVFnDz3/Zije0TDPN00R/Jc+ETDHKXomCdzjCyhynN1dXUhGcfJLjTHiC46AAAAAADgRD6fTzExMTp48KBat24tn8/XqB1dkcAYo6qqKn355ZeKiYmRz+c76xiKDgAAAAAAhFhMTIw6duyo0tJSHTx40O5wLJWYmKh27dopJubsD8iMyqJDfHy8nnjiCcXHx9sdimWYo3tEwzyjYY7hFg0/U+boHtEwz2iYY7hFw880GuYoRcc8mWP08vl8ateunWpqalRbW2t3OJbwer2KjY097y4Oj+EzQAAAAAAAgAXO3vsAAAAAAAAQAhQdAAAAAACAJSg6AAAAAAAAS0Rd0eHll19Wx44d1axZM/Xq1Ut/+9vf7A7pnJ588kl5PJ6AW2Zmpv9xY4yefPJJZWdnKyEhQQMHDtSmTZsCxqisrNSUKVN00UUXKSkpST/96U+1f//+gGOOHj2q22+/XSkpKUpJSdHtt9+uY8eOWTavVatWafTo0crOzpbH49G7774b8Hg457V3716NHj1aSUlJuuiiizR16lRVVVVZPseJEyee9dpeddVVETXHvLw8XXnllUpOTlZ6errGjBmjrVu3BhzjhtcyUpHnyHP1yHPBI885G3mOPFePPBc88hzCxkSRRYsWmbi4OPPaa6+ZzZs3m/vvv98kJSWZPXv22B3aWZ544gnTrVs3U1pa6r+VlZX5H3/22WdNcnKyefvtt83GjRvNLbfcYrKyskxFRYX/mHvvvde0adPGLF++3Kxbt84MGjTI9OzZ09TU1PiPGT58uMnNzTVr1qwxa9asMbm5uWbUqFGWzWvJkiXmscceM2+//baRZAoKCgIeD9e8ampqTG5urhk0aJBZt26dWb58ucnOzjaTJ0+2fI4TJkwww4cPD3htjxw5EnCM0+c4bNgwk5+fbz777DOzYcMGM3LkSNOuXTtz4sQJ/zFueC0jEXmOPFePPNc05DnnIs+R5+qR55qGPIdwiaqiw49+9CNz7733BtzXpUsX88gjj9gU0fk98cQTpmfPnud8rK6uzmRmZppnn33Wf9+ZM2dMSkqKeeWVV4wxxhw7dszExcWZRYsW+Y85cOCAiYmJMcuWLTPGGLN582YjyRQVFfmP+eijj4wk889//tOCWQX6bgIP57yWLFliYmJizIEDB/zHLFy40MTHx5vy8nLL5mjM14vU9ddff97nRNocjTGmrKzMSDIrV640xrjztYwU5DnyHHmOPOd25DnyHHmOPIfIEjXtFVVVVSopKdHQoUMD7h86dKjWrFljU1QXtn37dmVnZ6tjx44aN26cPv/8c0nSrl27dOjQoYC5xMfHa8CAAf65lJSUqLq6OuCY7Oxs5ebm+o/56KOPlJKSoj59+viPueqqq5SSkmLLzySc8/roo4+Um5ur7Oxs/zHDhg1TZWWlSkpKLJ2nJBUWFio9PV2dO3fWPffco7KyMv9jkTjH8vJySVJqaqqk6HotnYQ8R55z0r8N8lxkztPpyHPkOSf92yDPReY8EX5RU3Q4fPiwamtrlZGREXB/RkaGDh06ZFNU59enTx+98cYbev/99/Xaa6/p0KFD6tevn44cOeKP90JzOXTokHw+n1q1anXBY9LT0886d3p6ui0/k3DO69ChQ2edp1WrVvL5fJbPfcSIEXrzzTf14Ycf6je/+Y2Ki4t13XXXqbKy0h9bJM3RGKNp06bp6quvVm5urv/c9TFfaA6RNM9IQJ479zHkuW+Q54JDnnMO8ty5jyHPfYM8FxzyHKwUa3cA4ebxeAK+N8acdZ8TjBgxwv919+7d1bdvX1188cWaP3++/yI1wczlu8ec63i7fybhmpddc7/lllv8X+fm5qp3795q3769/u///k9jx4497/OcOsfJkyfr008/1erVq896zO2vpVOR55z/34fb/22Q59zzWjoVec75/324/d8Gec49ryWsFzU7HS666CJ5vd6zKmVlZWVnVdWcKCkpSd27d9f27dv9Vz2+0FwyMzNVVVWlo0ePXvCYL7744qxzffnll7b8TMI5r8zMzLPOc/ToUVVXV4d97llZWWrfvr22b9/ujy1S5jhlyhQtXrxYK1asUNu2bf33R+traTfy3LmPIc99gzzXeOQ5ZyHPnfsY8tw3yHONR56D1aKm6ODz+dSrVy8tX7484P7ly5erX79+NkXVcJWVldqyZYuysrLUsWNHZWZmBsylqqpKK1eu9M+lV69eiouLCzimtLRUn332mf+Yvn37qry8XB9//LH/mLVr16q8vNyWn0k459W3b1999tlnKi0t9R/zl7/8RfHx8erVq5el8/yuI0eOaN++fcrKypIUGXM0xmjy5Ml655139OGHH6pjx44Bj0fra2k38hx5zqn/Nshzzp1npCHPkeec+m+DPOfcecIBQnxhSker/4ilefPmmc2bN5sHHnjAJCUlmd27d9sd2lkefPBBU1hYaD7//HNTVFRkRo0aZZKTk/2xPvvssyYlJcW88847ZuPGjebWW28958fXtG3b1nzwwQdm3bp15rrrrjvnx9f06NHDfPTRR+ajjz4y3bt3t/Qjlo4fP27Wr19v1q9fbySZ559/3qxfv97/MVfhmlf9x/IMHjzYrFu3znzwwQembdu2IflYngvN8fjx4+bBBx80a9asMbt27TIrVqwwffv2NW3atImoOf785z83KSkpprCwMOCjok6dOuU/xg2vZSQiz5Hn6pHnmoY851zkOfJcPfJc05DnEC5RVXQwxpj/+q//Mu3btzc+n89cccUV/o+EcZr6z8CNi4sz2dnZZuzYsWbTpk3+x+vq6swTTzxhMjMzTXx8vLn22mvNxo0bA8Y4ffq0mTx5sklNTTUJCQlm1KhRZu/evQHHHDlyxIwfP94kJyeb5ORkM378eHP06FHL5rVixQoj6azbhAkTwj6vPXv2mJEjR5qEhASTmppqJk+ebM6cOWPpHE+dOmWGDh1qWrdubeLi4ky7du3MhAkTzorf6XM81/wkmfz8fP8xbngtIxV5jjxXjzwXPPKcs5HnyHP1yHPBI88hXDzGGBPKnRMAAAAAAABSFF3TAQAAAAAAhBdFBwAAAAAAYAmKDgAAAAAAwBIUHQAAAAAAgCUoOgAAAAAAAEtQdAAAAAAAAJag6AAAAAAAACxB0QEAAAAAAFiCogNcr7CwUB6PR8eOHbM7FACwBHkOgNuR54DI5THGGLuDAEJp4MCBuuyyyzR79mxJUlVVlb766itlZGTI4/HYGxwAhAB5DoDbkecA94i1OwDAaj6fT5mZmXaHAQCWIc8BcDvyHBC5aK+Aq0ycOFErV67UCy+8II/HI4/Ho9dffz1gO97rr7+uli1b6r333tMll1yixMRE3XTTTTp58qTmz5+vDh06qFWrVpoyZYpqa2v9Y1dVVemhhx5SmzZtlJSUpD59+qiwsNCeiQKIWuQ5AG5HngPchZ0OcJUXXnhB27ZtU25urp566ilJ0qZNm8467tSpU3rxxRe1aNEiHT9+XGPHjtXYsWPVsmVLLVmyRJ9//rluvPFGXX311brlllskSXfeead2796tRYsWKTs7WwUFBRo+fLg2btyoTp06hXWeAKIXeQ6A25HnAHeh6ABXSUlJkc/nU2Jion8L3j//+c+zjquurtacOXN08cUXS5Juuukm/f73v9cXX3yh5s2bq2vXrho0aJBWrFihW265RTt37tTChQu1f/9+ZWdnS5J++ctfatmyZcrPz9esWbPCN0kAUY08B8DtyHOAu1B0QFRKTEz0L1CSlJGRoQ4dOqh58+YB95WVlUmS1q1bJ2OMOnfuHDBOZWWl0tLSwhM0ADQCeQ6A25HngMhA0QFRKS4uLuB7j8dzzvvq6uokSXV1dfJ6vSopKZHX6w047tsLGwA4BXkOgNuR54DIQNEBruPz+QIuGBQKl19+uWpra1VWVqZrrrkmpGMDQGOR5wC4HXkOcA8+vQKu06FDB61du1a7d+/W4cOH/dXtpujcubPGjx+vO+64Q++884527dql4uJiPffcc1qyZEkIogaAhiPPAXA78hzgHhQd4Dq//OUv5fV61bVrV7Vu3Vp79+4Nybj5+fm644479OCDD+qSSy7RT3/6U61du1Y5OTkhGR8AGoo8B8DtyHOAe3iMMcbuIAAAAAAAgPuw0wEAAAAAAFiCogMAAAAAALAERQcAAAAAAGAJig4AAAAAAMASFB0AAAAAAIAlKDoAAAAAAABLUHQAAAAAAACWoOgAAAAAAAAsQdEBAAAAAABYgqIDAAAAAACwBEUHAAAAAABgCYoOAAAAAADAEv8frvvT42z6qrcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff.sel(id=plid)['rh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAF8CAYAAAD4sQEKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRaklEQVR4nO3de3xU1bn/8e+e3BNCIEBIUkiIF1ABY0REFCWoBaKiFG+oBwG16pFLbawXtCq0Cl5aDlZ+orYaaKtieypItQellYAchEIwBxTkZhAQYgQhgdxmMrN+f0BGA0lIJjPZmcnnfV5zyuy9Zq9n7ZDH5GGttS1jjBEAAAAAAEArc9gdAAAAAAAAaJ8oSgAAAAAAAFtQlAAAAAAAALagKAEAAAAAAGxBUQIAAAAAANiCogQAAAAAALAFRQkAAAAAAGALihIAAAAAAMAW4XYHAAAAAABAqHK73XK5XHaHYZvIyEg5HA3Ph6AoAQAAAACAnxljVFxcrMOHD9sdiq0cDocyMjIUGRlZ73nLGGNaOSYAAAAAAELa/v37dfjwYSUlJSk2NlaWZdkdUqvzeDzat2+fIiIilJaWVu89YKYEAAAAAAB+5Ha7vQWJLl262B2Orbp166Z9+/appqZGERERJ51no0sAAAAAAPyodg+J2NhYmyOxX+2yDbfbXe95ihIAAAAAAARAe1yycaJT3QOKEgAAAAAAwBYUJQAAAAAACHLZ2dm6//77Gzzfq1cvzZkzp9XiaSo2ugQAAAAAIMi988479W4k2dZRlAAAAAAAIMglJibaHYJPWL4BAAAAAECQ++HyjZKSEo0aNUoxMTHKyMjQG2+8YW9wjWCmBAAAAAAAIWTChAnas2ePPvroI0VGRmrq1KkqKSmxO6x6UZQAAAAAACBEbNu2Tf/zP/+jNWvWaNCgQZKk1157TWeffbbNkdWP5RsAAAAAAISILVu2KDw8XBdccIH32FlnnaVOnTrZF1QjKEoAAAAAABAijDGSJMuybI6kaShKAAAAAAAQIs4++2zV1NRo/fr13mNbt27V4cOH7QuqERQlAAAAAAAIEX369NHIkSP105/+VGvXrlVBQYHuuusuxcTE2B1avShKAAAAAAAQQvLy8tSzZ08NHTpUY8aM0d13362kpCS7w6qXZWoXnAAAAAAAgBarqqpSUVGRMjIyFB0dbXc4tjrVvWCmBAAAAAAAsAVFCQAAAAAAYAuKEgAAAAAAwBYUJQAAAAAAgC0oSgAAAAAAAFtQlAAAAAAAALagKAEAAAAAAGxBUQIAAAAAANiCogQAAAAAALAFRQkAAAAAAGALihIAAAAAAECSdOTIEd1///1KT09XTEyMLr74Yq1bty5g/VGUAAAAAAAAkqS77rpLy5Yt05/+9Cdt2rRJw4cP15VXXqmvv/46IP1ZxhgTkCsDAAAAANAOVVVVqaioSBkZGYqOjpYxRpUuty2xxESEybKsJrWtrKxUfHy83n33XV199dXe4+edd56uueYaPfXUU83u/8R7caLwZl8RAAAAAAA0WaXLrXOe+MCWvjf/aoRiI5v2q39NTY3cbvdJxYOYmBitWrUqEOGxfAMAAAAAAEjx8fEaPHiwfv3rX2vfvn1yu93685//rLVr12r//v0B6ZOZEgAAAAAABFBMRJg2/2qEbX03x5/+9Cfdcccd+tGPfqSwsDCdf/75uvXWW7Vhw4aAxEdRAgAAAACAALIsq8lLKOx2+umna8WKFSovL1dZWZlSUlJ08803KyMjIyD9sXwDAAAAAADUERcXp5SUFB06dEgffPCBrrvuuoD0ExylGgAAAAAAEHAffPCBjDHq06ePduzYoQcffFB9+vTRxIkTA9IfMyUAAAAAAIAkqbS0VJMmTdJZZ52l22+/XUOGDNGHH36oiIiIgPRnGWNMQK4MAAAAAEA7VFVVpaKiImVkZJz0eM325lT3gpkSAAAAAADAFhQlAAAAAACALShKAAAAAAAAW1CUAAAAAAAAtqAoAQAAAAAAbEFRAgAAAAAA2IKiBAAAAAAAsAVFCQAAAAAAYAuKEgAAAAAAwBYUJRA0/vu//1v9+/dXTEyMunTpoiuvvFLl5eWSpAkTJmj06NGaMWOGkpKS1LFjR91zzz1yOp3ezy9dulRDhgxRp06d1KVLF11zzTXauXNnnT727t2rsWPHKjExUXFxcbrgggu0du1a7/m///3vGjBggKKjo3XaaadpxowZqqmpCch4jTG68sorNXLkSBljJEmHDx9WWlqaHnvssYD0CaB1tbe8tmvXLjkcDq1fv77O8RdffFHp6eneXAcgeLW3vCZJvXr1kmVZJ72AYLVy5UqNGjVKqampsixLixcvrnPeGKPp06crNTVVMTExys7O1ueff+5zfxQlEBT279+vW265RXfccYe2bNmi/Px8jRkzps4PsP/617+0ZcsWLV++XG+99ZYWLVqkGTNmeM+Xl5crNzdX69at07/+9S85HA795Cc/kcfjkSQdPXpUQ4cO1b59+7RkyRL93//9nx566CHv+Q8++ED/8R//oalTp2rz5s165ZVXNH/+fD399NMNxv3GG2+oQ4cOjb7eeOONej9rWZYWLFigf//73/rd734nSbr33nvVvXt3TZ8+vaW3FIDN2mNe69Wrl6688krl5eXVOZ6Xl6cJEybwQzwQ5NpjXpOkdevWaf/+/dq/f7/27t2riy66SJdeemlLbydgm/LycmVmZmru3Ln1nn/uuec0e/ZszZ07V+vWrVNycrJ+/OMf68iRI751aNqhFStWmGuuucakpKQYSWbRokVtor/NmzebUaNGmY4dO5oOHTqYQYMGma+++iqgsQWLgoICI8ns2rWr3vPjx483iYmJpry83Hts3rx5pkOHDsbtdtf7mZKSEiPJbNq0yRhjzCuvvGLi4+PNwYMH621/6aWXmpkzZ9Y59qc//cmkpKQ0GHdZWZnZvn17o6+ysrJGx/6Xv/zFREVFmWnTppnY2FizdevWRtsDCA7tNa+9/fbbpnPnzqaqqsoYY0xhYaGxLMsUFRU1+BkAwaG95rUfmjp1qklPTzclJSVNao/QVVlZaTZv3mwqKyvtDqVFTvz91ePxmOTkZPPMM894j1VVVZmEhATz8ssv13uNU92LcN9KGcGttvIzceJEXX/99W2iv507d2rIkCG68847NWPGDCUkJGjLli2Kjo4OeHzBIDMzU1dccYX69++vESNGaPjw4brhhhvUuXPnOm1iY2O97wcPHqyjR49qz549Sk9P186dO/X4449rzZo1OnDggLeivnv3bvXr10+FhYXKyspSYmJivTEUFBRo3bp1dSrtbrdbVVVVqqioqNN3rfj4eMXHx7do7DfeeKMWLVqkWbNmad68eerdu3eLrgegbWiveW306NGaPHmyFi1apLFjx+r111/XsGHD1KtXL5+vCaBtaK95rdarr76q1157Tf/7v/+rbt26tfh6CDHGSK4Ke/qOiJX8NBuxqKhIxcXFGj58uPdYVFSUhg4dqtWrV+uee+5p9jXbZVEiJydHOTk5DZ53Op365S9/qTfeeEOHDx9Wv3799Oyzzyo7Ozsg/UnSY489pquuukrPPfec99hpp53mU3+hKCwsTMuWLdPq1av14Ycf6sUXX9Rjjz2mtWvXKiMjo9HP1k4HHjVqlHr27Knf//73Sk1NlcfjUb9+/bzrGGNiYhq9jsfj0YwZMzRmzJiTzjVUPHrjjTdO+Y35yiuv6LbbbmvwfEVFhQoKChQWFqbt27c3ei0AwaO95rXIyEiNGzdOeXl5GjNmjN58803NmTOn0esBCA7tNa9JUn5+vqZMmaK33npLmZmZjV4L7ZSrQpqZak/fj+6TIuP8cqni4mJJUvfu3esc7969u7766iufrtkuixKnMnHiRO3atUsLFy5UamqqFi1apJEjR2rTpk0688wz/d6fx+PR+++/r4ceekgjRozQp59+qoyMDE2bNk2jR4/2e3/ByrIsXXLJJbrkkkv0xBNPKD09XYsWLVJubq4k6f/+7/9UWVnp/Y/VmjVr1KFDB/Xo0UMHDx7Uli1b9Morr3jX+K1atarO9c8991z94Q9/0HfffVdv9f3888/X1q1bdcYZZzQ55muvvVaDBg1qtM2J39AneuCBB+RwOPQ///M/uuqqq3T11Vfr8ssvb3IMANqu9prX7rrrLvXr108vvfSSXC5Xvb88AAhO7TGv7dixQ9dff70effRR8hnajRP3gTLG+Lw3FEWJE+zcuVNvvfWW9u7dq9TUY5WsX/ziF1q6dKny8vI0c+ZMv/dZUlKio0eP6plnntFTTz2lZ599VkuXLtWYMWO0fPlyDR061O99Bpu1a9fqX//6l4YPH66kpCStXbtW3377rc4++2xvG6fTqTvvvFO//OUv9dVXX+nJJ5/U5MmT5XA41LlzZ3Xp0kWvvvqqUlJStHv3bj3yyCN1+rjllls0c+ZMjR49WrNmzVJKSoo+/fRTpaamavDgwXriiSd0zTXXqGfPnrrxxhvlcDi0ceNGbdq0SU899VS9cbd0OuD777+v119/XZ988onOP/98PfLIIxo/frw2btxYZyokgODTXvOaJJ199tm66KKL9PDDD+uOO+445b98AggO7TGvVVZWatSoUTrvvPN09913e/8VWZKSk5N9uiZCVETssRkLdvXtJ7V/r4uLi5WSkuI9XlJScsp/lGiQPzfBCEY6YeOOv/zlL0aSiYuLq/MKDw83N910kzHGmKKiIiOp0dekSZOa1J8xxnz99ddGkrnlllvqHB81apQZO3asX8cbrDZv3mxGjBhhunXrZqKiokzv3r3Niy++6D0/fvx4c91115knnnjCdOnSxXTo0MHcdddd3o3UjDFm2bJl5uyzzzZRUVHm3HPPNfn5+Sd9PXbt2mWuv/5607FjRxMbG2suuOACs3btWu/5pUuXmosvvtjExMSYjh07mgsvvNC8+uqrARlzSUmJ6d69e53Nmlwul7nwwgu9fxcBBK/2mNd+6LXXXjOSzL///e+A9wWgdbTHvNbY7wVo30J9o8tnn33We6y6urpFG11axztqtyzL0qJFi7zLJN5++23ddttt+vzzzxUWFlanbYcOHZScnCyXy3XS85JP1Llz53orRSf2Jx2rGMfFxenJJ5/UL3/5S+/xhx9+WKtWrdL//u//+j7AdmLChAk6fPjwSc/QBYBgFep57emnn9bChQu1adMmu0MB0EpCPa8BP1RVVaWioiJlZGQE3cMLjh49qh07dkiSsrKyNHv2bA0bNkyJiYlKS0vTs88+q1mzZikvL09nnnmmZs6cqfz8fG3durXeWUenuhcs3zhBVlaW3G63SkpKGny+cEREhM466yy/9RkZGamBAwdq69atdY5v27ZN6enpfusHAAC7HT16VFu2bNGLL76oX//613aHAwAATrB+/XoNGzbM+752T5jx48dr/vz5euihh1RZWan77rtPhw4d0qBBg/Thhx/6vAyqXRYlflj5kY491qSwsFCJiYnq3bu3brvtNt1+++367W9/q6ysLB04cEAfffSR+vfvr6uuusqv/aWlpUmSHnzwQd1888267LLLNGzYMC1dulR///vflZ+f3+LxAgDQVkyePFlvvfWWRo8erTvuuMPucAAAwAmys7PV2IIKy7I0ffp0TZ8+3S/9tcvlG/n5+XUqP7VqKz8ul0tPPfWU/vjHP+rrr79Wly5dNHjwYM2YMUP9+/f3e3+1Xn/9dc2aNUt79+5Vnz59NGPGDF133XXN7g8AAAAAYJ9gXr7hb6e6F+2yKAEAAAAAQKBQlPjeqe6Fw4aYAAAAAAAAKEoAAAAAAAB7tKuNLj0ej/bt26f4+HhZlmV3OADaIWOMjhw5otTUVDkcLasLk9MAtAXkNQChxJ85DU3TrooS+/btU8+ePe0OAwC0Z88e9ejRo0XXIKcBaEvIawBCiT9yGpqmXRUlap+bumfPHnXs2NHmaAC0R2VlZerZs6fPz3H+IXIagLaAvAYglPgzp6Fp2lVRonYaYMeOHfkPHQBb+WNaMjkNQFtCXgMQSlhC1nqCZpHMrFmzNHDgQMXHxyspKUmjR4/W1q1b7Q4LAAAAAAD4KGiKEitWrNCkSZO0Zs0aLVu2TDU1NRo+fLjKy8vtDg0AAAAAAPggaIoSS5cu1YQJE9S3b19lZmYqLy9Pu3fvVkFBgd2hAQAAAAAQElauXKlRo0YpNTVVlmVp8eLFdc6/8847GjFihLp27SrLslRYWNii/oKmKHGi0tJSSVJiYmKDbaqrq1VWVlbnBQDBipwGINSQ1wCg7SkvL1dmZqbmzp3b4PlLLrlEzzzzjF/6C8qNLo0xys3N1ZAhQ9SvX78G282aNUszZsxoxcgAIHDIaQBCDXkNANqenJwc5eTkNHh+3LhxkqRdu3b5pb+gLEpMnjxZGzdu1KpVqxptN23aNOXm5nrf1z7eBQCCETkNQKghryFQ9m37Qgf27GpS2/7DhstyBO0EcgQJY4wqaypt6TsmPKZNP00k6IoSU6ZM0ZIlS7Ry5Ur16NGj0bZRUVGKiopqpcgAILDIaQBCDXkNgVBRVqq3pz8ij7umSe37Dfux2u6vawgVlTWVGvTmIFv6XnvrWsVGxNrSd1METVHCGKMpU6Zo0aJFys/PV0ZGht0hAQAAAGhjSkuK5XHXKDwqSun9z7M7HACnEDRFiUmTJunNN9/Uu+++q/j4eBUXF0uSEhISFBMTY3N0AAAAANqCitLDkqQuP+qp0Q8+bm8wwHEx4TFae+ta2/puy4KmKDFv3jxJUnZ2dp3jeXl5mjBhQusHBAAAAKDNKT98WJIUm9DJ1jiAH7Isq00vobBT0BQljDF2hwAAAACgjaudKUFRAvDN0aNHtWPHDu/7oqIiFRYWKjExUWlpafruu++0e/du7du3T5K0detWSVJycrKSk5Ob3R/bzAIAAAAIGbVFiTiKEoBP1q9fr6ysLGVlZUmScnNzlZWVpSeeeEKStGTJEmVlZenqq6+WJI0dO1ZZWVl6+eWXfeovaGZKAAAAAMCplHtnSnS2NxAgSGVnZze6UmHChAl+3UKBmRIAAAAAQkZF6SFJUmynTvYGAqBJKEoAAAAACBkVxze6ZPkGEBwoSgAAAAAIGWx0CQQXihIAAAAAQoK7xqWq8qOSKEoAwYKiBAAAAICQUFFaKkmyHA7FdIi3ORoATcHTNwAAAAC0mhV/fl27CgsCcu0al1PSsVkSloN/fwWCAUUJAAAAAK3CVV2l9X9/J+D9JKVnBLwPAP5BUQIAAABAq3BVVXn/fMMvn5JlWX7vw7IsJZ/e2+/XBRAYFCUAAAAAtArn8aJEeFSU0vufZ28wANoEFloBAAAAaBWu6mNFiYioaJsjAdBWUJQAAAAA0CpcVZWSpMhoihJAW7Vy5UqNGjVKqampsixLixcv9p5zuVx6+OGH1b9/f8XFxSk1NVW333679u3b53N/FCUAAAAAtApXVbUkZkoAbVl5ebkyMzM1d+7ck85VVFRow4YNevzxx7Vhwwa988472rZtm6699lqf+2NPCQAAAACtwll9bKZEBDMlgDYrJydHOTk59Z5LSEjQsmXL6hx78cUXdeGFF2r37t1KS0trdn8UJQAAAAC0ipoq9pRA+2SMkamstKVvKyYmIE+6qVVaWirLstSpUyefPk9RAgAAAECrqH36RkR0jM2RAK3LVFZq6/kDbOm7z4YCWbGxAbl2VVWVHnnkEd16663q2LGjT9dgTwkAAAAAraL26RtsdAkEP5fLpbFjx8rj8eill17y+TrMlAAAAADQKlws30A7ZcXEqM+GAtv69jeXy6WbbrpJRUVF+uijj3yeJSFRlAAAAADQSmpnSrDRJdoby7ICtoSitdUWJLZv367ly5erS5cuLboeRQkAAAAAreL7PSUoSgBt1dGjR7Vjxw7v+6KiIhUWFioxMVGpqam64YYbtGHDBr333ntyu90qLi6WJCUmJioyMrLZ/VGUAAAAANAqaqpZvgG0devXr9ewYcO873NzcyVJ48eP1/Tp07VkyRJJ0nnnnVfnc8uXL1d2dnaz+6MoAQAAAKBVOKuOPRKRmRJA25WdnS1jTIPnGzvnC56+AQAAAKBVuKqrJUmRPBIUwHEUJQAAAAC0ClftTImoKJsjAdBWUJQAAAAA0CpcVcdmSkQwUwLAcRQlAAAAALQKVzV7SgCoi6IEAAAAgFbhquLpGwDqoigBAAAAoFU4jxclIpkpAeA4ihIAAAAAAs4YI1c1MyUA1EVRAgAAAEDAuV0uGY9HEhtdAvgeRQkAAAAAAVc7S0KSIqJ5JCiAY8LtDgAAAABoicPfFGv1X9+Qs7LS7lDQiBrnsceBhkdEyuEIszkaAG0FRQkAAAAEtU3/WqotHy+3Oww0UXzXbnaHAKARK1eu1PPPP6+CggLt379fixYt0ujRo73np0+froULF2rPnj2KjIzUgAED9PTTT2vQoEE+9RdURYlT3RwAAAC0P67qY/8Cn5F1gc644CKbo8Gp9Ozb3+4QADSivLxcmZmZmjhxoq6//vqTzvfu3Vtz587VaaedpsrKSv3Xf/2Xhg8frh07dqhbt+YXHYOqKHGqmwMAAID2x13jkiSlnNFH51450uZoACC45eTkKCcnp8Hzt956a533s2fP1muvvaaNGzfqiiuuaHZ/QVWUONXNAQAAQPvjrqmRJDnC2KcAQNtkjFGN02NL3+GRDlmWFZBrO51Ovfrqq0pISFBmZqZP1wiqokRzVVdXq/r4dD5JKisrszEaAGgZchqAUOOvvOY5XpQICw/pH20BBLEap0ev/myFLX3f/cJQRUT5t2j73nvvaezYsaqoqFBKSoqWLVumrl27+nStkH4k6KxZs5SQkOB99ezZ0+6QAMBn5DQAocZfec3tdkuSHOER/gwPANCAYcOGqbCwUKtXr9bIkSN10003qaSkxKdrhXQ5edq0acrNzfW+Lysr44d4AEGLnAYg1Pgrr3mO7ynBTAkAbVV4pEN3vzDUtr79LS4uTmeccYbOOOMMXXTRRTrzzDP12muvadq0ac2Pz+/RtSFRUVGKioqyOwwA8AtyGoBQ46+85mb5BoA2zrIsvy+haEuMMXWW4zUHmRsAAABBzbvRJUUJAGixo0ePaseOHd73RUVFKiwsVGJiorp06aKnn35a1157rVJSUnTw4EG99NJL2rt3r2688Uaf+guqzN3YzUlLS7MxMgAAANjF42amBAD4y/r16zVs2DDv+9plduPHj9fLL7+sL774QgsWLNCBAwfUpUsXDRw4UB9//LH69u3rU39Blbkbuznz58+3KSoAAADYiZkSAOA/2dnZMsY0eP6dd97xa39BlblPdXMAAADQ/vBIUAAIXiH9SFAAAACEPu9MiTCKEgAQbChKAAAAIKgxUwIAghdFCQAAAAQ1NxtdAkDQoigBAACAoOb2zpSIsDkSAEBzUZQAAABAUPN495QIszkSAEBzUZQAAABAUPPwSFAACFoUJQAAABDUWL4BAMGLogQAAACCmpunbwBA0KIoAQAAgKBljJGHp28AQNCiKAEAAICg5XG7vX92hFGUAICWWrlypUaNGqXU1FRZlqXFixc32Paee+6RZVmaM2eOz/1RlAAAAEDQqt3kUmKmBAD4Q3l5uTIzMzV37txG2y1evFhr165Vampqi/ojcwMAACBouX9QlODpGwDQcjk5OcrJyWm0zddff63Jkyfrgw8+0NVXX92i/sjcAAAACFruGpf3z46wMBsjAYCGGWNUU11tS9/hUVGyLMtv1/N4PBo3bpwefPBB9e3bt8XXoygBAACAoFW7p4QjLNyvP3QDgD/VVFfrd+NvsKXvqQv+WxHR0X673rPPPqvw8HBNnTrVL9ejKAEAAICgxeNAAaD1FBQU6IUXXtCGDRv8VggmewMAACBo1S7foCgBoC0Lj4rS1AX/bVvf/vLxxx+rpKREaWlp3mNut1sPPPCA5syZo127djU/Pr9FBwAAALSy2qdvsMklgLbMsiy/LqGwy7hx43TllVfWOTZixAiNGzdOEydO9OmaZG8AAAAELe+eEhQlAMAvjh49qh07dnjfFxUVqbCwUImJiUpLS1OXLl3qtI+IiFBycrL69OnjU39kbwAAAAQtlm8AgH+tX79ew4YN877Pzc2VJI0fP17z58/3e39kbwAAAAQt70aXYfxYCwD+kJ2dLWNMk9v7so/EDzla9GkAAADARjx9AwCCG0UJAAAABC2Pm40uASCYUZQAAABA0HLz9A0ACGoUJQAAABC0PCzfAICgRlECAAAAQev7PSUibI4EAOALihIAAAAIWsyUgJ2MMdo7Zaq29OuvLf36q3ztv+0OCQg6ZG8AAAAELe+eEmFhNkeC9qj6iy90ZNmy7w804zGKAI5hpgQAAACC1vcbXbJ8A62vdMnfJUkxFwzQGSvyFXN+ls0RAcGHmRIAAAAIWp4alySWb7RV5WvW6Jtnn5NxOu0OJSBcX38tSeoycaIiune3ORogOJG9AQAAELTcbrckihKtwblnj1x79jS5vXG5tO+RaXIfOhTAqOwX3r27Olx6qd1hAEGL7A0AAICg5WFPCb+qWLdOFQUFJx13ffONDr/9F8njafY1o846S92nTZMsf0QYWE6n0c4dLjmdUkyspd69T70sKOqMM2RFRrZCdEDrWLlypZ5//nkVFBRo//79WrRokUaPHu09P2HCBC1YsKDOZwYNGqQ1a9b41B9FCQAAAAQtHgn6PU9FhZy7d8u1b/8p27r27lXFun/L1Li//3xlpSpO8UtF5Omny2rGrBRHhw5K+fWvFXVaRpM/4wtnVY2Kd5bK7W7aRpOeGo/2bj2ko4eq6xw/sOeIjh46ttSka88OyvqPC/0eK9DWlZeXKzMzUxMnTtT1119fb5uRI0cqLy/P+z6yBYU5ihIAAAAIWu7je0o4QmD5hjFGNd98I7nd9ZyTqj77TFWbN8tUV8njdMpUVavm229V802xXN+UyFNW5pc44n/8Y4V1Sqh70HKoQ/ZQxV9+uV/6cLs8+urzg9q+/hvt235YxtOyp1ZUV9bIU+OfJ1/Ed4lWz7M6q0NitF+uBwSbnJwc5eTkNNomKipKycnJfukv+LM3AAAA2i1PEO0p4T5yROVr1kjHZ3f8kHF79N2f/qiq/9vYoj4cCQmK7NlTCmv8IXuO2Fh1uOQSORLqFh+izzpbMf37tSiGWke+q9I3RWWqPOLU7s3fqcZ57GtljPTt7iNyVp58H1qiY9doxcQ3/V9rE1Pi1D2joyzr+3Ul4ZEO9Tq3qyKj2/7fJwQXY4yMq/nLn/zBinDU+XvuD/n5+UpKSlKnTp00dOhQPf3000pKSvLpWkH33fbSSy/p+eef1/79+9W3b1/NmTNHl7KxDAAAQLvkfSRomD0/1h5ZvlwVa/996obGqGzp0mMzIRoTFiYrov6lKOHduiluyCUK69BBVkSkrKgohXftqvDk7opITlZ49+4K69BBNU63jnxXJdOEiQMnNqmUVLm//KR2bpdHe774ThVlJz9Fo7qiRns+Pyhndd0ZHq6qk2d8ePs1LkXHViq9X1f1OKuzImNa9vULC7fUITFaDodDnbqnNHuPkW+KdmrrJx/LU1OjfV/U3+b8q65Tx67dWhQn2i/j8mjfE6tt6Tv1VxfLivTfvjs5OTm68cYblZ6erqKiIj3++OO6/PLLVVBQoKioqGZfL6iKEm+//bbuv/9+vfTSS7rkkkv0yiuvKCcnR5s3b1ZaWprd4QEAAKCV2flI0JpDh7R36s8kl6ve80ZSdVQn1YTHHj8SprBe5yqie/1TnsO7dVPif9ym8G7d5K7x6FBxhb7bXy7X8V/2a5zlclaWynKEKyquixyOMKlG0l5Ju2tUXLRFh/aXq6bGc3K1IYCMqZbH9ZWMqb0PRsZdIo/7W0VEOuQIsxQVEy5HuKP2Ayo7sFelh6u1cZ+08UP/xhPTMUGdU37UnAGoeOc276ybhpw9JJuiBCDp5ptv9v65X79+uuCCC5Senq73339fY8aMafb1gqooMXv2bN1555266667JElz5szRBx98oHnz5mnWrFk2RwcAAIDW9v1Gl03/sdZ4jFzVblVX1qi63KnKcqeqy6tVXeFSdUW1qiqccrvcqnE65XZWq8ZZLXdNjWpcTjkrq1RdXiVnVZWcZeVyZY2TcYRLJ02N9si4S2VUc7zPChnPkWOnXKUy7sOSTpjKvadImtXYrIsTp343vkTD76z6H6BhjFFDVRDn8RUaVUdOPhcZE+P3GS41Lqcqy0pVWVba7M+edv5AdemZ3uD52IROLYgM7Z0V4VDqry62re9ASklJUXp6urZv3+7T54OmKOF0OlVQUKBHHnmkzvHhw4dr9erATIMpO3xYn67+V0CujVOLkOSwIvy+/qm1WBFRshyWYiMj5HDUkwjCrbprGCPC6/w8ExkWLSs6WpZlKdoRdaxtVHTQ3o9WExGtZj1zLMKhyMjIFt3XmIiwoPm6kNfsE6Fjv7NYCu68FhcdWX9Ok+rktRNzmvR9XosJ+0EuI6+dWnPy2vF1wxERvv89C6acVllRoZ2b1kuSCv71F31e+J4kj4wx8ng8Mqb2l3hT5/8b1T3ebGGS4iRHnNT8icq1fHtaiEMOmeP/15DI6mollh1olSdwGmNJshQVE6vIuBjv8fDIaHVITJTlqH/KeER0lKLjE+op5jRPTcceciZlet973DX6bu9euaqrG/nU9yyHFBUmxcR3UGLPnmrse63kwNcqOfB1nWNRjihZMdGKCY8Jmu+bNqGZeS0UflazLMuvSyjakoMHD2rPnj1KSUnx6fNBU5Q4cOCA3G63unfvXud49+7dVVxcXO9nqqurVf2DhFTWzB2JP139L61fkHfqhkAjhm/6Uo4GdpT+4dETJ346JY17IEzVkZau23Wdwk3QfLsGnT9Xna8a+f4fic2/GqHYyMB/fVqa0yTyGlqusZwmfZ/X6pvMXpvXRu4bTU4LsJbktdbKaVLL89q2Lz6V62h3SWWqLh0kd/V5/g0wSLkknbwrRIBVHn/90L7W6vzwCe87NfmTQ1f+XGGeY/tknLxbxqmR11pHsPysFiqOHj2qHTt2eN8XFRWpsLBQiYmJSkxM1PTp03X99dcrJSVFu3bt0qOPPqquXbvqJz/5iU/9tfKcr5Y7scJljGmw6jVr1iwlJCR4Xz179myNEAEgIMhpAEKNP/KaIzxFjogzZIV18n+AANAOrV+/XllZWcrKypIk5ebmKisrS0888YTCwsK0adMmXXfdderdu7fGjx+v3r1765NPPlF8fLxP/VnGNGVfXvs5nU7Fxsbqr3/9a50KzM9+9jMVFhZqxYoVJ32mvup7z549VVpaqo4dO56yT6Y524vlGyzf8EkbX75RVlamhISEJuehH2ppTpPIa3Zi+QbLN3zWxpdv2JnXKisqtO2LT5vVZ6ib8Y1LmyqNpnYJ12Xxgf/3R8uy73s4Kf8hRR74TAcufERVaZf5dA1HuBQd4VCzfnb4AZZv+KgNL99oSU77oaqqKhUVFSkjI0PR0dE+XycUnOpetGgOi9PpVElJiTyeupvuBOJJGJGRkRowYICWLVtWpyixbNkyXXfddfV+JioqyqdHktTq2KmThl51vc+fBwB/amlOk8hrANqWlua1mNhYZZ5/iR8jCn6xG7/UkYNl6tq7pzJTutgdTmBt+5F0eJ06J0VL551vdzQAfORTUWL79u264447TtpgsnYphfsUj9PxVW5ursaNG6cLLrhAgwcP1quvvqrdu3fr3nvvDUh/AAAAQDCJPP6vwa5G9n4JGR2Sjv1veYm9cQBoEZ+KEhMmTFB4eLjee+89paSktNpUpZtvvlkHDx7Ur371K+3fv1/9+vXTP/7xD6WnN/zoHgAAAKC9CHcc+7ncGRwrtFsm7nhR4ihFCSCY+VSUKCwsVEFBgc466yx/x3NK9913n+67775W7xcAAABo69rXTIlux/63/Ft74wDQIj7tfnPOOefowIED/o4FAAAAQAtEHJ8pUcNMCQBBoslFibKyMu/r2Wef1UMPPaT8/HwdPHiwzrnmPl8aAAAAgH9EHJ8p4WwXMyUoSgChoMnLNzp16lRn7whjjK644oo6bQK90SUAAACAhtUWJVztYabEDze6NEYnPYcYQFBoclFi+fLlko49T3rmzJm65ZZbbNlTAgAAAED9apdvtIuZErXLN9xOqapUiulkazgAfNPkosTQoUO9f77tttt0+eWX68wzzwxIUAAAAACar3ajy3axp0REtBTVUaouO7bZJUUJICj59PSN22+/Xa+99pqeeeYZf8cDAAAAwEcR7emRoJIU1+1YUeJPY44VKZqjQ3dpwnuBiQtAk/lUlHA6nfrDH/6gZcuW6YILLlBcXFyd87Nnz/ZLcAAAAACazrunhMdjcyStJOVc6budUunu5n/WWeH/eIAQsHLlSj3//PMqKCjQ/v37tWjRIo0ePbpOmy1btujhhx/WihUr5PF41LdvX/3lL39RWlpas/vzqSjx2Wef6fzzz5ckbdu2rc45iw1mAAAAAFtEOI49XK/dzJT4ySvSoHsljw8b7YdH+T8eIASUl5crMzNTEydO1PXXX3/S+Z07d2rIkCG68847NWPGDCUkJGjLli2Kjm7mbKXjfCpK1G56CQAAAKDt8O4p0R42upSOFRbSLrI7CiCk5OTkKCcnp8Hzjz32mK666io999xz3mOnnXaaz/05fP4kAAAAgDal3e0pAQQJY4ycTqctL+PHfODxePT++++rd+/eGjFihJKSkjRo0CAtXrzY52v6NFMCAAAAQNvz/Z4SFCWAtsTlcmnmzJm29P3oo48qMjLSL9cqKSnR0aNH9cwzz+ipp57Ss88+q6VLl2rMmDFavnx5nad2NhVFCQAAACBE1M6UcDFTAkAAeI5vonvdddfp5z//uSTpvPPO0+rVq/Xyyy9TlAAAAADas9o9JZzMlADalIiICD366KO29e0vXbt2VXh4uM4555w6x88++2ytWrXKp2tSlAAAAABCRO1MiRpmSgBtimVZfltCYafIyEgNHDhQW7durXN827ZtSk9P9+maFCUAAACAEBHBTAkALXT06FHt2LHD+76oqEiFhYVKTExUWlqaHnzwQd1888267LLLNGzYMC1dulR///vflZ+f71N/FCUAAACAEOHd6JKZEgB8tH79eg0bNsz7Pjc3V5I0fvx4zZ8/Xz/5yU/08ssva9asWZo6dar69Omjv/3tbxoyZIhP/VGUAAAAAEKE95GgzJQA4KPs7OxTPkb0jjvu0B133OGX/hx+uQoAAAAA29VudMmeEgCCBUUJAAAAIESE186UMB6bIwGApqEoAQAAAISI2pkSLpZvAAgSFCUAAACAEBHhOPbjPRtdAggWFCUAAACAEBHBTAkAQYaiBAAAABAiap++wUwJAMGCogQAAAAQImr3lOCRoACCBUUJAAAAIETUzpTwSHIzWwJAEKAoAQAAAISI2j0lJPaVABAcKEoAAAAAIaJOUYKZEgCCAEUJAAAAIETULt+Q2FcCgG9WrlypUaNGKTU1VZZlafHixXXOW5ZV7+v555/3qT+KEgAAAECICLMshR2vSzBTAoAvysvLlZmZqblz59Z7fv/+/XVer7/+uizL0vXXX+9Tf+EtCRYAAABA2xJhWXIbI6fHY3coAIJQTk6OcnJyGjyfnJxc5/27776rYcOG6bTTTvOpP4oSAAAAQAiJsCxVyaiGiRJAm2GMkcdTaUvfDkeMrB/sN+NP33zzjd5//30tWLDA52tQlAAAAABCSITDktyS0zBTAmgrPJ5K5a/ob0vf2UM3KSwsNiDXXrBggeLj4zVmzBifr8GeEgAAAEAIibSO/YjPI0EBBNrrr7+u2267TdHR0T5fg5kSAAAAQAgJP/7PjhQlgLbD4YhR9tBNtvUdCB9//LG2bt2qt99+u0XXoSgBAAAAhBDvTAmevgG0GZZlBWwJhV1ee+01DRgwQJmZmS26TtAs33j66ad18cUXKzY2Vp06dbI7HAAAAKBNinAc29COogQAXxw9elSFhYUqLCyUJBUVFamwsFC7d+/2tikrK9Nf//pX3XXXXS3uL2iKEk6nUzfeeKP+8z//0+5QAAAAgDYr8vgu+06WbwDwwfr165WVlaWsrCxJUm5urrKysvTEE0942yxcuFDGGN1yyy0t7i9olm/MmDFDkjR//nx7AwEAAADasHCLmRIAfJednS1zivxx99136+677/ZLf0FTlPBFdXW1qqurve/LyspsjAYAWoacBiDUkNcCI7J2+QYzJQAEgaBZvuGLWbNmKSEhwfvq2bOn3SEBgM/IaQBCDXktMCKYKQEgiNhalJg+fbosy2r0tX79ep+vP23aNJWWlnpfe/bs8WP0ANC6yGkAQg15LTBqN7pkTwkAwcDW5RuTJ0/W2LFjG23Tq1cvn68fFRWlqKgonz8PAG0JOQ1AqCGvBQYzJQAEE1uLEl27dlXXrl3tDAEAAAAIKbUzJYqrXfqqsvoUrZtwPctSanRki68DAPUJmo0ud+/ere+++067d++W2+32PjP1jDPOUIcOHewNDgAAAGgjah8J+ptdxfrNrmK/XPPBXsl6ICPZL9cCgB8KmqLEE088oQULFnjf1z4zdfny5crOzrYpKgAAAKBtubpbJ604dETVfthTosYYVXuMCo9U+CEyADhZ0BQl5s+fr/nz59sdBgAAANCmXZPUSdckdfLLtf67+DtN3rKbx4sCCJiQfiQoAAAAAN+FH18K4mTTTAABQlECAAAAQL0ij2+aWUNRAkCAUJQAAAAAUK/ax4s6Wb4BtBsrV67UqFGjlJqaKsuytHjx4jrnjx49qsmTJ6tHjx6KiYnR2WefrXnz5vncH0UJAAAAAPWKdBz7dcFlPDZHAqC1lJeXKzMzU3Pnzq33/M9//nMtXbpUf/7zn7Vlyxb9/Oc/15QpU/Tuu+/61F/QbHQJAAAAoHWFH5sowUwJoB3JyclRTk5Og+c/+eQTjR8/3vsUzLvvvluvvPKK1q9fr+uuu67Z/VGUAAAAAFCv72dKUJQAWsIYowqPPTOOYh0OWceXYvnDkCFDtGTJEt1xxx1KTU1Vfn6+tm3bphdeeMGn61GUAAAAAFCv2j0leCQo0DIVHo9OX7nJlr53XtZfcWFhfrve7373O/30pz9Vjx49FB4eLofDoT/84Q8aMmSIT9ejKAEAAACgXrVP32CmBIBav/vd77RmzRotWbJE6enpWrlype677z6lpKToyiuvbPb1KEoAAAAAqFc4MyUAv4h1OLTzsv629e0vlZWVevTRR7Vo0SJdffXVkqRzzz1XhYWF+s1vfkNRAgAAAID/1M6UcDJTAmgRy7L8uoTCLi6XSy6XS44TCh1hYWHy+LhnBkUJAAAAAPWq3VOihqIE0G4cPXpUO3bs8L4vKipSYWGhEhMTlZaWpqFDh+rBBx9UTEyM0tPTtWLFCv3xj3/U7NmzfeqPogQAAACAenlnSniMjDF+3cEfQNu0fv16DRs2zPs+NzdXkjR+/HjNnz9fCxcu1LRp03Tbbbfpu+++U3p6up5++mnde++9PvVHUQIAAABAvWpnShhJbiOFU5MAQl52drZMI7OjkpOTlZeX57f+/LfjBQAAAICQEvGDmRHsKwEgEChKAAAAAKhXhOP7ogT7SgAIBIoSAAAAAOpVZ6YEjwUFEAAUJQAAAADUy7Isb2HCZXx73B8ANIaiBAAAAIAGhVvfP4EDAPyNogQAAACABtU+FpQ9JQAEAkUJAAAAAA2KYKYEgACiKAEAAACgQbUzJVzMlAAQABQlAAAAADSodk8JFzMlAAQARQkAAAAADaqdKeFkpgSAAKAoAQAAAKBBtXtK1DBTAmgXVq5cqVGjRik1NVWWZWnx4sV1zn/zzTeaMGGCUlNTFRsbq5EjR2r79u0+90dRAgAAAECDIpgpAbQr5eXlyszM1Ny5c086Z4zR6NGj9eWXX+rdd9/Vp59+qvT0dF155ZUqLy/3qb/wlgYMAAAAIHRFsKcE0K7k5OQoJyen3nPbt2/XmjVr9Nlnn6lv376SpJdeeklJSUl66623dNdddzW7P4oSAAAAABrkfSQoMyUAnxljVOly29J3TESYrOPfxy1VXV0tSYqOjvYeCwsLU2RkpFatWkVRAgAAAIB/1W50WUNRAvBZpcutc574wJa+N/9qhGIj/fOr/1lnnaX09HRNmzZNr7zyiuLi4jR79mwVFxdr//79Pl2TPSUAAAAANCjCOvYrg9PjsTkSAHaLiIjQ3/72N23btk2JiYmKjY1Vfn6+cnJyFBYW5tM1mSkBAAAAoEG1MyVczJQAfBYTEabNvxphW9/+NGDAABUWFqq0tFROp1PdunXToEGDdMEFF/h0PYoSAAAAABoUXrunBBtdAj6zLMtvSyjaioSEBEnHNr9cv369fv3rX/t0ndC6KwAAAAD8ij0lgPbl6NGj2rFjh/d9UVGRCgsLlZiYqLS0NP31r39Vt27dlJaWpk2bNulnP/uZRo8ereHDh/vUH0UJAAAAAA2KYKYE0K6sX79ew4YN877Pzc2VJI0fP17z58/X/v37lZubq2+++UYpKSm6/fbb9fjjj/vcH0UJAAAAAA1iTwmgfcnOzpZp5Pt96tSpmjp1qt/6C4qnb+zatUt33nmnMjIyFBMTo9NPP11PPvmknE6n3aEBAAAAIa12TwkXMyUABEBQzJT44osv5PF49Morr+iMM87QZ599pp/+9KcqLy/Xb37zG7vDAwAAAEJWBDMlAARQUBQlRo4cqZEjR3rfn3baadq6davmzZtHUQIAAAAIoEhmSgAIoKAoStSntLRUiYmJjbaprq5WdXW1931ZWVmgwwKAgCGnAQg15LXgUDtTwslMCQABEBR7Spxo586devHFF3Xvvfc22m7WrFlKSEjwvnr27NlKEQKA/5HTAIQa8lpwiGCmBIAAsrUoMX36dFmW1ehr/fr1dT6zb98+jRw5UjfeeKPuuuuuRq8/bdo0lZaWel979uwJ5HAAIKDIaQBCDXktOHgfCWo8NkcCIBTZunxj8uTJGjt2bKNtevXq5f3zvn37NGzYMA0ePFivvvrqKa8fFRWlqKioloYJAG0COQ1AqCGvBYfaR4LWMFECQADYWpTo2rWrunbt2qS2X3/9tYYNG6YBAwYoLy9PDkdQrjwBAAAAgkrE8Z+7nR5mSgDwv6DY6HLfvn3Kzs5WWlqafvOb3+jbb7/1nktOTrYxMgAAACC0efeUYKNLAAEQFEWJDz/8UDt27NCOHTvUo0ePOucMyREAAAAIGO+eEmx0CSAAgmINxIQJE2SMqfcFAAAAIHC+31OCn72BUDdr1iwNHDhQ8fHxSkpK0ujRo7V169Y6bYwxmj59ulJTUxUTE6Ps7Gx9/vnnPvcZFEUJAAAAAPbgkaBA+7FixQpNmjRJa9as0bJly1RTU6Phw4ervLzc2+a5557T7NmzNXfuXK1bt07Jycn68Y9/rCNHjvjUZ1As3wAAAABgj9qZEk5mSgAhb+nSpXXe5+XlKSkpSQUFBbrssstkjNGcOXP02GOPacyYMZKkBQsWqHv37nrzzTd1zz33NLtPihIAAAAAGhR+fKbEZ0cqNXjNZu/x3/ftpX7xsXaFBQQXYyRXhT19R8RKx7+Pm6u0tFSSlJiYKEkqKipScXGxhg8f7m0TFRWloUOHavXq1RQlAAAAAPjXabFRcujYTImiSqf3OBtfAs3gqpBmptrT96P7pMi4Zn/MGKPc3FwNGTJE/fr1kyQVFxdLkrp3716nbffu3fXVV1/5FB5FCQAAAAAN6hUTpXWDz9G+aled473jom2KCEBrmDx5sjZu3KhVq1addM46YeaFMeakY01FUQIAAABAo34UHakfRUfaHQYQvCJij81YsKvvZpoyZYqWLFmilStXqkePHt7jycnJko7NmEhJSfEeLykpOWn2RFNRlAAAAAAAIJAsy6clFK3NGKMpU6Zo0aJFys/PV0ZGRp3zGRkZSk5O1rJly5SVlSVJcjqdWrFihZ599lmf+qQoAQAAAAAANGnSJL355pt69913FR8f791DIiEhQTExMbIsS/fff79mzpypM888U2eeeaZmzpyp2NhY3XrrrT71SVECAAAAAABo3rx5kqTs7Ow6x/Py8jRhwgRJ0kMPPaTKykrdd999OnTokAYNGqQPP/xQ8fHxPvVJUQIAAAAAAMiYUz9Vx7IsTZ8+XdOnT/dLnw6/XAUAAAAAAKCZKEoAAAAAAABbtKvlG7VTUcrKymyOBEB7VZt/mjI17lTIaQDaAvIagFDiz5yGpmlXRYkjR45Iknr27GlzJADauyNHjighIaHF15DIaQDaBvIagFDij5yGpmlXRYnU1FTt2bNH8fHxsiyrSZ8pKytTz549tWfPHnXs2DHAEdoj1McY6uOTQn+MoTQ+Y4yOHDmi1NTUFl/Ll5wmhdb9rE+oj08K/TGG+vik0Bqj3XktlO5lQxhj8Av18UmhM0Z/5jQ0TbsqSjgcDvXo0cOnz3bs2DGov7maItTHGOrjk0J/jKEyPn9V3VuS06TQuZ8NCfXxSaE/xlAfnxQ6Y2wLeS1U7mVjGGPwC/XxSaExRmZItC42ugQAAAAAALagKAEAAAAAAGxBUeIUoqKi9OSTTyoqKsruUAIm1McY6uOTQn+MoT6+1hbq9zPUxyeF/hhDfXxS+xhja2kP95IxBr9QH5/UPsaIwLAMzzoBAAAAAMBvqqqqVFRUpIyMDEVHR9sdjq1OdS+YKQEAAAAAADRr1iwNHDhQ8fHxSkpK0ujRo7V169Y6bd555x2NGDFCXbt2lWVZKiwsbFGfFCUAAAAAAIBWrFihSZMmac2aNVq2bJlqamo0fPhwlZeXe9uUl5frkksu0TPPPOOXPtvVI0EBAAAAAED9li5dWud9Xl6ekpKSVFBQoMsuu0ySNG7cOEnSrl27/NInRQkAAAAAAALIGKPKmkpb+o4Jj5FlWT59trS0VJKUmJjoz5DqYPlGI1566SXvZhwDBgzQxx9/bHdI9Zo+fbosy6rzSk5O9p43xmj69OlKTU1VTEyMsrOz9fnnn9e5RnV1taZMmaKuXbsqLi5O1157rfbu3VunzaFDhzRu3DglJCQoISFB48aN0+HDhwMyppUrV2rUqFFKTU2VZVlavHhxnfOtOabdu3dr1KhRiouLU9euXTV16lQ5nc6Ajm/ChAknfU0vuuiioBlfU9aiBfvXMFiR1+zJa6Ge05oyRvJa2x9jsAqGvBZqOU0ir0nktWAYY1tSWVOpQW8OsuXlazHEGKPc3FwNGTJE/fr18/Md+R5FiQa8/fbbuv/++/XYY4/p008/1aWXXqqcnBzt3r3b7tDq1bdvX+3fv9/72rRpk/fcc889p9mzZ2vu3Llat26dkpOT9eMf/1hHjhzxtrn//vu1aNEiLVy4UKtWrdLRo0d1zTXXyO12e9vceuutKiws1NKlS7V06VIVFhZ6p+74W3l5uTIzMzV37tx6z7fWmNxut66++mqVl5dr1apVWrhwof72t7/pgQceCOj4JGnkyJF1vqb/+Mc/6pxvy+Nrylq0YP8aBiPymn15LdRzWlPGKJHX2voYg1Ew5bVQymkSea0Wea1tjxEtM3nyZG3cuFFvvfVWYDsyqNeFF15o7r333jrHzjrrLPPII4/YFFHDnnzySZOZmVnvOY/HY5KTk80zzzzjPVZVVWUSEhLMyy+/bIwx5vDhwyYiIsIsXLjQ2+brr782DofDLF261BhjzObNm40ks2bNGm+bTz75xEgyX3zxRQBG9T1JZtGiRbaM6R//+IdxOBzm66+/9rZ56623TFRUlCktLQ3I+IwxZvz48ea6665r8DPBND5jjCkpKTGSzIoVK4wxofc1DBbktbaR10I9p9U3RmPIa8E4xmAQLHktlHOaMeS1hgTbGMlr/lVZWWk2b95sKisrjTHH7me5s9yWl8fjaXb8kydPNj169DBffvllg22KioqMJPPpp582616ciJkS9XA6nSooKNDw4cPrHB8+fLhWr15tU1SN2759u1JTU5WRkaGxY8fqyy+/lCQVFRWpuLi4zliioqI0dOhQ71gKCgrkcrnqtElNTVW/fv28bT755BMlJCRo0KBB3jYXXXSREhISWv2etOaYPvnkE/Xr10+pqaneNiNGjFB1dbUKCgoCOs78/HwlJSWpd+/e+ulPf6qSkhLvuWAb34lr0drL17AtIa+13bzWnr4fyGvBNca2LtjyWnvJaVL7+n4grwXXGO1kWZZiI2JteTVnPwljjCZPnqx33nlHH330kTIyMgJ4V46hKFGPAwcOyO12q3v37nWOd+/eXcXFxTZF1bBBgwbpj3/8oz744AP9/ve/V3FxsS6++GIdPHjQG29jYykuLlZkZKQ6d+7caJukpKST+k5KSmr1e9KaYyouLj6pn86dOysyMjKg487JydEbb7yhjz76SL/97W+1bt06XX755aqurvbGFSzjM/WsRWsPX8O2hrxWf5u2kNfay/cDeS24xhgMgimvtaecVhtLbXw/FGrfD+S14BojmmbSpEn685//rDfffFPx8fEqLi5WcXGxKiu/35fiu+++U2FhoTZv3ixJ2rp1qwoLC33+WvH0jUacWFEyxvi8a2kg5eTkeP/cv39/DR48WKeffroWLFjg3WzHl7Gc2Ka+9nbek9Yakx3jvvnmm71/7tevny644AKlp6fr/fff15gxYxr8XFscX+1atFWrVp10LpS/hm0Vea3t/t0I9e8H8lrD2uIYg0kw5LX2mNOk0P9+IK81rC2OEU0zb948SVJ2dnad43l5eZowYYIkacmSJZo4caL33NixYyVJTz75pKZPn97sPpkpUY+uXbsqLCzspEpPSUnJSRW8tiguLk79+/fX9u3bvTs7NzaW5ORkOZ1OHTp0qNE233zzzUl9ffvtt61+T1pzTMnJySf1c+jQIblcrlYdd0pKitLT07V9+3ZvXMEwvilTpmjJkiVavny5evTo4T3eHr+GdiOv1d+mLeS19vr9QF5ruI3dYwwWwZzXQjmn1cYitb/vB/Jaw23sHiOazhhT76u2ICEde/JMfW18KUhIFCXqFRkZqQEDBmjZsmV1ji9btkwXX3yxTVE1XXV1tbZs2aKUlBRlZGQoOTm5zlicTqdWrFjhHcuAAQMUERFRp83+/fv12WefedsMHjxYpaWl+ve//+1ts3btWpWWlrb6PWnNMQ0ePFifffaZ9u/f723z4YcfKioqSgMGDAjoOH/o4MGD2rNnj1JSUiS1/fGdai1ae/wa2o281nbzWnv9fiCvtb0xBptgzmuhnNOk9vv9QF5re2NEkGh0m8x2bOHChSYiIsK89tprZvPmzeb+++83cXFxZteuXXaHdpIHHnjA5Ofnmy+//NKsWbPGXHPNNSY+Pt4b6zPPPGMSEhLMO++8YzZt2mRuueUWk5KSYsrKyrzXuPfee02PHj3MP//5T7NhwwZz+eWXm8zMTFNTU+NtM3LkSHPuueeaTz75xHzyySemf//+5pprrgnImI4cOWI+/fRT8+mnnxpJZvbs2ebTTz81X331VauOqaamxvTr189cccUVZsOGDeaf//yn6dGjh5k8eXLAxnfkyBHzwAMPmNWrV5uioiKzfPlyM3jwYPOjH/0oaMb3n//5nyYhIcHk5+eb/fv3e18VFRXeNsH+NQxG5DX78lqo57RTjZG8FhxjDEbBktdCLacZQ14jrwXHGO10qidOtCenuhcUJRrx//7f/zPp6ekmMjLSnH/++d7H47Q1N998s0lJSTEREREmNTXVjBkzxnz++efe8x6Pxzz55JMmOTnZREVFmcsuu8xs2rSpzjUqKyvN5MmTTWJioomJiTHXXHON2b17d502Bw8eNLfddpuJj4838fHx5rbbbjOHDh0KyJiWL19uJJ30Gj9+fKuP6auvvjJXX321iYmJMYmJiWby5MmmqqoqYOOrqKgww4cPN926dTMREREmLS3NjB8//qTY2/L46hubJJOXl+dtE+xfw2BFXrMnr4V6TjvVGMlrwTHGYBUMeS3Ucpox5DXyWnCM0U4UJb53qnthGWOMP2deAAAAAADQnlVVVamoqEgZGRmKjo62OxxbnepesKcEAAAAAACwBUUJAAAAAABgC4oSAAAAAADAFhQlAAAAAACALShKAAAAAAAAW1CUAAAAAAAAtqAoAQAAAAAANGvWLA0cOFDx8fFKSkrS6NGjtXXrVu95l8ulhx9+WP3791dcXJxSU1N1++23a9++fT73SVECOC4/P1+WZenw4cN2hwIALUZOAxBqyGtA4K1YsUKTJk3SmjVrtGzZMtXU1Gj48OEqLy+XJFVUVGjDhg16/PHHtWHDBr3zzjvatm2brr32Wp/7tIwxxl8DAIJJdna2zjvvPM2ZM0eS5HQ69d1336l79+6yLMve4ACgmchpAEINeQ3BrKqqSkVFRcrIyFB0dLTd4fjs22+/VVJSklasWKHLLrus3jbr1q3ThRdeqK+++kppaWknnT/VvQj3e9RAkIqMjFRycrLdYQCAX5DTAIQa8hqCmTFGprLSlr6tmBifC3mlpaWSpMTExEbbWJalTp06+dQHyzfQLk2YMEErVqzQCy+8IMuyZFmW5s+fX2dK4Pz589WpUye999576tOnj2JjY3XDDTeovLxcCxYsUK9evdS5c2dNmTJFbrfbe22n06mHHnpIP/rRjxQXF6dBgwYpPz/fnoECaBfIaQBCDXkNocZUVmrr+QNseflaDDHGKDc3V0OGDFG/fv3qbVNVVaVHHnlEt956qzp27OhTP8yUQLv0wgsvaNu2berXr59+9atfSZI+//zzk9pVVFTod7/7nRYuXKgjR45ozJgxGjNmjDp16qR//OMf+vLLL3X99ddryJAhuvnmmyVJEydO1K5du7Rw4UKlpqZq0aJFGjlypDZt2qQzzzyzVccJoH0gpwEINeQ1wH6TJ0/Wxo0btWrVqnrPu1wujR07Vh6PRy+99JLP/VCUQLuUkJCgyMhIxcbGeqcBfvHFFye1c7lcmjdvnk4//XRJ0g033KA//elP+uabb9ShQwedc845GjZsmJYvX66bb75ZO3fu1FtvvaW9e/cqNTVVkvSLX/xCS5cuVV5enmbOnNl6gwTQbpDTAIQa8hpCjRUToz4bCmzru7mmTJmiJUuWaOXKlerRo8dJ510ul2666SYVFRXpo48+8nmWhERRAmhUbGys9z9yktS9e3f16tVLHTp0qHOspKREkrRhwwYZY9S7d+8616murlaXLl1aJ2gAaAA5DUCoIa8hWFiWJSs21u4wTskYoylTpmjRokXKz89XRkbGSW1qCxLbt2/X8uXLW/y9Q1ECaERERESd95Zl1XvM4/FIkjwej8LCwlRQUKCwsLA67X74H0cAsAM5DUCoIa8B/jVp0iS9+eabevfddxUfH6/i4mJJx2YvxcTEqKamRjfccIM2bNig9957T26329smMTFRkZGRze6TogTarcjIyDqbHvlDVlaW3G63SkpKdOmll/r12gDQGHIagFBDXgNa37x58yQdeyTvD+Xl5WnChAnau3evlixZIkk677zz6rRZvnz5SZ9rCooSaLd69eqltWvXateuXerQoYO3gt4SvXv31m233abbb79dv/3tb5WVlaUDBw7oo48+Uv/+/XXVVVf5IXIAOBk5DUCoIa8Brc8Y0+j5Xr16nbJNc/FIULRbv/jFLxQWFqZzzjlH3bp10+7du/1y3by8PN1+++164IEH1KdPH1177bVau3atevbs6ZfrA0B9yGkAQg15DWgfLOPvMgcAAAAAAO1YVVWVioqKlJGRoejoaLvDsdWp7gUzJQAAAAAAgC0oSgAAAAAAAFtQlAAAAAAAALagKAEAAAAAAGxBUQIAAAAAANiCogQAAAAAALAFRQkAAAAAAGALihIAAAAAAMAWFCUAAAAAAIBmzZqlgQMHKj4+XklJSRo9erS2bt1ap8306dN11llnKS4uTp07d9aVV16ptWvX+twnRQkAAAAAAKAVK1Zo0qRJWrNmjZYtW6aamhoNHz5c5eXl3ja9e/fW3LlztWnTJq1atUq9evXS8OHD9e233/rUp2WMMf4aAAAAAAAA7V1VVZWKioqUkZGh6Ohou8Px2bfffqukpCStWLFCl112Wb1tysrKlJCQoH/+85+64oorTjp/qnsR7veoAQAAAACAlzFGNU6PLX2HRzpkWZZPny0tLZUkJSYm1nve6XTq1VdfVUJCgjIzM32Lz6dPAQAAAACAJqlxevTqz1bY0vfdLwxVRFRYsz9njFFubq6GDBmifv361Tn33nvvaezYsaqoqFBKSoqWLVumrl27+hQfe0oAAAAAAIA6Jk+erI0bN+qtt9466dywYcNUWFio1atXa+TIkbrppptUUlLiUz/sKQEAAAAAgB+duI9CsC3fmDJlihYvXqyVK1cqIyPjlO3PPPNM3XHHHZo2bdpJ59hTAgAAAAAAG1mW5dMSitZmjNGUKVO0aNEi5efnN6kgUfu56upqn/qkKAEAAAAAADRp0iS9+eabevfddxUfH6/i4mJJUkJCgmJiYlReXq6nn35a1157rVJSUnTw4EG99NJL2rt3r2688Uaf+qQoAQAAAAAANG/ePElSdnZ2neN5eXmaMGGCwsLC9MUXX2jBggU6cOCAunTpooEDB+rjjz9W3759feqTogQAAAAAANCptpyMjo7WO++849c+efoGAAAAAACwBUUJAAAAAABgC4oSAAAAAADAFhQlAAAAAACALShKAAAAAAAAW1CUAAAAAAAAtqAoAQAAAAAAbEFRAgAAAAAA2IKiBAAAAAAAsAVFCQAAAAAAYAuKEgAAAAAAQLNmzdLAgQMVHx+vpKQkjR49Wlu3bm2w/T333CPLsjRnzhyf+6QoAQAAAAAAtGLFCk2aNElr1qzRsmXLVFNTo+HDh6u8vPyktosXL9batWuVmpraoj7DW/RpAAAAAAAQEpYuXVrnfV5enpKSklRQUKDLLrvMe/zrr7/W5MmT9cEHH+jqq69uUZ8UJQAAAAAACCBjjGqqq23pOzwqSpZl+fTZ0tJSSVJiYqL3mMfj0bhx4/Tggw+qb9++LY+vxVcAAAAAAAANqqmu1u/G32BL31MX/LcioqOb/TljjHJzczVkyBD169fPe/zZZ59VeHi4pk6d6pf4KEoAAAAAAIA6Jk+erI0bN2rVqlXeYwUFBXrhhRe0YcMGn2dfnMgyxhi/XAkAAAAAAKiqqkpFRUXKyMhQdHR00C3fmDJlihYvXqyVK1cqIyPDe3zOnDnKzc2Vw/H9MzPcbrccDod69uypXbt2nXStE+/FSfE1KzIAAAAAANAslmX5tISitRljNGXKFC1atEj5+fl1ChKSNG7cOF155ZV1jo0YMULjxo3TxIkTfeqTogQAAAAAANCkSZP05ptv6t1331V8fLyKi4slSQkJCYqJiVGXLl3UpUuXOp+JiIhQcnKy+vTp41OfjlM3AQAAAAAAoW7evHkqLS1Vdna2UlJSvK+33347YH0yUwIAAAAAAMiXLSfr20eiOZgpAQAAAAAAbEFRAgAAAAAA2IKiBAAAAAAAsAVFCQAAAAAAYAuKEgAAAAAABIAvG0eGmlPdA4oSAAAAAAD4UUREhCSpoqLC5kjs53Q6JUlhYWH1nueRoAAAAAAA+FFYWJg6deqkkpISSVJsbKwsy7I5qtbn8Xj07bffKjY2VuHh9ZcfKEoAAAAAAOBnycnJkuQtTLRXDodDaWlpDRZlLMMiFwAAAAAAAsLtdsvlctkdhm0iIyPlcDS8cwRFCQAAAAAAYAs2ugQAAAAAALagKAEAAAAAAGxBUQIAAAAAANiCogQAAAAAALAFRQkAAAAAAGALihIAAAAAAMAWFCUAAAAAAIAt/j9ks5r9Lam6pwAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -94,27 +134,57 @@ } ], "source": [ - "swiftdiff['rh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + "swiftdiff.sel(id=tpid)['rh'].plot(x=\"time\",hue=\"id\",col=\"space\")" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBMAAAEiCAYAAACr2JryAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA61klEQVR4nO3deXRU9f3/8ddkmclCCARIyEgCuABCkFJQiIiAWDAshaL9Un4UAxYsVbQYLRXtV8Qq2NPWgxsIfjksomC/RShWQFEJ1C+LhEUREEHCIiSmoBAIkPXz+8NmTEyAmWQmd5bn45w5h7nb5/1JyPtm3nnfe23GGCMAAAAAAAA3hVkdAAAAAAAACCwUEwAAAAAAgEcoJgAAAAAAAI9QTAAAAAAAAB6hmAAAAAAAADxCMQEAAAAAAHiEYgIAAAAAAPAIxQQAAAAAAOARigkAAAAAAMAjFBMAAAAAAIBHKCYAAAAAAACPUEwAAAAAAAAeoZiAgPH3v/9dnTt3VnR0tJo1a6bbb79dRUVFkqSxY8dq+PDhmj59uhITE9W4cWP9+te/VklJiWv/tWvX6pZbblGTJk3UrFkzDRkyRF9++WW1Mb766iv94he/UEJCgmJjY9W9e3dt3brVtf7tt99Wt27dFBUVpauvvlrTp09XWVmZT+ZrjNHtt9+uO+64Q8YYSdLp06eVmpqqxx9/3CdjAvCtUMtjhw8fVlhYmHJycqotf/HFF9W6dWtXbgMQWEItl0lSmzZtZLPZaryAUEYxAQEhLy9Po0aN0j333KN9+/YpOztbI0aMqPaL6AcffKB9+/Zp/fr1Wrp0qVasWKHp06e71hcVFSkrK0vbtm3TBx98oLCwMP3sZz9TRUWFJOncuXPq06ePTpw4oVWrVumTTz7RlClTXOvfffdd/fKXv9SDDz6ovXv3au7cuVq4cKGeeeaZS8b9+uuvq1GjRpd9vf7667Xua7PZtGjRIn388cd64YUXJEkTJ05UUlKSnnzyyfp+SQE0sFDMY23atNHtt9+uBQsWVFu+YMECjR07ll/EgQAUirlMkrZt26a8vDzl5eXpq6++Us+ePdW7d+/6fjmBwGZC2IYNG8yQIUNMcnKykWRWrFhh+Xhnz541999/v7nqqqtMVFSU6dChg5k9e7ZP4woE27dvN5LM4cOHa12fmZlpEhISTFFRkWvZnDlzTKNGjUx5eXmt+xQUFBhJZvfu3cYYY+bOnWvi4uLMqVOnat2+d+/eZsaMGdWWvfbaayY5OfmScRcWFpoDBw5c9lVYWHjZuf/tb38zDofDTJ061cTExJj9+/dfdnsA/ilU89ibb75pmjZtai5evGiMMWbXrl3GZrOZ3NzcS+4DwH+Fai6r6sEHHzStW7c2BQUFbm0PBKuQLiasXr3aPP7442b58uUNUkxwZ7zx48eba665xqxfv97k5uaauXPnmvDwcLNy5UqfxubvysrKTP/+/U1cXJy56667zLx588w333zjWp+ZmWn69etXbZ9du3ZVO9kdPHjQjBo1yrRt29bExcWZ2NhYI8m88847xhhjfvOb35hbb731kjHExMSYqKgoExsb63pFRUUZSdVOmL4watQoI8nMmTPHp+MA8J1QzWPFxcWmRYsWZunSpcaY734Jv+2223wyFgDfC9VcVmnu3LkmNjbW7Nq1y6fjAIEgpC9zyMjI0NNPP60RI0bUur6kpERTpkzRVVddpdjYWPXo0UPZ2dk+G0+SNm/erMzMTPXt21dt2rTRvffeqy5dutS43jTUhIeHa926dVqzZo06duyoF198Ue3bt1dubu4V961sox06dKhOnTqlV199VVu3bnVdd1d5DV90dPRlj1NRUaHp06dr165drtfu3bt14MABRUVF1bpPfVvqJOn8+fPavn27wsPDdeDAgSvOF4B/CtU8ZrfbNWbMGC1YsEAlJSV64403dM8991xxzgD8U6jmMknKzs7WAw88oMWLF6tLly5XnC8Q7CKsDsCfjRs3TocPH9ayZcvkdDq1YsUK3XHHHdq9e7euu+46n4x5yy23aNWqVbrnnnvkdDqVnZ2tL774Qs8//7xPxgskNptNvXr1Uq9evfTEE0+odevWWrFihbKysiRJn3zyiS5cuOA6AW3ZskWNGjVSq1atdOrUKe3bt09z5851Xd/20UcfVTv+DTfcoP/5n//RN998o4SEhBrj//jHP9b+/ft17bXXuh3zT3/6U/Xo0eOy2yQlJV12/cMPP6ywsDCtWbNGgwYN0uDBg3Xbbbe5HQMA/xGqeWz8+PFKS0vT7NmzVVpaetmiOgD/F4q57ODBg7rzzjv12GOPkcOASla3RvgL/eCyg4MHDxqbzWaOHz9ebbv+/fubqVOnen28SsXFxebuu+82kkxERISx2+1m8eLF9R4v0G3ZssU888wzZtu2bebIkSPmb3/7m7Hb7Wb16tXGmO9a6ho1amRGjRpl9uzZY1avXm2SkpLMo48+aowxpry83DRr1sz88pe/NAcOHDAffPCBufHGG6t9H4qLi027du1M7969zUcffWS+/PJL8/e//91s2rTJGGPM2rVrTUREhJk2bZr57LPPzN69e82yZcvM448/7rN5//Of/zR2u91s377dGGPMH/7wB9OqVatq7YQAAkOo5rFKN998s7Hb7WbixIk+HwuA74RiLjt//rzp0KGDue2228yJEydMXl6e6wWEMooJ//HDD/d/+9vfjKRq12LFxsaaiIgI81//9V/GGGNyc3ONpMu+7r//frfGq/TnP//ZtGvXzqxatcp88skn5sUXXzSNGjUy69at88W0A8bevXvNwIEDTYsWLYzD4TDt2rUzL774omt9ZmamGTZsmHniiSdMs2bNTKNGjcz48eNdN/wyxph169aZ66+/3jgcDnPDDTeY7OzsGt+Hw4cPmzvvvNM0btzYxMTEmO7du5utW7e61q9du9bcfPPNJjo62jRu3NjcdNNNZt68eT6Zc0FBgUlKSqp2g6HS0lJz0003uf4PAggcoZjHqpo/f76RZD7++GOfjwXAd0Ixl13ud34glNmM4SHP0nftWitWrNDw4cMlSW+++aZGjx6tPXv2KDw8vNq2jRo1UsuWLVVaWlrjmbg/1LRp01pbpn44niRduHBB8fHxWrFihQYPHuxaPn78eH311Vdau3Zt3ScY5MaOHavTp09r5cqVVocCAHUS7HnsmWee0bJly7R7926rQwHgQ8GeywB8j3smXELXrl1VXl6ugoKCSz5DNjIyUh06dPDamKWlpSotLVVYWPX7YoaHh7ueqwsAQCA5d+6c9u3bpxdffFF//OMfrQ4HAAB4SUgXE86dO6eDBw+63ufm5mrXrl1KSEhQu3btNHr0aN19993661//qq5du+rkyZP68MMP1blzZw0aNMir46Wmpqpx48bq06ePfve73yk6OlqtW7fWhg0btHjxYj333HNemTMAAA1p0qRJWrp0qYYPH85THAAACCIhfZlDdna2+vXrV2N5ZmamFi5cqNLSUj399NNavHixjh8/rmbNmik9PV3Tp09X586dvT6eJOXn52vq1Kl677339M0336h169a699579dBDD7kepwMAAAAAgJVCupgAAAAAAAA8F3blTQAAAAAAAL5HMQEAAAAAAHgk5G7AWFFRoRMnTiguLo57EABoMMYYnT17Vk6ns8YTW+qCXAagoZHHAAQDb+eyUBZyxYQTJ04oJSXF6jAAhKhjx46pVatW9T4OuQyAVchjAIKBt3JZKAu5YkJcXJyk7/7zNG7c2OJoAISKwsJCpaSkuHJQfZHLADQ08hiAYODtXBbKQq6YUNlG17hxY05cABqct1p5yWUArEIeAxAMuLyq/rhIBAAAAAAAeIRiAgAAAAAA8AjFBAAAAAAA4BGKCQAAAAAAwCMUEwAAAAAAgEdC7mkOAAAAocIYo7y8PBUVFdW63ul0KjY2toGjAgAEA4oJAAAAQerLL7/UkiVLLrl+9OjRuu666xowIgBAsKCYAAAAEKS+/fZbSZLD4VBCQkKN9Q6Ho6FDAgAECYoJAAAAQcoYI0m6+uqrNXLkSIujAQAEE27ACAAAEKQqKiokSWFh/MoHAPAuziwAAABBqrIzwWazWRwJACDYUEwAAAAIUnQmAAB8hTMLAABAkKIzAQDgKxQTAAAAghSdCQAAX+HMAgAAEKQqiwl0JgAAvI1iAgAAQJCqvMyBzgQAgLdFWB0AAAAAfIPLHADAd4wxKisrU3l5udWh+ER4eLgiIiIu2d1GMQEAACBIcQNGAPCNkpIS5eXl6fz581aH4lMxMTFKTk6W3W6vsY5iAgAAQJCiMwEAvK+iokK5ubkKDw+X0+mU3W4PuqKtMUYlJSX697//rdzcXF133XU1ziUUEwAAAIIUnQkA4H0lJSWqqKhQSkqKYmJirA7HZ6KjoxUZGakjR46opKREUVFR1dZTpgYAAAhSdCYAgO+EQm693ByDf/YAAAAhis4EAICvUEwAAAAIUnQmAEBg6tu3ryZPnnzJ9W3atNGsWbMaLJ7acM8EAACAIEVnAgAEprfeekuRkZFWh3FZFBMAAACCFJ0JABCYEhISrA7hijizAAAABCk6EwAgMFW9zKGgoEBDhw5VdHS02rZtq9dff93a4P6DzgQAAIAgRWcCAAS+sWPH6tixY/rwww9lt9v14IMPqqCgwOqwKCYAAAAEKzoTACCwffHFF1qzZo22bNmiHj16SJLmz5+v66+/3uLILL7MYePGjRo6dKicTqdsNptWrlx52e2zs7Nls9lqvD7//POGCRgAACCA0JkAAIFt3759ioiIUPfu3V3LOnTooCZNmlgX1H9Y2plQVFSkLl26aNy4cbrzzjvd3m///v1q3Lix632LFi18ER4AAEBAozMBAAKbP+dxS4sJGRkZysjI8Hi/xMREv6jEAAAA+DM6EwAgsF1//fUqKytTTk6ObrrpJknf/XH99OnT1gamAL1nQteuXXXx4kV17NhRf/jDH9SvX79LbltcXKzi4mLX+8LCwoYIEQC8ilwGoC786S9a5DEA8Fz79u11xx13aMKECZo3b54iIiI0efJkRUdHWx1aYD0aMjk5WfPmzdPy5cv11ltvqX379urfv782btx4yX1mzpyp+Ph41yslJaUBIwYA7yCXAagLf+pMII8BQN0sWLBAKSkp6tOnj0aMGKF7771XiYmJVoclm6ksWVvMZrNpxYoVGj58uEf7DR06VDabTatWrap1fW1V8JSUFJ05c6bafRcAwJcKCwsVHx9f59xDLgNQF0uWLNHBgwc1bNgwde3atV7HIo8BCAb1zWWSdPHiReXm5qpt27aKiorycoT+5XJzDcjLHKrq2bOnlixZcsn1DodDDoejASMCAO8jlwGoC3/qTCCPAUBwsf7MUk87d+5UcnKy1WEAAAD4HX+6ZwIAILhY2plw7tw5HTx40PU+NzdXu3btUkJCglJTUzV16lQdP35cixcvliTNmjVLbdq0UadOnVRSUqIlS5Zo+fLlWr58uVVTAAAA8Fv+1JkAAAgulhYTcnJyqj2JISsrS5KUmZmphQsXKi8vT0ePHnWtLykp0SOPPKLjx48rOjpanTp10jvvvKNBgwY1eOwAAAD+js4EAICvWFpM6Nu3ry53/8eFCxdWez9lyhRNmTLFx1EBAAAEBzoTAAC+wpkFAAAgSFX+0YZiAgDA2zizAAAABKnKzgQucwAAeBvFBAAAgCBFZwIAwFc4swAAAAQpOhMAAL5CMQEAACBIcQNGAICvcGYBAAAIUjwaEgBQ1caNGzV06FA5nU7ZbDatXLmyzseimAAAABCk6EwAAFRVVFSkLl266KWXXqr3sSK8EA8AAAD8EJ0JAICqMjIylJGR4ZVjUUwAAAAIUnQmAIDvGWN0obTckrGjI8MtKxhTTAAAAAhSdCYAgO9dKC1XxyfetWTsvU8NVIzdmo/1lKkBAACCFJ0JAABfoTMBAAAgSNGZAAC+Fx0Zrr1PDbRsbKtQTAAAAAhSdCYAgO/ZbDbLLjWwUujNGAAAIETQmQAAqOrcuXM6ePCg631ubq527dqlhIQEpaamenQsigkAAABBis4EAEBVOTk56tevn+t9VlaWJCkzM1MLFy706FgUEwAAAIIUnQkAgKr69u3rOjfUF2VqAACAIEVnAgDAVzizAAAABCk6EwAAvkIxAQAAIAgZY1zFBDoTAADexpkFAAAgCFW9JpbOBACAt1FMAAAACEKV90uQ6EwAAHgfZxYAAIAgVLUzgWICAMDbOLMAAAAEoaqdCVzmAADwNooJAAAAQYjOBACAL3FmAQAACEJ0JgAAfIliAgAAQBCiMwEA4EucWQAAAIIQnQkAgKpmzpypG2+8UXFxcUpMTNTw4cO1f//+Oh+PYgIAAEAQquxMoCsBACBJGzZs0P33368tW7Zo3bp1Kisr04ABA1RUVFSn40V4OT4AAAD4gcrOBLoSAACStHbt2mrvFyxYoMTERG3fvl233nqrx8ejmAAAABCE6EwAgAZijFR63pqxI2OkOhaNz5w5I0lKSEio0/4UEwAAAIIQnQkA0EBKz0sznNaM/dgJyR7r8W7GGGVlZemWW25RWlpanYammAAAABCE6EwAAFzKpEmT9Omnn+qjjz6q8zEoJgAAAAQhOhMAoIFExnzXIWDV2B564IEHtGrVKm3cuFGtWrWq89AUEwAAAIJQZTGBzgQA8DGbrU6XGjQ0Y4weeOABrVixQtnZ2Wrbtm29jkcxAQAAIAhVXuZAZwIAQJLuv/9+vfHGG/rHP/6huLg45efnS5Li4+MVHR3t8fEoVQMAAAQhOhMAAFXNmTNHZ86cUd++fZWcnOx6vfnmm3U6Hp0JAAAAQYjOBABAVZXnBW+hVA0AABCE6EwAAPgSZxcAAIAgRGcCAMCXKCYAAAAEIToTAAC+xNkFAAAgCNGZAADwJYoJAAAAQYjOBACAL3F2AQAACEKVnQkUEwAAvmDp2WXjxo0aOnSonE6nbDabVq5cecV9NmzYoG7duikqKkpXX321XnnlFd8HCgAAEGAqOxO4zAEA4AuWFhOKiorUpUsXvfTSS25tn5ubq0GDBql3797auXOnHnvsMT344INavny5jyMFAAAILHQmAAB8KcLKwTMyMpSRkeH29q+88opSU1M1a9YsSdL111+vnJwc/eUvf9Gdd97poygBAAACD50JAABfCqhS9ebNmzVgwIBqywYOHKicnByVlpZaFBUAAID/oTMBAOBLAXV2yc/PV1JSUrVlSUlJKisr08mTJ2vdp7i4WIWFhdVeABBoyGUAPOVvnQnkMQCw1pw5c3TDDTeocePGaty4sdLT07VmzZo6Hy+giglSzRPilZ6hPHPmTMXHx7teKSkpPo8RALyNXAbAU/7WmUAeAwBrtWrVSs8++6xycnKUk5Oj2267TcOGDdOePXvqdDz/OLu4qWXLlsrPz6+2rKCgQBEREWrWrFmt+0ydOlVnzpxxvY4dO9YQoQKAV5HLAHjK3zoTyGMAYK2hQ4dq0KBBateundq1a6dnnnlGjRo10pYtW+p0PEtvwOip9PR0vf3229WWvffee+revbsiIyNr3cfhcMjhcDREeADgM+QyAJ7yt84E8hiAYGWM0YWyC5aMHR0RXaeicXl5uf73f/9XRUVFSk9Pr9PYlhYTzp07p4MHD7re5+bmateuXUpISFBqaqqmTp2q48ePa/HixZKkiRMn6qWXXlJWVpYmTJigzZs3a/78+Vq6dKlVUwAAAPBL/taZAADB6kLZBfV4o4clY2/9f1sVExnj9va7d+9Wenq6Ll68qEaNGmnFihXq2LFjnca2tJiQk5Ojfv36ud5nZWVJkjIzM7Vw4ULl5eXp6NGjrvVt27bV6tWr9dBDD+nll1+W0+nUCy+8wGMhAQAAfsDfOhMAANZr3769du3apdOnT2v58uXKzMzUhg0b6lRQsLSY0LdvX9eJrjYLFy6ssaxPnz7asWOHD6MCAAAIfHQmAEDDiI6I1tb/t9WysT1ht9t17bXXSpK6d++ubdu26fnnn9fcuXM9Hjug7pkAAAAA99CZAAANw2azeXSpgT8xxqi4uLhO+1JMAAAACEJ0JgAAqnrssceUkZGhlJQUnT17VsuWLVN2drbWrl1bp+NRTAAAAAhCdCYAAKr6+uuvNWbMGOXl5Sk+Pl433HCD1q5dq5/85Cd1Op7HxYSioiI9++yz+uCDD1RQUOCqelc6dOhQnQIBAACA99CZAACoav78+V49nsfFhPHjx2vDhg0aM2aMkpOTOUEBAAD4IToTAAC+5HExYc2aNXrnnXfUq1cvX8QDAAAAL6AzAQDgSx4XE5o2baqEhARfxALAzxljalzaFMrCwsL4JR2AS0VFxWUfed3QysvLJdGZAADwDY+LCX/84x/1xBNPaNGiRYqJCczHXwDwnDFGr732GvdFqeLxxx9XZGSk1WEA8AN5eXlatGiRLl68aHUoNVD0BAD4glvFhK5du1Y7ER08eFBJSUlq06ZNjV+kd+zY4d0IAfiFsrIyCgkAcAmHDx/220JCamqq1WEAAIKQW8WE4cOH+zgMAP6u6uUNWVlZiojgybJ8DQBUKikpkSR16dJFAwcOtDia74WHh8vhcFgdBgAgCLn1m/C0adNc/x43bpxGjx6t/v370zYHhJCqxYSYmBg+SANAFZXFhJiYGC4DBQCEBI/vyHPq1CkNGTJErVq10iOPPKJdu3b5ICwA/qbqTcUoJAJAdZXFBLvdbnEkAAA0DI+LCatWrVJ+fr6mTZumnJwcdevWTR07dtSMGTN0+PBhH4QIwB9U7UzgzuAAUB3FBABAqKnTJ4ImTZro3nvvVXZ2to4cOaJx48bptdde07XXXuvt+AD4CToTAODSiouLJVFMAACEjnr9ebG0tFQ5OTnaunWrDh8+rKSkJG/FBcDPVHYm0JUAADXRmQAACDQzZ86UzWbT5MmT67R/nT4VrF+/XhMmTFBSUpIyMzMVFxent99+W8eOHatTEAD8X2VnAsUEAKiJYgIAIJBs27ZN8+bN0w033FDnY3h8O/ZWrVrp1KlTGjhwoObOnauhQ4cqKiqqzgEACAyVnQlc4gAANVFMAAAEinPnzmn06NF69dVX9fTTT9f5OB4XE5544gn9/Oc/V9OmTes8KIDAQ2cCAFwaxQQACF3GGJkLFywZ2xYd7fEf++6//34NHjxYt99+e8MWE+699946DwYgcNGZAACXRjEBAEKXuXBB+3/czZKx2+/YLltMjNvbL1u2TDt27NC2bdvqPbbHxQQAoYnOBAC4NIoJAAB/d+zYMf32t7/Ve++955VbFVBMAOAWOhMAoHbl5eUqKyuTRDEBAEKRLTpa7Xdst2xsd23fvl0FBQXq1u37Lory8nJt3LhRL730koqLixUeHu728SgmAHALnQkAULvS0lLXvykmAEDosdlsHl1qYJX+/ftr9+7d1ZaNGzdOHTp00O9//3uPCgkSxQQAbqIzAQBqV3mJg81mU0QEv1oBAPxTXFyc0tLSqi2LjY1Vs2bNaix3B39iBOAWOhMAoHaVxQSHw0HBFQAQMiifA3ALnQkAUDtuvggACFTZ2dl13pc/MQJwC50JAFA7igkAgFDEpwIAbqEzAQBqRzEBABCKKCYAcAudCQBQO4oJAIBQxKcCAG6hMwEAaldcXCyJYgIAILRQTADgFjoTAKB2dCYAAEIRT3MA4BY6EwCEgry8POXm5kr67tnbaWlpCg8Pd60/dOiQ8vPzq+1z6NAhSRQTAAChhWICALfQmQAgFLzxxhs6e/as673D4VCHDh0kSUVFRXrttddc+fCHoqOjGyRGAAD8AcUEAG6hMwFAsKuoqHAVEmJjY1VUVFStsHDu3DkZYxQREaGOHTtW29dut+vGG29s0HgBALASxQQAbqEzAUCwKy0tdf27devW2rt3r+t+CNL390Zo1KiRRowY0eDxAQDgT/hUAMAtdCYACHZVCwcxMTE1lnGjRQAAvkcxAYBb6EwAEOyqFgscDoek7x/7+MP1AAAEmieffFI2m63aq2XLlnU+Hpc5AHALnQkAgl3VYkFlwYDOBABAMOnUqZPef/991/uqTyzyFMUEAG6hMwFAsKOYAAAIdhEREfXqRqh2LK8cBUDQozMBQLCrLBY4HI7LFhMqL4EAAED67o9uZSUVlowdYQ/z6PfzAwcOyOl0yuFwqEePHpoxY4auvvrquo1dp70AhJzKYgKdCQCCFZ0JAIC6KCup0LzfbrBk7Huf76NIh3uXKvTo0UOLFy9Wu3bt9PXXX+vpp5/WzTffrD179qhZs2Yej00xAYBbKi9zoDMBQLCimAAACGYZGRmuf3fu3Fnp6em65pprtGjRImVlZXl8PIoJANxCZwKAYEcxAQBQFxH2MN37fB/Lxq6r2NhYde7cWQcOHKjb2HUeGUBIoTMBQLCrfAzkpYoJVdcDAFDJZrO5famBPykuLta+ffvUu3fvOu1v+Z8YZ8+erbZt2yoqKkrdunXTv/71r0tum52dXeO5mDabTZ9//nkDRgyEJjoTAAQ7OhMAAMHskUce0YYNG5Sbm6utW7fqrrvuUmFhoTIzM+t0PEs7E958801NnjxZs2fPVq9evTR37lxlZGRo7969Sk1NveR++/fvV+PGjV3vW7Ro0RDhAiGNR0MCCHaXKiYYY2Sz2SgmAAAC2ldffaVRo0bp5MmTatGihXr27KktW7aodevWdTqepcWE5557Tr/61a80fvx4SdKsWbP07rvvas6cOZo5c+Yl90tMTFSTJk0aKEoAEo+GBBD8qhYLKh//WFFRofLyckVERFBMAAAEtGXLlnn1eJb9ibGkpETbt2/XgAEDqi0fMGCANm3adNl9u3btquTkZPXv31/r16/3ZZgA/oPOBADBrmqxIDIyssZyigkAAHzPss6EkydPqry8XElJSdWWJyUlKT8/v9Z9kpOTNW/ePHXr1k3FxcV67bXX1L9/f2VnZ+vWW2+tdZ/i4mLXDZMkqbCw0HuTAEIInQnWIpcBvle1WBAeHq6IiAiVlZWppKREMTExFBPqiTwGAMHF8qc5/PCDSeV1ibVp37692rdv73qfnp6uY8eO6S9/+csliwkzZ87U9OnTvRcwEKLoTLAWuQzwvR8WC+x2u6uYUNt6eIY8BgDBxbJPBc2bN1d4eHiNLoSCgoIa3QqX07Nnz8s+F3Pq1Kk6c+aM63Xs2LE6xwyEMjoTrEUuA3yvtmJC1eUUE+qHPAYAwcWyzgS73a5u3bpp3bp1+tnPfuZavm7dOg0bNszt4+zcuVPJycmXXO9wOFw3UQJQd3QmWItcBvjepYoJxcXFKisrcxVVKSbUDXkMAIKLpZc5ZGVlacyYMerevbvS09M1b948HT16VBMnTpT0XQX7+PHjWrx4saTvnvbQpk0bderUSSUlJVqyZImWL1+u5cuXWzkNICTQmQAg2F2uM6FyXdXlAACEMkuLCSNHjtSpU6f01FNPKS8vT2lpaVq9erXrOZd5eXk6evSoa/uSkhI98sgjOn78uKKjo9WpUye98847GjRokFVTAEIGnQkAgl1lwaDyr+e1FRMiIiIUHh5uTYAAAPgRy2/AeN999+m+++6rdd3ChQurvZ8yZYqmTJnSAFEB+CE6EwAEs4qKCrc6E+hKAADgO5YXEwAEBjoTAASD7du361//+pcrp1Wq+v6HxYQPPvhAERER1ZYBABDqKCYAcAudCQCCQU5Ojk6fPn3J9U2bNlVkZKQkqWXLlvr000918eJF13pPnjgFAEAwo5gAwC10JgAIBpWXKwwdOlQtW7assb558+auoml6erquvfZalZaWSvqumEoxAQAQyI4fP67f//73WrNmjS5cuKB27dpp/vz56tatm8fHopgAwC10JgAIBsXFxZIkp9N52UdLS9/lu8TExIYICwAAn/v222/Vq1cv9evXT2vWrFFiYqK+/PJLNWnSpE7Ho5gAwC10JgAIBtxIEQAQqv70pz8pJSVFCxYscC1r06ZNnY9HMQGAW+hMABDojDEUEwAAXmeMUdl/Ot8aWoTD4fbv56tWrdLAgQP185//XBs2bNBVV12l++67TxMmTKjb2HXaC0DIoTMBQKCrvPeBRDEBAOA9ZcXFeiHzLkvGfnDR3xUZFeXWtocOHdKcOXOUlZWlxx57TB9//LEefPBBORwO3X333R6PTTEBgFvoTAAQ6Cq7EiS5ntgAAECoqKioUPfu3TVjxgxJUteuXbVnzx7NmTOHYgIA36EzAUCgqywmREZGkssAAF4T4XDowUV/t2xsdyUnJ6tjx47Vll1//fVavnx53cau014AQg6dCQACHfdLAAD4gs1mc/tSAyv16tVL+/fvr7bsiy++UOvWret0PMryANxCZwKAQEcxAQAQyh566CFt2bJFM2bM0MGDB/XGG29o3rx5uv/+++t0PD4VAHALnQkAAh3FBABAKLvxxhu1YsUKLV26VGlpafrjH/+oWbNmafTo0XU6Hpc5AHALnQkAAh3FBABAqBsyZIiGDBnilWPxqQCAW+hMABDoKosJDg9uVgUAAGpHMQGAWyqLCXQmAAhUdCYAAOA9fCoA4JbKyxzoTAAQqCgmAADgPRQTALiFzgQAgY5iAgAA3sOnAgBu4QaMAAIdxQQAALyHTwUA3MINGAEEuuLiYkkUEwAA8AaKCQDcQmcCgEBHZwIAAN7DpwIAbqEzAUCgo5gAAID3UEwA4BY6EwAEOooJAAB4D58KALiFzgQAgY5iAgAA3hNhdQAAAgOdCQCsVl5ersOHD7uKAp46d+6cJIoJAIDQ1KZNGx05cqTG8vvuu08vv/yyx8ejmADALXQmALDa5s2b9f7779f7OA6HwwvRAAAQWLZt26by8nLX+88++0w/+clP9POf/7xOx6OYAMAtdCYAsFpBQYEkqUmTJoqLi6vTMVq0aKGkpCRvhgUAQEBo0aJFtffPPvusrrnmGvXp06dOx6OYAMAtdCYAsFrlZQr9+vVTly5dLI4GAIDvGGNkSissGdsWGVan389LSkq0ZMkSZWVl1fn3e4oJANxCZwIAq1UWExo1amRxJAAAfM+UVujEE5ssGdv51M2y2cM93m/lypU6ffq0xo4dW+ex+VQAwC10JgCw2tmzZyVRTAAAoL7mz5+vjIwMOZ3OOh+DzgQAbqEzAYCVysrKdOHCBUkUEwAA/sUWGSbnUzdbNranjhw5ovfff19vvfVWvcammADALXQmALBSUVGRpO8KmtHR0RZHAwDA92w2W50uNbDKggULlJiYqMGDB9frOPyJEYBb6EwAYKXK+yXExsaShwAAqKOKigotWLBAmZmZioioX28BZ2MAbqEzAYCVKosJdX0kJAAAkN5//30dPXpU99xzT72PxWUOANxCZwIAK/EkBwAA6m/AgAGu3+vri08FANxCZwIAK1FMAADAv9CZAMAtdCYA8KULFy5o27ZtKi4urnV9bm6uJIoJAAD4C4oJAK6ositBojMBgG9s2bJFGzZsuOJ2TZo08X0wAADgiigmALiiqtdV0ZkAwBeOHz8uSbr22mvVvHnzWreJiYlRWlpaQ4YFAAAugWICgCuiMwGAr+Xn50uS+vTpo5SUFIujAQAAV8KfGAFcEZ0JAHzp7NmzrhssJiUlWRwNAABwB58KAFwRnQkAfKmyK6FZs2ay2+0WRwMAANzBZQ4ArqhqMYHOBADe8Mknn+jLL7+UJJ06dUqSlJycbGVIAADAAxQTAFxR1csc6EwAUF+FhYVauXJltdwiSa1atbIoIgAA4CmKCQCuiM4EAN706aefyhijxMRE/ehHP5IkRUVFqXPnztYGBgAA3GZ5MWH27Nn685//rLy8PHXq1EmzZs1S7969L7n9hg0blJWVpT179sjpdGrKlCmaOHFiA0YMhJ7Kvx7SlQDAXfv379fOnTtrdB9I0rFjxyRJPXv21I9//OOGDg0AgJBUVlamJ598Uq+//rry8/OVnJyssWPH6g9/+EOd/mBoaTHhzTff1OTJkzV79mz16tVLc+fOVUZGhvbu3avU1NQa2+fm5mrQoEGaMGGClixZov/7v//TfffdpxYtWujOO++0YAZAaKjsTKArAYA7ysrK9I9//EPnz5+/5DYOh0OdOnVqwKgAAAhtf/rTn/TKK69o0aJF6tSpk3JycjRu3DjFx8frt7/9rcfHs7SY8Nxzz+lXv/qVxo8fL0maNWuW3n33Xc2ZM0czZ86ssf0rr7yi1NRUzZo1S5J0/fXXKycnR3/5y198VkwoKS5WwZcHfXJsIFCcPVckSbJJ+mrvHmuD8SOJ11wru8NhdRhuIZehIR3+6rjOnz+v6CiHuqWl1bpNYkIz/Zv/k5YLpDxWfPGiTnyx3+owAPgZZ7v2ckRFWR1GQNi8ebOGDRumwYMHS5LatGmjpUuXKicnp07Hs6yYUFJSou3bt+vRRx+ttnzAgAHatGlTrfts3rxZAwYMqLZs4MCBmj9/vkpLSxUZGVljn+LiYhUXF7veFxYWehRnwZcH9enrO3Us7KRH+wHBpMxWIYVJtjJJi7+xOhy/UXD3QbXq2DB/Wa1vLtvxfx/pk/W7vR0WUKvCsAuSTbruXEt1+KjJJbYql0Q+sVog5bETX+zXordWeDssAAEuc8TP1PaGLpbGYIxRaWmpJWNHRka6fSnyLbfcoldeeUVffPGF2rVrp08++UQfffSR64/1nrKsmHDy5EmVl5crKSmp2vKkpCTX86Z/KD8/v9bty8rKdPLkyVofKTVz5kxNnz69XrGetp1XXvjpeh0DCAaxJjD+ehWM6pvLLlwsJo+hQdmMTe3KedQjvueN38kAwB+VlpZqxowZloz92GOPyW63u7Xt73//e505c0YdOnRQeHi4ysvL9cwzz2jUqFF1GtvyGzD+sIpijLlsZaW27WtbXmnq1KnKyspyvS8sLFRKSorb8SVec62uv+MbtSqqeQ8HINQkNWsuxURbHYbfSLzm2gYbq765rFO3bgoP554XaDiNGzVSk6ZNrQ4DVxBIeczZrr0yR/zMF6EBCGDOdu2tDiFgvPnmm1qyZIneeOMNderUSbt27dLkyZPldDqVmZnp8fEsKyY0b95c4eHhNboQCgoKanQfVGrZsmWt20dERKhZs2a17uNwOOSox7WAdodDXW+59NMlAKAh1DeXJSYlKXHgHV6MCAA8U9885oiKsryVGQBqExkZqccee8yysd31u9/9To8++qh+8YtfSJI6d+6sI0eOaObMmYFVTLDb7erWrZvWrVunn/3s+yrzunXrNGzYsFr3SU9P19tvv11t2Xvvvafu3bt79EUEAAAAAMAbbDab25caWOn8+fM1ns4WHh7uenKbpyy9zCErK0tjxoxR9+7dlZ6ernnz5uno0aOaOHGipO/a4Y4fP67FixdLkiZOnKiXXnpJWVlZmjBhgjZv3qz58+dr6dKlVk4DAAAAAAC/NnToUD3zzDNKTU1Vp06dtHPnTj333HO655576nQ8S4sJI0eO1KlTp/TUU08pLy9PaWlpWr16tVq3bi1JysvL09GjR13bt23bVqtXr9ZDDz2kl19+WU6nUy+88ILPHgsJAAAAAEAwePHFF/Xf//3fuu+++1RQUCCn06lf//rXeuKJJ+p0PJupvINhiCgsLFR8fLzOnDmjxo0bWx0OgBDh7dxDLgPQ0MhjAIKBN3LPxYsXlZubq7Zt2yoqKsrLEfqXy82VW3sDAAAAAACPUEwAAAAAAAAesfSeCVaovKqjsLDQ4kgAhJLKnOOtK8vIZQAaGnkMQDDwdi4LZSFXTDh79qwkKSUlxeJIAISis2fPKj4+3ivHkchlABoeeQxAMPBWLgtlIVdMcDqdOnbsmOLi4mSz2dzap7CwUCkpKTp27FhQ3iCI+QU25hcYjDE6e/asnE6nV45HLquJ+QWuYJ6bFDzzI4/5HvMLbMwvMHg7l4WykCsmhIWFqVWrVnXat3HjxgH9g3MlzC+wMT//583qN7ns0phf4ArmuUnBMT/yWMNgfoGN+fk/b+WyiooKrxzHn11ujiFXTAAAAAAAoK7sdrvCwsJ04sQJtWjRQna73e0Oq0BhjFFJSYn+/e9/KywsTHa7vcY2FBMAAAAAAHBTWFiY2rZtq7y8PJ04ccLqcHwqJiZGqampCgur+SBIiglucDgcmjZtmhwOh9Wh+ATzC2zMD+4K9q8l8wtcwTw3Kfjn15CC/WvJ/AIb8wstdrtdqampKisrU3l5udXh+ER4eLgiIiIu2XVhMzwTAwAAAAAAeKBmrwIAAAAAAMBlUEwAAAAAAAAeoZgAAAAAAAA8QjHBDbNnz1bbtm0VFRWlbt266V//+pfVIdXw5JNPymazVXu1bNnStd4YoyeffFJOp1PR0dHq27ev9uzZU+0YxcXFeuCBB9S8eXPFxsbqpz/9qb766qtq23z77bcaM2aM4uPjFR8frzFjxuj06dNen8/GjRs1dOhQOZ1O2Ww2rVy5str6hpzP0aNHNXToUMXGxqp58+Z68MEHVVJS4tP5jR07tsb3s2fPngExv5kzZ+rGG29UXFycEhMTNXz4cO3fv7/aNoH+/QtE5LHvkcfIY1dCHvNP5LHvkcfIY1dCHkODMLisZcuWmcjISPPqq6+avXv3mt/+9rcmNjbWHDlyxOrQqpk2bZrp1KmTycvLc70KCgpc65999lkTFxdnli9fbnbv3m1GjhxpkpOTTWFhoWubiRMnmquuusqsW7fO7Nixw/Tr18906dLFlJWVuba54447TFpamtm0aZPZtGmTSUtLM0OGDPH6fFavXm0ef/xxs3z5ciPJrFixotr6hppPWVmZSUtLM/369TM7duww69atM06n00yaNMmn88vMzDR33HFHte/nqVOnqm3jr/MbOHCgWbBggfnss8/Mrl27zODBg01qaqo5d+6ca5tA//4FGvIYeYw85hnymP8hj5HHyGOeIY+hIVBMuIKbbrrJTJw4sdqyDh06mEcffdSiiGo3bdo006VLl1rXVVRUmJYtW5pnn33WtezixYsmPj7evPLKK8YYY06fPm0iIyPNsmXLXNscP37chIWFmbVr1xpjjNm7d6+RZLZs2eLaZvPmzUaS+fzzz30wq+/8MLk35HxWr15twsLCzPHjx13bLF261DgcDnPmzBmfzM+Y705ew4YNu+Q+gTS/goICI8ls2LDBGBN8379AQB4jj5HH6oc8Zj3yGHmMPFY/5DH4Apc5XEZJSYm2b9+uAQMGVFs+YMAAbdq0yaKoLu3AgQNyOp1q27atfvGLX+jQoUOSpNzcXOXn51ebh8PhUJ8+fVzz2L59u0pLS6tt43Q6lZaW5tpm8+bNio+PV48ePVzb9OzZU/Hx8Q369WjI+WzevFlpaWlyOp2ubQYOHKji4mJt377dp/PMzs5WYmKi2rVrpwkTJqigoMC1LpDmd+bMGUlSQkKCpND5/vkL8hh5jDxWf+Qxa5HHyGPksfojj8EXKCZcxsmTJ1VeXq6kpKRqy5OSkpSfn29RVLXr0aOHFi9erHfffVevvvqq8vPzdfPNN+vUqVOuWC83j/z8fNntdjVt2vSy2yQmJtYYOzExsUG/Hg05n/z8/BrjNG3aVHa73adzzsjI0Ouvv64PP/xQf/3rX7Vt2zbddtttKi4udsUVCPMzxigrK0u33HKL0tLSXGNWxnq52ANhfoGAPEYeI4/VD3nMeuQx8hh5rH7IY/CVCKsDCAQ2m63ae2NMjWVWy8jIcP27c+fOSk9P1zXXXKNFixa5bhRTl3n8cJvatrfq69FQ87FiziNHjnT9Oy0tTd27d1fr1q31zjvvaMSIEZfcz9/mN2nSJH366af66KOPaqwL5u+fPyKP+ef/iWD+OSCP1c7f5hdIyGP++X8imH8OyGO187f5wTp0JlxG8+bNFR4eXqNiVlBQUKO65m9iY2PVuXNnHThwwHUX4cvNo2XLliopKdG333572W2+/vrrGmP9+9//btCvR0POp2XLljXG+fbbb1VaWtqgc05OTlbr1q114MABV1z+Pr8HHnhAq1at0vr169WqVSvX8lD8/lmJPEYe85efA/KY/80vUJDHyGP+8nNAHvO/+cFaFBMuw263q1u3blq3bl215evWrdPNN99sUVTuKS4u1r59+5ScnKy2bduqZcuW1eZRUlKiDRs2uObRrVs3RUZGVtsmLy9Pn332mWub9PR0nTlzRh9//LFrm61bt+rMmTMN+vVoyPmkp6frs88+U15enmub9957Tw6HQ926dfPpPKs6deqUjh07puTkZEn+PT9jjCZNmqS33npLH374odq2bVttfSh+/6xEHiOP+cvPAXnMf+YXaMhj5DF/+Tkgj/nP/OAnvHxDx6BT+Sii+fPnm71795rJkyeb2NhYc/jwYatDq+bhhx822dnZ5tChQ2bLli1myJAhJi4uzhXns88+a+Lj481bb71ldu/ebUaNGlXro19atWpl3n//fbNjxw5z22231frolxtuuMFs3rzZbN682XTu3NknjyI6e/as2blzp9m5c6eRZJ577jmzc+dO1yOgGmo+lY+y6d+/v9mxY4d5//33TatWrer9KJvLze/s2bPm4YcfNps2bTK5ublm/fr1Jj093Vx11VUBMb/f/OY3Jj4+3mRnZ1d7lNL58+dd2wT69y/QkMfIY+Qxz5DH/A95jDxGHvMMeQwNgWKCG15++WXTunVrY7fbzY9//GPXI1X8SeVzYSMjI43T6TQjRowwe/bsca2vqKgw06ZNMy1btjQOh8PceuutZvfu3dWOceHCBTNp0iSTkJBgoqOjzZAhQ8zRo0erbXPq1CkzevRoExcXZ+Li4szo0aPNt99+6/X5rF+/3kiq8crMzGzw+Rw5csQMHjzYREdHm4SEBDNp0iRz8eJFn83v/PnzZsCAAaZFixYmMjLSpKammszMzBqx++v8apuXJLNgwQLXNoH+/QtE5LHvkcfIY1dCHvNP5LHvkcfIY1dCHkNDsBljjDc7HQAAAAAAQHDjngkAAAAAAMAjFBMAAAAAAIBHKCYAAAAAAACPUEwAAAAAAAAeoZgAAAAAAAA8QjEBAAAAAAB4hGICAAAAAADwCMUEAAAAAADgEYoJgKTs7GzZbDadPn3a6lAAoM7IZQACHXkMCBw2Y4yxOgigofXt21c/+tGPNGvWLElSSUmJvvnmGyUlJclms1kbHAC4iVwGINCRx4DAFWF1AIA/sNvtatmypdVhAEC9kMsABDryGBA4uMwBIWfs2LHasGGDnn/+edlsNtlsNi1cuLBaS93ChQvVpEkT/fOf/1T79u0VExOju+66S0VFRVq0aJHatGmjpk2b6oEHHlB5ebnr2CUlJZoyZYquuuoqxcbGqkePHsrOzrZmogCCGrkMQKAjjwGBjc4EhJznn39eX3zxhdLS0vTUU09Jkvbs2VNju/Pnz+uFF17QsmXLdPbsWY0YMUIjRoxQkyZNtHr1ah06dEh33nmnbrnlFo0cOVKSNG7cOB0+fFjLli2T0+nUihUrdMcdd2j37t267rrrGnSeAIIbuQxAoCOPAYGNYgJCTnx8vOx2u2JiYlxtdJ9//nmN7UpLSzVnzhxdc801kqS77rpLr732mr7++ms1atRIHTt2VL9+/bR+/XqNHDlSX375pZYuXaqvvvpKTqdTkvTII49o7dq1WrBggWbMmNFwkwQQ9MhlAAIdeQwIbBQTgEuIiYlxnbQkKSkpSW3atFGjRo2qLSsoKJAk7dixQ8YYtWvXrtpxiouL1axZs4YJGgB+gFwGINCRxwD/RDEBuITIyMhq7202W63LKioqJEkVFRUKDw/X9u3bFR4eXm27qic7AGhI5DIAgY48BvgnigkISXa7vdpNeryha9euKi8vV0FBgXr37u3VYwNAbchlAAIdeQwIXDzNASGpTZs22rp1qw4fPqyTJ0+6Ktn10a5dO40ePVp333233nrrLeXm5mrbtm3605/+pNWrV3shagCojlwGINCRx4DARTEBIemRRx5ReHi4OnbsqBYtWujo0aNeOe6CBQt099136+GHH1b79u3105/+VFu3blVKSopXjg8AVZHLAAQ68hgQuGzGGGN1EAAAAAAAIHDQmQAAAAAAADxCMQEAAAAAAHiEYgIAAAAAAPAIxQQAAAAAAOARigkAAAAAAMAjFBMAAAAAAIBHKCYAAAAAAACPUEwAAAAAAAAeoZgAAAAAAAA8QjEBAAAAAAB4hGICAAAAAADwCMUEAAAAAADgkf8PxuKdV/0SKioAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "swiftdiff.sel(id=plid)['vh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAIjCAYAAABCuxM+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABezklEQVR4nO3deXhU5d3/8c/JnkAIBMgyZYsLiICIYBFFCJUGI6AI7j4YcH9kKcYVrIpWQWv1QqWg9KEsVcD2gSBVi8ZKEnkEJEAqFUTQIAiJEZRAQvbcvz/8ZUpMyDKcmclM3q/ryqVztu99JuSezCffc8YyxhgBAAAAAADYJMDbAwAAAAAAAP6FsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANgqyNsDAAAAAADA11RVVamiosLbw/Ca4OBgBQYGnnY9YQMAAAAAAE1kjFF+fr6OHTvm7aF4Xfv27RUXFyfLsuqsI2wAAAAAAKCJaoKGmJgYRURE1PtG298ZY3Ty5EkVFBRIkuLj4+tsQ9gAAAAAAEATVFVVOYOGjh07ens4XhUeHi5JKigoUExMTJ1LKrhBJAAAAAAATVBzj4aIiAgvj6RlqHke6rt3BWEDAAAAAADN0BovnahPQ88DYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAAC1YYmKiZsyYcdr1PXr00Lx58zw2nqbg0ygAAAAAAGjB1qxZo+DgYG8Po1kIGwAAAAAAaMGio6O9PYRm4zIKAAAAAABasFMvoygoKNDYsWMVHh6uhIQEvfnmm94d3GnQ2QAAAAAAgI+YNGmSDh48qI8++kghISGaPn26CgoKvD2sOggbAAAAAADwAV9++aX+8Y9/aPPmzRo8eLAkafHixerdu7eXR1YXl1EAAAAAAOADdu/eraCgIA0aNMi57LzzzlP79u29N6jTIGwAAAAAAMAHGGMkSZZleXkkjSNsAAAAAADAB/Tu3VuVlZXKzs52LtuzZ4+OHTvmvUGdBmEDAAAAAAA+oFevXrryyit11113acuWLdq2bZvuvPNOhYeHe3todRA2AAAAAADgI5YsWaKuXbtq+PDhGj9+vO6++27FxMR4e1h1WKbmog8AAAAAAHBapaWlys3NVUJCgsLCwrw9HK9r6PmgswEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAANiKsAEAAAAAAD+XlZWlsWPHyuFwyLIsrV271q31CBsAAAAAAPBzxcXF6t+/v+bPn++RekEeqQIAAAAAgB8yxqikosrjdcODA2VZVpO3T05OVnJyshtHVBthAwAAAAAALiqpqNL5T7zv8bq7nh6liJCW+5aeyygAAAAAAICtWm4MAgAAAABACxceHKhdT4/ySt2WjLABAAAAAAAXWZbVoi9n8BYuowAAAAAAALYifgEAAAAAwM8VFRVp3759zse5ubnKyclRdHS0unXrZns9wgYAAAAAAPxcdna2RowY4XycmpoqSUpJSdHSpUttr0fYAAAAAACAn0tMTJQxxmP1uGcDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAAB+bu7cubr44osVGRmpmJgYjRs3Tnv27HFbPcIGAAAAAAD8XGZmpqZMmaLNmzcrPT1dlZWVSkpKUnFxsVvqBbnlqAAAAAAAoMVYv359rcdLlixRTEyMtm3bpmHDhtlej7ABAAAAAABXGSNVnPR83eAIybJc3r2wsFCSFB0dbdeIaiFsAAAAAADAVRUnpTkOz9eddVgKaePSrsYYpaamaujQoerbt6/NA/sJYQMAAAAAAK3I1KlT9dlnn2njxo1uq0HYAAAAAACAq4Ijfuoy8EZdF0ybNk3r1q1TVlaWunTpYvOg/oOwAQAAAAAAV1mWy5czeJIxRtOmTVNaWpoyMjKUkJDg1nqEDQAAAAAA+LkpU6ZoxYoVevvttxUZGan8/HxJUlRUlMLDw22vF2D7EQEAAAAAQIuycOFCFRYWKjExUfHx8c6vt956yy316GwAAAAAAMDPGWM8Wo/OBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAAYCvCBgAAAAAA/NzChQt1wQUXqF27dmrXrp2GDBmif/zjH26rR9gAAAAAAICf69Kli5577jllZ2crOztbv/rVr3TNNdfo888/d0s9yxhj3HJkAAAAAAD8SGlpqXJzc5WQkKCwsDBvD+eMRUdH64UXXtAdd9zh0v4NPR9BdgwQAAAAAIDWyBijksoSj9cNDwqXZVku7VtVVaW//e1vKi4u1pAhQ2we2U8IGwAAAAAAcFFJZYkGrxjs8bpbbtmiiOCIZu2zc+dODRkyRKWlpWrbtq3S0tJ0/vnnu2V83LMBAAAAAIBWoFevXsrJydHmzZv13//930pJSdGuXbvcUot7NgAAAAAA0AT13aPAFy+jqDFy5EidffbZev31113an3s2AAAAAADgBpZlNftyhpbCGKOysjK3HJuwAQAAAAAAPzdr1iwlJyera9euOnHihFatWqWMjAytX7/eLfUIGwAAAAAA8HPfffedJk6cqLy8PEVFRemCCy7Q+vXr9etf/9ot9QgbAAAAAADwc4sXL/ZoPT6NAgAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAACAVmTu3LmyLEszZsxwWw3CBgAAAAAAWomtW7dq0aJFuuCCC9xah7ABAAAAAIBWoKioSLfeeqv+9Kc/qUOHDm6tFeTWowMAAAAA4MeMMTIlJR6va4WHy7KsZu0zZcoUjR49WiNHjtQzzzzjppH9hLABAAAAAAAXmZIS7blooMfr9tq+TVZERJO3X7VqlbZv366tW7e6cVT/QdgAAAAAAIAfO3jwoH7zm9/ogw8+UFhYmEdqWsYY45FKAAAAAAD4sNLSUuXm5iohIcH5pt0XLqNYu3atrr32WgUGBjqXVVVVybIsBQQEqKysrNa6pqrv+ahBZwMAAAAAAC6yLKtZlzN4wxVXXKGdO3fWWjZ58mSdd955euSRR1wKGhpD2AAAAAAAgB+LjIxU3759ay1r06aNOnbsWGe5XfjoSwAAAAAAYCs6GwAAAAAAaGUyMjLcenw6GwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAAgK0IGwAAAAAA8HOzZ8+WZVm1vuLi4txWL8htRwYAAAAAAC1Gnz599OGHHzofBwYGuq0WYQMAAAAAAK1AUFCQW7sZatXySBUAAAAAAPyQMUaV5dUerxsUEiDLspq1z969e+VwOBQaGqrBgwdrzpw5Ouuss9wzPrccFQAAAACAVqCyvFqLfpPp8bp3vzxcwaFNvwxi8ODBWr58uXr27KnvvvtOzzzzjC699FJ9/vnn6tixo+3jI2wAAAAAAMDPJScnO/+/X79+GjJkiM4++2wtW7ZMqampttcjbAAAAAAAwEVBIQG6++XhXql7Jtq0aaN+/fpp7969No2oNsIGAAAAAABcZFlWsy5naCnKysq0e/duXX755W45/plFIQAAAAAAoMV78MEHlZmZqdzcXG3ZskXXXXedjh8/rpSUFLfUo7MBAAAAAAA/9+233+rmm2/WkSNH1LlzZ11yySXavHmzunfv7pZ6hA0AAAAAAPi5VatWebQel1EAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETYAAAAAAABbETbAZ/zv//6v+vXrp/DwcHXs2FEjR45UcXGxJGnSpEkaN26cnnrqKcXExKhdu3a65557VF5e7tx//fr1Gjp0qNq3b6+OHTtqzJgx+uqrr2rV+Pbbb3XTTTcpOjpabdq00aBBg7Rlyxbn+r///e8aOHCgwsLCdNZZZ+mpp55SZWWlW87XGKORI0fqyiuvlDFGknTs2DF169ZNjz32mFtqAnCv1jaP7d+/XwEBAcrOzq61/NVXX1X37t2dcxsA39Ha5jFJ6tGjhyzLqvMF+KJDhw7pv/7rv9SxY0dFRETowgsv1LZt29xSi7ABPiEvL08333yzbr/9du3evVsZGRkaP358rV9U//nPf2r37t3asGGDVq5cqbS0ND311FPO9cXFxUpNTdXWrVv1z3/+UwEBAbr22mtVXV0tSSoqKtLw4cN1+PBhrVu3Tv/617/08MMPO9e///77+q//+i9Nnz5du3bt0uuvv66lS5fq2WefPe2433zzTbVt27bBrzfffLPefS3L0rJly/Tpp5/qlVdekSTde++9io2N1ezZs8/0KQXgYa1xHuvRo4dGjhypJUuW1Fq+ZMkSTZo0iV/WAR/TGucxSdq6davy8vKUl5enb7/9Vpdccokuv/zyM306AY/78ccfddlllyk4OFj/+Mc/tGvXLr344otq3769ewqaViwzM9OMGTPGxMfHG0kmLS3N6/VOnDhhpkyZYn7xi1+YsLAwc95555kFCxa4dVy+YNu2bUaS2b9/f73rU1JSTHR0tCkuLnYuW7hwoWnbtq2pqqqqd5+CggIjyezcudMYY8zrr79uIiMjzdGjR+vd/vLLLzdz5syptewvf/mLiY+PP+24jx8/bvbu3dvg1/Hjxxs897/+9a8mNDTUzJw500RERJg9e/Y0uD2Alqm1zmNvvfWW6dChgyktLTXGGJOTk2MsyzK5ubmn3QdAy9Ra57FTTZ8+3XTv3t0UFBQ0aXv4n5KSErNr1y5TUlLi7aE02yOPPGKGDh1q6zEbej6C3BNh+Ibi4mL1799fkydP1oQJE1pEvfvvv18bNmzQG2+8oR49euiDDz7QfffdJ4fDoWuuucbtY2yp+vfvryuuuEL9+vXTqFGjlJSUpOuuu04dOnSotU1ERITz8ZAhQ1RUVKSDBw+qe/fu+uqrr/T4449r8+bNOnLkiDMhP3DggPr27aucnBwNGDBA0dHR9Y5h27Zt2rp1a63kvKqqSqWlpTp58mSt2jUiIyMVGRl5Rud+/fXXKy0tTXPnztXChQvVs2fPMzoeAO9orfPYuHHjNHXqVKWlpemmm27Sn//8Z40YMUI9evRw+ZgAvKO1zmM1Fi1apMWLF+v//u//1Llz5zM+HvyHMUaVZWUerxsUGtqsLsF169Zp1KhRuv7665WZmalf/OIXuu+++3TXXXe5Z3xuOaqPSE5OVnJy8mnXl5eX67e//a3efPNNHTt2TH379tXzzz+vxMREt9STpE2bNiklJcVZ4+6779brr7+u7OzsVh02BAYGKj09XZ988ok++OADvfrqq3rssce0ZcsWJSQkNLhvzQ/g2LFj1bVrV/3pT3+Sw+FQdXW1+vbt67yOMDw8vMHjVFdX66mnntL48ePrrAsLC6t3nzfffFP33HNPg8d9/fXXdeutt552/cmTJ7Vt2zYFBgZq7969DR4LQMvVWuexkJAQTZw4UUuWLNH48eO1YsUKzZs3r8HjAWiZWus8JkkZGRmaNm2aVq5cqf79+zd4LLQ+lWVleiXlOo/Xnb7sfxV8mn/39fn666+1cOFCpaamatasWfr00081ffp0hYaG6rbbbrN9fK06bGjM5MmTtX//fq1atUoOh0NpaWm68sortXPnTp177rluqTl06FCtW7dOt99+uxwOhzIyMvTll1/q5Zdfdks9X2JZli677DJddtlleuKJJ9S9e3elpaUpNTVVkvSvf/1LJSUlzhepzZs3q23bturSpYuOHj2q3bt36/XXX3deY7dx48Zax7/gggv0P//zP/rhhx/qTdMvuugi7dmzR+ecc06Tx3z11Vdr8ODBDW4TGxvb4PoHHnhAAQEB+sc//qGrrrpKo0eP1q9+9asmjwFAy9Fa57E777xTffv21YIFC1RRUVHvmwQAvqE1zmP79u3ThAkTNGvWLOYv+LTq6moNGjRIc+bMkSQNGDBAn3/+uRYuXEjY4ElfffWVVq5cqW+//VYOh0OS9OCDD2r9+vVasmSJ8xtkt1deeUV33XWXunTpoqCgIAUEBOh//ud/NHToULfU8xVbtmzRP//5TyUlJSkmJkZbtmzR999/r969ezu3KS8v1x133KHf/va3+uabb/Tkk09q6tSpCggIUIcOHdSxY0ctWrRI8fHxOnDggB599NFaNW6++WbNmTNH48aN09y5cxUfH68dO3bI4XBoyJAheuKJJzRmzBh17dpV119/vQICAvTZZ59p586deuaZZ+od95m27b377rv685//rE2bNumiiy7So48+qpSUFH322We1WhYBtHytdR6TpN69e+uSSy7RI488ottvv73Rv1wCaJla4zxWUlKisWPH6sILL9Tdd9+t/Px857q4uDiXjgn/ExQaqunL/tcrdZsjPj5e559/fq1lvXv31urVq+0c1n/YencIH6af3bDxr3/9q5Fk2rRpU+srKCjI3HDDDcYYY3Jzc42kBr+mTJnSpHo1XnjhBdOzZ0+zbt06869//cu8+uqrpm3btiY9Pd0dp+0zdu3aZUaNGmU6d+5sQkNDTc+ePc2rr77qXJ+SkmKuueYa88QTT5iOHTuatm3bmjvvvNN5QzJjjElPTze9e/c2oaGh5oILLjAZGRl1vg/79+83EyZMMO3atTMRERFm0KBBZsuWLc7169evN5deeqkJDw837dq1M7/85S/NokWL3HLOBQUFJjY2ttZNkCoqKswvf/lL579BAL6jNc5jp1q8eLGRZD799FO31wLgHq1xHmvo9320Tr58g8ibb765zg0iZ8yYYYYMGeLyMRt6Pixj+JBr6aeWsLS0NI0bN06S9NZbb+nWW2/V559/rsDAwFrbtm3bVnFxcaqoqKjzucA/16FDh3rbsn5eT/opOY2KilJaWppGjx7tXH7nnXfq22+/1fr1610/QT83adIkHTt2TGvXrvX2UADAJf4+jz377LNatWqVdu7c6e2hAHATf5/HAEkqLS1Vbm6uEhISTnufkJZq69atuvTSS/XUU0/phhtu0Keffqq77rpLixYtavCeJQ1p6PngMorTGDBggKqqqlRQUHDaz9ENDg7WeeedZ1vNiooKVVRUKCAgoNbywMBA5516AQDwJUVFRdq9e7deffVV/e53v/P2cAAAaLUuvvhipaWlaebMmXr66aeVkJCgefPmuRw0NKZVhw1FRUXat2+f83Fubq5ycnIUHR2tnj176tZbb9Vtt92mF198UQMGDNCRI0f00UcfqV+/frrqqqtsrdetWze1a9dOw4cP10MPPaTw8HB1795dmZmZWr58uV566SVbzhkAAE+aOnWqVq5cqXHjxun222/39nAAAGjVxowZozFjxnikVqu+jCIjI0MjRoyoszwlJUVLly5VRUWFnnnmGS1fvlyHDh1Sx44dNWTIED311FPq16+f7fUkKT8/XzNnztQHH3ygH374Qd27d9fdd9+t+++/v1mfoQoAAAAAsJcvX0bhDg09H606bAAAAAAAoKkIG2pr6PkIOM0+AAAAAAAALml192yorq7W4cOHFRkZyWUJADzGGKMTJ07I4XDUuQlsczGPAfAG5jEAvs7OeQyNa3Vhw+HDh9W1a1dvDwNAK3Xw4EF16dLljI7BPAbAm5jHAPg6O+YxNK7VhQ2RkZGSfvoH1q5dOy+PBkBrcfz4cXXt2tU5B50J5jEA3sA8BsDX2TmPoXGtLmyoadVr164dL24APM6OdmHmMQDexDwGwNdx+ZZncKEKAAAAAACwFWEDAAAAAACwFWEDAAAAAACwFWEDAAAAAAB+rkePHrIsq87XlClT3FKv1d0gEgAAAACA1mbr1q2qqqpyPv73v/+tX//617r++uvdUo+wAQAAwE9VVlbq6NGj9a4LCQlRhw4dPDwiAIC3dO7cudbj5557TmeffbaGDx/ulnqEDQAAAH5q+fLlOnDgQL3rEhISlJKS4uERAYD/McbIVFR7vK4VHODyx3iWl5frjTfeUGpqqts+CpSwAQAAwE/l5+dLksLDw+v8MhkWFuaNIQGA3zEV1Tr8xCcer+t4+lJZIYEu7bt27VodO3ZMkyZNsndQpyBsAAAA8FPGGEnS3XffzSUTAACnxYsXKzk5WQ6Hw201CBsAAAD8VHX1T2297mqRBQD8dDmD4+lLvVLXFd98840+/PBDrVmzxuYR1UbYAAAA4KdqOhsCAvi0cwBwF8uyXL6cwRuWLFmimJgYjR492q11eOUBAADwU3Q2AABOVV1drSVLliglJUVBQe7tPSBsAAAA8EM1XQ0SnQ0AgJ98+OGHOnDggG6//Xa31+IyCgAAAD9U09Ug0dkAAPhJUlJSrTDanYi5AQAA/BCdDQAAb+KVBwAAwA/R2QAA8CbCBgAAAD9EZwMAwJt45QEAAPBDdDYAALyJsAEAAMAPnRo20NkAAPA0XnkAAAD80KmXUdDZAADwNMIGAAAAP1TT2WBZFmEDAMDjCBsAAAD8UE1nA5dQAAC8gVcfAAAAP3RqZwMAAJ5G2AAAAOCH6GwAAHgTrz4AAAB+iM4GAMCpKisr9dvf/lYJCQkKDw/XWWedpaeffrrWpxfZKcgtRwUAAIBX0dkAADjV888/r9dee03Lli1Tnz59lJ2drcmTJysqKkq/+c1vbK9H2AAAAOCH6GwAAM8wxqiiosLjdYODg5s1x2/atEnXXHONRo8eLUnq0aOHVq5cqezsbLeMj7ABAADAD9HZAACeUVFRoTlz5ni87qxZsxQSEtLk7YcOHarXXntNX375pXr27Kl//etf2rhxo+bNm+eW8Xn11ScrK0tjx46Vw+GQZVlau3Ztg9tnZGQ4Pyv61K8vvvjCMwMGAADwEXQ2AABO9cgjj+jmm2/Weeedp+DgYA0YMEAzZszQzTff7JZ6Xu1sKC4uVv/+/TV58mRNmDChyfvt2bNH7dq1cz7u3LmzO4YHAADgs+hsAADPCA4O1qxZs7xStzneeustvfHGG1qxYoX69OmjnJwczZgxQw6HQykpKbaPz6thQ3JyspKTk5u9X0xMjNq3b2//gAAAAPwEnQ0A4BmWZTXrcgZveeihh/Too4/qpptukiT169dP33zzjebOnet/YYOrBgwYoNLSUp1//vn67W9/qxEjRpx227KyMpWVlTkfHz9+3BNDBADbMI8BcEVL6mxgHgMA7zt58mSd14TAwEC3ffSl9199miE+Pl6LFi3S6tWrtWbNGvXq1UtXXHGFsrKyTrvP3LlzFRUV5fzq2rWrB0cMAGeOeQyAK1pSZwPzGAB439ixY/Xss8/q3Xff1f79+5WWlqaXXnpJ1157rVvqWaYm9vYyy7KUlpamcePGNWu/sWPHyrIsrVu3rt719SXpXbt2VWFhYa37PgCAOx0/flxRUVEuzT3MYwBckZubq2XLlqlz586aMmXKGR+PeQyArzuTeaxGaWmpcnNzlZCQoLCwMJtH6F4nTpzQ448/rrS0NBUUFMjhcOjmm2/WE0884fJlIA09Hz55GcWpLrnkEr3xxhunXR8aGqrQ0FAPjggA7MU8BsAVLamzgXkMALwvMjJS8+bNc9tHXf6cT11GUZ8dO3YoPj7e28MAAABoUWrChpZwzwYAQOvj1c6GoqIi7du3z/k4NzdXOTk5io6OVrdu3TRz5kwdOnRIy5cvlyTNmzdPPXr0UJ8+fVReXq433nhDq1ev1urVq711CgAAAC1SzZWyLaGzAQDQ+ng1bMjOzq71SRKpqamSpJSUFC1dulR5eXk6cOCAc315ebkefPBBHTp0SOHh4erTp4/effddXXXVVR4fOwAAQEtGZwMAwJu8GjYkJiaqoftTLl26tNbjhx9+WA8//LCbRwUAAOD7WtJHXwIAWh9efQAAAPxQS7pBJACg9SFsAAAA8EN0NgAAvIlXHwAAAD9EZwMAwJsIGwAAAPwQnQ0AAG/i1QcAAMAP0dkAAPAmwgYAAAA/RGcDAMCbePUBAADwQ3Q2AABOdeLECc2YMUPdu3dXeHi4Lr30Um3dutVt9QgbAAAA/BCdDQCAU915551KT0/XX/7yF+3cuVNJSUkaOXKkDh065JZ6QW45KgAAALyKzgYA8AxjjKqrSzxeNyAgvMlzfElJiVavXq23335bw4YNkyTNnj1ba9eu1cKFC/XMM8/YPj7CBgAAAD9EZwMAeEZ1dYkyMvt5vG7i8J0KDIxo0raVlZWqqqpSWFhYreXh4eHauHGjO4bHZRQAAAD+iM4GAECNyMhIDRkyRL/73e90+PBhVVVV6Y033tCWLVuUl5fnlpp0NgAAAPghOhsAwDMCAsKVOHynV+o2x1/+8hfdfvvt+sUvfqHAwEBddNFFuuWWW7R9+3a3jI+wAQAAwA/R2QAAnmFZVpMvZ/Cms88+W5mZmSouLtbx48cVHx+vG2+8UQkJCW6pR9QNAADgh2rCBjobAACnatOmjeLj4/Xjjz/q/fff1zXXXOOWOnQ2AAAA+KGayyjobAAASNL7778vY4x69eqlffv26aGHHlKvXr00efJkt9Qj6gYAAPBDdDYAAE5VWFioKVOm6LzzztNtt92moUOH6oMPPlBwcLBb6tHZAAAA4Ie4QSQA4FQ33HCDbrjhBo/V49UHAADAD3GDSACANxE2AAAA+CE6GwAA3sSrDwAAgB+iswEA4E2EDQAAAH6IzgYAgDfx6gMAAOCH6GwAAHgTYQMAAIAforMBAOBNvPoAAAD4ITobAADeRNgAAADgh+hsAAB4E68+AAAAfojOBgCANxE2AAAA+CE6GwAA3sSrDwAAgB+iswEAcKqsrCyNHTtWDodDlmVp7dq1tdYbYzR79mw5HA6Fh4crMTFRn3/+ucv1CBsAAAD8EJ0NAIBTFRcXq3///po/f36963//+9/rpZde0vz587V161bFxcXp17/+tU6cOOFSvaAzGSwAAABaJjobAMAzjDE6+f/nXE+KCAho1hyfnJys5OTketcZYzRv3jw99thjGj9+vCRp2bJlio2N1YoVK3TPPfc0e3yEDQAAAH6oJmygswEA3OtkdbXOztrp8bpfDeunNoGBthwrNzdX+fn5SkpKci4LDQ3V8OHD9cknn7gUNvDqAwAA4IdqLqOgswEA0Jj8/HxJUmxsbK3lsbGxznXNRWcDAACAH6KzAQA8IyIgQF8N6+eVunb7eUBtjHE5tCZsAAAA8EPcIBIAPMOyLNsuZ/CWuLg4ST91OMTHxzuXFxQU1Ol2aCpefQAAAPwQN4gEADRVQkKC4uLilJ6e7lxWXl6uzMxMXXrppS4dk84GAAAAP0RnAwDgVEVFRdq3b5/zcW5urnJychQdHa1u3bppxowZmjNnjs4991yde+65mjNnjiIiInTLLbe4VI+wAQAAwA/R2QAAOFV2drZGjBjhfJyamipJSklJ0dKlS/Xwww+rpKRE9913n3788UcNHjxYH3zwgSIjI12qR9gAAADgh+hsAACcKjEx0fnaUB/LsjR79mzNnj3blnq8+gAAAPghOhsAAN5E2AAAAOCH6GwAAHgTrz4AAAB+iM4GAIA3ETYAAAD4ITobAADexKsPAACAH6KzAQDgTV4NG7KysjR27Fg5HA5ZlqW1a9c2uk9mZqYGDhyosLAwnXXWWXrttdfcP1AAAAAfQ2cDAMCbvPrqU1xcrP79+2v+/PlN2j43N1dXXXWVLr/8cu3YsUOzZs3S9OnTtXr1ajePFAAAwLfQ2QAA8KYgbxZPTk5WcnJyk7d/7bXX1K1bN82bN0+S1Lt3b2VnZ+sPf/iDJkyY4KZRAgAA+B46GwAA3uTVsKG5Nm3apKSkpFrLRo0apcWLF6uiokLBwcF19ikrK1NZWZnz8fHjx90+TgCwE/MYAFe0pM4G5jEAaH18KurOz89XbGxsrWWxsbGqrKzUkSNH6t1n7ty5ioqKcn517drVE0MFANswjwFwRU3Y0BI6G5jHAKD18f6rTzP9PJ2vaRE8XWo/c+ZMFRYWOr8OHjzo9jECgJ2YxwC4orHfkTyJeQwAvK+xD2hYs2aNRo0apU6dOsmyLOXk5JxRPZ+6jCIuLk75+fm1lhUUFCgoKEgdO3asd5/Q0FCFhoZ6YngA4BbMYwBc0ZI6G5jHAMD7aj6gYfLkyfXe87C4uFiXXXaZrr/+et11111nXM+nwoYhQ4bo73//e61lH3zwgQYNGlTv/RoAAABaK24QCQCeYYxRSUWVx+uGBwc2q3utsQ9omDhxoiRp//79Zzo0SV4OG4qKirRv3z7n49zcXOXk5Cg6OlrdunXTzJkzdejQIS1fvlySdO+992r+/PlKTU3VXXfdpU2bNmnx4sVauXKlt04BAACgRWpJN4gEAH9WUlGl85943+N1dz09ShEhLbd/wKsjy87O1ogRI5yPU1NTJUkpKSlaunSp8vLydODAAef6hIQEvffee7r//vv1xz/+UQ6HQ6+88gofewkAAPAzdDYAALzJq2FDYmKi84WwPkuXLq2zbPjw4dq+fbsbRwUAAOD76GwAAM8IDw7UrqdHeaVuS9Zyey4AAADgMjobAMAzLMtq0ZczeAuvPgAAAH6IzgYAgDcRvwAAAPghOhsAAKdq7AMafvjhBx04cECHDx+WJO3Zs0eSFBcXp7i4uGbXa3bYUFxcrOeee07//Oc/VVBQ4EzNa3z99dfNHgQAAADsc+rvZ3Q2AACkxj+gYd26dZo8ebJz/U033SRJevLJJzV79uxm12t22HDnnXcqMzNTEydOVHx8PC9gAAAALcypN+CmswEAIDX+AQ2TJk3SpEmTbKvX7LDhH//4h959911ddtlltg0CAAAA9qGzAQDgbc0OGzp06KDo6Gh3jAVAC1ZdXa2srCz98MMP3h5KixAdHa3ExERvDwNAC/HFF19o165d3h6G06lhA50NAABvaHbY8Lvf/U5PPPGEli1bpoiICHeMCUALlJ+fr4yMDG8Po8Xo0qULYQMAp3feeUdFRUXeHkYdISEhCgxs2Z/DDgDwT00KGwYMGFCrBW/fvn2KjY1Vjx49FBwcXGvb7du32ztCAC1CRUWFJCkiIkJDhw718mi8r23btt4eAoAW5OTJk5KkYcOGKSwszMuj+Y+uXbsSNgAAvKJJYcO4cePcPAwALV1NS25ERIQuvfRSL48GAFqOyspK5xw5ZMgQhYeHe3lEAAB4X5PChieffNL5/5MnT9att96qK664ghsOAa0In9cOAPUrLy93/n9ISIgXRwIAQMvR7HcNR48e1ZgxY9SlSxc9+OCDysnJccOwALQ0NX+1I2QEgNpqwobAwEAuWQAA4P9rdtiwbt065efn68knn1R2drYGDhyo888/X3PmzNH+/fvdMEQALUFN2EBnAwDUVhM20NUAAMB/uPSuoX379rr77ruVkZGhb775RpMnT9Zf/vIXnXPOOXaPD0ALUXMZBZ0NAFAbYQMAAHWd0Z8oKyoqlJ2drS1btmj//v2KjY21a1wAWhg6GwCgfoQNAADU5dK7hg0bNuiuu+5SbGysUlJSFBkZqb///e86ePCg3eMD0EJwg0gAqF9N2BAaGurlkQAAcHpZWVkaO3asHA6HLMvS2rVrnesqKir0yCOPqF+/fmrTpo0cDoduu+02HT582OV6zX7X0KVLF1111VX6/vvv9frrr+u7777TkiVLNHLkSN6EAH6MG0QCQP3obAAA+ILi4mL1799f8+fPr7Pu5MmT2r59ux5//HFt375da9as0Zdffqmrr77a5XpN+ujLUz3xxBO6/vrr1aFDB5eLAvA9dDYAQP3KysokETYAQKtljFRx0vN1gyOkZvwhMDk5WcnJyfWui4qKUnp6eq1lr776qn75y1/qwIED6tatW7OH1+yw4e677252EQC+j84GAKgfnQ0A0MpVnJTmODxfd9ZhKaSN2w5fWFgoy7LUvn17l/bnT5QAmoTOBgCoH2EDAMDflJaW6tFHH9Utt9yidu3auXSMZnc2AGid6GwAgPoRNgBAKxcc8VOXgTfqukFFRYVuuukmVVdXa8GCBS4fh7ABQJPQ2QAA9SNsAIBWzrLcejmDJ1VUVOiGG25Qbm6uPvroI5e7GiTCBgBNRGcDANSPsAEA4A9qgoa9e/dqw4YN6tix4xkdj7ABQJPQ2QAA9asJG0JDQ708EgAATq+oqEj79u1zPs7NzVVOTo6io6PlcDh03XXXafv27XrnnXdUVVWl/Px8SVJ0dLRLgTphA4AmobMBAOpHZwMAwBdkZ2drxIgRzsepqamSpJSUFM2ePVvr1q2TJF144YW19tuwYYMSExObXY+wAUCT0NkAAPUrKyuTRNgAAGjZEhMTnb/T16ehda7gXQOAJqGzAQDqR2cDAAB1ETYAaBI6GwCgfoQNAADUxbsGAE1CZwMA1I+wAQCAurhnA4AmqQkb6GwA4K+MMfruu+9UUlIiy7LkcDhqBQjV1dU6dOiQKisra+1H2AAAQF2EDQCapOYyCjobAPir3bt3669//avzcffu3TV58mTn48zMTGVmZp52fz76EgCA/yBsANAkdDYA8HdHjhyRJAUFBamystL5+Ofr27Ztq7CwsFrrzjrrrDrLAABozQgbADQJN4gE4O9qLoc455xz9MUXXzgf/3z9FVdcoQEDBnh8fAAA+BLeNQBoEm4QCcDf1YQJbdu2lSRVVFQ4575T13NvBgAAGkfYAKBJ6GwA4O/Kysok/SdskH4KHH6+nrABAIDG8a4BQJPQ2QDA39V0LkRERDjnupqA4dT1hA0AADSOsAFAk9DZAMDfnRom1AQKp963gbABAODLsrKyNHbsWDkcDlmWpbVr19ZaP3v2bJ133nlq06aNOnTooJEjR2rLli0u1+NdA4AmobMBgL8jbAAA+LPi4mL1799f8+fPr3d9z549NX/+fO3cuVMbN25Ujx49lJSUpO+//96lenwaBYAmobMBgL+rCRNCQ0PrhA3GmFrrAQCoYYxRSWWJx+uGB4U36w+BycnJSk5OPu36W265pdbjl156SYsXL9Znn32mK664otnjI2wA0CR0NgDwdw11Npx6o0g6GwAApyqpLNHgFYM9XnfLLVsUERzhlmOXl5dr0aJFioqKUv/+/V06BmEDgCahswGAv2sobDj1coqgIH59AgD4p3feeUc33XSTTp48qfj4eKWnp6tTp04uHYtXSwBNQmcDAH93athQc6nEz8OGkJAQQlcAQC3hQeHacovrN1I8k7p2GzFihHJycnTkyBH96U9/0g033KAtW7YoJiam2ccibADQJHQ2APBn1dXVzkslGups4BIKAMDPWZbltssZPK1NmzY655xzdM455+iSSy7Rueeeq8WLF2vmzJnNPpbX3zUsWLBACQkJCgsL08CBA/Xxxx+fdtuMjAxZllXn64svvvDgiIHWic4GAP7s1Msk6gsbysrKnOsAAGgtjDHO18Dm8mpnw1tvvaUZM2ZowYIFuuyyy/T6668rOTlZu3btUrdu3U673549e9SuXTvn486dO3tiuECrRmcDAH9WEypYlqWgoCA6GwAAfqeoqEj79u1zPs7NzVVOTo6io6PVsWNHPfvss7r66qsVHx+vo0ePasGCBfr22291/fXXu1TPq+8aXnrpJd1xxx2688471bt3b82bN09du3bVwoULG9wvJiZGcXFxzq/AwEAPjRhovehsAODPTg0TLMtyhgo1f80hbAAA+Lrs7GwNGDBAAwYMkCSlpqZqwIABeuKJJxQYGKgvvvhCEyZMUM+ePTVmzBh9//33+vjjj9WnTx+X6nmts6G8vFzbtm3To48+Wmt5UlKSPvnkkwb3HTBggEpLS3X++efrt7/9rUaMGHHabcvKymq1fRw/fvzMBg60UjVhA50Nnsc8Brjfz8MEOhvsxTwGAN6XmJjo7Fauz5o1a2yt57V3DUeOHFFVVZViY2NrLY+NjVV+fn69+8THx2vRokVavXq11qxZo169eumKK65QVlbWaevMnTtXUVFRzq+uXbvaeh5Aa1EzMdHZ4HnMY4D7ETa4F/MYALQ+Xv8T5c/fuBhjTvtmplevXrrrrrt00UUXaciQIVqwYIFGjx6tP/zhD6c9/syZM1VYWOj8OnjwoK3jB1oLOhu8h3kMcL+aMKHmIy9PFzbUrEfzMI8BQOvjtcsoOnXqpMDAwDpdDAUFBXW6HRpyySWX6I033jjt+tDQUH4xAGzADSK9h3kMcL+fdy7U/MzR2WAP5jEAaH289q4hJCREAwcOVHp6eq3l6enpuvTSS5t8nB07dig+Pt7u4QH4GW4QCcCfcRkFAAD28upHX6ampmrixIkaNGiQhgwZokWLFunAgQO69957Jf3Ucnfo0CEtX75ckjRv3jz16NFDffr0UXl5ud544w2tXr1aq1ev9uZpAK0CnQ0A/FnNzQtPFzb8fD0AAGiYV8OGG2+8UUePHtXTTz+tvLw89e3bV++99566d+8uScrLy9OBAwec25eXl+vBBx/UoUOHFB4erj59+ujdd9/VVVdd5a1TAFoNOhsA+DM6GwAAsJdXwwZJuu+++3TffffVu27p0qW1Hj/88MN6+OGHPTAqAD9HZwMAX1dWVqaVK1eqsLCwzrqSkhJJdcOG4uJivfzyyzpx4kSt5QAAoGFeDxsA+AY6GwD4uoMHD2r//v0NbhMTEyNJatu2rcLDw1VSUqIff/yxznoAANAwwgYATUJnAwBfV3Pfhbi4OI0ePbrO+tDQUHXu3FmSFBwcrGnTpuno0aPO9W3btlWHDh08M1gAAHwcYQOAJqGzAYCvq7nvQmRkpLp27dro9hEREYqIiHD3sAAA8Ev8iRJAk9DZAMDXcZNHAEBrlpWVpbFjx8rhcMiyLK1du/a0295zzz2yLEvz5s1zuR7vGgA0CZ0NAHwdYQMAoDUrLi5W//79NX/+/Aa3W7t2rbZs2SKHw3FG9biMAkCT0NkAwNcRNgAA3MEYI/P/P9XIk6zw8Gb9ITA5OVnJyckNbnPo0CFNnTpV77//fr33N2oOwgYATUJnAwBfV3ODSMIGAICdTEmJ9lw00ON1e23fJsvGewtVV1dr4sSJeuihh9SnT58zPh5/ogTQJHQ2APB1dDYAAHB6zz//vIKCgjR9+nRbjkdnA4AmobMBgK8jbAAAuIMVHq5e27d5pa5dtm3bppdfflnbt2+37fd9wgYATVITNtDZAMBX1YQNoaGhXh4JAMCfWJZl6+UM3vDxxx+roKBA3bp1cy6rqqrSAw88oHnz5mn//v3NPiZhA4Am4TIKAL6OzgYAAOo3ceJEjRw5stayUaNGaeLEiZo8ebJLxyRsANAkXEYBwNcRNgAAWrOioiLt27fP+Tg3N1c5OTmKjo5Wt27d1LFjx1rbBwcHKy4uTr169XKpHmEDgCahswGAryNsAAC0ZtnZ2RoxYoTzcWpqqiQpJSVFS5cutb0eYQOAJqGzAYCvI2wAALRmiYmJzj8gNoUr92k4FX+iBNAkdDYA8HWEDQAAeA7vGgA0CZ0NAHxZdXU1YQMAAB5E2ACgSehsAODLKioqnP9P2AAAgPvxrgFAk9DZAMCX1XQ1WJal4OBgL48GAAD/R9gAoFGn3kiGzgYAvujUSygITQEAcD/eNQBoVE1Xg0RnAwDfxP0aAADwLMIGAI2iswGAryNsAADAs3jXAKBRdDYA8HWEDQAAeFaQtwcAoOWjswFAS5Cbm6uDBw+6tG9BQYEkwgYAADyFsAFAo+hsAOBtlZWVWrFiRa2PsHRFRESETSMCAAANIWwA0Cg6GwB4W1FRkSoqKhQQEKALL7zQpWMEBgZq0KBB9g4MAAAfkZWVpRdeeEHbtm1TXl6e0tLSNG7cOOf6SZMmadmyZbX2GTx4sDZv3uxSPcIGAI2iswGAt504cUKSFBkZqauvvtrLowEAwPcUFxerf//+mjx5siZMmFDvNldeeaWWLFnifHwmlx8SNgBoVE3YYFkWYQMArygqKpIktW3b1ssjAQCgNmOMKsurG9/QZkEhAc363Tw5OVnJyckNbhMaGqq4uLgzHZokwgYATVBzGQWXUADwFsIGAEBLVVlerUW/yfR43btfHq7g0EBbj5mRkaGYmBi1b99ew4cP17PPPquYmBiXjsU7BwCNOrWzAQC8oSZsiIyM9PJIAADwT8nJyXrzzTf10Ucf6cUXX9TWrVv1q1/9SmVlZS4dj84GAI2iswGAt9HZAABoqYJCAnT3y8O9UtdON954o/P/+/btq0GDBql79+569913NX78+GYfj7ABQKPobADgbTU3iCRsAAC0NJZl2X45Q0sQHx+v7t27a+/evS7tz58pATSKzgYA3kZnAwAAnnX06FEdPHhQ8fHxLu1PZwOARtHZAMDbuGcDAABnpqioSPv27XM+zs3NVU5OjqKjoxUdHa3Zs2drwoQJio+P1/79+zVr1ix16tRJ1157rUv1CBsANIrOBgDeVF1dTWcDAABnKDs7WyNGjHA+Tk1NlSSlpKRo4cKF2rlzp5YvX65jx44pPj5eI0aM0FtvveVy0E/YAKBRdDYAcLePP/5Yu3fvrnedMcY5D7Vp08aTwwIAwG8kJiY6/4hYn/fff9/WeoQNABpFZwMAdyovL9dHH33U4C9AktSpUycFBfGrCwAAvoBXbACNorMBgDt99913MsYoIiJC48aNO+12DofDc4MCAABnhLABQKPobADgTnl5eZJ+ChN69uzp5dEAAAA78M4BQKPobADgTvn5+ZLk8kdrAQCAloewAUCj6GwA4E41nQ1xcXFeHgkAALALl1EAaBSdDQDsdPjwYa1atUplZWWS5PwvYQMAAP6DsAFAo2rCBjobANjh//7v/3T8+PFayzp16qQOHTp4aUQAAMBuhA0AGlVzGQWdDQCaqrq6ut6PsiwpKdEXX3whSbr11lsVHR0tSYqKiiLQBADAj3g9bFiwYIFeeOEF5eXlqU+fPpo3b54uv/zy026fmZmp1NRUff7553I4HHr44Yd17733enDEQOtDZwOA5igsLNSf/vQnFRUVnXabuLg4nXvuuR4cFQAA8CSvvnN46623NGPGDD322GPasWOHLr/8ciUnJ+vAgQP1bp+bm6urrrpKl19+uXbs2KFZs2Zp+vTpWr16tYdHDrQu3CASQHNs27atwaDBsixdeumlHhwRAADwNK92Nrz00ku64447dOedd0qS5s2bp/fff18LFy7U3Llz62z/2muvqVu3bpo3b54kqXfv3srOztYf/vAHTZgwwS1jPPLdd8p4/2/Ov+zCcwIsSwqU5KOd+20VKCskQrLqf4MepGCp5rKEkOA65xkaIJnAMIUGWrIsS9WBp2zvYXnff69qVam4qFCfbf7EK2NoilA1/SkKDJKCQkLO6NKQ2HP6KDQszOX9PYV5zHta6zz26b9yVF1VrgHn9VRMjEOhQT/NYyYwRLIsBVoB0smiFj2feAvzWP2O/fCD1n30riI/SlePz3bJqvSj+awFzA91L3hq/hYtlSVLh5N6qbJdeJ11gQqQIyDyP9u2j5AVGFhrm4AASQHBCgs0sizpROxgmaC6x8J/hMiz89gvel6gsHC+J02RlZWlF154Qdu2bVNeXp7S0tI0bty4Wtvs3r1bjzzyiDIzM1VdXa0+ffror3/9q7p169bsel4LG8rLy7Vt2zY9+uijtZYnJSXpk0/q/+Vj06ZNSkpKqrVs1KhRWrx4sSoqKhQcHFxnn7KyMuddriXVuSFVYzb+c7X6zPljs/YBavS6Lk8BQa6/QPcu/bNS9v9FwabSxlG5JkpShaT0Tz7y9lBs0+/2LxQYfCa/QP1V3foOtG08p8M8Bm9yZR67QvopZNkr9d7ZcuYxf9Ra5rFN/7deYX+aq/O+8d03vfCeK068r1CrqvENC+tfXPv3sU9tHRtsmMe+XKGz+w+2b0B+rLi4WP3799fkyZPr/WP9V199paFDh+qOO+7QU089paioKO3evVthLobSXgsbjhw5oqqqKsXGxtZaHhsbq/z8/Hr3yc/Pr3f7yspKHTlyRPHx8XX2mTt3rp566in7Bg4AHsY8BsDXnek8FhAYrP2x0rmHpb9eHqDc2Mb3ASTpj/nfKzjCjzph0CIZY1R5SqDqKUGhoc3qCklOTlZycvJp1z/22GO66qqr9Pvf/9657KyzznJ9fC7vaZOfPznGmAafsPq2r295jZkzZyo1NdX5+Pjx4+ratWuTxzf0igna2OStYafAAN9uP45QoPJC2so6TftxqBX8n1MLDqrTbxYcaGl5YJhCg6b+1H4cEOa1yyh8hTfajz2Becx3MY8xjzUX81j9Bl9yhaqPFejDc7/V4fadFMy/I4/pddY5PjuHSdIOq1qBAc17Q3aq0EBpcUCYQoPu51O5msgbl1F4W2VZmV5Juc7jdacv+18F23QpXHV1td599109/PDDGjVqlHbs2KGEhATNnDmzzqUWTeW1sKFTp04KDAys08VQUFBQp3uhRlxcXL3bBwUFqWPHjvXuExoaqtDQUNfHGRurcbfc5/L+AHCmmMcA+LozncfaR0cr+ZYp+lVZmXbu3Kny8nIbR4eGcDNXoHUoKChQUVGRnnvuOT3zzDN6/vnntX79eo0fP14bNmzQ8OHDm31Mr4UNISEhGjhwoNLT03Xttdc6l6enp+uaa66pd58hQ4bo73//e61lH3zwgQYNGlTv/RoAAADgP0JDQzVo0CBvDwMAagkKDdX0Zf/rlbp2qbmR+DXXXKP7779fknThhRfqk08+0WuvveZbYYMkpaamauLEiRo0aJCGDBmiRYsW6cCBA7r33nsl/dRyd+jQIS1fvlySdO+992r+/PlKTU3VXXfdpU2bNmnx4sVauXKlN08DAAAAANBKWZZl2+UM3tKpUycFBQXp/PPPr7W8d+/e2rjRtQtyvRo23HjjjTp69Kiefvpp5eXlqW/fvnrvvffUvXt3SVJeXp4OHDjg3D4hIUHvvfee7r//fv3xj3+Uw+HQK6+84raPvQQAAAAAwN+FhITo4osv1p49e2ot//LLL53vz5vL6zeIvO+++3TfffVfS7x06dI6y4YPH67t27e7eVQAAAAAAPiPoqIi7du3z/k4NzdXOTk5io6OVrdu3fTQQw/pxhtv1LBhwzRixAitX79ef//735WRkeFSPa+HDQAAAAAAwL2ys7M1YsQI5+OaTwlKSUnR0qVLde211+q1117T3LlzNX36dPXq1UurV6/W0KFDXapH2AAAAAAAgJ9LTEyUMabBbW6//XbdfvvtttSr/4OzAQAAAAAAXNTqOhtqkpzjx497eSQAWpOaOaexNLkpmMcAeAPzGABfZ+c8hsa1urDhxIkTkqSuXbt6eSQAWqMTJ04oKirqjI8hMY8B8A7mMQC+zo55DI1rdWGDw+HQwYMHFRkZKcuymrTP8ePH1bVrVx08eFDt2rVz8wg9j/PzbZyfbzDG6MSJE3I4HGd8LOaxujg/38b5+QbmMffi/Hwb5+cb7JzH0LhWFzYEBASoS5cuLu3brl07n/7hagzn59s4v5bPrgSdeez0OD/fxvm1fMxj7sf5+TbOr+Wjo8FzuEEkAAAAAACwFWEDAAAAAACwFWFDE4SGhurJJ59UaGiot4fiFpyfb+P80BT+/jxyfr6N80NT+PvzyPn5Ns4PqMsyfO4HAAAAAACNKi0tVW5urhISEhQWFubt4XhdQ88HnQ0AAAAAAMBWhA0AAAAAAPi5rKwsjR07Vg6HQ5Zlae3atbXWW5ZV79cLL7zgUj3CBgAAAAAA/FxxcbH69++v+fPn17s+Ly+v1tef//xnWZalCRMmuFQv6EwGCwAAAABAa2aMkamo9nhdKzhAlmU1efvk5GQlJyefdn1cXFytx2+//bZGjBihs846y6Xx0dnQBAsWLHDe8GLgwIH6+OOPvT2kOmbPnl2n3eXUfyzGGM2ePVsOh0Ph4eFKTEzU559/XusYZWVlmjZtmjp16qQ2bdro6quv1rfffltrmx9//FETJ05UVFSUoqKiNHHiRB07dsz282msxceT53PgwAGNHTtWbdq0UadOnTR9+nSVl5e77dwmTZpU53t5ySWX+MS5SdLcuXN18cUXKzIyUjExMRo3bpz27NlTaxtf/v75Kuax//DEPObPc1hTzo95rOWfoy9iHvsP5jHmscYwj3mWqajW4Sc+8fiXOwOO7777Tu+++67uuOMOl49B2NCIt956SzNmzNBjjz2mHTt26PLLL1dycrIOHDjg7aHV0adPn1ptLzt37nSu+/3vf6+XXnpJ8+fP19atWxUXF6df//rXOnHihHObGTNmKC0tTatWrdLGjRtVVFSkMWPGqKqqyrnNLbfcopycHK1fv17r169XTk6OJk6caPu5NNbi46nzqaqq0ujRo1VcXKyNGzdq1apVWr16tR544AG3nZskXXnllbW+l++9916t9S313CQpMzNTU6ZM0ebNm5Wenq7KykolJSWpuLjYuY0vf/98EfOY5+cxf57DmnJ+EvNYSz9HX8M8xjzGPNY8zGM4U8uWLVNkZKTGjx/v+kEMGvTLX/7S3HvvvbWWnXfeeebRRx/10ojq9+STT5r+/fvXu666utrExcWZ5557zrmstLTUREVFmddee80YY8yxY8dMcHCwWbVqlXObQ4cOmYCAALN+/XpjjDG7du0ykszmzZud22zatMlIMl988YUbzuonkkxaWppXzue9994zAQEB5tChQ85tVq5caUJDQ01hYaHt52aMMSkpKeaaa6457T6+cm41CgoKjCSTmZlpjPGv75+vYB7z7jzmz3NYfednDPOYL55jS8c8xjzGPHZmmMfsU1JSYnbt2mVKSkqcy6qrq01VWaXHv6qrq10+j/r+3Z+qV69eZurUqS49HzXobGhAeXm5tm3bpqSkpFrLk5KS9Mknn3hpVKe3d+9eORwOJSQk6KabbtLXX38tScrNzVV+fn6t8wgNDdXw4cOd57Ft2zZVVFTU2sbhcKhv377ObTZt2qSoqCgNHjzYuc0ll1yiqKgojz4fnjyfTZs2qW/fvnI4HM5tRo0apbKyMm3bts1t55iRkaGYmBj17NlTd911lwoKCpzrfO3cCgsLJUnR0dGSWsf3ryVhHmt581hr+RlgHvOtc2zJmMeYx5jHzhzzmHtZlqWAkECPfzXnfg3N8fHHH2vPnj268847z+g4hA0NOHLkiKqqqhQbG1treWxsrPLz8700qvoNHjxYy5cv1/vvv68//elPys/P16WXXqqjR486x9rQeeTn5yskJEQdOnRocJuYmJg6tWNiYjz6fHjyfPLz8+vU6dChg0JCQtx2zsnJyXrzzTf10Ucf6cUXX9TWrVv1q1/9SmVlZc4x+cq5GWOUmpqqoUOHqm/fvs66NeNtaPy+co4tHfNYy5vHWsPPAPOYb51jS8c8xjzGPHZmmMfQXIsXL9bAgQPVv3//MzoOn0bRBD9PjIwxbkuRXHXqXUX79eunIUOG6Oyzz9ayZcucN7Nx5Tx+vk1923vr+fDU+Xj6nG+88Ubn//ft21eDBg1S9+7d9e677zZ4zVRLPLepU6fqs88+08aNG+us89fvX0vFPNby/k34888A89jptcRz9BXMYy3v34Q//wwwj51eSzxHNE1RUZH27dvnfJybm6ucnBxFR0erW7dukqTjx4/rb3/7m1588cUzrkdnQwM6deqkwMDAOolbQUFBnXSupWnTpo369eunvXv3Ou+C3NB5xMXFqby8XD/++GOD23z33Xd1an3//fcefT48eT5xcXF16vz444+qqKjw2DnHx8ere/fu2rt3r3NMvnBu06ZN07p167RhwwZ16dLFuby1ff+8jXms5c1jrfFngHns9Nt4+xx9AfMY81hL+BlgHjv9Nt4+RzRddna2BgwYoAEDBkiSUlNTNWDAAD3xxBPObVatWiVjjG6++eYzrkfY0ICQkBANHDhQ6enptZanp6fr0ksv9dKomqasrEy7d+9WfHy8EhISFBcXV+s8ysvLlZmZ6TyPgQMHKjg4uNY2eXl5+ve//+3cZsiQISosLNSnn37q3GbLli0qLCz06PPhyfMZMmSI/v3vfysvL8+5zQcffKDQ0FANHDjQredZ4+jRozp48KDi4+MltfxzM8Zo6tSpWrNmjT766CMlJCTUWt/avn/exjzW8uax1vgzwDzW8s7RlzCPMY+1hJ8B5rGWd45ovsTERBlj6nwtXbrUuc3dd9+tkydPKioq6swLNnp7yVZu1apVJjg42CxevNjs2rXLzJgxw7Rp08bs37/f20Or5YEHHjAZGRnm66+/Nps3bzZjxowxkZGRznE+99xzJioqyqxZs8bs3LnT3HzzzSY+Pt4cP37ceYx7773XdOnSxXz44Ydm+/bt5le/+pXp37+/qaysdG5z5ZVXmgsuuMBs2rTJbNq0yfTr18+MGTPG9vM5ceKE2bFjh9mxY4eRZF566SWzY8cO880333j0fCorK03fvn3NFVdcYbZv324+/PBD06VLlybdmdWVcztx4oR54IEHzCeffGJyc3PNhg0bzJAhQ8wvfvELnzg3Y4z57//+bxMVFWUyMjJMXl6e8+vkyZPObXz5++eLmMc8P4/58xzW2Pkxj/nGOfoa5jHmMeax5mEec5+GPn2hNWro+SBsaII//vGPpnv37iYkJMRcdNFFzo+MaUluvPFGEx8fb4KDg43D4TDjx483n3/+uXN9dXW1efLJJ01cXJwJDQ01w4YNMzt37qx1jJKSEjN16lQTHR1twsPDzZgxY8yBAwdqbXP06FFz6623msjISBMZGWluvfVW8+OPP9p+Phs2bDCS6nylpKR4/Hy++eYbM3r0aBMeHm6io6PN1KlTTWlpqVvO7eTJkyYpKcl07tzZBAcHm27dupmUlJQ6426p52aMqffcJJklS5Y4t/Hl75+vYh77D0/MY/48hzV2fsxjvnGOvoh57D+Yx5jHGsM85j6EDbU19HxYxhhz5v0RAAAAAAD4t9LSUuXm5iohIUFhYWHeHo7XNfR8cM8GAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAABgK8IGAAAAAAD8XFZWlsaOHSuHwyHLsrR27dpa64uKijR16lR16dJF4eHh6t27txYuXOhyPcIGQFJGRoYsy9KxY8e8PRQAcAnzGABfxzwGuFdxcbH69++v+fPn17v+/vvv1/r16/XGG29o9+7duv/++zVt2jS9/fbbLtWzjDHmTAYM+KLExERdeOGFmjdvniSpvLxcP/zwg2JjY2VZlncHBwBNwDwGwNcxj8EXlZaWKjc3VwkJCQoLC5MkGWNUUVHh8bEEBwe7/LNiWZbS0tI0btw457K+ffvqxhtv1OOPP+5cNnDgQF111VX63e9+V+9x6ns+agS5NDLAz4SEhCguLs7bwwAAlzGPAfB1zGPwVRUVFZozZ47H686aNUshISG2HW/o0KFat26dbr/9djkcDmVkZOjLL7/Uyy+/7NLxuIwCrc6kSZOUmZmpl19+WZZlybIsLV26tFbb3tKlS9W+fXu988476tWrlyIiInTdddepuLhYy5YtU48ePdShQwdNmzZNVVVVzmOXl5fr4Ycf1i9+8Qu1adNGgwcPVkZGhndOFIDfYh4D4OuYx4CW55VXXtH555+vLl26KCQkRFdeeaUWLFigoUOHunQ8OhvQ6rz88sv68ssv1bdvXz399NOSpM8//7zOdidPntQrr7yiVatW6cSJExo/frzGjx+v9u3b67333tPXX3+tCRMmaOjQobrxxhslSZMnT9b+/fu1atUqORwOpaWl6corr9TOnTt17rnnevQ8Afgv5jEAvo55DP4kODhYs2bN8kpdO73yyivavHmz1q1bp+7duysrK0v33Xef4uPjNXLkyGYfj7ABrU5UVJRCQkIUERHhbNX74osv6mxXUVGhhQsX6uyzz5YkXXfddfrLX/6i7777Tm3bttX555+vESNGaMOGDbrxxhv11VdfaeXKlfr222/lcDgkSQ8++KDWr1+vJUuWeKW1CoB/Yh4D4OuYx+BPLMuy9XIGbygpKdGsWbOUlpam0aNHS5IuuOAC5eTk6A9/+ANhA2CniIgI5wubJMXGxqpHjx5q27ZtrWUFBQWSpO3bt8sYo549e9Y6TllZmTp27OiZQQPAKZjHAPg65jHAMyoqKlRRUaGAgNp3WggMDFR1dbVLxyRsAE7j521JlmXVu6zmh6+6ulqBgYHatm2bAgMDa2136gsiAHgK8xgAX8c8BtinqKhI+/btcz7Ozc1VTk6OoqOj1a1bNw0fPlwPPfSQwsPD1b17d2VmZmr58uV66aWXXKpH2IBWKSQkpNaNhOwwYMAAVVVVqaCgQJdffrmtxwaAn2MeA+DrmMcAz8rOztaIESOcj1NTUyVJKSkpWrp0qVatWqWZM2fq1ltv1Q8//KDu3bvr2Wef1b333utSPcIGtEo9evTQli1btH//frVt29bl1qBT9ezZU7feeqtuu+02vfjiixowYICOHDmijz76SP369dNVV11lw8gB4CfMYwB8HfMY4FmJiYkyxpx2fVxcnJYsWWJbPT76Eq3Sgw8+qMDAQJ1//vnq3LmzDhw4YMtxlyxZottuu00PPPCAevXqpauvvlpbtmxR165dbTk+ANRgHgPg65jHAP9mmYaiDQAAAAAAIEkqLS1Vbm6uEhISFBYW5u3heF1DzwedDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAAwFaEDQAAAAAA+LmsrCyNHTtWDodDlmVp7dq1tdZ/9913mjRpkhwOhyIiInTllVdq7969LtcjbAAAAAAAwM8VFxerf//+mj9/fp11xhiNGzdOX3/9td5++23t2LFD3bt318iRI1VcXOxSvaAzHTAAAAAAAK2VMUbV1SUerxsQEC7Lspq8fXJyspKTk+tdt3fvXm3evFn//ve/1adPH0nSggULFBMTo5UrV+rOO+9s9vgIGwAAAAAAcFF1dYkyMvt5vG7i8J0KDIyw5VhlZWWSpLCwMOeywMBAhYSEaOPGjS6FDVxGAQAAAABAK3beeeepe/fumjlzpn788UeVl5frueeeU35+vvLy8lw6Jp0NAAAAAAC4KCAgXInDd3qlrl2Cg4O1evVq3XHHHYqOjlZgYKBGjhx52ssumoKwAQAAAAAAF1mWZdvlDN40cOBA5eTkqLCwUOXl5ercubMGDx6sQYMGuXQ8LqMAAAAAAACSpKioKHXu3Fl79+5Vdna2rrnmGpeOQ2cDAAAAAAB+rqioSPv27XM+zs3NVU5OjqKjo9WtWzf97W9/U+fOndWtWzft3LlTv/nNbzRu3DglJSW5VI+wAQAAAAAAP5edna0RI0Y4H6empkqSUlJStHTpUuXl5Sk1NVXfffed4uPjddttt+nxxx93uZ5ljDFnPGoAAAAAAPxcaWmpcnNzlZCQUOtjIlurhp4P7tkAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAICfmzt3ri6++GJFRkYqJiZG48aN0549e2ptY4zR7Nmz5XA4FB4ersTERH3++ecu1SNsAAAAAADAz2VmZmrKlCnavHmz0tPTVVlZqaSkJBUXFzu3+f3vf6+XXnpJ8+fP19atWxUXF6df//rXOnHiRLPrWcYYY+cJAAAAAADgj0pLS5Wbm6uEhASFhYV5ezhn5Pvvv1dMTIwyMzM1bNgwGWPkcDg0Y8YMPfLII5KksrIyxcbG6vnnn9c999xT5xgNPR9BHjkLAAAAAAD8kDFGJ6urPV43IiBAlmW5vH9hYaEkKTo6WpKUm5ur/Px8JSUlObcJDQ3V8OHD9cknn9QbNjSEsAEAAAAAABedrK7W2Vk7PV73q2H91CYw0KV9jTFKTU3V0KFD1bdvX0lSfn6+JCk2NrbWtrGxsfrmm2+aXYOwAQAAAACAVmTq1Kn67LPPtHHjxjrrft4tYYxxqYOCsAEAAAAAABdFBAToq2H9vFLXFdOmTdO6deuUlZWlLl26OJfHxcVJ+qnDIT4+3rm8oKCgTrdDUxA2AAAAAADgIsuyXL6cwZOMMZo2bZrS0tKUkZGhhISEWusTEhIUFxen9PR0DRgwQJJUXl6uzMxMPf/8882uR9gAAAAAAICfmzJlilasWKG3335bkZGRzns0REVFKTw8XJZlacaMGZozZ47OPfdcnXvuuZozZ44iIiJ0yy23NLseYQMAAAAAAH5u4cKFkqTExMRay5csWaJJkyZJkh5++GGVlJTovvvu048//qjBgwfrgw8+UGRkZLPrWcYYc6aDBgAAAADA35WWlio3N1cJCQkKCwvz9nC8rqHnw7U7SgAAAAAAAJwGYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAALAVYQMAAAAAAH5u7ty5uvjiixUZGamYmBiNGzdOe/bsqbXNmjVrNGrUKHXq1EmWZSknJ8fleoQNAAAAAAD4uczMTE2ZMkWbN29Wenq6KisrlZSUpOLiYuc2xcXFuuyyy/Tcc8+dcb2gMz4CAAAAAABo0davX1/r8ZIlSxQTE6Nt27Zp2LBhkqSJEydKkvbv33/G9QgbAAAAAABwkTFGJRVVHq8bHhwoy7Jc3r+wsFCSFB0dbdeQaiFsAAAAAADARSUVVTr/ifc9XnfX06MUEeLaW3pjjFJTUzV06FD17dvX5pH9hLABAAAAAIBWZOrUqfrss8+0ceNGt9UgbAAAAAAAwEXhwYHa9fQor9R1xbRp07Ru3TplZWWpS5cuNo/qPwgbAAAAAABwkWVZLl/O4EnGGE2bNk1paWnKyMhQQkKCW+u1/GcEAAAAAACckSlTpmjFihV6++23FRkZqfz8fElSVFSUwsPDJUk//PCDDhw4oMOHD0uS9uzZI0mKi4tTXFxcs+oF2Dh2AAAAAADQAi1cuFCFhYVKTExUfHy88+utt95ybrNu3ToNGDBAo0ePliTddNNNGjBggF577bVm17OMMca20QMAAAAA4KdKS0uVm5urhIQEhYWFeXs4XtfQ80FnAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAsBVhAwAAAAAAfm7u3Lm6+OKLFRkZqZiYGI0bN0579uxxrq+oqNAjjzyifv36qU2bNnI4HLrtttt0+PBhl+oRNgAAAAAA4OcyMzM1ZcoUbd68Wenp6aqsrFRSUpKKi4slSSdPntT27dv1+OOPa/v27VqzZo2+/PJLXX311S7Vs4wxxs4TAAAAAADAH5WWlio3N1cJCQkKCwvz9nDOyPfff6+YmBhlZmZq2LBh9W6zdetW/fKXv9Q333yjbt261Vnf0PMR5JZRAwAAAADQGhgjVZz0fN3gCMmyXN69sLBQkhQdHd3gNpZlqX379s0+PmEDAAAAAACuqjgpzXF4vu6sw1JIG5d2NcYoNTVVQ4cOVd++fevdprS0VI8++qhuueUWtWvXrtk1CBsAAAAAAGhFpk6dqs8++0wbN26sd31FRYVuuukmVVdXa8GCBS7VIGwAAAAAAMBVwRE/dRl4o64Lpk2bpnXr1ikrK0tdunSps76iokI33HCDcnNz9dFHH7nU1SARNgAAAAAA4DrLcvlyBk8yxmjatGlKS0tTRkaGEhIS6mxTEzTs3btXGzZsUMeOHV2uR9gAAAAAAICfmzJlilasWKG3335bkZGRys/PlyRFRUUpPDxclZWVuu6667R9+3a98847qqqqcm4THR2tkJCQZtXjoy8BAAAAAGgCX/7oS+s0n1yxZMkSTZo0Sfv376+320GSNmzYoMTExDrL+ehLAAAAAABascb6DHr06NHoNs0RYNuRAAAAAAAARNgAAAAAAABsRtgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAABsRdgAAAAAAICfmzt3ri6++GJFRkYqJiZG48aN0549e2ptM3v2bJ133nlq06aNOnTooJEjR2rLli0u1SNsAAAAAADAz2VmZmrKlCnavHmz0tPTVVlZqaSkJBUXFzu36dmzp+bPn6+dO3dq48aN6tGjh5KSkvT99983u55ljDF2ngAAAAAAAP6otLRUubm5SkhIUFhYmLeHc0a+//57xcTEKDMzU8OGDat3m+PHjysqKkoffvihrrjiijrrG3o+gtwyagAAAAAAWgFjjEoqSzxeNzwoXJZlubx/YWGhJCk6Orre9eXl5Vq0aJGioqLUv3//Zh+fsAEAAAAAABeVVJZo8IrBHq+75ZYtigiOcGlfY4xSU1M1dOhQ9e3bt9a6d955RzfddJNOnjyp+Ph4paenq1OnTs2uwT0bAAAAAABoRaZOnarPPvtMK1eurLNuxIgRysnJ0SeffKIrr7xSN9xwgwoKCppdg3s2AAAAAADQBPXdo8DXLqOYNm2a1q5dq6ysLCUkJDS6/bnnnqvbb79dM2fOrLOOezYAAAAAAOAGlmW5fDmDJxljNG3aNKWlpSkjI6NJQUPNfmVlZc2uR9gAAAAAAICfmzJlilasWKG3335bkZGRys/PlyRFRUUpPDxcxcXFevbZZ3X11VcrPj5eR48e1YIFC/Ttt9/q+uuvb3Y9wgYAAAAAAPzcwoULJUmJiYm1li9ZskSTJk1SYGCgvvjiCy1btkxHjhxRx44ddfHFF+vjjz9Wnz59ml2PsAEAAAAAAD/X2O0aw8LCtGbNGtvq8WkUAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAADAVoQNAAAAAAD4ublz5+riiy9WZGSkYmJiNG7cOO3Zs+e0299zzz2yLEvz5s1zqR5hAwAAAAAAfi4zM1NTpkzR5s2blZ6ersrKSiUlJam4uLjOtmvXrtWWLVvkcDhcrhd0JoMFAAAAAAAt3/r162s9XrJkiWJiYrRt2zYNGzbMufzQoUOaOnWq3n//fY0ePdrleoQNAAAAAAC4yBgjU1Li8bpWeLgsy3J5/8LCQklSdHS0c1l1dbUmTpyohx56SH369Dmj8RE2AAAAAADgIlNSoj0XDfR43V7bt8mKiHBpX2OMUlNTNXToUPXt29e5/Pnnn1dQUJCmT59+xuMjbAAAAAAAoBWZOnWqPvvsM23cuNG5bNu2bXr55Ze1ffv2M+qYqEHYAAAAAACAi6zwcPXavs0rdV0xbdo0rVu3TllZWerSpYtz+ccff6yCggJ169bNuayqqkoPPPCA5s2bp/379zerDmEDAAAAAAAusizL5csZPMkYo2nTpiktLU0ZGRlKSEiotX7ixIkaOXJkrWWjRo3SxIkTNXny5GbXI2wAAAAAAMDPTZkyRStWrNDbb7+tyMhI5efnS5KioqIUHh6ujh07qmPHjrX2CQ4OVlxcnHr16tXsegG2jBoAAAAAALRYCxcuVGFhoRITExUfH+/8euutt9xSj84GAAAAAAD8nDGm2fs09z4Np6KzAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAACAZnDlZov+qKHngbABAAAAAIAmCA4OliSdPHnSyyNpGWqeh5rn5VR89CUAAAAAAE0QGBio9u3bq6CgQJIUEREhy7K8PCrPM8bo5MmTKigoUPv27RUYGFhnG8vQ/wEAAAAAQJMYY5Sfn69jx455eyhe1759e8XFxdUbuBA2AAAAAADQTFVVVaqoqPD2MLwmODi43o6GGoQNAAAAAADAVtwgEgAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2IqwAQAAAAAA2Or/AdWd9EgAauThAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCQAAAF8CAYAAAAXc2o0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQ40lEQVR4nO3deXhV1aH+8XdnHgiBAJkKhKiAaBAjIKAoQWkgMooDiD8ErFMZvBiHOlwRrYLaykWlILQ04AB4bwXEoWCqhEgZSgIpKIigQRASIwgBQuas3x80R2JCIIeTM+X7eZ7zPJy9195rrROyCG/WXssyxhgBAAAAAAA4kY+rGwAAAAAAAJoeAgkAAAAAAOB0BBIAAAAAAMDpCCQAAAAAAIDTEUgAAAAAAACnI5AAAAAAAABORyABAAAAAACcjkACAAAAAAA4nZ+rGwAAAAAAgDeqrKxUeXm5q5vhMv7+/vL19T3reQIJAAAAAAAcyBij/Px8HTt2zNVNcbkWLVooOjpalmXVOkcgAQAAAACAA1WHEZGRkQoJCanzP+PezhijU6dOqaCgQJIUExNTqwyBBAAAAAAADlJZWWkLI1q1auXq5rhUcHCwJKmgoECRkZG1Ht9gUUsAAAAAABykes2IkJAQF7fEPVR/DnWtpUEgAQAAAACAgzXFxzTqUt/nQCABAAAAAACcjkACAAAAAAAPl5SUpKlTp571fIcOHTR79mynted8sKglAAAAAAAebvny5fL393d1MxqEQAIAAAAAAA8XERHh6iY0GI9sAAAAAADg4c58ZKOgoEBDhw5VcHCw4uPj9c4777i2cWfBDAkAAAAAALzI+PHjdeDAAX322WcKCAjQgw8+qIKCAlc3qxYCCQAAAAAAvMTXX3+tv//979q0aZN69eolSVq4cKG6dOni4pbVxiMbAAAAAAB4iV27dsnPz089evSwHbv00kvVokUL1zXqLAgkAAAAAADwEsYYSZJlWS5uybkRSAAAAAAA4CW6dOmiiooKZWVl2Y7t3r1bx44dc12jzoJAAgAAAAAAL9G5c2cNGjRI9957rzZv3qzs7Gzdc889Cg4OdnXTaiGQAAAAAADAi6Slpaldu3bq16+fRo4cqfvuu0+RkZGublYtlql+wAQAAAAAAFyQkpIS5ebmKj4+XkFBQa5ujsvV93kwQwIAAAAAADgdgQQAAAAAAHA6AgkAAAAAAOB0BBIAAAAAAMDpCCQAAAAAAIDTEUgAAAAAAACnI5AAAAAAAABORyABAAAAAACcjkACAAAAAAA4HYEEAAAAAABwOgIJAAAAAACgEydOaOrUqYqLi1NwcLCuueYabdmypdHqI5AAAAAAAAC65557lJ6errfeeks7duxQcnKyBgwYoIMHDzZKfZYxxjTKnQEAAAAAaGJKSkqUm5ur+Ph4BQUFyRij4vJKl7Ql2N9XlmWdV9ni4mKFhYXp/fff1+DBg23Hr7zySg0ZMkTPP/+8XW345edxJj+77ggAAAAAAM6puLxSl01b45K6dz43UCEB5/ff/oqKClVWVtYKDYKDg7V+/frGaB6PbAAAAAAA0NSFhYWpT58++v3vf69Dhw6psrJSb7/9tjZv3qy8vLxGqZMZEgAAAAAANJJgf1/tfG6gy+puiLfeekt33323fvWrX8nX11dXXXWVxowZo61btzZK+wgkAAAAAABoJJZlnfdjE6528cUXa926dSoqKtLx48cVExOjUaNGKT4+vlHq45ENAAAAAABgExoaqpiYGB09elRr1qzR8OHDG6Uez4hpAAAAAABAo1qzZo2MMercubP27t2rRx99VJ07d9aECRMapT5mSAAAAAAAABUWFmrSpEm69NJLddddd6lv37765JNP5O/v3yj1WcYY0yh3BgAAAACgiSkpKVFubq7i4+NrbaHZFNX3eTBDAgAAAAAAOB2BBAAAAAAAcDoCCQAAAAAA4HQEEgAAAAAAwOkIJAAAAAAAgNMRSAAAAAAAAKcjkAAAAAAAAE5HIAEAAAAAAJyOQAIAAAAAADgdgQS80t/+9jd17dpVwcHBatWqlQYMGKCioiJJ0vjx4zVixAg9++yzioyMVPPmzXX//ferrKzMdv3q1avVt29ftWjRQq1atdKQIUP0zTff1Kjj+++/1+jRoxUREaHQ0FD16NFDmzdvtp3/4IMP1L17dwUFBemiiy7Ss88+q4qKikbprzFGAwYM0KBBg2SMkSQdO3ZM7du311NPPdUodQJoPE1tDNu3b598fHyUlZVV4/jrr7+uuLg427gGwDM0tTFMkjp06CDLsmq9AE+TmZmpoUOHKjY2VpZlaeXKlTXOG2M0ffp0xcbGKjg4WElJSfryyy/tro9AAl4nLy9Pd9xxh+6++27t2rVLGRkZGjlyZI0faD/99FPt2rVLa9eu1dKlS7VixQo9++yztvNFRUVKTU3Vli1b9Omnn8rHx0c333yzqqqqJEknT55Uv379dOjQIa1atUr//ve/9dhjj9nOr1mzRv/v//0/Pfjgg9q5c6fmz5+vRYsW6YUXXjhru9955x01a9as3tc777xT57WWZWnx4sX617/+pddee02S9MADDygqKkrTp0+/0I8UgBM1xTGsQ4cOGjBggNLS0mocT0tL0/jx4/mhHvAgTXEMk6QtW7YoLy9PeXl5+v7779W7d29dd911F/pxAk5XVFSkbt26ac6cOXWef/nllzVr1izNmTNHW7ZsUXR0tH7961/rxIkT9lVocN7WrVtnhgwZYmJiYowks2LFikatb8aMGaZHjx6mWbNmpk2bNmb48OHmq6++qlGmqqrKPPPMMyYmJsYEBQWZfv36mS+++KJR2+XusrOzjSSzb9++Os+PGzfOREREmKKiItuxefPmmWbNmpnKyso6rykoKDCSzI4dO4wxxsyfP9+EhYWZI0eO1Fn+uuuuMzNmzKhx7K233jIxMTFnbffx48fNnj176n0dP3683r7/7//+rwkMDDRPPPGECQkJMbt37663PAD301THsHfffde0bNnSlJSUGGOMycnJMZZlmdzc3LNeA8D9NNUx7EwPPvigiYuLMwUFBedVHt6nuLjY7Ny50xQXF7u6KRfkl//nraqqMtHR0ebFF1+0HSspKTHh4eHmjTfeOOt96vs8CCQa4OOPPzZPPfWUee+995wSSAwcONCkpaWZL774wuTk5JjBgweb9u3bm5MnT9rKvPjiiyYsLMy89957ZseOHWbUqFEmJibmvAdMb1RRUWFuvPFGExYWZm699VazYMEC89NPP9nOjxs3zvTv37/GNTk5OTX+8dy7d6+54447THx8vAkLCzOhoaFGkvnoo4+MMcb89re/Nddff/1Z2xASEmKCgoJMaGio7RUUFGQk1fgHuDHccccdRpKZN29eo9YDoHE01TGstLTUtGnTxixdutQYc/oH+htuuKFR6gLQeJrqGFZt/vz5JjQ01OTk5DRqPXBvtf4DXlVlTOlJ17yqquzuxy//z/vNN98YSWbr1q01yg0bNszcdddd5/95nMHPvnkVTVNKSopSUlLOer6srEz//d//rXfeeUfHjh1TQkKCXnrpJSUlJdlV3+rVq2u8T0tLU2RkpLKzs3X99dfLGKPZs2frqaee0siRIyVJixcvVlRUlJYsWaL777/frno9na+vr9LT07VhwwZ98sknev311/XUU09p8+bNio+Pr/fa6mnBQ4cOVbt27fTnP/9ZsbGxqqqqUkJCgu35xuDg4HrvU1VVpWeffdb2dTlTUFBQnde888475/yazZ8/X3feeedZz586dUrZ2dny9fXVnj176r0XAPfUVMewgIAAjR07VmlpaRo5cqSWLFmi2bNn13s/AO6nqY5hkpSRkaEpU6Zo6dKl6tatW733QhNTfkqaEeuaup88JAWEOuRW+fn5kqSoqKgax6OiovTdd9/ZdU8CCQeaMGGC9u3bp2XLlik2NlYrVqzQoEGDtGPHDnXs2PGC719YWChJioiIkCTl5uYqPz9fycnJtjKBgYHq16+fNmzY0GQDCen0P2jXXnutrr32Wk2bNk1xcXFasWKFUlNTJUn//ve/VVxcbPsHbdOmTWrWrJnatm2rI0eOaNeuXZo/f77t2b/169fXuP8VV1yhv/zlL/rpp59sX48zXXXVVdq9e7cuueSS827zsGHD1KtXr3rL/PKb/5cefvhh+fj46O9//7tuuukmDR48WDfccMN5twGAe2iqY9g999yjhIQEzZ07V+Xl5XX+ZwKA+2uKY9jevXt1yy236Mknn2Tsgtf75dpOxhi713sikHCQb775RkuXLtX333+v2NjT6dcjjzyi1atXKy0tTTNmzLig+xtjlJqaqr59+yohIUFS4yRU3mDz5s369NNPlZycrMjISG3evFk//vijunTpYitTVlam3/zmN/rv//5vfffdd3rmmWc0efJk+fj4qGXLlmrVqpUWLFigmJgY7d+/X48//niNOu644w7NmDFDI0aM0MyZMxUTE6Nt27YpNjZWffr00bRp0zRkyBC1a9dOt912m3x8fLR9+3bt2LFDzz//fJ3tDgsLU1hYmN39/uijj/TXv/5VGzdu1FVXXaXHH39c48aN0/bt29WyZUu77wvAuZrqGCZJXbp0Ue/evfW73/1Od9999zl/CwrA/TTFMay4uFhDhw7VlVdeqfvuu8/2M7okRUdH23VPeBn/kNMzFVxVt4NU/33Oz89XTEyM7XhBQcE5f+lwVnY/UNLE6RfP0/zv//6vkVTjWbXQ0FDj5+dnbr/9dmOMMbm5uUZSva9JkybVWd/EiRNNXFycOXDggO3YP//5TyPJHDp0qEbZe+65xwwcONDxnfYQO3fuNAMHDjRt2rQxgYGBplOnTub111+3nR83bpwZPny4mTZtmmnVqpVp1qyZueeee2wLqRljTHp6uunSpYsJDAw0V1xxhcnIyKj1Nd+3b5+55ZZbTPPmzU1ISIjp0aOH2bx5s+386tWrzTXXXGOCg4NN8+bNzdVXX20WLFjQKH0uKCgwUVFRNRZwKi8vN1dffbXt7x8Az9AUx7AzLVy40Egy//rXvxq9LgCO1xTHsPp+xkfT5O2LWr700ku2Y6WlpRe0qKX1n4rQQJZlacWKFRoxYoQk6d1339Wdd96pL7/8Ur6+vjXKNmvWTNHR0SovL6+1h/IvtWzZsla6NGXKFK1cuVKZmZk1nr379ttvdfHFF2vr1q1KTEy0HR8+fLhatGihxYsXX2AvvdP48eN17NixWnvqAoAn8PYx7IUXXtCyZcu0Y8cOVzcFQCPw9jEMkKSSkhLl5uYqPj7+rOuWuKuTJ09q7969kqTExETNmjVL/fv3V0REhNq3b6+XXnpJM2fOVFpamjp27KgZM2YoIyNDu3fvPusso/o+Dx7ZcJDExERVVlaqoKDgrHsO+/v769JLLz3vexpjNGXKFK1YsUIZGRm1FgKKj49XdHS00tPTbYFEWVmZ1q1bp5deesn+zgAA4GQnT57Url279Prrr+v3v/+9q5sDAECTlJWVpf79+9veV6/9Mm7cOC1atEiPPfaYiouLNXHiRB09elS9evXSJ598YvcjTwQSDXBmWiSdXlQyJydHERER6tSpk+68807dddddeuWVV5SYmKjDhw/rs88+U9euXXXTTTc1uL5JkyZpyZIlev/99xUWFmZ7Hi08PFzBwcGyLEtTp07VjBkz1LFjR1tCFRISojFjxjis3wAANLbJkydr6dKlGjFihO6++25XNwcAgCYpKSlJ9T1EYVmWpk+frunTpzukPh7ZaICMjIwaaVG16rSovLxczz//vN58800dPHhQrVq1Up8+ffTss8+qa9euDa7vbCuVpqWlafz48ZJOz6J49tlnNX/+fFtC9ac//cm28CUAAAAAwHk8+ZGNxlDf50EgAQAAAACAgxBI1FTf5+HjojYBAAAAAIAmjEACAAAAAAA4HYtankNVVZUOHTqksLCws67pAACOYozRiRMnFBsbKx+fC8+MGcMAOBNjGABP5ugxDOdGIHEOhw4dUrt27VzdDABNzIEDB9S2bdsLvg9jGABXYAwD4MkcNYbh3AgkzqF6P9UDBw6oefPmLm4NAG93/PhxtWvXzu69nH+JMQyAMzGGAfBkjh7DcG4EEudQPT2wefPm/EMIwGkcNTWZMQyAKzCGAfBkPCLmPDwYAwAAAAAAnI5AAgAAAAAAOB2BBAAAAAAAUGZmpoYOHarY2FhZlqWVK1fWOL98+XINHDhQrVu3lmVZysnJuaD6CCQAAAAAAICKiorUrVs3zZkz56znr732Wr344osOqY9FLQEAAAAAgFJSUpSSknLW82PHjpUk7du3zyH1EUgAAADAqxw5eEAHv/rS1c2AB+nQ7So1bx3p6mbASxljVFxR7JK6g/2C3XrXEAIJAAAAeJW/Pf/fOvnTEVc3Ax7k5t89QyCBRlNcUaxeS3q5pO7NYzYrxD/EJXWfDwIJAAAAeJWTR3+SJMVdkSi/gAAXtwaeICS8haubADRJBBIAAADwGqaqSjJGknTTlEcU0jzcxS0C0NQF+wVr85jNLqvbnRFIAAAAwGtUVlba/uzj6+vClgDAaZZlufVjE65EIAEAAACvYc4IJHx9+VEXABri5MmT2rt3r+19bm6ucnJyFBERofbt2+unn37S/v37dejQIUnS7t27JUnR0dGKjo5ucH0+jmk2AAAA4HqVlRW2P1vMkACABsnKylJiYqISExMlSampqUpMTNS0adMkSatWrVJiYqIGDx4sSRo9erQSExP1xhtv2FUfsTEAAAC8RlWNGRIEEgDQEElJSTL/WYenLuPHj9f48eMdVh8zJAAAAOA1qgMJy/KR5cOPugDgzhilAQAA4DWqAwkfX37MBQB3x0gNAAAAr/FzIMGTyQDg7ggkAAAA4DV+DiRYPwIA3B2BBAAAALxG1X922SCQAAD3RyABAAAAr8EMCQDwHB4XSMydO1fx8fEKCgpS9+7d9fnnn5+1bEZGhizLqvX66quvnNhiAAAAOAtrSACA5/CoQOLdd9/V1KlT9dRTT2nbtm267rrrlJKSov3799d73e7du5WXl2d7dezY0UktBgAAgDP9/MiGR/2YCwBNkkeN1LNmzdJvfvMb3XPPPerSpYtmz56tdu3aad68efVeFxkZqejoaNvLlyl8AAAAXokZEgDgOTwmkCgrK1N2draSk5NrHE9OTtaGDRvqvTYxMVExMTG68cYbtXbt2nrLlpaW6vjx4zVeAOApGMMAeDJHjGGsIQEAnsNjAonDhw+rsrJSUVFRNY5HRUUpPz+/zmtiYmK0YMECvffee1q+fLk6d+6sG2+8UZmZmWetZ+bMmQoPD7e92rVr59B+AEBjYgwD4MkcMYYRSACA/TIzMzV06FDFxsbKsiytXLnSdq68vFy/+93v1LVrV4WGhio2NlZ33XWXDh06ZHd9HhNIVLMsq8Z7Y0ytY9U6d+6se++9V1dddZX69OmjuXPnavDgwfrjH/941vs/8cQTKiwstL0OHDjg0PYDQGNiDAPgyRwxhhFIAID9ioqK1K1bN82ZM6fWuVOnTmnr1q16+umntXXrVi1fvlxff/21hg0bZnd9HvNwXevWreXr61trNkRBQUGtWRP16d27t95+++2zng8MDFRgYKDd7QQAV2IMA+DJHDGGEUgAgP1SUlKUkpJS57nw8HClp6fXOPb666/r6quv1v79+9W+ffsG1+cxgURAQIC6d++u9PR03Xzzzbbj6enpGj58+HnfZ9u2bYqJiWmMJgIAAMDFft5lw2N+zAXg5YwxMsXFLqnbCg4+6xMFjlBYWCjLstSiRQu7rveokTo1NVVjx45Vjx491KdPHy1YsED79+/XAw88IOn0NL+DBw/qzTfflCTNnj1bHTp00OWXX66ysjK9/fbbeu+99/Tee++5shsAAABoJMyQAOBuTHGxdl/V3SV1d96aLSskpFHuXVJSoscff1xjxoxR8+bN7bqHRwUSo0aN0pEjR/Tcc88pLy9PCQkJ+vjjjxUXFydJysvL0/79+23ly8rK9Mgjj+jgwYMKDg7W5Zdfro8++kg33XSTq7oAAACARkQgAQCNr7y8XKNHj1ZVVZXmzp1r9308KpCQpIkTJ2rixIl1nlu0aFGN94899pgee+wxJ7QKAAAA7oBAAoC7sYKD1XlrtsvqdrTy8nLdfvvtys3N1WeffWb37AjJAwMJAAAA4Gx+DiT4MReAe7Asq9Eem3C26jBiz549Wrt2rVq1anVB92OkBgAAgNf4OZDwuN3tAcDlTp48qb1799re5+bmKicnRxEREYqNjdWtt96qrVu36sMPP1RlZaVtF8yIiAgFBAQ0uD4CCQAAAHgNdtkAAPtlZWWpf//+tvepqamSpHHjxmn69OlatWqVJOnKK6+scd3atWuVlJTU4PoYqQEAAOA1WEMCAOyXlJQkY8xZz9d3zh7MZQMAAIDXqCSQAACPQSABAAAAr2EIJADAYxBIAAAAwGtUsssGAHgMAgkAAAB4DVPFLhsA4CkYqQEAAOA1KivYZQMAPAWBBAAAALzGzzMkWEMCANwdgQQAAAC8RmXF6UDCl0ACANwegQQAAAC8RlUlj2wAgKcgkAAAAIDXqKqsksQjGwDgCQgkAAAA4DV+niFBIAEA7o65bAAAAPAaVZWn15AwRUUqz8tzcWvQFPm2aiWfgABXNwOwS2Zmpv7whz8oOztbeXl5WrFihUaMGGE7P336dC1btkwHDhxQQECAunfvrhdeeEG9evWyqz4CCQAAAHiN6kDiyJw/ae+zM13cGjRFHZYtVfCVV7q6GYBdioqK1K1bN02YMEG33HJLrfOdOnXSnDlzdNFFF6m4uFj/8z//o+TkZO3du1dt2rRpcH0EEgAAAPAa1Y9sWJIsf3/JslzbIDQ9/J2DB0tJSVFKSspZz48ZM6bG+1mzZmnhwoXavn27brzxxgbXRyABAAAAr1FZViZJ8jFGnbb8Sz5BQS5uEYCmzhijirIql9TtF+Ajq5FCsrKyMi1YsEDh4eHq1q2bXfcgkAAAAIDXqCovlyRZsmQFBrq4NQAgVZRVacF/rXNJ3fe92k/+gY5d5PfDDz/U6NGjderUKcXExCg9PV2tW7e2617ssgEAAACvUT1DwjcgoNF+KwgATVn//v2Vk5OjDRs2aNCgQbr99ttVUFBg172YIQEAAACvUT1Dgl0OALgLvwAf3fdqP5fV7WihoaG65JJLdMkll6h3797q2LGjFi5cqCeeeKLh7XN46wAAAAAXqQ4kfAkkALgJy7Ic/tiEOzHGqLS01K5rCSQAAADgNaoqTu+y4cv6EQDQYCdPntTevXtt73Nzc5WTk6OIiAi1atVKL7zwgoYNG6aYmBgdOXJEc+fO1ffff6/bbrvNrvoIJAAAAOA1Km2BBLtrAEBDZWVlqX///rb3qampkqRx48bpjTfe0FdffaXFixfr8OHDatWqlXr27KnPP/9cl19+uV31EUgAAADAa1RVng4kfIKYIQEADZWUlCRjzFnPL1++3KH1scsGAAAAvEZVZaUkyTeIGRIA4O6YIQEAAACvYQskgkNc3JLGc2jPUZ06Xu7qZjQJliVdfFWkq5sBeC0CCQAAAHiNnwMJ75whsTe7QGv+/IWrm9Fk+PhZ+i2BBNBoCCQAAADgNapMlSSpoCJSX23KkyQFBvupfUIr+fo67mllU2W0f9dPKj5R5rB7no9/rcqVJEXEhioo1N+pdTdFPr6Wq5sAeDUCCQAAAHiNqqrTgcT2vCLtmP+/tuMto0MVFd+8zmuCmrVUm7iGrRC/b/th5f77sP0NvQDNWgbq1sd7yD/A1yX1A4CjEEgAAADAa1RVnV4dvrJkU43jP357+nU2/qHD5BtwSYPq8vGz1LZTS8mJv0T38bF05YD2hBEAvAKBBAAAALxG1Rnb1V10VU9JUllxpQp/LLbNnjhTZdkJlZw8JFWuV3hE7fPVAoJbKzSio+29r7+vrrihrWIvaeG4xgNAE0MgAQAAAK9RpdOBhOUTopt/98w5y5cVn9Jfp96vomM/6YdvPqy37O3PzFS7y7o6pJ2N4dTxQu3Z/E9VlLEDR2O5uPvVahEd4+pmAF6DQAIAAABew/wnkAgIqHu9iF8KCA7RkP/6nf79j7/XOYNCkgp/yNMP3+7Vpwvn6fJ+NzqsrY5kjFHOJx/pxOEfXd0Ur9Y8MpJAAnAgAgkAAAB4BXPG4xpBoeHnfV3byxLU9rKEs54/dbxQCx+8V0e+36/Md9IuqI2NLTwqWjGXdHZ1M7xWWERrVzcB8CoeF0jMnTtXf/jDH5SXl6fLL79cs2fP1nXXXXfW8uvWrVNqaqq+/PJLxcbG6rHHHtMDDzzgxBYDAADAGcwZMxxCwyMcdt+Q5uEa+tDj+uqfmZLMOcu7SnDzcPUcdotCmp9/GAMAZ8rMzNQf/vAHZWdnKy8vTytWrNCIESPqLHv//fdrwYIF+p//+R9NnTrVrvo8KpB49913NXXqVM2dO1fXXnut5s+fr5SUFO3cuVPt27evVT43N1c33XST7r33Xr399tv65z//qYkTJ6pNmza65ZZbXNADAAAANJbKygrbn1u0aePQe3fodpU6dLvKofcEAHdTVFSkbt26acKECfX+n3nlypXavHmzYmNjL6g+jwokZs2apd/85je65557JEmzZ8/WmjVrNG/ePM2cObNW+TfeeEPt27fX7NmzJUldunRRVlaW/vjHPzZaIHH82DFt2/Bpo9wbZ+cvycfyl2U5cd8tB7L8AxUaFCAfH5+6C/hZtr75+fvpl90M8A2SFRSkYN+g0+UCgzz2s3Aa/yCd9z5t/j4KCAiw+zMN9vf1mK8HY5hrMIYxhjUYY1idio6fsP25JLBK/976Txe2pmmxLM/9vvXxk4L8fWTv/q2BPoGygoMU7BfssZ+B0zGGua2UlBSlpKTUW+bgwYOaPHmy1qxZo8GDB19QfR4TSJSVlSk7O1uPP/54jePJycnasGFDndds3LhRycnJNY4NHDhQCxcuVHl5ufz9/WtdU1paqtLSUtv748ePN6id2zZ8qqzF7v1sIdxT8o5v5VN19mmg1WfqWje7TNLYh3016NAI+RmP+bb2KG+XXKUK2bfn+87nBiokwDlfF8YwuApjmHtrKmPYv7estf354PZLdGhHaT2l4Vie+1n3y3xIZVVldl/PGNb4PGUMOxtjjCpKXfM94hcY6NBApqqqSmPHjtWjjz6qyy+//ILv5zHfMYcPH1ZlZaWioqJqHI+KilJ+fn6d1+Tn59dZvqKiQocPH1ZMTO0VcmfOnKlnn33WcQ0HACdiDAPgyS50DDv54+Ez3vEbUQDuoaK0VK+Nu9UldT+4+G/yDwpy2P1eeukl+fn56cEHH3TI/TwmkKj2y3THGFNv4lNX+bqOV3viiSeUmppqe3/8+HG1a9fuvNuXeI17bgXl7bxhurNPUIB0AdOdVzLduWEaOFXw0QucKugsjGGeiTGMMazBGMPq1O6iBAW0/IuMLEVN8JH8AxqjmaiDJz+yYU1OU8AFPrLxPo9sNIyXjmHeLjs7W6+++qq2bt3qsL/rHhNItG7dWr6+vrVmQxQUFNSaBVEtOjq6zvJ+fn5q1apVndcEBgYqMDDQ7nY2b9FC/W5iwUwArsEYBsCTXegY1q5HD13xj72SpIqEbvILddxOGwBgL7/AQD24+G8uq9tRPv/8cxUUFNTYUKKyslIPP/ywZs+erX379jW8fQ5rXSMLCAhQ9+7dlZ6erptvvtl2PD09XcOHD6/zmj59+uiDDz6oceyTTz5Rjx496lw/AgAAAJ7L1y9AFfKVnypVVX5KEoEEANezLMuhj024ytixYzVgwIAaxwYOHKixY8dqwoQJdt3TYwIJSUpNTdXYsWPVo0cP9enTRwsWLND+/fv1wAMPSDo9ze/gwYN68803JUkPPPCA5syZo9TUVN17773auHGjFi5cqKVLl7qyGwAAAGgEPpKKfQMVVnlKprzE1c0BAI9z8uRJ7d271/Y+NzdXOTk5ioiIUPv27Ws9aeDv76/o6Gh17tzZrvo8KpAYNWqUjhw5oueee055eXlKSEjQxx9/rLi4OElSXl6e9u/fbysfHx+vjz/+WA899JD+9Kc/KTY2Vq+99lqjbfkJAAAA17EsqdjndCBRVV7s6uYAgMfJyspS//79be+r1/UZN26cFi1a5PD6PCqQkKSJEydq4sSJdZ6r6wPq16+ftm7d2sitAgAAgKv5yFKJz+nnpQ2BBAA0WFJSkm0jiPNhz7oRZzrLctgAAACAZ/G1pGLf0ztrmAoe2QAAd0cgAQAAAK/ADAkA8CwEEgAAAPAKPpYIJADAgxBIAAAAwCtYOiOQqCCQAAB3RyABAAAAr2BZlkp9Tq8hIbb9BAC3RyABAAAAr1Hie3qGBIEEALg/AgkAAAB4jVLbGhKnXNwSAMC5EEgAAADAa5RWz5Bg208AcHsEEgAAAPAaJb5Bp//ALhsA4PYIJAAAAOA1qh/ZsJghAQBuj0ACAAAAXsP2yAYzJACgwTIzMzV06FDFxsbKsiytXLmyxvnx48fLsqwar969e9tdH4EEAAAAvEaZD2tIAIC9ioqK1K1bN82ZM+esZQYNGqS8vDzb6+OPP7a7Pj+7rwQAAADcTPUMCR7ZAICGS0lJUUpKSr1lAgMDFR0d7ZD6CCQAAADgNcr+s6ilxSMbANyEMUamvMoldVv+PrIsy6H3zMjIUGRkpFq0aKF+/frphRdeUGRkpF33IpAAAACA1yhj208AbsaUV+nQtA0uqTv2uWtkBfg67H4pKSm67bbbFBcXp9zcXD399NO64YYblJ2drcDAwAbfj0ACAAAAXqP8P4GED4EEADjcqFGjbH9OSEhQjx49FBcXp48++kgjR45s8P0IJAAAAOA1ytj2E4Cbsfx9FPvcNS6ruzHFxMQoLi5Oe/bsset6AgkAAAB4jVI/AgkA7sWyLIc+NuFOjhw5ogMHDigmJsau6wkkAAAA4DXKfU4vaulTwaKWANBQJ0+e1N69e23vc3NzlZOTo4iICEVERGj69Om65ZZbFBMTo3379unJJ59U69atdfPNN9tVH4EEAAAAvEY5234CgN2ysrLUv39/2/vU1FRJ0rhx4zRv3jzt2LFDb775po4dO6aYmBj1799f7777rsLCwuyqr8GBRFFRkV588UV9+umnKigoUFVVze1Lvv32W7saAgAAAFyoMt9gSSxqCQD2SEpKkjHmrOfXrFnj0PoaHEjcc889WrduncaOHauYmBiH72kKAAAA2Kvc74xdNoyR+FkVANxWgwOJv//97/roo4907bXXNkZ7AAAAALuV/WcNCctUSZVl0n8CCgCA+2lwINGyZUtFREQ0RlsAAACAC1JxZgBxeI8UaN9zzWhCwmIkvwBXtwJokhocSPz+97/XtGnTtHjxYoWEhDRGmwAAAAC7VPn4qVI+8lWV9AYzenEeHvinFJ3g6lYATdJ5BRKJiYk11orYu3evoqKi1KFDB/n7+9cou3XrVse2EAAAADhPPpaP/i8qWbcdWSdf1o/A+eDvCeAy5xVIjBgxopGbAQAAAFw4y5KmXvqEWl/xZw1o1dzVzQEA1OO8AolnnnnG9ucJEybozjvv1I033sgOGwAAAHArvjr982lVPdvWAQDcg09DLzhy5IiGDBmitm3b6pFHHlFOTk4jNAsAAABoOJ///L6MOAIA3F+DA4lVq1YpPz9fzzzzjLKystS9e3dddtllmjFjhvbt29cITQQAAADOj89/ZkhUMkMCANxegwMJSWrRooXuu+8+ZWRk6LvvvtOECRP01ltv6ZJLLnF0+wAAAIDz5vufGRJV5BEA4PbsCiSqlZeXKysrS5s3b9a+ffsUFRXlqHYBAAAADVa9xFmVa5sBAB4pMzNTQ4cOVWxsrCzL0sqVK2uV2bVrl4YNG6bw8HCFhYWpd+/e2r9/v1312RVIrF27Vvfee6+ioqI0btw4hYWF6YMPPtCBAwfsagQAAADgCD4sagkAdisqKlK3bt00Z86cOs9/88036tu3ry699FJlZGTo3//+t55++mkFBQXZVd957bJxprZt2+rIkSMaOHCg5s+fr6FDh9pdOQAAAOBIPsyQAAC7paSkKCUl5aznn3rqKd100016+eWXbccuuugiu+trcCAxbdo03XbbbWrZsqXdlQIAAACNgW0/AbgbY4zKy8tdUre/v7+s6mfZLlBVVZU++ugjPfbYYxo4cKC2bdum+Ph4PfHEExoxYoRd92xwIHHffffZVREAAADQ2FhDAoC7KS8v14wZM1xS95NPPqmAgACH3KugoEAnT57Uiy++qOeff14vvfSSVq9erZEjR2rt2rXq169fg+95QYtaOtPRo0c1duxYhYeHKzw8XGPHjtWxY8fqvWb8+PGyLKvGq3fv3s5pMAAAAJyObT8BoHFUVZ2OeocPH66HHnpIV155pR5//HENGTJEb7zxhl33bPAMCVcZM2aMvv/+e61evVrS6ZkaY8eO1QcffFDvdYMGDVJaWprtvaPSIQAAALif6jUkyCMAuAt/f389+eSTLqvbUVq3bi0/Pz9ddtllNY536dJF69evt+ueHhFI7Nq1S6tXr9amTZvUq1cvSdKf//xn9enTR7t371bnzp3Pem1gYKCio6Od1VQAAAC4kC+PbABwM5ZlecUvxgMCAtSzZ0/t3r27xvGvv/5acXFxdt3TIwKJjRs3Kjw83BZGSFLv3r0VHh6uDRs21BtIZGRkKDIyUi1atFC/fv30wgsvKDIy8qzlS0tLVVpaant//Phxx3QCAJyAMQyAJ3PEGMa2nwBgv5MnT2rv3r2297m5ucrJyVFERITat2+vRx99VKNGjdL111+v/v37a/Xq1frggw+UkZFhV30esYZEfn5+nSFCZGSk8vPzz3pdSkqK3nnnHX322Wd65ZVXtGXLFt1www01/qH7pZkzZ9rWqQgPD1e7du0c0gcAcAbGMACezBFjWPWilpUObhsANAVZWVlKTExUYmKiJCk1NVWJiYmaNm2aJOnmm2/WG2+8oZdfflldu3bVX/7yF7333nvq27evXfW5NJCYPn16rUUnf/nKysqSpDq3KjHG1LuFyahRozR48GAlJCRo6NCh+vvf/66vv/5aH3300VmveeKJJ1RYWGh7HThw4MI7CgBOwhgGwJM5YgxjhgQA2C8pKUnGmFqvRYsW2crcfffd2rNnj4qLi5WTk6Phw4fbXZ9LH9mYPHmyRo8eXW+ZDh06aPv27frhhx9qnfvxxx8VFRV13vXFxMQoLi5Oe/bsOWuZwMBABQYGnvc9AcCdMIYB8GSOGMOq15AgjgAA9+fSQKJ169Zq3br1Ocv16dNHhYWF+te//qWrr75akrR582YVFhbqmmuuOe/6jhw5ogMHDigmJsbuNgMAAMB9+VjMkAAAT+ERa0h06dJFgwYN0r333qtNmzZp06ZNuvfeezVkyJAaC1peeumlWrFihaTTi3E88sgj2rhxo/bt26eMjAwNHTpUrVu31s033+yqrgAAAKARVf9wW0keAQBuzyMCCUl655131LVrVyUnJys5OVlXXHGF3nrrrRpldu/ercLCQkmSr6+vduzYoeHDh6tTp04aN26cOnXqpI0bNyosLMwVXQAAAEAjs9j2EwA8hkds+ylJERERevvtt+stY86YmhccHKw1a9Y0drMAAADgRnxZ1BIAPIbHzJAAAAAAzsWHRS0BwGMQSAAAAMBrVG/7WckMCQBwewQSAAAA8BrVMySqyCMAwO0RSAAAAMBr2Lb95KENAHB7BBIAAADwGtU/3DJDAgDcH4EEAAAAvIYP234CgN0yMzM1dOhQxcbGyrIsrVy5ssZ5y7LqfP3hD3+wqz4CCQAAAHgNtv0EAPsVFRWpW7dumjNnTp3n8/Lyarz++te/yrIs3XLLLXbV53chjQUAAADcicUMCQCwW0pKilJSUs56Pjo6usb7999/X/3799dFF11kV30EEgAAAPAabPsJwN0YY1RVVeySun18gmVVJ7UO9sMPP+ijjz7S4sWL7b4HgQQAAAC8RvUaEuQRANxFVVWxMtZ1dUndSf12yNc3pFHuvXjxYoWFhWnkyJF234M1JAAAAOA1fNn2EwCc4q9//avuvPNOBQUF2X0PZkgAAADAa7DtJwB34+MTrKR+O1xWd2P4/PPPtXv3br377rsXdB8CCQAAAHiN6kelK13bDACwsSyr0R6bcJWFCxeqe/fu6tat2wXdh0ACAAAAXsOHbT8BwG4nT57U3r17be9zc3OVk5OjiIgItW/fXpJ0/Phx/d///Z9eeeWVC66PQAIAAABew7d6UUvXNgMAPFJWVpb69+9ve5+amipJGjdunBYtWiRJWrZsmYwxuuOOOy64PgIJAAAAeA1mSACA/ZKSkmTOMX7ed999uu+++xxSH7tsAAAAwGtUb/tZSR4BAG6PQAIAAABew4dtPwHAYxBIAAAAwGuw7ScAeA4CCQAAAHiN6kc2qlzbDADAeSCQAAAAgNdgUUsA8BwEEgAAAPAazJAAAM9BIAEAAACvYVvUkhkSAOD2CCQAAADgNVjUEgA8B4EEAAAAvMbPj2yQSACAuyOQAAAAgNfwtS1q6eKGAADOiUACAAAAXsNiUUsAsFtmZqaGDh2q2NhYWZallStX1jh/8uRJTZ48WW3btlVwcLC6dOmiefPm2V0fgQQAAAC8RvW2n5UsagkADVZUVKRu3bppzpw5dZ5/6KGHtHr1ar399tvatWuXHnroIU2ZMkXvv/++XfX5XUhjAQAAAHdSvYYEeQQANFxKSopSUlLOen7jxo0aN26ckpKSJEn33Xef5s+fr6ysLA0fPrzB9RFIAAAAwGv4Vm/7yaKWANyEMUanqlzzIFmIj4+s6mfZHKBv375atWqV7r77bsXGxiojI0Nff/21Xn31VbvuRyABAAAAr8G2nwDczamqKl2cucMldX9zfVeF+vo67H6vvfaa7r33XrVt21Z+fn7y8fHRX/7yF/Xt29eu+xFIAAAAwGtU/yKwkhkSAOBwr732mjZt2qRVq1YpLi5OmZmZmjhxomJiYjRgwIAG349AAgAAAF6jelFL1pAA4C5CfHz0zfVdXVa3oxQXF+vJJ5/UihUrNHjwYEnSFVdcoZycHP3xj38kkAAAAEDT5su2nwDcjGVZDn1swlXKy8tVXl4un1+EHL6+vqqyc40MAgkAAAB4Dbb9BAD7nTx5Unv37rW9z83NVU5OjiIiItS+fXv169dPjz76qIKDgxUXF6d169bpzTff1KxZs+yqz3HzNxrZCy+8oGuuuUYhISFq0aLFeV1jjNH06dMVGxur4OBgJSUl6csvv2zchgIAAMBlqrf9ZFFLAGi4rKwsJSYmKjExUZKUmpqqxMRETZs2TZK0bNky9ezZU3feeacuu+wyvfjii3rhhRf0wAMP2FWfx8yQKCsr02233aY+ffpo4cKF53XNyy+/rFmzZmnRokXq1KmTnn/+ef3617/W7t27FRYW1sgtBgAAgLP5sO0nANgtKSlJpp4ZZtHR0UpLS3NYfR4zQ+LZZ5/VQw89pK5dz28xEGOMZs+eraeeekojR45UQkKCFi9erFOnTmnJkiWN3FoAAAC4QvUPtzyxAQDuz2NmSDRUbm6u8vPzlZycbDsWGBiofv36acOGDbr//vvrvK60tFSlpaW298ePH2/0tgKAozCGAfBkjhjDfNj2EwA8hsfMkGio/Px8SVJUVFSN41FRUbZzdZk5c6bCw8Ntr3bt2jVqOwHAkRjDAHgyR4xh1YtasoYEALg/lwYS06dPl2VZ9b6ysrIuqA7rP88RVjPG1Dp2pieeeEKFhYW214EDBy6ofgBwJsYwAJ7MEWOYD9t+AoDHcOkjG5MnT9bo0aPrLdOhQwe77h0dHS3p9EyJmJgY2/GCgoJasybOFBgYqMDAQLvqBABXYwwD4MkcMYb9PEOCKRIA4O5cGki0bt1arVu3bpR7x8fHKzo6Wunp6bYtS8rKyrRu3Tq99NJLjVInAAAAXIttPwHAc3jMGhL79+9XTk6O9u/fr8rKSuXk5CgnJ0cnT560lbn00ku1YsUKSacf1Zg6dapmzJihFStW6IsvvtD48eMVEhKiMWPGuKobAAAAaETVP9yy7ScAuD+P2WVj2rRpWrx4se199ayHtWvXKikpSZK0e/duFRYW2so89thjKi4u1sSJE3X06FH16tVLn3zyicLCwpzadgAAADiHr8WilgDgKTwmkFi0aJEWLVpUbxnzi2cFLcvS9OnTNX369MZrGAAAANyGxaKWAOAxPOaRDQAAAOBcWNQSADwHgQQAAAC8Btt+AoD9MjMzNXToUMXGxsqyLK1cubLG+R9++EHjx49XbGysQkJCNGjQIO3Zs8fu+ggkAAAA4DV+XkOCGRIA0FBFRUXq1q2b5syZU+ucMUYjRozQt99+q/fff1/btm1TXFycBgwYoKKiIrvq85g1JAAAAIBzqf5tWyV5BAA0WEpKilJSUuo8t2fPHm3atElffPGFLr/8cknS3LlzFRkZqaVLl+qee+5pcH0EEgAAAPAaPy9qSSIBwD0YY1RcXumSuoP9fWVVD4wXqLS0VJIUFBRkO+br66uAgACtX7+eQAIAAABNm+9/FrXkiQ0A7qK4vFKXTVvjkrp3PjdQIQGO+W//pZdeqri4OD3xxBOaP3++QkNDNWvWLOXn5ysvL8+ue7KGBAAAALwGi1oCQOPw9/fXe++9p6+//loREREKCQlRRkaGUlJS5Ovra9c9mSEBAAAAr1G97WclUyQAuIlgf1/tfG6gy+p2pO7duysnJ0eFhYUqKytTmzZt1KtXL/Xo0cOu+xFIAAAAwGswQwKAu7Esy2GPTbiL8PBwSacXuszKytLvf/97u+7jXZ8KAAAAmjQfq3oNCWZIAEBDnTx5Unv37rW9z83NVU5OjiIiItS+fXv93//9n9q0aaP27dtrx44d+q//+i+NGDFCycnJdtVHIAEAAACvUb1AGjMkAKDhsrKy1L9/f9v71NRUSdK4ceO0aNEi5eXlKTU1VT/88INiYmJ011136emnn7a7PgIJAAAAeI3qRzZYQwIAGi4pKaneGWYPPvigHnzwQYfVxy4bAAAA8BrVi1pWkUcAgNsjkAAAAIDXYFFLAPAcBBIAAADwGtUzJCQWtgQAd0cgAQAAAK/h83MeoUryCABwawQSAAAA8Bpn/nBbJRIJAHBnBBIAAADwGr7Wz1MkWNgSANwbgQQAAAC8hnXmIxvMkAAAt0YgAQAAAK9Rc1FLFzYEAHBOBBIAAADwGmcuasnWnwDg3ggkAAAA4DV8deYaEkyRAAB3RiABAAAAr8G2nwBgn5kzZ6pnz54KCwtTZGSkRowYod27d9coY4zR9OnTFRsbq+DgYCUlJenLL7+0u04CCQAAAHiNM/IItv0EgAZYt26dJk2apE2bNik9PV0VFRVKTk5WUVGRrczLL7+sWbNmac6cOdqyZYuio6P161//WidOnLCrTj9HNR4AAABwNcuy5KPT60fwxAYAnL/Vq1fXeJ+WlqbIyEhlZ2fr+uuvlzFGs2fP1lNPPaWRI0dKkhYvXqyoqCgtWbJE999/f4PrJJAAAACAV/GxpCojDd66R37Wucvjwvyhczv1bRnm6mYA7ssYqfyUa+r2D6m5H3IDFBYWSpIiIiIkSbm5ucrPz1dycrKtTGBgoPr166cNGzYQSAAAAAAXBQfp61MlOlBS5uqmNAmnKtnPBKhX+SlpRqxr6n7ykBQQ2uDLjDFKTU1V3759lZCQIEnKz8+XJEVFRdUoGxUVpe+++86u5hFIAAAAwKt83L2jdhWVuLoZTcYlIYGubgIAB5s8ebK2b9+u9evX1zpn/WLGhTGm1rHzRSABAAAAr9LMz1c9wxv+G0EAaBT+IadnKriq7gaaMmWKVq1apczMTLVt29Z2PDo6WtLpmRIxMTG24wUFBbVmTZwvAgkAAAAAABqLZdn12ISzGWM0ZcoUrVixQhkZGYqPj69xPj4+XtHR0UpPT1diYqIkqaysTOvWrdNLL71kV50EEgAAAAAANHGTJk3SkiVL9P777yssLMy2ZkR4eLiCg4NlWZamTp2qGTNmqGPHjurYsaNmzJihkJAQjRkzxq46CSQAAAAAAGji5s2bJ0lKSkqqcTwtLU3jx4+XJD322GMqLi7WxIkTdfToUfXq1UuffPKJwsLs22mHQAIAAAAAgCbOGHPOMpZlafr06Zo+fbpD6vRxyF0AAAAAAAAagEACAAAAAAA4HY9snEP1tJXjx4+7uCUAmoLqseZ8psydD8YwAM7EGAbAkzl6DMO5EUicw4kTJyRJ7dq1c3FLADQlJ06cUHh4uEPuIzGGAXAuxjAAnsxRYxjOjUDiHGJjY3XgwAGFhYXJsqzzuub48eNq166dDhw4oObNmzdyC53Pm/vnzX2T6J8nMMboxIkTio2Ndcj9GMNq8+b+eXPfJPrnCRjDGp8398+b+ybRP0/g6DEM50YgcQ4+Pj5q27atXdc2b97cY78Zz4c398+b+ybRP3fnyESeMezsvLl/3tw3if65O8Yw5/Dm/nlz3yT65+6YGeFcLGoJAAAAAACcjkACAAAAAAA4HYFEIwgMDNQzzzyjwMBAVzelUXhz/7y5bxL9w/nx9s/Rm/vnzX2T6B/Oj7d/jt7cP2/um0T/gLpYhj1NAAAAAABwiJKSEuXm5io+Pl5BQUGubo7L1fd5MEMCAAAAAIAmbubMmerZs6fCwsIUGRmpESNGaPfu3TXKLF++XAMHDlTr1q1lWZZycnIuqE4CCQAAAAAAmrh169Zp0qRJ2rRpk9LT01VRUaHk5GQVFRXZyhQVFenaa6/Viy++6JA62fYTAAAAAIAmbvXq1TXep6WlKTIyUtnZ2br++uslSWPHjpUk7du3zyF1EkgAAAAAANBIjDEqrih2Sd3BfsGyLMuuawsLCyVJERERjmxSDTyy4WBz5861LdbRvXt3ff75565uUi3Tp0+XZVk1XtHR0bbzxhhNnz5dsbGxCg4OVlJSkr788ssa9ygtLdWUKVPUunVrhYaGatiwYfr+++9rlDl69KjGjh2r8PBwhYeHa+zYsTp27JjD+5OZmamhQ4cqNjZWlmVp5cqVNc47sz/79+/X0KFDFRoaqtatW+vBBx9UWVlZo/Vt/Pjxtb6WvXv39oi+nc8zap78tfNUjGE/YwxjDKsPY5h7Ygz7GWMYY1h9GMOcq7iiWL2W9HLJy94gxBij1NRU9e3bVwkJCQ7+RH5GIOFA7777rqZOnaqnnnpK27Zt03XXXaeUlBTt37/f1U2r5fLLL1deXp7ttWPHDtu5l19+WbNmzdKcOXO0ZcsWRUdH69e//rVOnDhhKzN16lStWLFCy5Yt0/r163Xy5EkNGTJElZWVtjJjxoxRTk6OVq9erdWrVysnJ8c2xceRioqK1K1bN82ZM6fO887qT2VlpQYPHqyioiKtX79ey5Yt03vvvaeHH3640fomSYMGDarxtfz4449rnHfXvp3PM2qe/LXzRIxhjGGMYeePMcz9MIYxhjGGnT/GMJzL5MmTtX37di1durRxKzJwmKuvvto88MADNY5deuml5vHHH3dRi+r2zDPPmG7dutV5rqqqykRHR5sXX3zRdqykpMSEh4ebN954wxhjzLFjx4y/v79ZtmyZrczBgweNj4+PWb16tTHGmJ07dxpJZtOmTbYyGzduNJLMV1991Qi9Ok2SWbFihUv68/HHHxsfHx9z8OBBW5mlS5eawMBAU1hY6PC+GWPMuHHjzPDhw896jaf0zRhjCgoKjCSzbt06Y4x3fe08BWMYYxhjmP0Yw1yPMYwxjDHMfoxhjlVcXGx27txpiouLjTGnP8+isiKXvKqqqhrc/smTJ5u2bduab7/99qxlcnNzjSSzbdu2Bn8eZ2KGhIOUlZUpOztbycnJNY4nJydrw4YNLmrV2e3Zs0exsbGKj4/X6NGj9e2330qScnNzlZ+fX6MfgYGB6tevn60f2dnZKi8vr1EmNjZWCQkJtjIbN25UeHi4evXqZSvTu3dvhYeHO/XzcGZ/Nm7cqISEBMXGxtrKDBw4UKWlpcrOzm60PmZkZCgyMlKdOnXSvffeq4KCAts5T+rbL59RawpfO3fCGMYYxhh2YRjDXIsxjDGMMezCMIY1LsuyFOIf4pJXQ9aPMMZo8uTJWr58uT777DPFx8c34qdyGoGEgxw+fFiVlZWKioqqcTwqKkr5+fkualXdevXqpTfffFNr1qzRn//8Z+Xn5+uaa67RkSNHbG2trx/5+fkKCAhQy5Yt6y0TGRlZq+7IyEinfh7O7E9+fn6telq2bKmAgIBG63NKSoreeecdffbZZ3rllVe0ZcsW3XDDDSotLbW1yRP6Zup4Rs3bv3buhjGMMYwxzH6MYa7HGMYYxhhmP8YwVJs0aZLefvttLVmyRGFhYcrPz1d+fr6Ki39eh+Knn35STk6Odu7cKUnavXu3cnJy7P5ascuGg/0ygTLG2L2qaWNJSUmx/blr167q06ePLr74Yi1evNi2EI89/fhlmbrKu+rzcFZ/nN3nUaNG2f6ckJCgHj16KC4uTh999JFGjhx51uvcrW/Vz6itX7++1jlv/dq5K8Yw9/w74a3fB4xhdXO3/nkSxjD3/Dvhrd8HjGF1c7f+4fzNmzdPkpSUlFTjeFpamsaPHy9JWrVqlSZMmGA7N3r0aEnSM888o+nTpze4TmZIOEjr1q3l6+tbKxkqKCiolfi5m9DQUHXt2lV79uyxrfJcXz+io6NVVlamo0eP1lvmhx9+qFXXjz/+6NTPw5n9iY6OrlXP0aNHVV5e7rQ+x8TEKC4uTnv27LG1yd37NmXKFK1atUpr165V27Ztbceb2tfO1RjDGMPc4fuAMcz9+ucpGMMYw9zh+4AxzP36h4YxxtT5qg4jpNO7y9RVxp4wQiKQcJiAgAB1795d6enpNY6np6frmmuucVGrzk9paal27dqlmJgYxcfHKzo6ukY/ysrKtG7dOls/unfvLn9//xpl8vLy9MUXX9jK9OnTR4WFhfrXv/5lK7N582YVFhY69fNwZn/69OmjL774Qnl5ebYyn3zyiQIDA9W9e/dG7We1I0eO6MCBA4qJiZHk3n071zNqTe1r52qMYYxh7vB9wBjmPv3zNIxhjGHu8H3AGOY+/YMHOeeSmDhvy5YtM/7+/mbhwoVm586dZurUqSY0NNTs27fP1U2r4eGHHzYZGRnm22+/NZs2bTJDhgwxYWFhtna++OKLJjw83Cxfvtzs2LHD3HHHHSYmJsYcP37cdo8HHnjAtG3b1vzjH/8wW7duNTfccIPp1q2bqaiosJUZNGiQueKKK8zGjRvNxo0bTdeuXc2QIUMc3p8TJ06Ybdu2mW3bthlJZtasWWbbtm3mu+++c2p/KioqTEJCgrnxxhvN1q1bzT/+8Q/Ttm1bM3ny5Ebp24kTJ8zDDz9sNmzYYHJzc83atWtNnz59zK9+9SuP6Ntvf/tbEx4ebjIyMkxeXp7tderUKVsZT/7aeSLGMMYwxrDzxxjmfhjDGMMYw84fY1jjqm9Xiaaovs+DQMLB/vSnP5m4uDgTEBBgrrrqKtvWOe5k1KhRJiYmxvj7+5vY2FgzcuRI8+WXX9rOV1VVmWeeecZER0ebwMBAc/3115sdO3bUuEdxcbGZPHmyiYiIMMHBwWbIkCFm//79NcocOXLE3HnnnSYsLMyEhYWZO++80xw9etTh/Vm7dq2RVOs1btw4p/fnu+++M4MHDzbBwcEmIiLCTJ482ZSUlDRK306dOmWSk5NNmzZtjL+/v2nfvr0ZN25crXa7a9/q6pckk5aWZivjyV87T8UY9jPGMMaw+jCGuSfGsJ8xhjGG1YcxrHERSNRU3+dhGWOMI2dcAAAAAADQVJWUlCg3N1fx8fEKCgpydXNcrr7PgzUkAAAAAACA0xFIAAAAAAAApyOQAAAAAAAATkcgAQAAAAAAnI5AAgAAAAAAOB2BBAAAAAAAcDoCCQAAAAAAmriZM2eqZ8+eCgsLU2RkpEaMGKHdu3fbzpeXl+t3v/udunbtqtDQUMXGxuquu+7SoUOH7K6TQAJooIyMDFmWpWPHjrm6KQDQYIxhADwZYxjQeNatW6dJkyZp06ZNSk9PV0VFhZKTk1VUVCRJOnXqlLZu3aqnn35aW7du1fLly/X1119r2LBhdtdpGWOMozoAeKOkpCRdeeWVmj17tiSprKxMP/30k6KiomRZlmsbBwDnwBgGwJMxhsETlZSUKDc3V/Hx8QoKCnJ1c+z2448/KjIyUuvWrdP1119fZ5ktW7bo6quv1nfffaf27dvXWaa+z8PP4a0GvFxAQICio6Nd3QwAsAtjGABPxhgGT2SMkSkudkndVnCw3eFdYWGhJCkiIqLeMpZlqUWLFnbVwSMbQD3Gjx+vdevW6dVXX5VlWbIsS4sWLaoxVXDRokVq0aKFPvzwQ3Xu3FkhISG69dZbVVRUpMWLF6tDhw5q2bKlpkyZosrKStu9y8rK9Nhjj+lXv/qVQkND1atXL2VkZLimowC8EmMYAE/GGAZvYYqLtfuq7i552RuEGGOUmpqqvn37KiEhoc4yJSUlevzxxzVmzBg1b97crnqYIQHU49VXX9XXX3+thIQEPffcc5KkL7/8sla5U6dO6bXXXtOyZct04sQJjRw5UiNHjlSLFi308ccf69tvv9Utt9yivn37atSoUZKkCRMmaN++fVq2bJliY2O1YsUKDRo0SDt27FDHjh2d2k8A3okxDIAnYwwDXGfy5Mnavn271q9fX+f58vJyjR49WlVVVZo7d67d9RBIAPUIDw9XQECAQkJCbNMDv/rqq1rlysvLNW/ePF188cWSpFtvvVVvvfWWfvjhBzVr1kyXXXaZ+vfvr7Vr12rUqFH65ptvtHTpUn3//feKjY2VJD3yyCNavXq10tLSNGPGDOd1EoDXYgwD4MkYw+AtrOBgdd6a7bK6G2rKlClatWqVMjMz1bZt21rny8vLdfvttys3N1efffaZ3bMjJAIJwCFCQkJs/whKUlRUlDp06KBmzZrVOFZQUCBJ2rp1q4wx6tSpU437lJaWqlWrVs5pNAD8B2MYAE/GGAZ3Z1mWrJAQVzfjnIwxmjJlilasWKGMjAzFx8fXKlMdRuzZs0dr16694O8ZAgnAAfz9/Wu8tyyrzmNVVVWSpKqqKvn6+io7O1u+vr41yp35jycAOANjGABPxhgGOMakSZO0ZMkSvf/++woLC1N+fr6k07OVgoODVVFRoVtvvVVbt27Vhx9+qMrKSluZiIgIBQQENLhOAgngHAICAmosguQIiYmJqqysVEFBga677jqH3hsAzsQYBsCTMYYBzjNv3jxJp7fbPVNaWprGjx+v77//XqtWrZIkXXnllTXKrF27ttZ154NAAjiHDh06aPPmzdq3b5+aNWtmS9cvRKdOnXTnnXfqrrvu0iuvvKLExEQdPnxYn332mbp27aqbbrrJAS0HAMYwAJ6NMQxwHmNMvec7dOhwzjINxbafwDk88sgj8vX11WWXXaY2bdpo//79DrlvWlqa7rrrLj388MPq3Lmzhg0bps2bN6tdu3YOuT8ASIxhADwbYxjg3Szj6IgDAAAAAIAmqqSkRLm5uYqPj1dQUJCrm+Ny9X0ezJAAAAAAAABORyABAAAAAACcjkACAAAAAAA4HYEEAAAAAABwOgIJAAAAAADgdAQSAAAAAADA6QgkAAAAAACA0xFIAAAAAAAApyOQAAAAAACgiZs5c6Z69uypsLAwRUZGasSIEdq9e3eNMtOnT9ell16q0NBQtWzZUgMGDNDmzZvtrpNAAgAAAACAJm7dunWaNGmSNm3apPT0dFVUVCg5OVlFRUW2Mp06ddKcOXO0Y8cOrV+/Xh06dFBycrJ+/PFHu+q0jDHGUR0AAAAAAKApKykpUW5uruLj4xUUFOTq5tjtxx9/VGRkpNatW6frr7++zjLHjx9XeHi4/vGPf+jGG2+ss0x9n4efw1sNAAAAAAAkScYYVZRVuaRuvwAfWZZl17WFhYWSpIiIiDrPl5WVacGCBQoPD1e3bt3sa59dVwEAAAAAgHOqKKvSgv9a55K673u1n/wDfRt8nTFGqamp6tu3rxISEmqc+/DDDzV69GidOnVKMTExSk9PV+vWre1qH2tIAAAAAAAAm8mTJ2v79u1aunRprXP9+/dXTk6ONmzYoEGDBun2229XQUGBXfWwhgQAAAAAAA7yyzUTPO2RjSlTpmjlypXKzMxUfHz8Oct37NhRd999t5544ok6z7OGBAAAAAAALmBZll2PTTibMUZTpkzRihUrlJGRcV5hRPV1paWldtVJIAEAAAAAQBM3adIkLVmyRO+//77CwsKUn58vSQoPD1dwcLCKior0wgsvaNiwYYqJidGRI0c0d+5cff/997rtttvsqpNAAgAAAACAJm7evHmSpKSkpBrH09LSNH78ePn6+uqrr77S4sWLdfjwYbVq1Uo9e/bU559/rssvv9yuOgkkAAAAAABo4s61vGRQUJCWL1/u0DrZZQMAAAAAADgdgQQAAAAAAHA6AgkAAAAAAOB0BBIAAAAAAMDpCCQAAAAAAIDTEUgAAAAAAACnI5AAAAAAAABORyABAAAAAACcjkACAAAAAAA4HYEEAAAAAABwOgIJAAAAAACauJkzZ6pnz54KCwtTZGSkRowYod27d5+1/P333y/LsjR79my76ySQAAAAAACgiVu3bp0mTZqkTZs2KT09XRUVFUpOTlZRUVGtsitXrtTmzZsVGxt7QXX6XdDVAAAAAADA461evbrG+7S0NEVGRio7O1vXX3+97fjBgwc1efJkrVmzRoMHD76gOgkkAAAAAABoJMYYVZSWuqRuv8BAWZZl17WFhYWSpIiICNuxqqoqjR07Vo8++qguv/zyC2/fBd8BAAAAAADUqaK0VK+Nu9UldT+4+G/yDwpq8HXGGKWmpqpv375KSEiwHX/ppZfk5+enBx980CHtI5AAAAAAAAA2kydP1vbt27V+/XrbsezsbL366qvaunWr3bMufskyxhiH3AkAAAAAgCaupKREubm5io+PV1BQkMc9sjFlyhStXLlSmZmZio+Ptx2fPXu2UlNT5ePz894YlZWV8vHxUbt27bRv37467/fLz6NG+xrUMgAAAAAAcN4sy7LrsQlnM8ZoypQpWrFihTIyMmqEEZI0duxYDRgwoMaxgQMHauzYsZowYYJddRJIAAAAAADQxE2aNElLlizR+++/r7CwMOXn50uSwsPDFRwcrFatWqlVq1Y1rvH391d0dLQ6d+5sV50+5y4CAAAAAAC82bx581RYWKikpCTFxMTYXu+++26j1ckMCQAAAAAAmjh7lpc827oR54sZEgAAAAAAwOkIJAAAAAAAgNMRSAAAAAAAAKcjkAAAAAAAAE5HIAEAAAAAgIPZs0ikN6rvcyCQAAAAAADAQfz9/SVJp06dcnFL3EP151D9uZyJbT8BAAAAAHAQX19ftWjRQgUFBZKkkJAQWZbl4lY5nzFGp06dUkFBgVq0aCFfX99aZSzDPBIAAAAAABzGGKP8/HwdO3bM1U1xuRYtWig6OrrOUIZAAgAAAACARlBZWany8nJXN8Nl/P3965wZUY1AAgAAAAAAOB2LWgIAAAAAAKcjkAAAAAAAAE5HIAEAAAAAAJyOQAIAAAAAADgdgQQAAAAAAHA6AgkAAAAAAOB0BBIAAAAAAMDp/j/wI+7NcdMSwwAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -124,7 +194,7 @@ } ], "source": [ - "swiftdiff['vh'].plot(x=\"time\",hue=\"id\",col=\"space\")" + "swiftdiff.sel(id=tpid)['vh'].plot(x=\"time\",hue=\"id\",col=\"space\")" ] }, { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6212ee56b..a0ae5a554 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,7 @@ # Add the source files SET(STRICT_MATH_FILES + ${SRC}/collision/collision_generate.f90 ${SRC}/fraggle/fraggle_generate.f90 ${SRC}/fraggle/fraggle_util.f90 ${SRC}/fraggle/fraggle_module.f90 @@ -59,7 +60,6 @@ SET(FAST_MATH_FILES ${SRC}/helio/helio_module.f90 ${SRC}/symba/symba_module.f90 ${SRC}/collision/collision_check.f90 - ${SRC}/collision/collision_generate.f90 ${SRC}/collision/collision_io.f90 ${SRC}/collision/collision_regime.f90 ${SRC}/collision/collision_resolve.f90 From 297dedf577f0a22955f488b8e64a9985cc676352 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 18 May 2023 09:28:37 -0400 Subject: [PATCH 069/149] Bug fixes uncovered when compiling in gfortran with coarrays turned off --- src/misc/solver_module.f90 | 15 +++++++++------ src/netcdf_io/netcdf_io_implementations.f90 | 4 ++++ src/swiftest/swiftest_driver.f90 | 2 ++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/misc/solver_module.f90 b/src/misc/solver_module.f90 index 410116b2a..936914561 100644 --- a/src/misc/solver_module.f90 +++ b/src/misc/solver_module.f90 @@ -424,22 +424,25 @@ end function f ! returns the minimum of two real numbers real(DP) Function Minimum(x1,x2) real(DP) x1,x2,resultat + if (x1 < x2) then - resultat = x1 + resultat = x1 else - resultat = x2 + resultat = x2 endif + Minimum = resultat end function Minimum ! TRUE if x1*x2 negative - integer Function RootBracketed(x1,x2) + logical Function RootBracketed(x1,x2) real(DP) x1,x2 - integer resultat + logical resultat + if ((x1 > 0.and.x2 > 0).or.(x1 < 0.and.x2 < 0)) then - resultat = 0 + resultat = .false. else - resultat = 1 + resultat = .true. endif RootBracketed = resultat end function RootBracketed diff --git a/src/netcdf_io/netcdf_io_implementations.f90 b/src/netcdf_io/netcdf_io_implementations.f90 index 40c561183..831b17902 100644 --- a/src/netcdf_io/netcdf_io_implementations.f90 +++ b/src/netcdf_io/netcdf_io_implementations.f90 @@ -40,8 +40,12 @@ module subroutine netcdf_io_close(self) character(namelen) :: message if (self%lfile_is_open) then +#ifdef COARRAY write(message,*) this_image() message = "netcdf_io_close on image " // trim(adjustl(message)) +#else + message = "netcdf_io_close" +#endif call netcdf_io_check( nf90_close(self%id), message) self%lfile_is_open = .false. end if diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index d8e2ae219..6cec24feb 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -33,7 +33,9 @@ program swiftest_driver param%integrator = trim(adjustl(integrator)) param%display_style = trim(adjustl(display_style)) call param%read_in(param_file_name) +#ifdef COARRAY if (.not.param%lcoarray .and. (this_image() /= 1)) stop ! Single image mode +#endif associate(t0 => param%t0, & tstart => param%tstart, & From 29afacf2e187dc1fdd866dfcf683173384d9c43f Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 18 May 2023 10:50:02 -0400 Subject: [PATCH 070/149] Changed how coarray preprocessor variable is defined to use the older CMAKE TARGET_COMPILE_DEFINITIONS instead of the newer ADD_DEFINITIONS --- CMakeLists.txt | 4 ---- src/CMakeLists.txt | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a9ed99c0..0aed20380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,10 +31,6 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) -IF (USE_COARRAY) - ADD_DEFINITIONS(-DCOARRAY) -ENDIF() - # Locate and set parallelization libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know # about Fortran. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a0ae5a554..4d731373d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -111,6 +111,8 @@ SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON) # # Uncomment if you need to link to BLAS and LAPACK TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${NETCDF_LIBRARIES} ${NETCDF_FORTRAN_LIBRARIES}) + + IF(USE_OPENMP) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}" @@ -118,6 +120,7 @@ IF(USE_OPENMP) ENDIF(USE_OPENMP) IF(USE_COARRAY) + TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DCOARRAY) SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES COMPILE_FLAGS "${Coarray_Fortran_FLAGS}" LINK_FLAGS "${Coarray_Fortran_FLAGS}") From db6ad98ada6f7ed11ca5a87a6d406b7fef01b58b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 18 May 2023 10:50:55 -0400 Subject: [PATCH 071/149] Fixed issue where the undefined nplm was being reported in non-SyMBA runs --- src/swiftest/swiftest_io.f90 | 2 +- src/swiftest/swiftest_module.f90 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 96f974362..e5485c24a 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -285,7 +285,7 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t call self%compact_output(param,integration_timer) end if - if (self%pl%nplm > 0) then + if (param%lmtiny_pl) then #ifdef COARRAY if (param%lcoarray) then write(param%display_unit, co_symbastatfmt) this_image(),self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index f7be030d1..e3664a9ed 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -248,7 +248,7 @@ module swiftest integer(I8B) :: nplpl !! Number of body-body comparisons in the flattened upper triangular matrix type(swiftest_kinship), dimension(:), allocatable :: kin !! Array of merger relationship structures that can account for multiple pairwise mergers in a single step logical, dimension(:), allocatable :: lmtiny !! flag indicating whether this body is below the GMTINY cutoff value - integer(I4B) :: nplm !! number of bodies above the GMTINY limit + integer(I4B) :: nplm = 0 !! number of bodies above the GMTINY limit integer(I8B) :: nplplm !! Number of body (all massive)-body (only those above GMTINY) comparisons in the flattened upper triangular matrix integer(I4B), dimension(:), allocatable :: nplenc !! number of encounters with other planets this time step integer(I4B), dimension(:), allocatable :: ntpenc !! number of encounters with test particles this time step From 43ae5edd07e5c3f2ced9ae46b88626d850ab2b58 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 18 May 2023 15:04:33 -0400 Subject: [PATCH 072/149] Switched variable from associated to integer based on an error that came up when compiling with ifx --- src/collision/collision_util.f90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 95152c0aa..74e634931 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -21,14 +21,15 @@ module subroutine collision_util_add_fragments_to_collider(self, nbody_system, p class(base_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object class(base_parameters), intent(in) :: param !! Current Swiftest run configuration parameters ! Internals - integer(I4B) :: i, npl_before, npl_after + integer(I4B) :: i, npl_before, npl_after, nfrag logical, dimension(:), allocatable :: lexclude select type(nbody_system) class is (swiftest_nbody_system) - associate(fragments => self%fragments, impactors => self%impactors, nfrag => self%fragments%nbody, pl => nbody_system%pl, cb => nbody_system%cb) + associate(fragments => self%fragments, impactors => self%impactors, pl => nbody_system%pl, cb => nbody_system%cb) npl_after = pl%nbody npl_before = npl_after - nfrag + nfrag = self%fragments%nbody allocate(lexclude(npl_after)) pl%status(npl_before+1:npl_after) = ACTIVE From 9d78839a97235e4253b97bdb21acd4186a52a41e Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 18 May 2023 15:18:05 -0400 Subject: [PATCH 073/149] Added a regex expression to catch compiler options that are not supported by ifx --- cmake/Modules/SetCompileFlag.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/Modules/SetCompileFlag.cmake b/cmake/Modules/SetCompileFlag.cmake index 4141c4773..d094009ed 100644 --- a/cmake/Modules/SetCompileFlag.cmake +++ b/cmake/Modules/SetCompileFlag.cmake @@ -45,6 +45,7 @@ FUNCTION(SET_COMPILE_FLAG FLAGVAR FLAGVAL LANG) SET(FAIL_REGEX "ignoring unknown option" # Intel "invalid argument" # Intel + "not supported" # Intel ifx "unrecognized .*option" # GNU "[Uu]nknown switch" # Portland Group "ignoring unknown option" # MSVC From 862888f64c490aedfc834712999531913edabccf Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 18 May 2023 15:19:21 -0400 Subject: [PATCH 074/149] Another variable that needed to be set to integer due to ifx rules on do concurrent --- src/collision/collision_util.f90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 74e634931..f4f8e341b 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -150,7 +150,7 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par class(base_parameters), intent(inout) :: param !! Current Swiftest run configuration parameters character(len=*), intent(in) :: phase !! One of "before" or "after", indicating which phase of the calculation this needs to be done ! Internals - integer(I4B) :: i, phase_val + integer(I4B) :: i, phase_val, nfrag select case(phase) case("before") @@ -166,7 +166,8 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par class is (swiftest_nbody_system) select type(param) class is (swiftest_parameters) - associate(fragments => self%fragments, impactors => self%impactors, nfrag => self%fragments%nbody, pl => nbody_system%pl, cb => nbody_system%cb) + associate(fragments => self%fragments, impactors => self%impactors, pl => nbody_system%pl, cb => nbody_system%cb) + nfrag = self%fragments%nbody if (phase_val == 1) then do concurrent(i = 1:2) impactors%ke_orbit(i) = 0.5_DP * impactors%mass(i) * dot_product(impactors%vc(:,i), impactors%vc(:,i)) From 4d527678850526806261d9a917861cae81b0d102 Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 18 May 2023 16:36:30 -0400 Subject: [PATCH 075/149] Fixed do concurrents to remove the associated variables from loop index --- src/collision/collision_resolve.f90 | 2 +- src/collision/collision_util.f90 | 2 +- src/fraggle/fraggle_generate.f90 | 15 ++++---- src/helio/helio_gr.f90 | 10 +++--- src/helio/helio_kick.f90 | 10 +++--- src/rmvs/rmvs_kick.f90 | 6 ++-- src/swiftest/swiftest_obl.f90 | 15 ++++---- src/swiftest/swiftest_util.f90 | 35 ++++++++++-------- src/whm/whm_gr.f90 | 13 ++++--- src/whm/whm_kick.f90 | 56 +++++++++++++++-------------- 10 files changed, 94 insertions(+), 70 deletions(-) diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index dadde1dfd..befc4ba8a 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -350,7 +350,7 @@ module subroutine collision_resolve_mergeaddsub(nbody_system, param, t, status) class is (swiftest_nbody_system) select type(param) class is (swiftest_parameters) - associate(pl => nbody_system%pl, pl_discards => nbody_system%pl_discards, info => nbody_system%pl%info, pl_adds => nbody_system%pl_adds, cb => nbody_system%cb, npl => nbody_system%pl%nbody, & + associate(pl => nbody_system%pl, pl_discards => nbody_system%pl_discards, info => nbody_system%pl%info, pl_adds => nbody_system%pl_adds, cb => nbody_system%cb, & collider => nbody_system%collider, impactors => nbody_system%collider%impactors,fragments => nbody_system%collider%fragments) ! Add the impactors%id bodies to the subtraction list diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index f4f8e341b..45f4f516d 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -536,7 +536,7 @@ module subroutine collision_util_set_coordinate_collider(self) ! Arguments class(collision_basic), intent(inout) :: self !! Collisional nbody_system - associate(fragments => self%fragments, impactors => self%impactors, nfrag => self%fragments%nbody) + associate(fragments => self%fragments, impactors => self%impactors) call impactors%set_coordinate_system() if (.not.allocated(self%fragments)) return diff --git a/src/fraggle/fraggle_generate.f90 b/src/fraggle/fraggle_generate.f90 index 93b4c3431..f1ba8d606 100644 --- a/src/fraggle/fraggle_generate.f90 +++ b/src/fraggle/fraggle_generate.f90 @@ -350,7 +350,7 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu real(DP), dimension(2) :: fragment_cloud_radius logical, dimension(collider%fragments%nbody) :: loverlap real(DP), dimension(collider%fragments%nbody) :: mass_rscale, phi, theta, u - integer(I4B) :: i, j, loop, istart + integer(I4B) :: i, j, loop, istart, nfrag, npl, ntp logical :: lsupercat, lhitandrun integer(I4B), parameter :: MAXLOOP = 10000 real(DP), parameter :: cloud_size_scale_factor = 3.0_DP ! Scale factor to apply to the size of the cloud relative to the distance from the impact point. @@ -358,8 +358,10 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu real(DP), parameter :: rbuffer = 1.01_DP ! Body radii are inflated by this scale factor to prevent secondary collisions real(DP), parameter :: pack_density = 0.5236_DP ! packing density of loose spheres - associate(fragments => collider%fragments, impactors => collider%impactors, nfrag => collider%fragments%nbody, & - pl => nbody_system%pl, tp => nbody_system%tp, npl => nbody_system%pl%nbody, ntp => nbody_system%tp%nbody) + associate(fragments => collider%fragments, impactors => collider%impactors, pl => nbody_system%pl, tp => nbody_system%tp) + nfrag = collider%fragments%nbody + npl = nbody_system%pl%nbody + ntp = nbody_system%tp%nbody lsupercat = (impactors%regime == COLLRESOLVE_REGIME_SUPERCATASTROPHIC) lhitandrun = (impactors%regime == COLLRESOLVE_REGIME_HIT_AND_RUN) @@ -508,7 +510,7 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i + integer(I4B) :: i, nfrag real(DP), parameter :: frag_rot_fac = 0.1_DP ! Fraction of projectile rotation magnitude to add as random noise to fragment rotation real(DP), parameter :: hitandrun_momentum_transfer = 0.01_DP ! Fraction of projectile momentum transfered to target in a hit and run real(DP) :: mass_fac @@ -516,7 +518,8 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) integer(I4B), parameter :: MAXLOOP = 10 logical :: lhitandrun - associate(fragments => collider%fragments, impactors => collider%impactors, nfrag => collider%fragments%nbody) + associate(fragments => collider%fragments, impactors => collider%impactors) + nfrag = collider%fragments%nbody lhitandrun = (impactors%regime == COLLRESOLVE_REGIME_HIT_AND_RUN) ! Initialize fragment rotations and velocities to be pre-impact rotation for body 1, and randomized for bodies >1 and scaled to the original rotation. @@ -805,7 +808,7 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu L_residual(:) = (collider%L_total(:,2) - collider%L_total(:,1)) call collision_util_velocity_torque(-L_residual(:), collider%fragments%mtot, impactors%rbcom, impactors%vbcom) - do concurrent(i = 1:collider%fragments%nbody) + do concurrent(i = 1:nfrag) collider%fragments%vb(:,i) = collider%fragments%vc(:,i) + impactors%vbcom(:) end do diff --git a/src/helio/helio_gr.f90 b/src/helio/helio_gr.f90 index 2092fca6a..def25224d 100644 --- a/src/helio/helio_gr.f90 +++ b/src/helio/helio_gr.f90 @@ -71,11 +71,12 @@ pure module subroutine helio_gr_p4_pl(self, nbody_system, param, dt) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters real(DP), intent(in) :: dt !! Step size ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl if (self%nbody == 0) return - associate(pl => self, npl => self%nbody) + associate(pl => self) + npl = self%nbody do concurrent(i = 1:npl, pl%lmask(i)) call swiftest_gr_p4_pos_kick(param, pl%rh(:, i), pl%vb(:, i), dt) end do @@ -99,11 +100,12 @@ pure module subroutine helio_gr_p4_tp(self, nbody_system, param, dt) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters real(DP), intent(in) :: dt !! Step size ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody do concurrent(i = 1:ntp, tp%lmask(i)) call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vb(:, i), dt) end do diff --git a/src/helio/helio_kick.f90 b/src/helio/helio_kick.f90 index 6fb4fad43..89c93a7ed 100644 --- a/src/helio/helio_kick.f90 +++ b/src/helio/helio_kick.f90 @@ -104,11 +104,12 @@ module subroutine helio_kick_vb_pl(self, nbody_system, param, t, dt, lbeg) real(DP), intent(in) :: dt !! Stepsize logical, intent(in) :: lbeg !! Logical flag indicating whether this is the beginning of the half step or not. ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl if (self%nbody == 0) return - associate(pl => self, npl => self%nbody) + associate(pl => self) + npl = self%nbody pl%ah(:, 1:npl) = 0.0_DP call pl%accel(nbody_system, param, t, lbeg) if (lbeg) then @@ -143,11 +144,12 @@ module subroutine helio_kick_vb_tp(self, nbody_system, param, t, dt, lbeg) real(DP), intent(in) :: dt !! Stepsize logical, intent(in) :: lbeg !! Logical flag indicating whether this is the beginning of the half step or not. ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody tp%ah(:, 1:ntp) = 0.0_DP call tp%accel(nbody_system, param, t, lbeg) do concurrent(i = 1:ntp, tp%lmask(i)) diff --git a/src/rmvs/rmvs_kick.f90 b/src/rmvs/rmvs_kick.f90 index 7d113f863..8fb6c14ce 100644 --- a/src/rmvs/rmvs_kick.f90 +++ b/src/rmvs/rmvs_kick.f90 @@ -29,11 +29,13 @@ module subroutine rmvs_kick_getacch_tp(self, nbody_system, param, t, lbeg) class(swiftest_parameters), allocatable :: param_planetocen real(DP), dimension(:, :), allocatable :: rh_original real(DP) :: GMcb_original - integer(I4B) :: i + integer(I4B) :: i, ntp, inner_index if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody, ipleP => self%ipleP, inner_index => self%index) + associate(tp => self, ipleP => self%ipleP) + ntp = self%nbody + inner_index = self%index select type(nbody_system) class is (rmvs_nbody_system) if (nbody_system%lplanetocentric) then ! This is a close encounter step, so any accelerations requiring heliocentric position values diff --git a/src/swiftest/swiftest_obl.f90 b/src/swiftest/swiftest_obl.f90 index 6bd7480fb..7a6463677 100644 --- a/src/swiftest/swiftest_obl.f90 +++ b/src/swiftest/swiftest_obl.f90 @@ -73,11 +73,12 @@ module subroutine swiftest_obl_acc_pl(self, nbody_system) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl if (self%nbody == 0) return - associate(pl => self, npl => self%nbody, cb => nbody_system%cb) + associate(pl => self, cb => nbody_system%cb) + npl = self%nbody call swiftest_obl_acc(npl, cb%Gmass, cb%j2rp2, cb%j4rp4, pl%rh, pl%lmask, pl%aobl, pl%Gmass, cb%aobl) do concurrent(i = 1:npl, pl%lmask(i)) @@ -103,11 +104,12 @@ module subroutine swiftest_obl_acc_tp(self, nbody_system) class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object ! Internals real(DP), dimension(NDIM) :: aoblcb - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody, cb => nbody_system%cb) + associate(tp => self, cb => nbody_system%cb) + ntp = self%nbody call swiftest_obl_acc(ntp, cb%Gmass, cb%j2rp2, cb%j4rp4, tp%rh, tp%lmask, tp%aobl) if (nbody_system%lbeg) then aoblcb = cb%aoblbeg @@ -139,10 +141,11 @@ module subroutine swiftest_obl_pot_system(self) ! Arguments class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody system object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl real(DP), dimension(self%pl%nbody) :: oblpot_arr - associate(nbody_system => self, pl => self%pl, npl => self%pl%nbody, cb => self%cb) + associate(nbody_system => self, pl => self%pl, cb => self%cb) + npl = self%pl%nbody if (.not. any(pl%lmask(1:npl))) return do concurrent (i = 1:npl, pl%lmask(i)) oblpot_arr(i) = swiftest_obl_pot_one(cb%Gmass, pl%Gmass(i), cb%j2rp2, cb%j4rp4, pl%rh(3,i), 1.0_DP / norm2(pl%rh(:,i))) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index b620a7f9d..1784a1252 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -277,10 +277,11 @@ module subroutine swiftest_util_coord_h2b_tp(self, cb) class(swiftest_tp), intent(inout) :: self !! Swiftest test particle object class(swiftest_cb), intent(in) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) tp%rb(:, i) = tp%rh(:, i) + cb%rb(:) tp%vb(:, i) = tp%vh(:, i) + cb%vb(:) @@ -303,11 +304,12 @@ module subroutine swiftest_util_coord_b2h_pl(self, cb) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_cb), intent(inout) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl if (self%nbody == 0) return - associate(pl => self, npl => self%nbody) + associate(pl => self) + npl = self%nbody do concurrent (i = 1:npl, pl%status(i) /= INACTIVE) pl%rh(:, i) = pl%rb(:, i) - cb%rb(:) pl%vh(:, i) = pl%vb(:, i) - cb%vb(:) @@ -330,11 +332,12 @@ module subroutine swiftest_util_coord_b2h_tp(self, cb) class(swiftest_tp), intent(inout) :: self !! Swiftest massive body object class(swiftest_cb), intent(in) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody do concurrent(i = 1:ntp, tp%status(i) /= INACTIVE) tp%rh(:, i) = tp%rb(:, i) - cb%rb(:) tp%vh(:, i) = tp%vb(:, i) - cb%vb(:) @@ -357,11 +360,12 @@ module subroutine swiftest_util_coord_vb2vh_pl(self, cb) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_cb), intent(inout) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl if (self%nbody == 0) return - associate(pl => self, npl => self%nbody) + associate(pl => self) + npl = self%nbody cb%vb(:) = 0.0_DP do i = npl, 1, -1 if (pl%status(i) /= INACTIVE) cb%vb(:) = cb%vb(:) - pl%Gmass(i) * pl%vb(:, i) / cb%Gmass @@ -413,12 +417,13 @@ module subroutine swiftest_util_coord_vh2vb_pl(self, cb) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_cb), intent(inout) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl real(DP) :: Gmtot if (self%nbody == 0) return - associate(pl => self, npl => self%nbody) + associate(pl => self) + npl = self%nbody Gmtot = cb%Gmass + sum(pl%Gmass(1:npl)) cb%vb(:) = 0.0_DP do i = 1, npl @@ -508,10 +513,11 @@ module subroutine swiftest_util_coord_rh2rb_tp(self, cb) class(swiftest_tp), intent(inout) :: self !! Swiftest test particle object class(swiftest_cb), intent(in) :: cb !! Swiftest central body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) tp%rb(:, i) = tp%rh(:, i) + cb%rb(:) end do @@ -1145,7 +1151,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) class(swiftest_nbody_system), intent(inout) :: self !! Swiftest nbody system object class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i,j + integer(I4B) :: i,j, npl real(DP) :: kecb, kespincb real(DP), dimension(self%pl%nbody) :: kepl, kespinpl real(DP), dimension(NDIM,self%pl%nbody) :: Lplorbit @@ -1153,7 +1159,8 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) real(DP), dimension(NDIM) :: Lcborbit, Lcbspin real(DP), dimension(NDIM) :: h - associate(nbody_system => self, pl => self%pl, npl => self%pl%nbody, cb => self%cb) + associate(nbody_system => self, pl => self%pl, cb => self%cb) + npl = self%pl%nbody nbody_system%L_orbit(:) = 0.0_DP nbody_system%L_spin(:) = 0.0_DP nbody_system%L_total(:) = 0.0_DP diff --git a/src/whm/whm_gr.f90 b/src/whm/whm_gr.f90 index b0891f006..8eb69abc6 100644 --- a/src/whm/whm_gr.f90 +++ b/src/whm/whm_gr.f90 @@ -84,10 +84,12 @@ pure module subroutine whm_gr_p4_pl(self, nbody_system, param, dt) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters real(DP), intent(in) :: dt !! Step size ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl - associate(pl => self, npl => self%nbody) - if (npl == 0) return + if (self%nbody == 0) return + + associate(pl => self) + npl = self%nbody do concurrent(i = 1:npl, pl%lmask(i)) call swiftest_gr_p4_pos_kick(param, pl%xj(:, i), pl%vj(:, i), dt) end do @@ -111,9 +113,10 @@ pure module subroutine whm_gr_p4_tp(self, nbody_system, param, dt) class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters real(DP), intent(in) :: dt !! Step size ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody if (ntp == 0) return do concurrent(i = 1:ntp, tp%lmask(i)) call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vh(:, i), dt) diff --git a/src/whm/whm_kick.f90 b/src/whm/whm_kick.f90 index 403678ed6..a1d4b0dc6 100644 --- a/src/whm/whm_kick.f90 +++ b/src/whm/whm_kick.f90 @@ -82,10 +82,12 @@ module subroutine whm_kick_getacch_tp(self, nbody_system, param, t, lbeg) real(DP), intent(in) :: t !! Current time logical, intent(in) :: lbeg !! Logical flag that determines whether or not this is the beginning or end of the step ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl, ntp real(DP), dimension(NDIM) :: ah0 - associate(tp => self, ntp => self%nbody, pl => nbody_system%pl, cb => nbody_system%cb, npl => nbody_system%pl%nbody) + associate(tp => self, pl => nbody_system%pl, cb => nbody_system%cb) + npl = nbody_system%pl%nbody + ntp = self%nbody if (ntp == 0 .or. npl == 0) return nbody_system%lbeg = lbeg @@ -151,16 +153,15 @@ pure subroutine whm_kick_getacch_ah1(cb, pl) class(swiftest_cb), intent(in) :: cb !! WHM central body object class(whm_pl), intent(inout) :: pl !! WHM massive body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl real(DP), dimension(NDIM) :: ah1h, ah1j - associate(npl => pl%nbody) - do concurrent (i = 2:npl, pl%lmask(i)) - ah1j(:) = pl%xj(:, i) * pl%ir3j(i) - ah1h(:) = pl%rh(:, i) * pl%ir3h(i) - pl%ah(:, i) = pl%ah(:, i) + cb%Gmass * (ah1j(:) - ah1h(:)) - end do - end associate + npl = pl%nbody + do concurrent (i = 2:npl, pl%lmask(i)) + ah1j(:) = pl%xj(:, i) * pl%ir3j(i) + ah1h(:) = pl%rh(:, i) * pl%ir3h(i) + pl%ah(:, i) = pl%ah(:, i) + cb%Gmass * (ah1j(:) - ah1h(:)) + end do return end subroutine whm_kick_getacch_ah1 @@ -178,22 +179,21 @@ pure subroutine whm_kick_getacch_ah2(cb, pl) class(swiftest_cb), intent(in) :: cb !! Swiftest central body object class(whm_pl), intent(inout) :: pl !! WHM massive body object ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl real(DP) :: etaj, fac real(DP), dimension(NDIM) :: ah2, ah2o - associate(npl => pl%nbody) - ah2(:) = 0.0_DP - ah2o(:) = 0.0_DP - etaj = cb%Gmass - do concurrent(i = 2:npl, pl%lmask(i)) - etaj = etaj + pl%Gmass(i - 1) - fac = pl%Gmass(i) * cb%Gmass * pl%ir3j(i) / etaj - ah2(:) = ah2o + fac * pl%xj(:, i) - pl%ah(:,i) = pl%ah(:, i) + ah2(:) - ah2o(:) = ah2(:) - end do - end associate + npl = pl%nbody + ah2(:) = 0.0_DP + ah2o(:) = 0.0_DP + etaj = cb%Gmass + do concurrent(i = 2:npl, pl%lmask(i)) + etaj = etaj + pl%Gmass(i - 1) + fac = pl%Gmass(i) * cb%Gmass * pl%ir3j(i) / etaj + ah2(:) = ah2o + fac * pl%xj(:, i) + pl%ah(:,i) = pl%ah(:, i) + ah2(:) + ah2o(:) = ah2(:) + end do return end subroutine whm_kick_getacch_ah2 @@ -215,9 +215,10 @@ module subroutine whm_kick_vh_pl(self, nbody_system, param, t, dt, lbeg) real(DP), intent(in) :: dt !! Stepsize logical, intent(in) :: lbeg !! Logical flag indicating whether this is the beginning of the half step or not. ! Internals - integer(I4B) :: i + integer(I4B) :: i, npl - associate(pl => self, npl => self%nbody, cb => nbody_system%cb) + associate(pl => self, cb => nbody_system%cb) + npl = self%nbody if (npl == 0) return if (lbeg) then if (pl%lfirst) then @@ -257,11 +258,12 @@ module subroutine whm_kick_vh_tp(self, nbody_system, param, t, dt, lbeg) real(DP), intent(in) :: dt !! Stepsize logical, intent(in) :: lbeg !! Logical flag indicating whether this is the beginning of the half step or not. ! Internals - integer(I4B) :: i + integer(I4B) :: i, ntp if (self%nbody == 0) return - associate(tp => self, ntp => self%nbody) + associate(tp => self) + ntp = self%nbody if (tp%lfirst) then do concurrent(i = 1:ntp, tp%lmask(i)) tp%ah(:, i) = 0.0_DP From 22bad3db298ddb0d34548c252ceb7af50dbb7c0c Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 19 May 2023 12:03:18 -0400 Subject: [PATCH 076/149] Cleaned up some unused variables --- src/collision/collision_generate.f90 | 1 - src/misc/solver_module.f90 | 2 +- src/whm/whm_gr.f90 | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/collision/collision_generate.f90 b/src/collision/collision_generate.f90 index f0fc6d9e7..6162122cf 100644 --- a/src/collision/collision_generate.f90 +++ b/src/collision/collision_generate.f90 @@ -46,7 +46,6 @@ module subroutine collision_generate_bounce(self, nbody_system, param, t) real(DP), intent(in) :: t !! The time of the collision ! Internals integer(I4B) :: i,j,nimp - real(DP), dimension(NDIM) :: rcom, vcom, rnorm logical, dimension(:), allocatable :: lmask select type(nbody_system) diff --git a/src/misc/solver_module.f90 b/src/misc/solver_module.f90 index 936914561..872db6428 100644 --- a/src/misc/solver_module.f90 +++ b/src/misc/solver_module.f90 @@ -16,7 +16,7 @@ module solver use lambda_function use, intrinsic :: ieee_exceptions private - public :: solve_linear_system, solve_rkf45, solve_roots + public :: solve_linear_system, solve_roots !, solve_rkf45 interface solve_linear_system module procedure solve_linear_system_dp diff --git a/src/whm/whm_gr.f90 b/src/whm/whm_gr.f90 index 8eb69abc6..1b1716800 100644 --- a/src/whm/whm_gr.f90 +++ b/src/whm/whm_gr.f90 @@ -25,8 +25,6 @@ pure module subroutine whm_gr_kick_getacch_pl(self, param) ! Internals integer(I4B) :: i real(DP), dimension(NDIM) :: suma - real(DP), dimension(:, :), allocatable :: aj - real(DP) :: beta, rjmag4 if (self%nbody == 0) return @@ -56,8 +54,6 @@ pure module subroutine whm_gr_kick_getacch_tp(self, param) class(whm_tp), intent(inout) :: self !! WHM massive body particle data structure class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i - real(DP) :: rjmag4, beta if (self%nbody == 0) return From 922c3f00aac8a56d5d97c732f5062bad2d642b52 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 19 May 2023 12:04:53 -0400 Subject: [PATCH 077/149] Reviewed compiler flags and improved the consistency, and added ability to turn off OpenMP SIMD directives when building for CPU-agnostic container --- CMakeLists.txt | 1 + cmake/Modules/FindOpenMP_Fortran.cmake | 31 ++- cmake/Modules/SetFortranFlags.cmake | 272 ++++++++++++++----------- 3 files changed, 167 insertions(+), 137 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0aed20380..a047c0163 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) # Set some options the user may choose OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) +OPTION(USE_SIMD "Use SIMD vectorization" ON) # Locate and set parallelization libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know diff --git a/cmake/Modules/FindOpenMP_Fortran.cmake b/cmake/Modules/FindOpenMP_Fortran.cmake index 32777569e..06d679e7c 100644 --- a/cmake/Modules/FindOpenMP_Fortran.cmake +++ b/cmake/Modules/FindOpenMP_Fortran.cmake @@ -25,24 +25,19 @@ INCLUDE (${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake) -SET (OpenMP_Fortran_FLAG_CANDIDATES - #Intel - "-qopenmp" - #Intel windows - "/Qopenmp" - #Gnu - "-fopenmp" - #Portland Group - "-mp" - #Empty, if compiler automatically accepts openmp - " " - #Sun - "-xopenmp" - #HP - "+Oopenmp" - #IBM XL C/c++ - "-qsmp" -) +IF (USE_SIMD) + SET (OpenMP_Fortran_FLAG_CANDIDATES + "-qopenmp" # Intel + "/Qopenmp" # Intel Windows + "-fopenmp" # GNU + ) +ELSE () + SET (OpenMP_Fortran_FLAG_CANDIDATES + "-qopenmp -qno-openmp-simd" # Intel + "/Qopenmp-simd-" # Intel Windows + "-fopenmp" # GNU + ) +ENDIF (USE_SIMD) IF (DEFINED OpenMP_Fortran_FLAGS) SET (OpenMP_Fortran_FLAG_CANDIDATES) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 76f23f5cf..ecf9c34e1 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -64,9 +64,51 @@ ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fort ### GENERAL FLAGS ### ##################### +# Free form +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-ffree-form" # GNU + ) + # Don't add underscores in symbols for C-compatability SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-fno-underscoring") + Fortran "-fno-underscoring" # GNU + ) + +# Determines whether the current Fortran Standard behavior of the compiler is fully implemented. +SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}" + Fortran "-standard-semantics" # Intel + "/standard-semantics" # Intel Windows + ) + +# Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018). +SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}" + Fortran "-stand f18" # Intel + "/stand:f18" # Intel Windows + "-fstd=f2018" # GNU + ) + +# Allows for lines longer than 80 characters without truncation +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-ffree-line-length-none" # GNU (gfortran) + ) + +# Disables right margin wrapping in list-directed output +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-no-wrap-margin" # Intel + "/wrap-margin-" # Intel Windows + ) + +# Aligns a variable to a specified boundary and offset +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-align all -align array64byte" # Intel + "/align:all /align:array64byte" # Intel Windows + ) + +# Enables changing the variable and array memory layout +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-pad" # Intel + "/Qpad" # Intel Windows + ) # There is some bug where -march=native doesn't work on Mac IF(APPLE) @@ -74,18 +116,10 @@ IF(APPLE) ELSE() SET(GNUNATIVE "-march=native") ENDIF() -# Optimize for the host's architecture -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-xhost" # Intel - "/QxHost" # Intel Windows - ${GNUNATIVE} # GNU - ) - ################### ### DEBUG FLAGS ### ################### -# NOTE: debugging symbols (-g or /debug:full) are already on by default # Disable optimizations SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" @@ -106,33 +140,34 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-traceback" # Intel Group "/traceback" # Intel Windows "-fbacktrace" # GNU (gfortran) - "-ftrace=full" # GNU (g95) ) # Sanitize SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fsanitize=address" # Gnu + Fortran "-fsanitize=address, undefined" # Gnu ) # Check everything SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-check" # Intel - "/check" # Intel Windows + Fortran "-check all" # Intel + "/check:all" # Intel Windows "-fcheck=all" # GNU ) SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fstack-check" # GNU + Fortran "-fstack-check" # GNU ) # Initializes matrices/arrays with NaN values SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-init=snan,arrays" # Intel + Fortran "-init=snan,arrays" # Intel + "/Qinit:snan,arrays" # Intel Windows ) # Does not generate an interface block for each routine in a source file SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-nogen-interfaces" # Intel + "/nogen-interfaces" # Intel Windows ) # Does not generate aposition independent executable @@ -143,74 +178,43 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" # Does not set denormal results from floating-point calculations to zero SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-no-ftz" # Intel + "/Qftz-" # Intel Windows ) # Enables floating-point invalid, divide-by-zero, and overflow exceptions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fpe-all=0" # Intel + Fortran "-fpe-all=0" # Intel + "/fpe-all:0" # Intel Windows "-ffpe-trap=zero,overflow,underflow" # GNU ) -# Improves floating-point precision and consistency -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-mp1" # Intel - ) - -# Strict model for floating-point calculations (precise and except) -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fp-model=strict" # Intel - ) - # Enables floating-point invalid, divide-by-zero, and overflow exceptions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fpe0" # Intel + Fortran "-fpe0" # Intel + "/fpe:0" # Intel Windows ) # Enables debug info SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-debug all" # Intel + "/debug:all" # Intel Windows ) -# Aligns a variable to a specified boundary and offset +# Disables additional interprocedural optimizations for a single file compilation SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-align all -align array64byte" # Intel + Fortran "-no-ip" # Intel + "/Qip-" # Intel Windows ) -# Enables changing the variable and array memory layout -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-pad" # Intel - ) - -# Enables additional interprocedural optimizations for a single file cimpilation -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-ip" # Intel - ) - -# Improves precision when dividing floating-points -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-prec-div" # Intel - ) - -# Improves precision when taking the square root of floating-points -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-prec-sqrt" # Intel - ) - -# Treat parentheses in accordance with the Fortran standard (ifort 10 only) -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-assume protect-parens" # Intel - ) - -# Checks the bounds of arrays at run-time +# Disables prefetch insertion optimization SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-CB" # Intel + Fortran "-qno-opt-prefetch" # Intel + "/Qopt-prefetch-" # Intel Windows ) - -# Allows for lines longer than 80 characters without truncation + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-no-wrap-margin" # Intel - "-ffree-line-length-none" # GNU (gfortran) - ) + Fortran "-fstack-check" # GNU + ) ##################### ### TESTING FLAGS ### @@ -218,8 +222,8 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" # Optimizations SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_TESTING "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran REQUIRED "-O3" # All compilers not on Windows - "/O3" # Intel Windows + Fortran REQUIRED "-O3" # All compilers not on Windows + "/O3" # Intel Windows ) ##################### @@ -229,9 +233,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_TESTING "${CMAKE_Fortran_FLAGS_DEBUG}" # Unroll loops SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-unroll" # Intel - "/unroll" # Intel Windows - "-funroll-loops" # GNU + Fortran "-unroll" # Intel + "/unroll" # Intel Windows + "-funroll-loops" # GNU ) # Inline functions @@ -241,91 +245,121 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" "-finline-functions" # GNU ) - -# Allows for lines longer than 80 characters without truncation -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-no-wrap-margin" # Intel - "-ffree-line-length-none" # GNU (gfortran) - ) - -# Disables prefetch insertion optimization -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-qopt-prefetch=0" # Intel - ) - # Calls the Matrix Multiply library SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-qopt-matmul" # Intel + Fortran "-qopt-matmul" # Intel + "/Qopt-matmul" # Intel Windows ) # Saves the compiler options and version number to the executable SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-sox" # Intel - ) - -# Enforces vectorization of loops -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-simd" # Intel + Fortran "-sox" # Intel ) # Aligns a variable to a specified boundary and offset SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-align all" # Intel - ) - -# Generate an extended set of vector functions -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-vecabi=cmdtarget" # Intel + Fortran "-align all" # Intel + "/align:all" # Intel Windows ) # No floating-point exceptions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-fp-model no-except" # Intel + Fortran "-fp-model no-except" # Intel + "/fp:no-except" # Intel Windows ) # Generate fused multiply-add instructions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-fma" # Intel - ) + Fortran "-fma" # Intel + "/Qfma" # Intel Windows + ) -# Generate fused multiply-add instructions +# Tells the compiler to link to certain libraries in the Intel oneAPI Math Kernel Library (oneMKL). SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-qmkl=cluster" # Intel - Fortran "-qmkl" # Intel - Fortran "-mkl" # Old Intel - ) + Fortran "-qmkl=cluster" # Intel + "-qmkl" # Intel + "/Qmkl:cluster" # Intel Windows + "/Qmkl" # Intel Windows + ) + +# Enables additional interprocedural optimizations for a single file compilation +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" + Fortran "-ip" # Intel + "/Qip" # Intel Windows + ) + ##################### ### MATH FLAGS ### ##################### # Some subroutines require more strict floating point operation optimizations for repeatability SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" - Fortran "-fp-model=precise -prec-div -prec-sqrt -assume protect-parens" # Intel - "/fp:precise /Qprec-div /Qprec-sqrt /assume:protect-parens" # Intel Windows - ) + Fortran "-fp-model=precise" # Intel + "/fp:precise" # Intel Windows + ) + +SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" + Fortran "-prec-div" # Intel + "/Qprec-div" # Intel Windows + ) + +SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" + Fortran "-prec-sqrt" # Intel + "/Qprec-sqrt" # Intel Windows + ) + +SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" + Fortran "-assume protect-parens" # Intel + "/assume:protect-parens" # Intel Windows + ) + +# Improves floating-point precision and consistency +SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" + Fortran "-mp1" # Intel + "/Qprec" # Intel Windows + ) # Most subroutines can use aggressive optimization of floating point operations without problems. SET_COMPILE_FLAG(FASTMATH_FLAGS "${FASTMATH_FLAGS}" - Fortran "-fp-model=fast" - "/fp:fast" - ) + Fortran "-fp-model=fast" # Intel + "/fp:fast" # Intel Windows + "-ffast-math" # GNU + ) + + +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" + Fortran ${STRICTMATH_FLAGS} + ) ##################### ### PROFILE FLAGS ### ##################### # Enables the optimization reports to be generated SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_RELEASE}" - Fortran "-O2 -pg -qopt-report=5 -traceback -p -g3" # Intel - "/O2 /Qopt-report:5 /traceback -g3" # Windows Intel - "-O2 -pg -fbacktrace" - ) - -# Sanitize -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fsanitize=address,undefined" # Gnu - ) - - -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" - Fortran "-fstack-check" # GNU - ) + Fortran "-O2 -pg -qopt-report=5 -traceback -p -g3" # Intel + "/O2 /Qopt-report:5 /traceback /Z7" # Intel Windows + "-O2 -pg -fbacktrace" # GNU + ) + +IF (USE_SIMD) + # Enables OpenMP SIMD compilation when OpenMP parallelization is disabled. + IF (NOT USE_OPENMP) + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-qno-openmp -qopenmp-simd" # Intel + Fortran "/Qopenmp- /Qopenmp-simd" # Intel Windows + ) + ENDIF (NOT USE_OPENMP) + + # Optimize for the host's architecture + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-xhost" # Intel + "/QxHost" # Intel Windows + ${GNUNATIVE} # GNU + ) + + # Generate an extended set of vector functions + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-vecabi=cmdtarget" # Intel + Fortran "/Qvecabi:cmdtarget" # Intel Windows + ) +ENDIF (USE_SIMD) \ No newline at end of file From 8a8e506ab5adc7c66a19232e1f955c755ccf82cc Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 19 May 2023 13:33:08 -0400 Subject: [PATCH 078/149] Simplified initial conditions of basic simulation to run from the Python script --- examples/Basic_Simulation/initial_conditions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/Basic_Simulation/initial_conditions.py b/examples/Basic_Simulation/initial_conditions.py index 564c2ebee..1e7cd5cd6 100755 --- a/examples/Basic_Simulation/initial_conditions.py +++ b/examples/Basic_Simulation/initial_conditions.py @@ -74,10 +74,10 @@ capm_tp = default_rng().uniform(0.0, 360.0, ntp) sim.add_body(name=name_tp, a=a_tp, e=e_tp, inc=inc_tp, capom=capom_tp, omega=omega_tp, capm=capm_tp) - -# Run the simulation. Arguments may be defined here or thorugh the swiftest.Simulation() method. -#sim.run(tstart=0.0, tstop=1.0e3, dt=0.01, istep_out=100, dump_cadence=10) sim.set_parameter(tstart=0.0, tstop=1.0e3, dt=0.01, istep_out=100, dump_cadence=0) # Display the run configuration parameters. sim.get_parameter() sim.save() + +# Run the simulation. Arguments may be defined here or thorugh the swiftest.Simulation() method. +sim.run() From a85d5adf3d0ae59d3e6a9d2ae2334304fa1b5300 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 19 May 2023 13:33:24 -0400 Subject: [PATCH 079/149] Fixed bug that prevented progress bar from updating --- src/swiftest/swiftest_io.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index e5485c24a..692a1a0b1 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -277,7 +277,7 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t if (this_image() == 1) then #endif write(pbarmessage,fmt=pbarfmt) self%t, param%tstop - call pbar%update(1_I8B,message=pbarmessage) + call pbar%update(param%iloop,message=pbarmessage) #ifdef COARRAY end if !(this_image() == 1) #endif From 20c7e4e971893fcee2f34c69517166b75b15c46f Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 19 May 2023 14:09:42 -0400 Subject: [PATCH 080/149] Fixed bug that prevented the tqdm progress bar to start --- src/swiftest/swiftest_io.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 692a1a0b1..0f2f6be2b 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -265,7 +265,7 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t if (param%display_style == "PROGRESS") then call pbar%reset(param%nloops) else if (param%display_style == "COMPACT") then - write(param%display_unit,*) "SWIFTEST START " // trim(adjustl(param%integrator)) + write(*,*) "SWIFTEST START " // trim(adjustl(param%integrator)) end if end if #ifdef COARRAY From cda911fc309516f61ed1878104664c95c1d131e8 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 19 May 2023 15:52:07 -0400 Subject: [PATCH 081/149] Added cmake option to test whether do concurrent accepts local-spec. Newer versions of ifort segfault without it. Old ones can't compile with it. --- src/CMakeLists.txt | 23 ++++++++++++++++++++++- src/encounter/encounter_check.f90 | 19 +++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d731373d..ef23fadc8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -150,4 +150,25 @@ IF(BT STREQUAL "DEBUG") ADD_DEFINITIONS(-DDEBUG) ELSEIF(BT STREQUAL "PROFILE") ADD_DEFINITIONS(-DPROFILE) -ENDIF() \ No newline at end of file +ENDIF() + +# Check to see if the compiler allows for local-spec in do concurrent statements. Set a preprocessor variable if it does +IF (USE_OPENMP) + SET(TESTFILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}") + SET(TESTFILE "${TESTFILE}/CMakeTmp/testFortranDoConcurrentLoc.f90") + FILE(WRITE "${TESTFILE}" +" +program TestDoConcurrentLoc +integer :: i +real,dimension(10) :: a +do concurrent(i = 1:10) shared(a) + a(i) = i +end do +end program TestDoConcurrentLoc +") + TRY_COMPILE(DOCONLOC_WORKS ${CMAKE_BINARY_DIR} ${TESTFILE} + COMPILE_DEFINITIONS "${CMAKE_Fortran_FLAGS}" OUTPUT_VARIABLE OUTPUT) + IF (DOCONLOC_WORKS) + TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DDOCONLOC) + ENDIF (DOCONLOC_WORKS) +ENDIF (USE_OPENMP) diff --git a/src/encounter/encounter_check.f90 b/src/encounter/encounter_check.f90 index 06d0a76d4..a539486b5 100644 --- a/src/encounter/encounter_check.f90 +++ b/src/encounter/encounter_check.f90 @@ -61,11 +61,6 @@ module subroutine encounter_check_all_plplm(param, nplm, nplt, rplm, vplm, rplt, integer(I4B), dimension(:), allocatable, intent(out) :: index2 !! List of indices for body 2 in each encounter logical, dimension(:), allocatable, intent(out) :: lvdotr !! Logical flag indicating the sign of v .dot. x ! Internals - ! type(interaction_timer), save :: itimer - logical, save :: lfirst = .true. - logical, save :: skipit = .false. - integer(I8B) :: nplplm = 0_I8B - integer(I4B) :: npl logical, dimension(:), allocatable :: plmplt_lvdotr !! Logical flag indicating the sign of v .dot. x in the plm-plt group integer(I4B), dimension(:), allocatable :: plmplt_index1 !! List of indices for body 1 in each encounter in the plm-plt group integer(I4B), dimension(:), allocatable :: plmplt_index2 !! List of indices for body 2 in each encounter in the plm-lt group @@ -387,7 +382,11 @@ subroutine encounter_check_all_triangular_one(i, n, xi, yi, zi, vxi, vyi, vzi, x real(DP) :: xr, yr, zr, vxr, vyr, vzr, renc12 logical, dimension(n) :: lencounteri, lvdotri +#ifdef DOCONLOC + do concurrent(j = i+1:n) shared(lencounteri, lvdotri) +#else do concurrent(j = i+1:n) +#endif xr = x(j) - xi yr = y(j) - yi zr = z(j) - zi @@ -605,11 +604,11 @@ module subroutine encounter_check_collapse_ragged_list(ragged_list, n1, nenc, in implicit none ! Arguments class(encounter_list), dimension(:), intent(in) :: ragged_list !! The ragged encounter list - integer(I4B), intent(in) :: n1 !! Number of bodies 1 - integer(I8B), intent(out) :: nenc !! Total number of encountersj - integer(I4B), dimension(:), allocatable, intent(out) :: index1 !! Array of indices for body 1 - integer(I4B), dimension(:), allocatable, intent(out) :: index2 !! Array of indices for body 1 - logical, dimension(:), allocatable, intent(out), optional :: lvdotr !! Array indicating which bodies are approaching + integer(I4B), intent(in) :: n1 !! Number of bodies 1 + integer(I8B), intent(out) :: nenc !! Total number of encountersj + integer(I4B), dimension(:), allocatable, intent(out) :: index1 !! Array of indices for body 1 + integer(I4B), dimension(:), allocatable, intent(out) :: index2 !! Array of indices for body 1 + logical, dimension(:), allocatable, intent(out), optional :: lvdotr !! Array indicating which bodies are approaching ! Internals integer(I4B) :: i integer(I8B) :: j1, j0, nenci From 9bd9aca223399882f347ab4a858667d37ff8e1c6 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 19 May 2023 17:51:12 -0400 Subject: [PATCH 082/149] Set locality-spec for a bunch of do concurrents --- src/fraggle/fraggle_generate.f90 | 10 +++++++++- src/swiftest/swiftest_obl.f90 | 4 ++++ src/swiftest/swiftest_util.f90 | 23 +++++++++++++++++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/fraggle/fraggle_generate.f90 b/src/fraggle/fraggle_generate.f90 index f1ba8d606..ee4466c11 100644 --- a/src/fraggle/fraggle_generate.f90 +++ b/src/fraggle/fraggle_generate.f90 @@ -299,7 +299,7 @@ module subroutine fraggle_generate_merge(self, nbody_system, param, t) real(DP), intent(in) :: t !! The time of the collision ! Internals - integer(I4B) :: i, j + integer(I4B) :: i real(DP), dimension(NDIM) :: L_spin_new, Ip, rot real(DP) :: rotmag, mass, volume, radius @@ -413,7 +413,11 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu end do ! Randomly place the n>2 fragments inside their cloud until none are overlapping +#ifdef DOCONLOC + do concurrent(i = istart:nfrag, loverlap(i)) shared(loverlap, mass_rscale, u, phi, theta, lhitandrun) local(j, direction) +#else do concurrent(i = istart:nfrag, loverlap(i)) +#endif j = fragments%origin_body(i) ! Scale the cloud size @@ -452,7 +456,11 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu ! Because body 1 and 2 are initialized near the original impactor positions, then if these bodies are still overlapping ! when the rest are not, we will randomly walk their position in space so as not to move them too far from their starting position if (all(.not.loverlap(istart:nfrag)) .and. any(loverlap(1:istart-1))) then +#ifdef DOCONLOC + do concurrent(i = 1:istart-1,loverlap(i)) shared(loverlap, u, theta, i) local(rwalk, dis) +#else do concurrent(i = 1:istart-1,loverlap(i)) +#endif dis = 0.1_DP * fragments%radius(i) * u(i)**(THIRD) rwalk(1) = fragments%rmag(i) * sin(theta(i)) * cos(phi(i)) rwalk(2) = fragments%rmag(i) * sin(theta(i)) * sin(phi(i)) diff --git a/src/swiftest/swiftest_obl.f90 b/src/swiftest/swiftest_obl.f90 index 7a6463677..85f88bc7d 100644 --- a/src/swiftest/swiftest_obl.f90 +++ b/src/swiftest/swiftest_obl.f90 @@ -147,7 +147,11 @@ module subroutine swiftest_obl_pot_system(self) associate(nbody_system => self, pl => self%pl, cb => self%cb) npl = self%pl%nbody if (.not. any(pl%lmask(1:npl))) return +#ifdef DOCONLOC + do concurrent (i = 1:npl, pl%lmask(i)) shared(oblpot_arr) +#else do concurrent (i = 1:npl, pl%lmask(i)) +#endif oblpot_arr(i) = swiftest_obl_pot_one(cb%Gmass, pl%Gmass(i), cb%j2rp2, cb%j4rp4, pl%rh(3,i), 1.0_DP / norm2(pl%rh(:,i))) end do nbody_system%oblpot = sum(oblpot_arr, pl%lmask(1:npl)) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 1784a1252..b3e43c7e2 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1178,7 +1178,11 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) nbody_system%be_cb = -3*cb%Gmass * cb%mass / (5 * cb%radius) Lcborbit(:) = cb%mass * (cb%rb(:) .cross. cb%vb(:)) +#ifdef DOCONLOC + do concurrent (i = 1:npl, pl%lmask(i)) local(h) shared(Lplorbit, kepl) +#else do concurrent (i = 1:npl, pl%lmask(i)) +#endif h(:) = pl%rb(:,i) .cross. pl%vb(:,i) ! Angular momentum from orbit @@ -1194,7 +1198,11 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) ! For simplicity, we always assume that the rotation pole is the 3rd principal axis Lcbspin(:) = cb%Ip(3) * cb%mass * cb%radius**2 * cb%rot(:) +#ifdef DOCONLOC + do concurrent (i = 1:npl, pl%lmask(i)) shared(Lplspin, kespinpl) +#else do concurrent (i = 1:npl, pl%lmask(i)) +#endif ! Currently we assume that the rotation pole is the 3rd principal axis ! Angular momentum from spin Lplspin(:,i) = pl%mass(i) * pl%Ip(3,i) * pl%radius(i)**2 * pl%rot(:,i) @@ -1226,7 +1234,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) end if nbody_system%ke_orbit = 0.5_DP * (kecb + sum(kepl(1:npl), pl%lmask(1:npl))) - + do concurrent (j = 1:NDIM) nbody_system%L_orbit(j) = Lcborbit(j) + sum(Lplorbit(j,1:npl), pl%lmask(1:npl)) end do @@ -1271,7 +1279,11 @@ module subroutine swiftest_util_get_potential_energy_flat(npl, nplpl, k_plpl, lm pecb(1:npl) = 0.0_DP end where +#ifdef DOCONLOC + do concurrent(i = 1:npl, lmask(i)) shared(pecb) +#else do concurrent(i = 1:npl, lmask(i)) +#endif pecb(i) = -GMcb * mass(i) / norm2(rb(:,i)) end do @@ -1318,7 +1330,11 @@ module subroutine swiftest_util_get_potential_energy_triangular(npl, lmask, GMcb pecb(1:npl) = 0.0_DP end where +#ifdef DOCONLOC + do concurrent(i = 1:npl, lmask(i)) shared(pecb) +#else do concurrent(i = 1:npl, lmask(i)) +#endif pecb(i) = -GMcb * mass(i) / norm2(rb(:,i)) end do @@ -1329,7 +1345,11 @@ module subroutine swiftest_util_get_potential_energy_triangular(npl, lmask, GMcb !$omp reduction(+:pe) do i = 1, npl if (lmask(i)) then +#ifdef DOCONLOC + do concurrent(j = i+1:npl, lmask(i) .and. lmask(j)) shared(pepl) +#else do concurrent(j = i+1:npl, lmask(i) .and. lmask(j)) +#endif pepl(j) = - (Gmass(i) * mass(j)) / norm2(rb(:, i) - rb(:, j)) end do pe = pe + sum(pepl(i+1:npl), lmask(i+1:npl)) @@ -1523,7 +1543,6 @@ module subroutine swiftest_util_peri(n,m, r, v, atp, q, isperi) integer(I4B) :: i real(DP), dimension(n) :: e !! Temporary, just to make use of the xv2aeq subroutine real(DP) :: vdotr - character(len=NAMELEN) :: message do i = 1,n vdotr = dot_product(r(:,i),v(:,i)) From eff54b56be0f8b3ac8752c8436792518c607dbee Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 19 May 2023 21:13:49 -0400 Subject: [PATCH 083/149] Added local-spec to every remaining do concurrent --- src/collision/collision_check.f90 | 9 +++++- src/collision/collision_regime.f90 | 4 +++ src/collision/collision_resolve.f90 | 4 +++ src/collision/collision_util.f90 | 16 ++++++++++ src/encounter/encounter_check.f90 | 40 ++++++++++++++++++++++-- src/fraggle/fraggle_generate.f90 | 38 ++++++++++++++++++++--- src/fraggle/fraggle_util.f90 | 16 ++++++++-- src/helio/helio_gr.f90 | 8 +++++ src/helio/helio_kick.f90 | 8 +++++ src/operator/operator_cross.f90 | 28 +++++++++++++++++ src/operator/operator_mag.f90 | 18 +++++++++-- src/operator/operator_unit.f90 | 12 ++++++++ src/rmvs/rmvs_kick.f90 | 12 ++++++++ src/swiftest/swiftest_drift.f90 | 4 +++ src/swiftest/swiftest_gr.f90 | 4 +++ src/swiftest/swiftest_kick.f90 | 41 +++++++++++++++++++++++-- src/swiftest/swiftest_obl.f90 | 14 ++++++++- src/swiftest/swiftest_orbel.f90 | 10 +++++- src/swiftest/swiftest_util.f90 | 47 +++++++++++++++++++++++++---- src/symba/symba_encounter_check.f90 | 9 +++++- src/symba/symba_kick.f90 | 9 +++++- src/whm/whm_gr.f90 | 8 +++++ src/whm/whm_kick.f90 | 34 ++++++++++++++++++++- 23 files changed, 366 insertions(+), 27 deletions(-) diff --git a/src/collision/collision_check.f90 b/src/collision/collision_check.f90 index 936963caa..0e69032bc 100644 --- a/src/collision/collision_check.f90 +++ b/src/collision/collision_check.f90 @@ -101,7 +101,11 @@ module subroutine collision_check_plpl(self, nbody_system, param, t, dt, irec, l lcollision(:) = .false. self%lclosest(:) = .false. +#ifdef DOCONLOC + do concurrent(k = 1:nenc, lmask(k)) shared(self,pl,lmask, dt, lcollision) local(i,j,xr,vr,rlim,Gmtot) +#else do concurrent(k = 1:nenc, lmask(k)) +#endif i = self%index1(k) j = self%index2(k) xr(:) = pl%rh(:, i) - pl%rh(:, j) @@ -204,8 +208,11 @@ module subroutine collision_check_pltp(self, nbody_system, param, t, dt, irec, l lcollision(:) = .false. self%lclosest(:) = .false. - +#ifdef DOCONLOC + do concurrent(k = 1:nenc, lmask(k)) shared(self,pl,tp,lmask, dt, lcollision) local(i,j,xr,vr) +#else do concurrent(k = 1:nenc, lmask(k)) +#endif i = self%index1(k) j = self%index2(k) xr(:) = pl%rh(:, i) - tp%rh(:, j) diff --git a/src/collision/collision_regime.f90 b/src/collision/collision_regime.f90 index a612eb37a..0c81eca14 100644 --- a/src/collision/collision_regime.f90 +++ b/src/collision/collision_regime.f90 @@ -111,7 +111,11 @@ subroutine collision_regime_LS12(collider, nbody_system, param) if (impactors%regime == COLLRESOLVE_REGIME_MERGE) then volume = 4._DP / 3._DP * PI * sum(impactors%radius(:)**3) radius = (3._DP * volume / (4._DP * PI))**(THIRD) +#ifdef DOCONLOC + do concurrent(i = 1:NDIM) shared(impactors,Ip,L_spin) +#else do concurrent(i = 1:NDIM) +#endif Ip(i) = sum(impactors%mass(:) * impactors%Ip(i,:)) L_spin(i) = sum(impactors%L_orbit(i,:) + impactors%L_spin(i,:)) end do diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index befc4ba8a..7aa76166c 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -387,7 +387,11 @@ module subroutine collision_resolve_mergeaddsub(nbody_system, param, t, status) ! plnew%tlag = pl%tlag(ibiggest) ! end if +#ifdef DOCONLOC + do concurrent(i = 1:nfrag) shared(plnew,fragments) local(volume) +#else do concurrent(i = 1:nfrag) +#endif volume = 4.0_DP/3.0_DP * PI * plnew%radius(i)**3 plnew%density(i) = fragments%mass(i) / volume end do diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 45f4f516d..179fb289c 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -36,7 +36,11 @@ module subroutine collision_util_add_fragments_to_collider(self, nbody_system, p pl%mass(npl_before+1:npl_after) = fragments%mass(1:nfrag) pl%Gmass(npl_before+1:npl_after) = fragments%mass(1:nfrag) * param%GU pl%radius(npl_before+1:npl_after) = fragments%radius(1:nfrag) +#ifdef DOCONLOC + do concurrent (i = 1:nfrag) shared(cb,pl,fragments) +#else do concurrent (i = 1:nfrag) +#endif pl%rb(:,npl_before+i) = fragments%rb(:,i) pl%vb(:,npl_before+i) = fragments%vb(:,i) pl%rh(:,npl_before+i) = fragments%rb(:,i) - cb%rb(:) @@ -169,7 +173,11 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par associate(fragments => self%fragments, impactors => self%impactors, pl => nbody_system%pl, cb => nbody_system%cb) nfrag = self%fragments%nbody if (phase_val == 1) then +#ifdef DOCONLOC + do concurrent(i = 1:2) shared(impactors) +#else do concurrent(i = 1:2) +#endif impactors%ke_orbit(i) = 0.5_DP * impactors%mass(i) * dot_product(impactors%vc(:,i), impactors%vc(:,i)) impactors%ke_spin(i) = 0.5_DP * impactors%mass(i) * impactors%radius(i)**2 * impactors%Ip(3,i) * dot_product(impactors%rot(:,i), impactors%rot(:,i)) impactors%be(i) = -3 * impactors%Gmass(i) * impactors%mass(i) / (5 * impactors%radius(i)) @@ -185,7 +193,11 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par call swiftest_util_get_potential_energy(2, [(.true., i = 1, 2)], 0.0_DP, impactors%Gmass, impactors%mass, impactors%rb, self%pe(phase_val)) self%te(phase_val) = self%ke_orbit(phase_val) + self%ke_spin(phase_val) + self%be(phase_val) + self%pe(phase_val) else if (phase_val == 2) then +#ifdef DOCONLOC + do concurrent(i = 1:nfrag) shared(fragments) +#else do concurrent(i = 1:nfrag) +#endif fragments%ke_orbit(i) = 0.5_DP * fragments%mass(i) * dot_product(fragments%vc(:,i), fragments%vc(:,i)) fragments%ke_spin(i) = 0.5_DP * fragments%mass(i) * fragments%radius(i)**2 * fragments%Ip(3,i) * dot_product(fragments%rot(:,i), fragments%rot(:,i)) fragments%L_orbit(:,i) = fragments%mass(i) * fragments%rc(:,i) .cross. fragments%vc(:,i) @@ -918,7 +930,11 @@ module subroutine collision_util_set_original_scale_factors(self) impactors%L_orbit = impactors%L_orbit * collider%Lscale impactors%ke_orbit = impactors%ke_orbit * collider%Escale impactors%ke_spin = impactors%ke_spin * collider%Escale +#ifdef DOCONLOC + do concurrent(i = 1:2) shared(impactors) +#else do concurrent(i = 1:2) +#endif impactors%rot(:,i) = impactors%L_spin(:,i) * (impactors%mass(i) * impactors%radius(i)**2 * impactors%Ip(3,i)) end do diff --git a/src/encounter/encounter_check.f90 b/src/encounter/encounter_check.f90 index a539486b5..b987f8799 100644 --- a/src/encounter/encounter_check.f90 +++ b/src/encounter/encounter_check.f90 @@ -174,7 +174,11 @@ subroutine encounter_check_all_sort_and_sweep_plpl(npl, r, v, renc, dt, nenc, in npl_last = npl end if +#ifdef DOCONLOC + do concurrent (i = 1:npl) shared(r,renc,rmin,rmax) local(rmag) +#else do concurrent (i = 1:npl) +#endif rmag = .mag.r(:,i) rmax(i) = rmag + RSWEEP_FACTOR * renc(i) rmin(i) = rmag - RSWEEP_FACTOR * renc(i) @@ -227,12 +231,20 @@ subroutine encounter_check_all_sort_and_sweep_plplm(nplm, nplt, rplm, vplm, rplt ntot_last = ntot end if +#ifdef DOCONLOC + do concurrent (i = 1:nplm) shared(rmin,rmax,rplm,rencm) local(rmag) +#else do concurrent (i = 1:nplm) +#endif rmag = .mag.rplm(:,i) rmax(i) = rmag + RSWEEP_FACTOR * rencm(i) rmin(i) = rmag - RSWEEP_FACTOR * rencm(i) end do +#ifdef DOCONLOC + do concurrent (i = 1:nplt) shared(rmin,rmax,rplt,renct) local(rmag) +#else do concurrent (i = 1:nplt) +#endif rmag = .mag.rplt(:,i) rmax(nplm+i) = rmag + RSWEEP_FACTOR * renct(i) rmin(nplm+i) = rmag - RSWEEP_FACTOR * renct(i) @@ -287,12 +299,20 @@ subroutine encounter_check_all_sort_and_sweep_pltp(npl, ntp, rpl, vpl, rtp, vtp, renctp(:) = 0.0_DP +#ifdef DOCONLOC + do concurrent (i = 1:npl) shared(rmin,rmax,rpl,rencpl) local(rmag) +#else do concurrent (i = 1:npl) +#endif rmag = .mag.rpl(:,i) rmax(i) = rmag + RSWEEP_FACTOR * rencpl(i) rmin(i) = rmag - RSWEEP_FACTOR * rencpl(i) end do - do concurrent (i = 1:ntp) +#ifdef DOCONLOC + do concurrent (i = 1:ntp) shared(rmin,rmax,rtp,renctp) local(rmag) +#else + do concurrent (i = 1:ntp) +#endif rmag = .mag.rtp(:,i) rmax(npl+i) = rmag + RSWEEP_FACTOR * renctp(i) rmin(npl+i) = rmag - RSWEEP_FACTOR * renctp(i) @@ -335,7 +355,11 @@ pure subroutine encounter_check_all_sweep_one(i, n, xi, yi, zi, vxi, vyi, vzi, x logical, dimension(n) :: lencounteri, lvdotri lencounteri(:) = .false. +#ifdef DOCONLOC + do concurrent(j = 1:n, lgood(j)) shared(lgood,lencounteri,lvdotri,x,y,z,vx,vy,vz,renci,renc) local(xr,yr,zr,vxr,vyr,vzr,renc12) +#else do concurrent(j = 1:n, lgood(j)) +#endif xr = x(j) - xi yr = y(j) - yi zr = z(j) - zi @@ -383,7 +407,7 @@ subroutine encounter_check_all_triangular_one(i, n, xi, yi, zi, vxi, vyi, vzi, x logical, dimension(n) :: lencounteri, lvdotri #ifdef DOCONLOC - do concurrent(j = i+1:n) shared(lencounteri, lvdotri) + do concurrent(j = i+1:n) shared(lencounteri, lvdotri, renci, renc) local(xr,yr,zr,vxr,vyr,vzr,renc12) #else do concurrent(j = i+1:n) #endif @@ -700,7 +724,11 @@ subroutine encounter_check_remove_duplicates(n, nenc, index1, index2, lvdotr) ! Sort on the second index and remove duplicates if (allocated(itmp)) deallocate(itmp) allocate(itmp, source=index2) +#ifdef DOCONLOC + do concurrent(i = 1:n, iend(i) - ibeg(i) > 0_I8B) shared(iend,ibeg,index2,lencounter,itmp) local(klo,khi,nenci,j) +#else do concurrent(i = 1:n, iend(i) - ibeg(i) > 0_I8B) +#endif klo = ibeg(i) khi = iend(i) nenci = khi - klo + 1_I8B @@ -747,7 +775,11 @@ pure module subroutine encounter_check_sort_aabb_1D(self, n, extent_arr) call util_sort(extent_arr, self%ind) +#ifdef DOCONLOC + do concurrent(k = 1_I8B:2_I8B * n) shared(self,n) local(i) +#else do concurrent(k = 1_I8B:2_I8B * n) +#endif i = self%ind(k) if (i <= n) then self%ibeg(i) = k @@ -940,7 +972,11 @@ module subroutine encounter_check_sweep_aabb_single_list(self, n, r, v, renc, dt call encounter_check_collapse_ragged_list(lenc, n, nenc, index1, index2, lvdotr) ! By convention, we always assume that index1 < index2, and so we must swap any that are out of order +#ifdef DOCONLOC + do concurrent(k = 1_I8B:nenc, index1(k) > index2(k)) shared(index1,index2) local(itmp) +#else do concurrent(k = 1_I8B:nenc, index1(k) > index2(k)) +#endif itmp = index1(k) index1(k) = index2(k) index2(k) = itmp diff --git a/src/fraggle/fraggle_generate.f90 b/src/fraggle/fraggle_generate.f90 index ee4466c11..0dec13fde 100644 --- a/src/fraggle/fraggle_generate.f90 +++ b/src/fraggle/fraggle_generate.f90 @@ -68,7 +68,11 @@ module subroutine fraggle_generate(self, nbody_system, param, t) ! Get the energy and momentum of the system before and after the collision call self%get_energy_and_momentum(nbody_system, param, phase="before") nfrag = fragments%nbody +#ifdef DOCONLOC + do concurrent(i = 1:2) shared(fragments,impactors) +#else do concurrent(i = 1:2) +#endif fragments%rc(:,i) = fragments%rb(:,i) - impactors%rbcom(:) fragments%vc(:,i) = fragments%vb(:,i) - impactors%vbcom(:) end do @@ -309,7 +313,11 @@ module subroutine fraggle_generate_merge(self, nbody_system, param, t) mass = sum(impactors%mass(:)) volume = 4._DP / 3._DP * PI * sum(impactors%radius(:)**3) radius = (3._DP * volume / (4._DP * PI))**(THIRD) +#ifdef DOCONLOC + do concurrent(i = 1:NDIM) shared(impactors, Ip, L_spin_new) +#else do concurrent(i = 1:NDIM) +#endif Ip(i) = sum(impactors%mass(:) * impactors%Ip(i,:)) L_spin_new(i) = sum(impactors%L_orbit(i,:) + impactors%L_spin(i,:)) end do @@ -414,7 +422,7 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu ! Randomly place the n>2 fragments inside their cloud until none are overlapping #ifdef DOCONLOC - do concurrent(i = istart:nfrag, loverlap(i)) shared(loverlap, mass_rscale, u, phi, theta, lhitandrun) local(j, direction) + do concurrent(i = istart:nfrag, loverlap(i)) shared(fragments, impactors, fragment_cloud_radius, fragment_cloud_center, loverlap, mass_rscale, u, phi, theta, lhitandrun) local(j, direction) #else do concurrent(i = istart:nfrag, loverlap(i)) #endif @@ -457,7 +465,7 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu ! when the rest are not, we will randomly walk their position in space so as not to move them too far from their starting position if (all(.not.loverlap(istart:nfrag)) .and. any(loverlap(1:istart-1))) then #ifdef DOCONLOC - do concurrent(i = 1:istart-1,loverlap(i)) shared(loverlap, u, theta, i) local(rwalk, dis) + do concurrent(i = 1:istart-1,loverlap(i)) shared(fragments,loverlap, u, theta, i) local(rwalk, dis) #else do concurrent(i = 1:istart-1,loverlap(i)) #endif @@ -519,7 +527,7 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i, nfrag - real(DP), parameter :: frag_rot_fac = 0.1_DP ! Fraction of projectile rotation magnitude to add as random noise to fragment rotation + real(DP), parameter :: FRAG_ROT_FAC = 0.1_DP ! Fraction of projectile rotation magnitude to add as random noise to fragment rotation real(DP), parameter :: hitandrun_momentum_transfer = 0.01_DP ! Fraction of projectile momentum transfered to target in a hit and run real(DP) :: mass_fac real(DP), dimension(NDIM) :: drot, dL @@ -568,9 +576,13 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) end if call random_number(fragments%rot(:,2:nfrag)) +#ifdef DOCONLOC + do concurrent (i = 2:nfrag) shared(fragments,impactors) local(mass_fac) +#else do concurrent (i = 2:nfrag) +#endif mass_fac = fragments%mass(i) / impactors%mass(2) - fragments%rot(:,i) = mass_fac**(5.0_DP/3.0_DP) * impactors%rot(:,2) + 2 * (fragments%rot(:,i) - 1.0_DP) * frag_rot_fac * .mag.impactors%rot(:,2) + fragments%rot(:,i) = mass_fac**(5.0_DP/3.0_DP) * impactors%rot(:,2) + 2 * (fragments%rot(:,i) - 1.0_DP) * FRAG_ROT_FAC * .mag.impactors%rot(:,2) end do fragments%rotmag(:) = .mag.fragments%rot(:,:) @@ -662,7 +674,11 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu ! Scale the magnitude of the velocity by the distance from the impact point ! This will reduce the chances of fragments colliding with each other immediately, and is more physically correct - do concurrent(i = 1:fragments%nbody) +#ifdef DOCONLOC + do concurrent(i = 1:fragments%nbody) shared(fragments,impactors,vscale) local(rimp) +#else + do concurrent(i = 1:fragments%nbody) +#endif rimp(:) = fragments%rc(:,i) - impactors%rcimp(:) vscale(i) = .mag. rimp(:) / sum(impactors%radius(1:2)) end do @@ -673,7 +689,11 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu ! Set the velocities of all fragments using all of the scale factors determined above if (istart > 1) fragments%vc(:,1) = impactors%vc(:,1) * impactors%mass(1) / fragments%mass(1) +#ifdef DOCONLOC + do concurrent(i = istart:fragments%nbody) shared(fragments,impactors,lhitandrun, vscale, vesc, vsign) local(j,vrot,vmag,vdisp,rimp,vimp_unit) +#else do concurrent(i = istart:fragments%nbody) +#endif j = fragments%origin_body(i) vrot(:) = impactors%rot(:,j) .cross. (fragments%rc(:,i) - impactors%rc(:,j)) if (lhitandrun) then @@ -704,7 +724,11 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu if (nsteps == 1) L_residual_best(:) = L_residual(:) ! Use equipartition of spin kinetic energy to distribution spin angular momentum +#ifdef DOCONLOC + do concurrent(i = istart:fragments%nbody) shared(DLi_mag, fragments) +#else do concurrent(i = istart:fragments%nbody) +#endif dLi_mag(i) = ((fragments%mass(i) / fragments%mass(istart)) * & (fragments%radius(i) / fragments%radius(istart))**2 * & (fragments%Ip(3,i) / fragments%Ip(3,istart)))**(1.5_DP) @@ -816,7 +840,11 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu L_residual(:) = (collider%L_total(:,2) - collider%L_total(:,1)) call collision_util_velocity_torque(-L_residual(:), collider%fragments%mtot, impactors%rbcom, impactors%vbcom) +#ifdef DOCONLOC + do concurrent(i = 1:nfrag) shared(collider, impactors) +#else do concurrent(i = 1:nfrag) +#endif collider%fragments%vb(:,i) = collider%fragments%vc(:,i) + impactors%vbcom(:) end do diff --git a/src/fraggle/fraggle_util.f90 b/src/fraggle/fraggle_util.f90 index 3b0c42d6a..351bd05fa 100644 --- a/src/fraggle/fraggle_util.f90 +++ b/src/fraggle/fraggle_util.f90 @@ -44,7 +44,11 @@ module subroutine fraggle_util_restructure(self, nbody_system, param, lfailure) new_fragments%Gmass(1) =sum(old_fragments%Gmass(1:2)) new_fragments%density(1) = new_fragments%mass(1) / volume new_fragments%radius(1) = (3._DP * volume / (4._DP * PI))**(THIRD) +#ifdef DOCONLOC + do concurrent(i = 1:NDIM) shared(old_fragments, new_fragments) +#else do concurrent(i = 1:NDIM) +#endif new_fragments%Ip(i,1) = sum(old_fragments%mass(1:2) * old_fragments%Ip(i,1:2)) end do new_fragments%Ip(:,1) = new_fragments%Ip(:,1) / new_fragments%mass(1) @@ -55,7 +59,11 @@ module subroutine fraggle_util_restructure(self, nbody_system, param, lfailure) new_fragments%Gmass(2:nnew) = old_fragments%Gmass(3:nold) new_fragments%density(2:nnew) = old_fragments%density(3:nold) new_fragments%radius(2:nnew) = old_fragments%radius(3:nold) +#ifdef DOCONLOC + do concurrent(i = 1:NDIM) shared(old_fragments,new_fragments) +#else do concurrent(i = 1:NDIM) +#endif new_fragments%Ip(i,2:nnew) = old_fragments%Ip(i,3:nold) end do new_fragments%origin_body(2:nnew) = old_fragments%origin_body(3:nold) @@ -87,10 +95,10 @@ module subroutine fraggle_util_set_mass_dist(self, param) class(collision_fraggle), intent(inout) :: self !! Fraggle collision system object class(swiftest_parameters), intent(in) :: param !! Current Swiftest run configuration parameters ! Internals - integer(I4B) :: i, j, k, jproj, jtarg, nfrag, istart, nfragmax, nrem + integer(I4B) :: i, j, jproj, jtarg, nfrag, istart, nfragmax real(DP), dimension(2) :: volume real(DP), dimension(NDIM) :: Ip_avg - real(DP) :: mremaining, mtot, mcumul, G, mass_noise, Mslr, x0, x1, ymid, y0, y1, y, yp, eps, Mrat + real(DP) :: mremaining, mtot, mcumul, G, mass_noise, Mslr, Mrat real(DP), dimension(:), allocatable :: mass real(DP) :: beta integer(I4B), parameter :: MASS_NOISE_FACTOR = 5 !! The number of digits of random noise that get added to the minimum mass value to prevent identical masses from being generated in a single run @@ -207,7 +215,11 @@ module subroutine fraggle_util_set_mass_dist(self, param) fragments%density(istart:nfrag) = fragments%mtot / sum(volume(:)) fragments%radius(istart:nfrag) = (3 * fragments%mass(istart:nfrag) / (4 * PI * fragments%density(istart:nfrag)))**(1.0_DP / 3.0_DP) +#ifdef DOCONLOC + do concurrent(i = istart:nfrag) shared(fragments,Ip_avg) +#else do concurrent(i = istart:nfrag) +#endif fragments%Ip(:, i) = Ip_avg(:) end do diff --git a/src/helio/helio_gr.f90 b/src/helio/helio_gr.f90 index def25224d..6de300cae 100644 --- a/src/helio/helio_gr.f90 +++ b/src/helio/helio_gr.f90 @@ -77,7 +77,11 @@ pure module subroutine helio_gr_p4_pl(self, nbody_system, param, dt) associate(pl => self) npl = self%nbody +#ifdef DOCONLOC + do concurrent(i = 1:npl, pl%lmask(i)) shared(param,pl,dt) +#else do concurrent(i = 1:npl, pl%lmask(i)) +#endif call swiftest_gr_p4_pos_kick(param, pl%rh(:, i), pl%vb(:, i), dt) end do end associate @@ -106,7 +110,11 @@ pure module subroutine helio_gr_p4_tp(self, nbody_system, param, dt) associate(tp => self) ntp = self%nbody +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(param,tp,dt) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vb(:, i), dt) end do end associate diff --git a/src/helio/helio_kick.f90 b/src/helio/helio_kick.f90 index 89c93a7ed..755a385d9 100644 --- a/src/helio/helio_kick.f90 +++ b/src/helio/helio_kick.f90 @@ -117,7 +117,11 @@ module subroutine helio_kick_vb_pl(self, nbody_system, param, t, dt, lbeg) else call pl%set_beg_end(rend = pl%rh) end if +#ifdef DOCONLOC + do concurrent(i = 1:npl, pl%lmask(i)) shared(pl,dt) +#else do concurrent(i = 1:npl, pl%lmask(i)) +#endif pl%vb(1, i) = pl%vb(1, i) + pl%ah(1, i) * dt pl%vb(2, i) = pl%vb(2, i) + pl%ah(2, i) * dt pl%vb(3, i) = pl%vb(3, i) + pl%ah(3, i) * dt @@ -152,7 +156,11 @@ module subroutine helio_kick_vb_tp(self, nbody_system, param, t, dt, lbeg) ntp = self%nbody tp%ah(:, 1:ntp) = 0.0_DP call tp%accel(nbody_system, param, t, lbeg) +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,dt) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%vb(:, i) = tp%vb(:, i) + tp%ah(:, i) * dt end do end associate diff --git a/src/operator/operator_cross.f90 b/src/operator/operator_cross.f90 index f784a1c98..ef80e1fb8 100644 --- a/src/operator/operator_cross.f90 +++ b/src/operator/operator_cross.f90 @@ -104,7 +104,11 @@ pure module function operator_cross_el_sp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_sp(A(:,i), B(:,i)) end do return @@ -118,7 +122,11 @@ pure module function operator_cross_el_dp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_dp(A(:,i), B(:,i)) end do return @@ -132,7 +140,11 @@ pure module function operator_cross_el_qp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_qp(A(:,i), B(:,i)) end do return @@ -146,7 +158,11 @@ pure module function operator_cross_el_i1b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_i1b(A(:,i), B(:,i)) end do return @@ -160,7 +176,11 @@ pure module function operator_cross_el_i2b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_i2b(A(:,i), B(:,i)) end do return @@ -174,7 +194,11 @@ pure module function operator_cross_el_i4b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_i4b(A(:,i), B(:,i)) end do return @@ -188,7 +212,11 @@ pure module function operator_cross_el_i8b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B,C) +#else do concurrent (i = 1:n) +#endif C(:,i) = operator_cross_i8b(A(:,i), B(:,i)) end do return diff --git a/src/operator/operator_mag.f90 b/src/operator/operator_mag.f90 index 8bf6bff5b..55f653fb9 100644 --- a/src/operator/operator_mag.f90 +++ b/src/operator/operator_mag.f90 @@ -44,7 +44,11 @@ pure module function operator_mag_el_sp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) - do concurrent (i=1:n) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B) +#else + do concurrent (i = 1:n) +#endif B(i) = norm2(A(:, i)) end do return @@ -59,7 +63,11 @@ pure module function operator_mag_el_dp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) - do concurrent (i=1:n) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B) +#else + do concurrent (i = 1:n) +#endif B(i) = norm2(A(:, i)) end do return @@ -74,7 +82,11 @@ pure module function operator_mag_el_qp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) - do concurrent (i=1:n) +#ifdef DOCONLOC + do concurrent (i = 1:n) shared(A,B) +#else + do concurrent (i = 1:n) +#endif B(i) = norm2(A(:, i)) end do return diff --git a/src/operator/operator_unit.f90 b/src/operator/operator_unit.f90 index 39f3fce53..a25ee1bb1 100644 --- a/src/operator/operator_unit.f90 +++ b/src/operator/operator_unit.f90 @@ -89,7 +89,11 @@ pure module function operator_unit_el_sp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) +#ifdef DOCONLOC + do concurrent (i=1:n) shared(A,B) +#else do concurrent (i=1:n) +#endif B(:,i) = operator_unit_sp(A(:,i)) end do @@ -109,7 +113,11 @@ pure module function operator_unit_el_dp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) +#ifdef DOCONLOC + do concurrent (i=1:n) shared(A,B) +#else do concurrent (i=1:n) +#endif B(:,i) = operator_unit_dp(A(:,i)) end do @@ -128,7 +136,11 @@ pure module function operator_unit_el_qp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) +#ifdef DOCONLOC + do concurrent (i=1:n) shared(A,B) +#else do concurrent (i=1:n) +#endif B(:,i) = operator_unit_qp(A(:,i)) end do diff --git a/src/rmvs/rmvs_kick.f90 b/src/rmvs/rmvs_kick.f90 index 8fb6c14ce..82d19463e 100644 --- a/src/rmvs/rmvs_kick.f90 +++ b/src/rmvs/rmvs_kick.f90 @@ -61,17 +61,29 @@ module subroutine rmvs_kick_getacch_tp(self, nbody_system, param, t, lbeg) ! Now compute any heliocentric values of acceleration if (tp%lfirst) then +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%rheliocentric(:,i) = tp%rh(:,i) + cb%inner(inner_index - 1)%x(:,1) end do else +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%rheliocentric(:,i) = tp%rh(:,i) + cb%inner(inner_index )%x(:,1) end do end if ! Swap the planetocentric and heliocentric position vectors and central body masses +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%rh(:, i) = tp%rheliocentric(:, i) end do GMcb_original = cb%Gmass diff --git a/src/swiftest/swiftest_drift.f90 b/src/swiftest/swiftest_drift.f90 index 5c0427c62..82239e452 100644 --- a/src/swiftest/swiftest_drift.f90 +++ b/src/swiftest/swiftest_drift.f90 @@ -82,7 +82,11 @@ module subroutine swiftest_drift_all(mu, x, v, n, param, dt, lmask, iflag) allocate(dtp(n)) if (param%lgr) then +#ifdef DOCONLOC + do concurrent(i = 1:n, lmask(i)) shared(param,lmask,x,v,mu,dtp,dt) local(rmag,vmag2,energy) +#else do concurrent(i = 1:n, lmask(i)) +#endif rmag = norm2(x(:, i)) vmag2 = dot_product(v(:, i), v(:, i)) energy = 0.5_DP * vmag2 - mu(i) / rmag diff --git a/src/swiftest/swiftest_gr.f90 b/src/swiftest/swiftest_gr.f90 index 1c37d3da4..1985f6dd1 100644 --- a/src/swiftest/swiftest_gr.f90 +++ b/src/swiftest/swiftest_gr.f90 @@ -73,7 +73,11 @@ pure module subroutine swiftest_gr_kick_getacch(mu, x, lmask, n, inv_c2, agr) real(DP) :: beta, rjmag4 agr(:,:) = 0.0_DP +#ifdef DOCONLOC + do concurrent (i = 1:n, lmask(i)) shared(lmask,x,mu,agr,inv_c2) local(rjmag4,beta) +#else do concurrent (i = 1:n, lmask(i)) +#endif rjmag4 = (dot_product(x(:, i), x(:, i)))**2 beta = -mu(i)**2 * inv_c2 agr(:, i) = 2 * beta * x(:, i) / rjmag4 diff --git a/src/swiftest/swiftest_kick.f90 b/src/swiftest/swiftest_kick.f90 index 751702fa2..82cefd26c 100644 --- a/src/swiftest/swiftest_kick.f90 +++ b/src/swiftest/swiftest_kick.f90 @@ -21,7 +21,6 @@ module subroutine swiftest_kick_getacch_int_pl(self, param) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_parameters), intent(inout) :: param !! Current swiftest run configuration parameters ! Internals - logical, save :: lfirst = .true. #ifdef PROFILE type(walltimer), save :: timer #endif @@ -136,7 +135,7 @@ module subroutine swiftest_kick_getacch_int_all_flat_norad_pl(npl, nplpl, k_plpl integer(I8B) :: k real(DP), dimension(NDIM,npl) :: ahi, ahj integer(I4B) :: i, j - real(DP) :: rji2, rlim2 + real(DP) :: rji2 real(DP) :: rx, ry, rz ahi(:,:) = 0.0_DP @@ -197,7 +196,11 @@ module subroutine swiftest_kick_getacch_int_all_tri_rad_pl(npl, nplm, r, Gmass, !$omp reduction(+:ahi) & !$omp reduction(-:ahj) do i = 1, nplm +#ifdef DOCONLOC + do concurrent(j = i+1:npl) shared(i,r,radius,ahi,ahj,Gmass) local(rx,ry,rz,rji2,rlim2) +#else do concurrent(j = i+1:npl) +#endif rx = r(1, j) - r(1, i) ry = r(2, j) - r(2, i) rz = r(3, j) - r(3, i) @@ -208,14 +211,22 @@ module subroutine swiftest_kick_getacch_int_all_tri_rad_pl(npl, nplm, r, Gmass, end do end do !$omp end parallel do +#ifdef DOCONLOC + do concurrent(i = 1:npl) shared(acc,ahi,ahj) +#else do concurrent(i = 1:npl) +#endif acc(:,i) = acc(:,i) + ahi(:,i) + ahj(:,i) end do else !$omp parallel do default(private) schedule(static)& !$omp shared(npl, nplm, r, Gmass, radius, acc) do i = 1, nplm +#ifdef DOCONLOC + do concurrent(j = 1:npl, i/=j) shared(i,r,radius,Gmass,acc) local(rx,ry,rz,rji2,rlim2,fac) +#else do concurrent(j = 1:npl, i/=j) +#endif rx = r(1,j) - r(1,i) ry = r(2,j) - r(2,i) rz = r(3,j) - r(3,i) @@ -235,7 +246,11 @@ module subroutine swiftest_kick_getacch_int_all_tri_rad_pl(npl, nplm, r, Gmass, !$omp parallel do default(private) schedule(static)& !$omp shared(npl, nplm, r, Gmass, radius, acc) do i = nplm+1,npl +#ifdef DOCONLOC + do concurrent(j = 1:nplm) shared(i,r,radius,Gmass,acc) local(rx,ry,rz,rji2,rlim2,fac) +#else do concurrent(j = 1:nplm) +#endif rx = r(1,j) - r(1,i) ry = r(2,j) - r(2,i) rz = r(3,j) - r(3,i) @@ -275,7 +290,7 @@ module subroutine swiftest_kick_getacch_int_all_tri_norad_pl(npl, nplm, r, Gmass real(DP), dimension(:,:), intent(inout) :: acc !! Acceleration vector array ! Internals integer(I4B) :: i, j, nplt - real(DP) :: rji2, rlim2, fac, rx, ry, rz + real(DP) :: rji2, fac, rx, ry, rz real(DP), dimension(NDIM,npl) :: ahi, ahj logical :: lmtiny @@ -290,7 +305,11 @@ module subroutine swiftest_kick_getacch_int_all_tri_norad_pl(npl, nplm, r, Gmass !$omp reduction(+:ahi) & !$omp reduction(-:ahj) do i = 1, nplm +#ifdef DOCONLOC + do concurrent(j = i+1:npl) shared(i,r,Gmass,ahi,ahj) local(rx,ry,rz,rji2) +#else do concurrent(j = i+1:npl) +#endif rx = r(1, j) - r(1, i) ry = r(2, j) - r(2, i) rz = r(3, j) - r(3, i) @@ -300,14 +319,22 @@ module subroutine swiftest_kick_getacch_int_all_tri_norad_pl(npl, nplm, r, Gmass end do end do !$omp end parallel do +#ifdef DOCONLOC + do concurrent(i = 1:npl) shared(acc,ahi,ahj) +#else do concurrent(i = 1:npl) +#endif acc(:,i) = acc(:,i) + ahi(:,i) + ahj(:,i) end do else !$omp parallel do default(private) schedule(static)& !$omp shared(npl, nplm, r, Gmass, acc) do i = 1, nplm +#ifdef DOCONLOC + do concurrent(j = 1:npl, j/=i) shared(i,r,Gmass, acc) local(rx,ry,rz,rji2,fac) +#else do concurrent(j = 1:npl, j/=i) +#endif rx = r(1,j) - r(1,i) ry = r(2,j) - r(2,i) rz = r(3,j) - r(3,i) @@ -324,7 +351,11 @@ module subroutine swiftest_kick_getacch_int_all_tri_norad_pl(npl, nplm, r, Gmass !$omp parallel do default(private) schedule(static)& !$omp shared(npl, nplm, r, Gmass, acc) do i = nplm+1,npl +#ifdef DOCONLOC + do concurrent(j = 1:nplm) shared(i,r,Gmass,acc) local(rx,ry,rz,rji2,fac) +#else do concurrent(j = 1:nplm) +#endif rx = r(1,j) - r(1,i) ry = r(2,j) - r(2,i) rz = r(3,j) - r(3,i) @@ -369,7 +400,11 @@ module subroutine swiftest_kick_getacch_int_all_tp(ntp, npl, rtp, rpl, GMpl, lma !$omp reduction(-:acc) do i = 1, ntp if (lmask(i)) then +#ifdef DOCONLOC + do concurrent (j = 1:npl) shared(rtp,rpl,GMpl,acc) local(rx,ry,rz,rji2) +#else do concurrent (j = 1:npl) +#endif rx = rtp(1, i) - rpl(1, j) ry = rtp(2, i) - rpl(2, j) rz = rtp(3, i) - rpl(3, j) diff --git a/src/swiftest/swiftest_obl.f90 b/src/swiftest/swiftest_obl.f90 index 85f88bc7d..21f4363ae 100644 --- a/src/swiftest/swiftest_obl.f90 +++ b/src/swiftest/swiftest_obl.f90 @@ -35,7 +35,11 @@ module subroutine swiftest_obl_acc(n, GMcb, j2rp2, j4rp4, rh, lmask, aobl, GMpl, if (n == 0) return aobl(:,:) = 0.0_DP +#ifdef DOCONLOC + do concurrent(i = 1:n, lmask(i)) shared(lmask,rh,aobl) local(r2,irh,rinv2,t0,t1,t2,t3,fac1,fac2) +#else do concurrent(i = 1:n, lmask(i)) +#endif r2 = dot_product(rh(:, i), rh(:, i)) irh = 1.0_DP / sqrt(r2) rinv2 = irh**2 @@ -81,7 +85,11 @@ module subroutine swiftest_obl_acc_pl(self, nbody_system) npl = self%nbody call swiftest_obl_acc(npl, cb%Gmass, cb%j2rp2, cb%j4rp4, pl%rh, pl%lmask, pl%aobl, pl%Gmass, cb%aobl) +#ifdef DOCONLOC + do concurrent(i = 1:npl, pl%lmask(i)) shared(cb,pl) +#else do concurrent(i = 1:npl, pl%lmask(i)) +#endif pl%ah(:, i) = pl%ah(:, i) + pl%aobl(:, i) - cb%aobl(:) end do end associate @@ -117,7 +125,11 @@ module subroutine swiftest_obl_acc_tp(self, nbody_system) aoblcb = cb%aoblend end if +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,aoblcb) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%ah(:, i) = tp%ah(:, i) + tp%aobl(:, i) - aoblcb(:) end do @@ -148,7 +160,7 @@ module subroutine swiftest_obl_pot_system(self) npl = self%pl%nbody if (.not. any(pl%lmask(1:npl))) return #ifdef DOCONLOC - do concurrent (i = 1:npl, pl%lmask(i)) shared(oblpot_arr) + do concurrent (i = 1:npl, pl%lmask(i)) shared(cb,pl,oblpot_arr) #else do concurrent (i = 1:npl, pl%lmask(i)) #endif diff --git a/src/swiftest/swiftest_orbel.f90 b/src/swiftest/swiftest_orbel.f90 index a3ec6f424..dafb3e293 100644 --- a/src/swiftest/swiftest_orbel.f90 +++ b/src/swiftest/swiftest_orbel.f90 @@ -26,7 +26,11 @@ module subroutine swiftest_orbel_el2xv_vec(self, cb) if (self%nbody == 0) return call self%set_mu(cb) +#ifdef DOCONLOC + do concurrent (i = 1:self%nbody) shared(self) +#else do concurrent (i = 1:self%nbody) +#endif call swiftest_orbel_el2xv(self%mu(i), self%a(i), self%e(i), self%inc(i), self%capom(i), & self%omega(i), self%capm(i), self%rh(:, i), self%vh(:, i)) end do @@ -887,7 +891,11 @@ module subroutine swiftest_orbel_xv2el_vec(self, cb) if (allocated(self%capom)) deallocate(self%capom); allocate(self%capom(self%nbody)) if (allocated(self%omega)) deallocate(self%omega); allocate(self%omega(self%nbody)) if (allocated(self%capm)) deallocate(self%capm); allocate(self%capm(self%nbody)) +#ifdef DOCONLOC + do concurrent (i = 1:self%nbody) shared(self) local(varpi,lam,f,cape,capf) +#else do concurrent (i = 1:self%nbody) +#endif call swiftest_orbel_xv2el(self%mu(i), self%rh(1,i), self%rh(2,i), self%rh(3,i), & self%vh(1,i), self%vh(2,i), self%vh(3,i), & self%a(i), self%e(i), self%inc(i), & @@ -932,7 +940,7 @@ pure module subroutine swiftest_orbel_xv2el(mu, rx, ry, rz, vx, vy, vz, a, e, in real(DP), intent(out) :: capf !! hyperbolic anomaly (hyperbolic orbits) ! Internals integer(I4B) :: iorbit_type - real(DP) :: hx, hy, hz, r, v2, h2, h, rdotv, energy, fac, u, w, cw, sw, face, tmpf, sf, cf, rdot, h_over_r2 + real(DP) :: hx, hy, hz, r, v2, h2, h, rdotv, energy, fac, u, w, cw, sw, face, tmpf, sf, cf, rdot a = 0.0_DP e = 0.0_DP diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index b3e43c7e2..b063cb418 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -282,7 +282,11 @@ module subroutine swiftest_util_coord_h2b_tp(self, cb) if (self%nbody == 0) return associate(tp => self) ntp = self%nbody +#ifdef DOCONLOC + do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) shared(cb,tp) +#else do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) +#endif tp%rb(:, i) = tp%rh(:, i) + cb%rb(:) tp%vb(:, i) = tp%vh(:, i) + cb%vb(:) end do @@ -310,7 +314,11 @@ module subroutine swiftest_util_coord_b2h_pl(self, cb) associate(pl => self) npl = self%nbody +#ifdef DOCONLOC + do concurrent (i = 1:npl, pl%status(i) /= INACTIVE) shared(cb,pl) +#else do concurrent (i = 1:npl, pl%status(i) /= INACTIVE) +#endif pl%rh(:, i) = pl%rb(:, i) - cb%rb(:) pl%vh(:, i) = pl%vb(:, i) - cb%vb(:) end do @@ -338,7 +346,11 @@ module subroutine swiftest_util_coord_b2h_tp(self, cb) associate(tp => self) ntp = self%nbody +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%status(i) /= INACTIVE) shared(cb,tp) +#else do concurrent(i = 1:ntp, tp%status(i) /= INACTIVE) +#endif tp%rh(:, i) = tp%rb(:, i) - cb%rb(:) tp%vh(:, i) = tp%vb(:, i) - cb%vb(:) end do @@ -370,7 +382,11 @@ module subroutine swiftest_util_coord_vb2vh_pl(self, cb) do i = npl, 1, -1 if (pl%status(i) /= INACTIVE) cb%vb(:) = cb%vb(:) - pl%Gmass(i) * pl%vb(:, i) / cb%Gmass end do +#ifdef DOCONLOC + do concurrent(i = 1:npl) shared(cb,pl) +#else do concurrent(i = 1:npl) +#endif pl%vh(:, i) = pl%vb(:, i) - cb%vb(:) end do end associate @@ -430,7 +446,11 @@ module subroutine swiftest_util_coord_vh2vb_pl(self, cb) cb%vb(:) = cb%vb(:) - pl%Gmass(i) * pl%vh(:, i) end do cb%vb(:) = cb%vb(:) / Gmtot +#ifdef DOCONLOC + do concurrent(i = 1:npl) shared(cb,pl) +#else do concurrent(i = 1:npl) +#endif pl%vb(:, i) = pl%vh(:, i) + cb%vb(:) end do end associate @@ -518,7 +538,11 @@ module subroutine swiftest_util_coord_rh2rb_tp(self, cb) if (self%nbody == 0) return associate(tp => self) ntp = self%nbody +#ifdef DOCONLOC + do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) shared(cb,tp) +#else do concurrent (i = 1:ntp, tp%status(i) /= INACTIVE) +#endif tp%rb(:, i) = tp%rh(:, i) + cb%rb(:) end do end associate @@ -1089,7 +1113,11 @@ module subroutine swiftest_util_flatten_eucl_plpl(self, param) if (err /=0) then ! An error occurred trying to allocate this big array. This probably means it's too big to fit in memory, and so we will force the run back into triangular mode param%lflatten_interactions = .false. else +#ifdef DOCONLOC + do concurrent (i=1:npl, j=1:npl, j>i) shared(self) local(k) +#else do concurrent (i=1:npl, j=1:npl, j>i) +#endif call swiftest_util_flatten_eucl_ij_to_k(self%nbody, i, j, k) self%k_plpl(1, k) = i self%k_plpl(2, k) = j @@ -1179,7 +1207,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) Lcborbit(:) = cb%mass * (cb%rb(:) .cross. cb%vb(:)) #ifdef DOCONLOC - do concurrent (i = 1:npl, pl%lmask(i)) local(h) shared(Lplorbit, kepl) + do concurrent (i = 1:npl, pl%lmask(i)) shared(pl,Lplorbit,kepl) local(h) #else do concurrent (i = 1:npl, pl%lmask(i)) #endif @@ -1199,7 +1227,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) Lcbspin(:) = cb%Ip(3) * cb%mass * cb%radius**2 * cb%rot(:) #ifdef DOCONLOC - do concurrent (i = 1:npl, pl%lmask(i)) shared(Lplspin, kespinpl) + do concurrent (i = 1:npl, pl%lmask(i)) shared(pl,Lplspin,kespinpl) #else do concurrent (i = 1:npl, pl%lmask(i)) #endif @@ -1213,7 +1241,11 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) nbody_system%ke_spin = 0.5_DP * (kespincb + sum(kespinpl(1:npl), pl%lmask(1:npl))) +#ifdef DOCONLOC + do concurrent (j = 1:NDIM) shared(nbody_system,pl,Lplspin,Lcbspin) +#else do concurrent (j = 1:NDIM) +#endif nbody_system%L_spin(j) = Lcbspin(j) + sum(Lplspin(j,1:npl), pl%lmask(1:npl)) end do else @@ -1234,8 +1266,11 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) end if nbody_system%ke_orbit = 0.5_DP * (kecb + sum(kepl(1:npl), pl%lmask(1:npl))) - +#ifdef DOCONLOC + do concurrent (j = 1:NDIM) shared(nbody_system,pl,Lcborbit,Lplorbit) +#else do concurrent (j = 1:NDIM) +#endif nbody_system%L_orbit(j) = Lcborbit(j) + sum(Lplorbit(j,1:npl), pl%lmask(1:npl)) end do @@ -1280,7 +1315,7 @@ module subroutine swiftest_util_get_potential_energy_flat(npl, nplpl, k_plpl, lm end where #ifdef DOCONLOC - do concurrent(i = 1:npl, lmask(i)) shared(pecb) + do concurrent(i = 1:npl, lmask(i)) shared(lmask,pecb,GMcb,mass,rb) #else do concurrent(i = 1:npl, lmask(i)) #endif @@ -1331,7 +1366,7 @@ module subroutine swiftest_util_get_potential_energy_triangular(npl, lmask, GMcb end where #ifdef DOCONLOC - do concurrent(i = 1:npl, lmask(i)) shared(pecb) + do concurrent(i = 1:npl, lmask(i)) shared(lmask, pecb, GMcb, mass, rb, lmask) #else do concurrent(i = 1:npl, lmask(i)) #endif @@ -1346,7 +1381,7 @@ module subroutine swiftest_util_get_potential_energy_triangular(npl, lmask, GMcb do i = 1, npl if (lmask(i)) then #ifdef DOCONLOC - do concurrent(j = i+1:npl, lmask(i) .and. lmask(j)) shared(pepl) + do concurrent(j = i+1:npl, lmask(i) .and. lmask(j)) shared(lmask, pepl, rb, mass, Gmass, lmask) #else do concurrent(j = i+1:npl, lmask(i) .and. lmask(j)) #endif diff --git a/src/symba/symba_encounter_check.f90 b/src/symba/symba_encounter_check.f90 index 895a9f34d..4bc92b79d 100644 --- a/src/symba/symba_encounter_check.f90 +++ b/src/symba/symba_encounter_check.f90 @@ -119,7 +119,11 @@ module function symba_encounter_check_list_plpl(self, param, nbody_system, dt, i eidx(:) = pack([(k, k = 1, self%nenc)], lencmask(:)) lencounter(:) = .false. +#ifdef DOCONLOC + do concurrent(lidx = 1:nenc_enc) shared(self,pl,eidx,lencounter,dt) local(i,j,k,xr,vr,rcrit12,rlim2,rji2) +#else do concurrent(lidx = 1:nenc_enc) +#endif k = eidx(lidx) i = self%index1(k) j = self%index2(k) @@ -188,8 +192,11 @@ module function symba_encounter_check_list_pltp(self, param, nbody_system, dt, i allocate(lencounter(nenc_enc)) eidx(:) = pack([(k, k = 1, self%nenc)], lencmask(:)) lencounter(:) = .false. - +#ifdef DOCONLOC + do concurrent(lidx = 1:nenc_enc) shared(self,pl,tp,eidx,lencounter,dt) local(i,j,k,xr,vr,rlim2,rji2) +#else do concurrent(lidx = 1:nenc_enc) +#endif k = eidx(lidx) i = self%index1(k) j = self%index2(k) diff --git a/src/symba/symba_kick.f90 b/src/symba/symba_kick.f90 index 0c75e1054..dc280ef1c 100644 --- a/src/symba/symba_kick.f90 +++ b/src/symba/symba_kick.f90 @@ -172,7 +172,11 @@ module subroutine symba_kick_list_plpl(self, nbody_system, dt, irec, sgn) allocate(good_idx(ngood)) good_idx(:) = pack([(i, i = 1, nenc)], lgoodlevel(:)) +#ifdef DOCONLOC + do concurrent (k = 1:ngood) shared(self,pl,good_idx) local(i,j) +#else do concurrent (k = 1:ngood) +#endif i = self%index1(good_idx(k)) j = self%index2(good_idx(k)) pl%ah(:,i) = 0.0_DP @@ -282,8 +286,11 @@ module subroutine symba_kick_list_pltp(self, nbody_system, dt, irec, sgn) allocate(good_idx(ngood)) good_idx(:) = pack([(i, i = 1, nenc)], lgoodlevel(:)) - +#ifdef DOCONLOC + do concurrent (k = 1_I8B:ngood) shared(self,tp,good_idx) local(j) +#else do concurrent (k = 1_I8B:ngood) +#endif j = self%index2(good_idx(k)) tp%ah(:,j) = 0.0_DP end do diff --git a/src/whm/whm_gr.f90 b/src/whm/whm_gr.f90 index 1b1716800..c46a5ce2d 100644 --- a/src/whm/whm_gr.f90 +++ b/src/whm/whm_gr.f90 @@ -86,7 +86,11 @@ pure module subroutine whm_gr_p4_pl(self, nbody_system, param, dt) associate(pl => self) npl = self%nbody +#ifdef DOCONLOC + do concurrent(i = 1:npl, pl%lmask(i)) shared(pl,dt) +#else do concurrent(i = 1:npl, pl%lmask(i)) +#endif call swiftest_gr_p4_pos_kick(param, pl%xj(:, i), pl%vj(:, i), dt) end do end associate @@ -114,7 +118,11 @@ pure module subroutine whm_gr_p4_tp(self, nbody_system, param, dt) associate(tp => self) ntp = self%nbody if (ntp == 0) return +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,dt) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vh(:, i), dt) end do end associate diff --git a/src/whm/whm_kick.f90 b/src/whm/whm_kick.f90 index a1d4b0dc6..bcddf0de7 100644 --- a/src/whm/whm_kick.f90 +++ b/src/whm/whm_kick.f90 @@ -93,13 +93,21 @@ module subroutine whm_kick_getacch_tp(self, nbody_system, param, t, lbeg) if (lbeg) then ah0(:) = whm_kick_getacch_ah0(pl%Gmass(1:npl), pl%rbeg(:, 1:npl), npl) +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,ah0) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%ah(:, i) = tp%ah(:, i) + ah0(:) end do call tp%accel_int(param, pl%Gmass(1:npl), pl%rbeg(:, 1:npl), npl) else ah0(:) = whm_kick_getacch_ah0(pl%Gmass(1:npl), pl%rend(:, 1:npl), npl) +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,ah0) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%ah(:, i) = tp%ah(:, i) + ah0(:) end do call tp%accel_int(param, pl%Gmass(1:npl), pl%rend(:, 1:npl), npl) @@ -157,7 +165,11 @@ pure subroutine whm_kick_getacch_ah1(cb, pl) real(DP), dimension(NDIM) :: ah1h, ah1j npl = pl%nbody +#ifdef DOCONLOC + do concurrent (i = 2:npl, pl%lmask(i)) shared(pl,cb) local(ah1j,ah1h) +#else do concurrent (i = 2:npl, pl%lmask(i)) +#endif ah1j(:) = pl%xj(:, i) * pl%ir3j(i) ah1h(:) = pl%rh(:, i) * pl%ir3h(i) pl%ah(:, i) = pl%ah(:, i) + cb%Gmass * (ah1j(:) - ah1h(:)) @@ -187,10 +199,14 @@ pure subroutine whm_kick_getacch_ah2(cb, pl) ah2(:) = 0.0_DP ah2o(:) = 0.0_DP etaj = cb%Gmass +#ifdef DOCONLOC + do concurrent(i = 2:npl, pl%lmask(i)) shared(pl,cb,ah2,ah2o) local(etaj,fac) +#else do concurrent(i = 2:npl, pl%lmask(i)) +#endif etaj = etaj + pl%Gmass(i - 1) fac = pl%Gmass(i) * cb%Gmass * pl%ir3j(i) / etaj - ah2(:) = ah2o + fac * pl%xj(:, i) + ah2(:) = ah2o(:) + fac * pl%xj(:, i) pl%ah(:,i) = pl%ah(:, i) + ah2(:) ah2o(:) = ah2(:) end do @@ -233,7 +249,11 @@ module subroutine whm_kick_vh_pl(self, nbody_system, param, t, dt, lbeg) call pl%accel(nbody_system, param, t, lbeg) call pl%set_beg_end(rend = pl%rh) end if +#ifdef DOCONLOC + do concurrent(i = 1:npl, pl%lmask(i)) shared(pl,dt) +#else do concurrent(i = 1:npl, pl%lmask(i)) +#endif pl%vh(:, i) = pl%vh(:, i) + pl%ah(:, i) * dt end do end associate @@ -265,19 +285,31 @@ module subroutine whm_kick_vh_tp(self, nbody_system, param, t, dt, lbeg) associate(tp => self) ntp = self%nbody if (tp%lfirst) then +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%ah(:, i) = 0.0_DP end do call tp%accel(nbody_system, param, t, lbeg=.true.) tp%lfirst = .false. end if if (.not.lbeg) then +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%ah(:, i) = 0.0_DP end do call tp%accel(nbody_system, param, t, lbeg) end if +#ifdef DOCONLOC + do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp) +#else do concurrent(i = 1:ntp, tp%lmask(i)) +#endif tp%vh(:, i) = tp%vh(:, i) + tp%ah(:, i) * dt end do end associate From a2c5e5465f5f2e2172cb24472c32f1c92a01aaae Mon Sep 17 00:00:00 2001 From: David A Minton Date: Fri, 19 May 2023 23:01:01 -0400 Subject: [PATCH 084/149] Fixed typo that prevented Ip from being automatically set when rotation vectors were passed --- python/swiftest/swiftest/init_cond.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/swiftest/swiftest/init_cond.py b/python/swiftest/swiftest/init_cond.py index efaa6429f..1b140f119 100644 --- a/python/swiftest/swiftest/init_cond.py +++ b/python/swiftest/swiftest/init_cond.py @@ -283,7 +283,7 @@ def vec2xr(param: Dict, **kwargs: Any): # Check for valid keyword arguments kwargs = {k:kwargs[k] for k,v in kwargs.items() if v is not None} - if "rotation" in param and param['rotation'] == True: + if "ROTATION" in param and param['ROTATION'] == True: if "rot" not in kwargs and "Gmass" in kwargs: kwargs['rot'] = np.zeros((len(kwargs['Gmass']),3)) if "Ip" not in kwargs and "Gmass" in kwargs: From b6e61d353e9cae167c606ff35574394317de8e2c Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sat, 20 May 2023 07:12:15 -0400 Subject: [PATCH 085/149] Fixed bug in example problem where escape velocity was miscalculated when setting test particle initial conditions --- .../rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py index d59c068d6..725c123b1 100755 --- a/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py +++ b/examples/rmvs_swifter_comparison/8pl_16tp_encounters/init_cond.py @@ -17,7 +17,7 @@ for i,n in enumerate(pl.name): pli = pl.sel(name=n) rstart = 2 * pli['radius'].data[0] # Start the test particles at a multiple of the planet radius away - vstart = 1.5 * np.sqrt(2 * pli['Gmass'].data[0]) / rstart # Start the test particle velocities at a multiple of the escape speed + vstart = 1.5 * np.sqrt(2 * pli['Gmass'].data[0] / rstart) # Start the test particle velocities at a multiple of the escape speed rstart_vec = np.array([rstart / np.sqrt(2.0), rstart / np.sqrt(2.0), 0.0]) vstart_vec = np.array([vstart, 0.0, 0.0]) rp = pli['rh'].data[0] From 4fa3506f561ff0589a9be5283c521c43eaaf331a Mon Sep 17 00:00:00 2001 From: David Minton Date: Sat, 20 May 2023 16:44:54 -0400 Subject: [PATCH 086/149] Cleaned up a few more unused variables caught during testing on fresh install --- src/encounter/encounter_util.f90 | 6 ++---- src/swiftest/swiftest_io.f90 | 10 +++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/encounter/encounter_util.f90 b/src/encounter/encounter_util.f90 index f935afe5a..7307eb9cb 100644 --- a/src/encounter/encounter_util.f90 +++ b/src/encounter/encounter_util.f90 @@ -22,7 +22,7 @@ module subroutine encounter_util_append_list(self, source, lsource_mask) class(encounter_list), intent(in) :: source !! Source object to append logical, dimension(:), intent(in) :: lsource_mask !! Logical mask indicating which elements to append to ! Internals - integer(I4B) :: nold, nsrc + integer(I4B) :: nold nold = int(self%nenc, kind=I4B) call util_append(self%tcollision, source%tcollision, nold, lsource_mask) @@ -100,8 +100,6 @@ module subroutine encounter_util_dealloc_bounding_box(self) implicit none ! Arguments class(encounter_bounding_box), intent(inout) :: self !! Bounding box structure - ! Internals - integer(I4B) :: i call self%aabb%dealloc() @@ -341,7 +339,7 @@ module subroutine encounter_util_setup_aabb(self, n, n_last) integer(I4B), intent(in) :: n !! Number of objects with bounding box extents integer(I4B), intent(in) :: n_last !! Number of objects with bounding box extents the previous time this was called ! Internals - integer(I4B) :: next, next_last, k, dim + integer(I4B) :: next, next_last, k integer(I4B), dimension(:), allocatable :: itmp next = 2 * n diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 0f2f6be2b..1e02a7645 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -414,7 +414,9 @@ module subroutine swiftest_io_dump_storage(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i +#ifdef COARRAY type(walltimer) :: iotimer +#endif if (self%iframe == 0) return call self%make_index_map() @@ -1043,7 +1045,7 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) real(DP), dimension(:,:), allocatable :: rh integer(I4B), dimension(:), allocatable :: body_status logical, dimension(:), allocatable :: lvalid - integer(I4B) :: idmax, status,i + integer(I4B) :: idmax, status call netcdf_io_check( nf90_inquire_dimension(self%id, self%name_dimid, len=idmax), "swiftest_io_netcdf_get_valid_masks nf90_inquire_dimension name_dimid" ) @@ -1615,11 +1617,14 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, idslot, old_mode, ntp + integer(I4B) :: i, j, idslot, old_mode integer(I4B), dimension(:), allocatable :: ind real(DP), dimension(NDIM) :: vh !! Temporary variable to store heliocentric velocity values when converting from pseudovelocity in GR-enabled runs real(DP) :: a, e, inc, omega, capom, capm, varpi, lam, f, cape, capf +#ifdef COARRAY + integer(I4B) :: ntp logical, dimension(:), allocatable :: tpmask, plmask +#endif call self%write_info(nc, param) @@ -1798,7 +1803,6 @@ module subroutine swiftest_io_netcdf_write_hdr_system(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - logical, dimension(:), allocatable :: tpmask, plmask integer(I4B) :: tslot call nc%find_tslot(self%t, tslot) From 9556d1812a71573db858a56b3703d39b2e2f41d8 Mon Sep 17 00:00:00 2001 From: David Minton Date: Mon, 22 May 2023 17:41:40 -0400 Subject: [PATCH 087/149] Draft Dockerfile for creating a container for Swftest. I've commented out everything after netcdf-c compilation to troubleshoot the location of mpiifort for compiling the rest. --- .gitignore | 7 +++++- Dockerfile | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ swiftest.def | 2 ++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 swiftest.def diff --git a/.gitignore b/.gitignore index 674a2f7f4..81a7fb376 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,11 @@ dump* !docs/*/*/* !README_figs/* +#Docker and Singularity files +!Dockerfile +!swiftest.def + bin/ build/* -disruption_headon/swiftest_driver.sh + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..69f0d1288 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,64 @@ +FROM debian:stable-slim as build + +# kick everything off +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev && \ + rm -rf /var/lib/apt/lists/* + +# Get CMAKE and install it +RUN mkdir -p cmake/build && \ + cd cmake/build && \ + curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ + /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license + +# Get the Intel compilers +RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - +RUN echo "deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y intel-oneapi-dev-utilities intel-oneapi-compiler-dpcpp-cpp intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mkl-devel intel-oneapi-mpi-devel +RUN . /opt/intel/oneapi/setvars.sh + +# Build the NetCDF libraries +RUN mkdir -p /opt/build && mkdir -p /opt/dist +ENV INDIR="/opt/dist//usr/local" +ENV INTEL_DIR="/opt/intel/oneapi/compiler/latest/linux" +ENV CC="${INTEL_DIR}/bin/icx-cc" +ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" + +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libhdf5-dev hdf5-tools zlib1g zlib1g-dev libxml2-dev libcurl-dev && \ + rm -rf /var/lib/apt/lists/* + + +#NetCDF-c library +RUN git clone https://github.com/Unidata/netcdf-c.git +RUN cd netcdf-c && mkdir build && cd build && \ + cmake .. -DCMAKE_PREFIX_PATH="${LD_LIBRARY_PATH}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ + make && make install + +#NetCDF-Fortran library +RUN git clone https://github.com/Unidata/netcdf-fortran.git +RUN cd netcdf-fortran && mkdir build && cd build && \ + cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ + make && make install + +#Swiftest +RUN git clone -b debug https://github.com/carlislewishard/swiftest.git +RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ + make && make install + +#Production container +FROM debian:stable-slim +COPY --from=build /opt/dist / + +# Get the Intel runtime libraries +RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - +RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran +RUN . /opt/intel/oneapi/setvars.sh + +CMD ["/usr/bin/swiftest_driver"] \ No newline at end of file diff --git a/swiftest.def b/swiftest.def new file mode 100644 index 000000000..9eeb9d3ef --- /dev/null +++ b/swiftest.def @@ -0,0 +1,2 @@ +Bootstrap: docker +From: daminton:swiftest:latest \ No newline at end of file From 5b5b5c130ea1b486ea049aeb84c288594d8f07e9 Mon Sep 17 00:00:00 2001 From: David Minton Date: Mon, 22 May 2023 18:01:55 -0400 Subject: [PATCH 088/149] Added missing components to Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 69f0d1288..4a82504e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,13 +22,13 @@ RUN . /opt/intel/oneapi/setvars.sh # Build the NetCDF libraries RUN mkdir -p /opt/build && mkdir -p /opt/dist ENV INDIR="/opt/dist//usr/local" -ENV INTEL_DIR="/opt/intel/oneapi/compiler/latest/linux" -ENV CC="${INTEL_DIR}/bin/icx-cc" +ENV INTEL_DIR="/opt/intel/oneapi" +ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libhdf5-dev hdf5-tools zlib1g zlib1g-dev libxml2-dev libcurl-dev && \ + libhdf5-dev hdf5-tools zlib1g zlib1g-dev libxml2-dev libcurl4-gnutls-dev m4 && \ rm -rf /var/lib/apt/lists/* From 8abdaec193811e103a78af0505bc975f4b52cb99 Mon Sep 17 00:00:00 2001 From: David Minton Date: Mon, 22 May 2023 20:06:11 -0400 Subject: [PATCH 089/149] Able to build the Docker container up to swiftest by using the ifx compiler, though I know this doesn't work for swiftest --- Dockerfile | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4a82504e0..33b92e209 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM debian:stable-slim as build # kick everything off RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev && \ + ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev && \ rm -rf /var/lib/apt/lists/* # Get CMAKE and install it @@ -13,10 +13,13 @@ RUN mkdir -p cmake/build && \ /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license # Get the Intel compilers -RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - -RUN echo "deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +# download the key to system keyring +RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ +| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null +# add signed entry to apt sources and configure the APT client to use Intel repository: +RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y intel-oneapi-dev-utilities intel-oneapi-compiler-dpcpp-cpp intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mkl-devel intel-oneapi-mpi-devel +RUN apt-get install -y intel-hpckit RUN . /opt/intel/oneapi/setvars.sh # Build the NetCDF libraries @@ -24,7 +27,7 @@ RUN mkdir -p /opt/build && mkdir -p /opt/dist ENV INDIR="/opt/dist//usr/local" ENV INTEL_DIR="/opt/intel/oneapi" ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" -ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -44,21 +47,21 @@ RUN cd netcdf-fortran && mkdir build && cd build && \ cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ make && make install -#Swiftest -RUN git clone -b debug https://github.com/carlislewishard/swiftest.git -RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ - cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ - make && make install +# #Swiftest +# RUN git clone -b debug https://github.com/carlislewishard/swiftest.git +# RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ +# cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ +# make && make install -#Production container -FROM debian:stable-slim -COPY --from=build /opt/dist / +# #Production container +# FROM debian:stable-slim +# COPY --from=build /opt/dist / -# Get the Intel runtime libraries -RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - -RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list -RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran -RUN . /opt/intel/oneapi/setvars.sh +# # Get the Intel runtime libraries +# RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - +# RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +# RUN apt-get -y update && apt-get upgrade -y +# RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran +# RUN . /opt/intel/oneapi/setvars.sh -CMD ["/usr/bin/swiftest_driver"] \ No newline at end of file +# ENTRYPOINT ["/usr/bin/swiftest_driver"] \ No newline at end of file From b1620a7ad4e7f36bd7010ed4066476efdeb6f0bf Mon Sep 17 00:00:00 2001 From: David Minton Date: Mon, 22 May 2023 22:41:17 -0400 Subject: [PATCH 090/149] Added another missing package --- Dockerfile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 33b92e209..fe92b3728 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM debian:stable-slim as build # kick everything off RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev && \ + ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev procps && \ rm -rf /var/lib/apt/lists/* # Get CMAKE and install it @@ -47,21 +47,21 @@ RUN cd netcdf-fortran && mkdir build && cd build && \ cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ make && make install -# #Swiftest -# RUN git clone -b debug https://github.com/carlislewishard/swiftest.git -# RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ -# cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ -# make && make install +#Swiftest +RUN git clone -b debug https://github.com/carlislewishard/swiftest.git +RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ + FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ + make && make install -# #Production container -# FROM debian:stable-slim -# COPY --from=build /opt/dist / +#Production container +FROM debian:stable-slim +COPY --from=build /opt/dist / -# # Get the Intel runtime libraries -# RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - -# RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list -# RUN apt-get -y update && apt-get upgrade -y -# RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran -# RUN . /opt/intel/oneapi/setvars.sh +# Get the Intel runtime libraries +RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - +RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran +RUN . /opt/intel/oneapi/setvars.sh -# ENTRYPOINT ["/usr/bin/swiftest_driver"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/swiftest_driver"] \ No newline at end of file From 3dfb89a100e89ee11f878f8d7de740c8a2e46649 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 08:51:36 -0400 Subject: [PATCH 091/149] More improvements to Dockerfile. Now testing. --- Dockerfile | 67 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe92b3728..a6572206b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,6 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list RUN apt-get -y update && apt-get upgrade -y RUN apt-get install -y intel-hpckit -RUN . /opt/intel/oneapi/setvars.sh # Build the NetCDF libraries RUN mkdir -p /opt/build && mkdir -p /opt/dist @@ -47,11 +46,65 @@ RUN cd netcdf-fortran && mkdir build && cd build && \ cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ make && make install -#Swiftest +# #Swiftest RUN git clone -b debug https://github.com/carlislewishard/swiftest.git -RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && \ - FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" cmake .. -DCMAKE_BUILD_TYPE=release -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ - make && make install +ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV NETCDF_HOME="${INDIR}" +ENV NETCDF_FORTRAN_HOME="${INDIR}" +ENV LANG=C.UTF-8 +ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' +ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' +ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' +ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' +ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' +ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' +ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' +ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' +ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' +ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' +ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' +ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DAL_MAJOR_BINARY='1' +ENV DAL_MINOR_BINARY='1' +ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' +ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' +ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' +ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' +ENV FPGA_VARS_ARGS='' +ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' +ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' +ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' +ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' +ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' +ENV IPPCP_TARGET_ARCH='intel64' +ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' +ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' +ENV IPP_TARGET_ARCH='intel64' +ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' +ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' +ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' +ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' +ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' +ENV ONEAPI_ROOT='/opt/intel/oneapi' +ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' +ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' +ENV SETVARS_COMPLETED='1' +ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' +ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' +ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' +ENV VT_MPI='impi4' +ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' +ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' + +RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCMAKE_BUILD_TYPE=release && make && make install #Production container FROM debian:stable-slim @@ -62,6 +115,8 @@ RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-P RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN apt-get -y update && apt-get upgrade -y RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran -RUN . /opt/intel/oneapi/setvars.sh +ENV NETCDF_HOME="/usr/local" +ENV LANG=C.UTF-8 +ENV LD_LIBRARY_PATH=/opt/intel/oneapi/lib ENTRYPOINT ["/usr/bin/swiftest_driver"] \ No newline at end of file From eb1eb50fc9e6d04040a125bae95dc77129288905 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 08:55:23 -0400 Subject: [PATCH 092/149] Fixed missing packages in final stage --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index a6572206b..9d4e8580e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,6 +111,11 @@ FROM debian:stable-slim COPY --from=build /opt/dist / # Get the Intel runtime libraries +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl gpg-agent software-properties-common gnupg pkg-config procps && \ + rm -rf /var/lib/apt/lists/* + RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN apt-get -y update && apt-get upgrade -y From 3dc18fe66ce2f6589e12179971b02e9d9ccb37b9 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 10:28:42 -0400 Subject: [PATCH 093/149] Fixed typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d4e8580e..eaef8474d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,7 +117,7 @@ RUN apt-get update && apt-get upgrade -y && \ rm -rf /var/lib/apt/lists/* RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - -RUN deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list +RUN echo "deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN apt-get -y update && apt-get upgrade -y RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran ENV NETCDF_HOME="/usr/local" From 3d8696b6e786b61c02ff54b99a71d51d093cfd5b Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 10:37:41 -0400 Subject: [PATCH 094/149] Added missing dependencies to Dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eaef8474d..ff24e3659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,5 +123,7 @@ RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl inte ENV NETCDF_HOME="/usr/local" ENV LANG=C.UTF-8 ENV LD_LIBRARY_PATH=/opt/intel/oneapi/lib +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y libhdf5-dev libxml2-dev -ENTRYPOINT ["/usr/bin/swiftest_driver"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file From 7aa1c81678e7abb76fc781b112b61f70f6ef4be3 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 11:09:50 -0400 Subject: [PATCH 095/149] Fixed typo in singularity definition file --- swiftest.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftest.def b/swiftest.def index 9eeb9d3ef..ce6c10f63 100644 --- a/swiftest.def +++ b/swiftest.def @@ -1,2 +1,2 @@ Bootstrap: docker -From: daminton:swiftest:latest \ No newline at end of file +From: daminton/swiftest:latest \ No newline at end of file From 9ada3e715d62746cd6b2dd21f234142d95d15c36 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 12:16:39 -0400 Subject: [PATCH 096/149] Adjusted flags to attempt to get cross compiling working with enough vector instructions --- Dockerfile | 2 +- cmake/Modules/SetFortranFlags.cmake | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff24e3659..f1532b7c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,7 +122,7 @@ RUN apt-get -y update && apt-get upgrade -y RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran ENV NETCDF_HOME="/usr/local" ENV LANG=C.UTF-8 -ENV LD_LIBRARY_PATH=/opt/intel/oneapi/lib +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/intel/oneapi/lib" RUN apt-get -y update && apt-get upgrade -y RUN apt-get install -y libhdf5-dev libxml2-dev diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index ecf9c34e1..5f0c94bf5 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -112,9 +112,9 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" # There is some bug where -march=native doesn't work on Mac IF(APPLE) - SET(GNUNATIVE "-mtune=native") + SET(GNUNATIVE "-mtune=CORE-AVX2 -xCORE-AVX2") ELSE() - SET(GNUNATIVE "-march=native") + SET(GNUNATIVE "-march=CORE-AVX2 -axCORE-AVX2") ENDIF() ################### @@ -350,13 +350,6 @@ IF (USE_SIMD) ) ENDIF (NOT USE_OPENMP) - # Optimize for the host's architecture - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-xhost" # Intel - "/QxHost" # Intel Windows - ${GNUNATIVE} # GNU - ) - # Generate an extended set of vector functions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-vecabi=cmdtarget" # Intel From ffb8f308f5857d8283386837e4dbf65ecba7b3cf Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 14:20:52 -0400 Subject: [PATCH 097/149] Updated structure of project to better handle containerization, including a new cmake option to build with a lower level of vector support --- .gitignore | 2 + CMakeLists.txt | 1 + cmake/Modules/SetFortranFlags.cmake | 93 ++++++++++++++++-------- docker/.gitignore | 2 + Dockerfile => docker/Dockerfile | 0 docker/bin/.gitignore | 1 + docker/bin/swiftest_driver | 2 + singularity/.gitignore | 2 + singularity/bin/.gitignore | 1 + singularity/bin/swiftest_driver | 2 + swiftest.def => singularity/swiftest.def | 0 11 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 docker/.gitignore rename Dockerfile => docker/Dockerfile (100%) create mode 100644 docker/bin/.gitignore create mode 100755 docker/bin/swiftest_driver create mode 100644 singularity/.gitignore create mode 100644 singularity/bin/.gitignore create mode 100755 singularity/bin/swiftest_driver rename swiftest.def => singularity/swiftest.def (100%) diff --git a/.gitignore b/.gitignore index 81a7fb376..e5c25df47 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ dump* !README_figs/* #Docker and Singularity files +!docker/ +!singularity/ !Dockerfile !swiftest.def diff --git a/CMakeLists.txt b/CMakeLists.txt index a047c0163..37cda5709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90) OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" OFF) OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) OPTION(USE_SIMD "Use SIMD vectorization" ON) +OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF) # Locate and set parallelization libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 5f0c94bf5..782fdc808 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -19,38 +19,40 @@ INCLUDE(${CMAKE_MODULE_PATH}/SetCompileFlag.cmake) # Make sure the build type is uppercase STRING(TOUPPER "${CMAKE_BUILD_TYPE}" BT) +SET(BUILD_TYPE_MSG "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING.") + IF(BT STREQUAL "RELEASE") SET(CMAKE_BUILD_TYPE RELEASE CACHE STRING - "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING." + ${BUILD_TYPE_MSG} FORCE) ELSEIF(BT STREQUAL "DEBUG") SET (CMAKE_BUILD_TYPE DEBUG CACHE STRING - "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING." + ${BUILD_TYPE_MSG} FORCE) ELSEIF(BT STREQUAL "TESTING") SET (CMAKE_BUILD_TYPE TESTING CACHE STRING - "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING." + ${BUILD_TYPE_MSG} FORCE) ELSEIF(BT STREQUAL "PROFILE") SET (CMAKE_BUILD_TYPE PROFILE CACHE STRING - "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING." - FORCE) + ${BUILD_TYPE_MSG} + FORCE) ELSEIF(NOT BT) SET(CMAKE_BUILD_TYPE RELEASE CACHE STRING - "Choose the type of build, options are DEBUG, RELEASE, PROFILE, or TESTING." + ${BUILD_TYPE_MSG} FORCE) MESSAGE(STATUS "CMAKE_BUILD_TYPE not given, defaulting to RELEASE") ELSE() - MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE not valid, choices are DEBUG, RELEASE, PROFILE, or TESTING") + MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE not valid! ${BUILD_TYPE_MSG}") ENDIF(BT STREQUAL "RELEASE") ######################################################### # If the compiler flags have already been set, return now ######################################################### -IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE) +IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER) RETURN () -ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE) +ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER) ######################################################################## # Determine the appropriate flags for this compiler for each build type. @@ -110,12 +112,57 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" "/Qpad" # Intel Windows ) -# There is some bug where -march=native doesn't work on Mac -IF(APPLE) - SET(GNUNATIVE "-mtune=CORE-AVX2 -xCORE-AVX2") -ELSE() - SET(GNUNATIVE "-march=CORE-AVX2 -axCORE-AVX2") -ENDIF() + + +IF (USE_SIMD) + # Enables OpenMP SIMD compilation when OpenMP parallelization is disabled. + IF (NOT USE_OPENMP) + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-qno-openmp -qopenmp-simd" # Intel + Fortran "/Qopenmp- /Qopenmp-simd" # Intel Windows + ) + ENDIF (NOT USE_OPENMP) + + IF (CONTAINERIZE) + # Optimize for an old enough processor that it should run on most computers + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-xSANDYBRIDGE" # Intel + "/QxSANDYBRIDGE" # Intel Windows + ${GNUNATIVE} # GNU + ) + ELSE + # Optimize for the host's architecture + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-xhost" # Intel + "/QxHost" # Intel Windows + ${GNUNATIVE} # GNU + ) + ENDIF (CONTAINERIZE) + + # Generate an extended set of vector functions + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-vecabi=cmdtarget" # Intel + Fortran "/Qvecabi:cmdtarget" # Intel Windows + ) +ENDIF (USE_SIMD) + +IF (CONTAINERIZE) + # There is some bug where -march=native doesn't work on Mac + IF(APPLE) + SET(GNUNATIVE "-mtune=sandybridge") + ELSE() + SET(GNUNATIVE "-march=sandybridge") + ENDIF() +ELSE () + # There is some bug where -march=native doesn't work on Mac + IF(APPLE) + SET(GNUNATIVE "-mtune=native") + ELSE() + SET(GNUNATIVE "-march=native") + ENDIF() +ENDIF (CONTAINERIZE) + + ################### ### DEBUG FLAGS ### @@ -340,19 +387,3 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_RELEASE}" "/O2 /Qopt-report:5 /traceback /Z7" # Intel Windows "-O2 -pg -fbacktrace" # GNU ) - -IF (USE_SIMD) - # Enables OpenMP SIMD compilation when OpenMP parallelization is disabled. - IF (NOT USE_OPENMP) - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-qno-openmp -qopenmp-simd" # Intel - Fortran "/Qopenmp- /Qopenmp-simd" # Intel Windows - ) - ENDIF (NOT USE_OPENMP) - - # Generate an extended set of vector functions - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-vecabi=cmdtarget" # Intel - Fortran "/Qvecabi:cmdtarget" # Intel Windows - ) -ENDIF (USE_SIMD) \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 000000000..a50979646 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +!Dockerfile +!bin/ diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/docker/bin/.gitignore b/docker/bin/.gitignore new file mode 100644 index 000000000..c1d7ed39d --- /dev/null +++ b/docker/bin/.gitignore @@ -0,0 +1 @@ +!swiftest_driver diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver new file mode 100755 index 000000000..0553553a7 --- /dev/null +++ b/docker/bin/swiftest_driver @@ -0,0 +1,2 @@ +#!/bin/sh -- +singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} diff --git a/singularity/.gitignore b/singularity/.gitignore new file mode 100644 index 000000000..c15d31235 --- /dev/null +++ b/singularity/.gitignore @@ -0,0 +1,2 @@ +!swiftest.def +!bin/ diff --git a/singularity/bin/.gitignore b/singularity/bin/.gitignore new file mode 100644 index 000000000..c1d7ed39d --- /dev/null +++ b/singularity/bin/.gitignore @@ -0,0 +1 @@ +!swiftest_driver diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver new file mode 100755 index 000000000..45765b4a7 --- /dev/null +++ b/singularity/bin/swiftest_driver @@ -0,0 +1,2 @@ +#!/bin/sh -- +docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:latest \ No newline at end of file diff --git a/swiftest.def b/singularity/swiftest.def similarity index 100% rename from swiftest.def rename to singularity/swiftest.def From 05aa725802cd37b31305c1afae3636b3cdb40277 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 16:55:43 -0400 Subject: [PATCH 098/149] Added ability to control whether to use the containerized executable or not via the Python code --- cmake/Modules/SetFortranFlags.cmake | 2 +- docker/Dockerfile | 2 +- docker/bin/swiftest_driver | 2 +- python/swiftest/swiftest/simulation_class.py | 38 ++++++++++++++++---- singularity/bin/swiftest_driver | 2 +- singularity/setenv.sh | 3 ++ 6 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 singularity/setenv.sh diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 782fdc808..ead3e1433 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -130,7 +130,7 @@ IF (USE_SIMD) "/QxSANDYBRIDGE" # Intel Windows ${GNUNATIVE} # GNU ) - ELSE + ELSE () # Optimize for the host's architecture SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-xhost" # Intel diff --git a/docker/Dockerfile b/docker/Dockerfile index f1532b7c3..c6d728b2d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ ENV VT_MPI='impi4' ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' -RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCMAKE_BUILD_TYPE=release && make && make install +RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release && make && make install #Production container FROM debian:stable-slim diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver index 0553553a7..d2cb42a90 100755 --- a/docker/bin/swiftest_driver +++ b/docker/bin/swiftest_driver @@ -1,2 +1,2 @@ #!/bin/sh -- -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} +docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:latest "$@" diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 804eee651..0ae7f540b 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -91,6 +91,8 @@ def __init__(self,read_param: bool = False, integrator : {"symba","rmvs","whm","helio"}, default "symba" Name of the n-body integrator that will be used when executing a run. Parameter input file equivalent: None + container : {"docker", "singularity"}, default None + Specify whether the driver exectuable is run from a Docker or Singularity container. Setting to `None` read_param : bool, default False Read the parameter file given by `param_file`. param_file : str, path-like, or file-lke, default "param.in" @@ -338,6 +340,7 @@ def __init__(self,read_param: bool = False, self.init_cond = xr.Dataset() self.encounters = xr.Dataset() self.collisions = xr.Dataset() + self.container = None # Set the location of the parameter input file, choosing the default if it isn't specified. self.simdir = Path.cwd() / Path(simdir) @@ -404,9 +407,11 @@ def _run_swiftest_driver(self): """ # Get current environment variables - env = os.environ.copy() + if self.container == "SINGULARITY" and "SWIFTEST_SIF" not in env: + env['SWIFTEST_SIF'] = Path(self.binary_source).parent.parent / "swiftest.sif" cmd = f"{env['SHELL']} -l {self.driver_script}" + def _type_scrub(output_data): int_vars = ["ILOOP","NPL","NTP","NPLM"] @@ -829,7 +834,8 @@ def set_parameter(self, verbose: bool = True, **kwargs): "restart": False, "encounter_save" : "NONE", "coarray" : False, - "simdir" : self.simdir + "simdir" : self.simdir, + "container" : None, } param_file = kwargs.pop("param_file",None) @@ -891,8 +897,9 @@ def get_parameter(self, **kwargs): return param_dict def set_integrator(self, - codename: Literal["Swiftest", "Swifter", "Swift"] | None = None, + codename: None | Literal["Swiftest", "Swifter", "Swift"] = "Swiftest", integrator: Literal["symba","rmvs","whm","helio"] | None = None, + container: Literal["docker", "singularity"] | None = None, mtiny: float | None = None, gmtiny: float | None = None, verbose: bool | None = None, @@ -905,6 +912,10 @@ def set_integrator(self, codename : {"swiftest", "swifter", "swift"}, optional integrator : {"symba","rmvs","whm","helio"}, optional Name of the n-body integrator that will be used when executing a run. + container : {"docker", "singularity"}, default None + Specify whether the driver exectuable is run from a Docker or Singularity container. + Setting to `None` uses the local executable. + *Note*: Only valid for Swiftest. mtiny : float, optional The minimum mass of fully interacting bodies. Bodies below this mass interact with the larger bodies, but not each other (SyMBA only). *Note.* Only mtiny or gmtiny is accepted, not both. @@ -926,7 +937,15 @@ def set_integrator(self, # TODO: Improve how it finds the executable binary update_list = [] - + + if container is not None: + valid_container = ["DOCKER", "SINGULARITY"] + if container.upper() not in valid_container: + warnings.warn(f"{container} is not a valid container type. Valid options are None, ",",".join(valid_container),stacklevel=2) + self.container = None + else: + self.container = container.upper() + if codename is not None: valid_codename = ["Swiftest", "Swifter", "Swift"] if codename.title() not in valid_codename: @@ -941,7 +960,13 @@ def set_integrator(self, self.param['! VERSION'] = f"{self.codename} input file" update_list.append("codename") if self.codename == "Swiftest": - self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" + if self.container is None: + self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" + elif self.container == "DOCKER": + self.binary_source = Path(_pyfile).parent.parent.parent.parent / "docker" / "bin" / "swiftest_driver" + elif self.container == "SINGULARITY": + self.binary_source = Path(_pyfile).parent.parent.parent.parent / "singularity" / "bin" / "swiftest_driver" + self.binary_path = self.simdir.resolve() self.driver_executable = self.binary_path / "swiftest_driver" if not self.binary_source.exists(): @@ -1016,7 +1041,8 @@ def get_integrator(self,arg_list: str | List[str] | None = None, verbose: bool | valid_instance_vars = {"codename": self.codename, "integrator": self.integrator, "param_file": str(self.param_file), - "driver_executable": str(self.driver_executable)} + "driver_executable": str(self.driver_executable), + "container": self.container} try: self.integrator diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver index 45765b4a7..dba0863a9 100755 --- a/singularity/bin/swiftest_driver +++ b/singularity/bin/swiftest_driver @@ -1,2 +1,2 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:latest \ No newline at end of file +singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh new file mode 100644 index 000000000..769339754 --- /dev/null +++ b/singularity/setenv.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# source this file to set the path to the swiftest.sif file +export SWIFTEST_SIF=${PWD} \ No newline at end of file From f8265e231c8640397359fae5f8770955ab2c116b Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 17:14:22 -0400 Subject: [PATCH 099/149] Adjusted flags to allow the containerized version to work on more CPUs --- cmake/Modules/SetFortranFlags.cmake | 35 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index ead3e1433..ce7f3149b 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -110,7 +110,23 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-pad" # Intel "/Qpad" # Intel Windows + ) +IF (CONTAINERIZE) + # There is some bug where -march=native doesn't work on Mac + IF(APPLE) + SET(GNUNATIVE "-mtune=generic") + ELSE() + SET(GNUNATIVE "-march=generic") + ENDIF() +ELSE () + # There is some bug where -march=native doesn't work on Mac + IF(APPLE) + SET(GNUNATIVE "-mtune=native") + ELSE() + SET(GNUNATIVE "-march=native") + ENDIF() +ENDIF (CONTAINERIZE) @@ -126,9 +142,7 @@ IF (USE_SIMD) IF (CONTAINERIZE) # Optimize for an old enough processor that it should run on most computers SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-xSANDYBRIDGE" # Intel - "/QxSANDYBRIDGE" # Intel Windows - ${GNUNATIVE} # GNU + Fortran ${GNUNATIVE} # GNU ) ELSE () # Optimize for the host's architecture @@ -146,21 +160,6 @@ IF (USE_SIMD) ) ENDIF (USE_SIMD) -IF (CONTAINERIZE) - # There is some bug where -march=native doesn't work on Mac - IF(APPLE) - SET(GNUNATIVE "-mtune=sandybridge") - ELSE() - SET(GNUNATIVE "-march=sandybridge") - ENDIF() -ELSE () - # There is some bug where -march=native doesn't work on Mac - IF(APPLE) - SET(GNUNATIVE "-mtune=native") - ELSE() - SET(GNUNATIVE "-march=native") - ENDIF() -ENDIF (CONTAINERIZE) From 6ce8e38d0732009ffa2e733928f7f4a40f18648f Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 17:56:35 -0400 Subject: [PATCH 100/149] Created docker pull script --- examples/Basic_Simulation/initial_conditions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Basic_Simulation/initial_conditions.py b/examples/Basic_Simulation/initial_conditions.py index 1e7cd5cd6..462063a89 100755 --- a/examples/Basic_Simulation/initial_conditions.py +++ b/examples/Basic_Simulation/initial_conditions.py @@ -37,7 +37,7 @@ # Initialize the simulation object as a variable. Arguments may be defined here or through the sim.run() method. #sim = swiftest.Simulation(fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8) -sim = swiftest.Simulation() +sim = swiftest.Simulation(container="docker") sim.clean() # Add the modern planets and the Sun using the JPL Horizons Database. From 11cd4b638992850fd2c58fae9b050d96a0f5522c Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 17:57:25 -0400 Subject: [PATCH 101/149] Actually put the pull script up this time. --- docker/.gitignore | 1 + docker/pull | 2 ++ examples/Basic_Simulation/initial_conditions.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100755 docker/pull diff --git a/docker/.gitignore b/docker/.gitignore index a50979646..825a38873 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,2 +1,3 @@ !Dockerfile !bin/ +!pull diff --git a/docker/pull b/docker/pull new file mode 100755 index 000000000..2367ba639 --- /dev/null +++ b/docker/pull @@ -0,0 +1,2 @@ +#!/bin/sh -- +docker pull daminton/swiftest:latest \ No newline at end of file diff --git a/examples/Basic_Simulation/initial_conditions.py b/examples/Basic_Simulation/initial_conditions.py index 462063a89..1e7cd5cd6 100755 --- a/examples/Basic_Simulation/initial_conditions.py +++ b/examples/Basic_Simulation/initial_conditions.py @@ -37,7 +37,7 @@ # Initialize the simulation object as a variable. Arguments may be defined here or through the sim.run() method. #sim = swiftest.Simulation(fragmentation=True, minimum_fragment_mass = 2.5e-11, mtiny=2.5e-8) -sim = swiftest.Simulation(container="docker") +sim = swiftest.Simulation() sim.clean() # Add the modern planets and the Sun using the JPL Horizons Database. From 76455226084e07bc7f821ecddc213953b2d955ab Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 23 May 2023 20:20:28 -0400 Subject: [PATCH 102/149] Added the pull script to the .gitignore --- singularity/.gitignore | 1 + singularity/pull | 3 +++ 2 files changed, 4 insertions(+) create mode 100755 singularity/pull diff --git a/singularity/.gitignore b/singularity/.gitignore index c15d31235..abc0e7f03 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -1,2 +1,3 @@ !swiftest.def !bin/ +!pull diff --git a/singularity/pull b/singularity/pull new file mode 100755 index 000000000..8946e4869 --- /dev/null +++ b/singularity/pull @@ -0,0 +1,3 @@ +#!/bin/sh -- +singularity pull swiftest.sif docker://daminton/swiftest:latest +source ./setenv.sh \ No newline at end of file From 571b4a2ffbf531ca88b6028cd8ec629de591c21c Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 24 May 2023 13:13:20 -0400 Subject: [PATCH 103/149] Fixed issue in the environment variable setter, as it was only pointing to the directory and not the swiftest.sif file itself --- singularity/pull | 3 +-- singularity/setenv.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) mode change 100644 => 100755 singularity/setenv.sh diff --git a/singularity/pull b/singularity/pull index 8946e4869..a62acaf6e 100755 --- a/singularity/pull +++ b/singularity/pull @@ -1,3 +1,2 @@ #!/bin/sh -- -singularity pull swiftest.sif docker://daminton/swiftest:latest -source ./setenv.sh \ No newline at end of file +singularity pull --force swiftest.sif docker://daminton/swiftest:latest diff --git a/singularity/setenv.sh b/singularity/setenv.sh old mode 100644 new mode 100755 index 769339754..79931a298 --- a/singularity/setenv.sh +++ b/singularity/setenv.sh @@ -1,3 +1,3 @@ #!/bin/sh # source this file to set the path to the swiftest.sif file -export SWIFTEST_SIF=${PWD} \ No newline at end of file +export SWIFTEST_SIF="${PWD}/swiftest.sif" From 0605144509a074209d833721bddfb5b0f5b1d505 Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 24 May 2023 16:41:05 -0400 Subject: [PATCH 104/149] Moved swiftest_particle_info further up in the code before it is used in the swiftest_body type definition. --- src/swiftest/swiftest_module.f90 | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index e3664a9ed..3d35749ee 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -97,6 +97,25 @@ module swiftest end type swiftest_kinship + type, extends(base_particle_info) :: swiftest_particle_info + character(len=NAMELEN) :: name !! Non-unique name + character(len=NAMELEN) :: particle_type !! String containing a description of the particle type (e.g. Central Body, Massive Body, Test Particle) + character(len=NAMELEN) :: origin_type !! String containing a description of the origin of the particle (e.g. Initial Conditions, Supercatastrophic, Disruption, etc.) + real(DP) :: origin_time !! The time of the particle's formation + integer(I4B) :: collision_id !! The ID of the collision that formed the particle + real(DP), dimension(NDIM) :: origin_rh !! The heliocentric distance vector at the time of the particle's formation + real(DP), dimension(NDIM) :: origin_vh !! The heliocentric velocity vector at the time of the particle's formation + real(DP) :: discard_time !! The time of the particle's discard + character(len=NAMELEN) :: status !! Particle status description: Active, Merged, Fragmented, etc. + real(DP), dimension(NDIM) :: discard_rh !! The heliocentric distance vector at the time of the particle's discard + real(DP), dimension(NDIM) :: discard_vh !! The heliocentric velocity vector at the time of the particle's discard + integer(I4B) :: discard_body_id !! The id of the other body involved in the discard (0 if no other body involved) + contains + procedure :: copy => swiftest_util_copy_particle_info !! Copies one set of information object components into another, component-by-component + procedure :: set_value => swiftest_util_set_particle_info !! Sets one or more values of the particle information metadata object + end type swiftest_particle_info + + !> An abstract class for a generic collection of Swiftest bodies type, abstract, extends(base_object) :: swiftest_body !! Superclass that defines the generic elements of a Swiftest particle @@ -167,25 +186,6 @@ module swiftest end type swiftest_body - type, extends(base_particle_info) :: swiftest_particle_info - character(len=NAMELEN) :: name !! Non-unique name - character(len=NAMELEN) :: particle_type !! String containing a description of the particle type (e.g. Central Body, Massive Body, Test Particle) - character(len=NAMELEN) :: origin_type !! String containing a description of the origin of the particle (e.g. Initial Conditions, Supercatastrophic, Disruption, etc.) - real(DP) :: origin_time !! The time of the particle's formation - integer(I4B) :: collision_id !! The ID of the collision that formed the particle - real(DP), dimension(NDIM) :: origin_rh !! The heliocentric distance vector at the time of the particle's formation - real(DP), dimension(NDIM) :: origin_vh !! The heliocentric velocity vector at the time of the particle's formation - real(DP) :: discard_time !! The time of the particle's discard - character(len=NAMELEN) :: status !! Particle status description: Active, Merged, Fragmented, etc. - real(DP), dimension(NDIM) :: discard_rh !! The heliocentric distance vector at the time of the particle's discard - real(DP), dimension(NDIM) :: discard_vh !! The heliocentric velocity vector at the time of the particle's discard - integer(I4B) :: discard_body_id !! The id of the other body involved in the discard (0 if no other body involved) - contains - procedure :: copy => swiftest_util_copy_particle_info !! Copies one set of information object components into another, component-by-component - procedure :: set_value => swiftest_util_set_particle_info !! Sets one or more values of the particle information metadata object - end type swiftest_particle_info - - type, abstract, extends(base_object) :: swiftest_cb !> An abstract class for a generic central body in a Swiftest simulation class(swiftest_particle_info), allocatable :: info !! Particle metadata information From 5245e2ba30ccaf48a1f001a8169bf4ed6356b02f Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 24 May 2023 17:23:59 -0400 Subject: [PATCH 105/149] Use more robust way to check for valid bodies in a NetCDF file with the ieee_arithetic intrinsic module --- src/swiftest/swiftest_io.f90 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 1e02a7645..c3061858b 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1035,6 +1035,8 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) !! !! Given an open NetCDF, returns logical masks indicating which bodies in the body arrays are active pl and tp type at the current time. !! Uses the value of tslot stored in the NetCDF parameter object as the definition of current time + use, intrinsic :: ieee_exceptions + use, intrinsic :: ieee_arithmetic implicit none ! Arguments class(swiftest_netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset @@ -1045,7 +1047,11 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) real(DP), dimension(:,:), allocatable :: rh integer(I4B), dimension(:), allocatable :: body_status logical, dimension(:), allocatable :: lvalid - integer(I4B) :: idmax, status + integer(I4B) :: idmax, status, i + logical, dimension(size(IEEE_ALL)) :: fpe_halting_modes + + call ieee_get_halting_mode(IEEE_ALL,fpe_halting_modes) ! Save the current halting modes so we can turn them off temporarily + call ieee_set_halting_mode(IEEE_ALL,.false.) call netcdf_io_check( nf90_inquire_dimension(self%id, self%name_dimid, len=idmax), "swiftest_io_netcdf_get_valid_masks nf90_inquire_dimension name_dimid" ) @@ -1067,20 +1073,20 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) if (status == NF90_NOERR) then allocate(rh(NDIM,idmax)) call netcdf_io_check( nf90_get_var(self%id, self%rh_varid, rh, start=[1, 1, tslot], count=[NDIM,idmax,1]), "swiftest_io_netcdf_get_valid_masks nf90_getvar rh_varid" ) - lvalid(:) = rh(1,:) == rh(1,:) + lvalid(:) = ieee_is_normal(rh(1,:)) else status = nf90_inq_varid(self%id, self%a_varname, self%a_varid) if (status == NF90_NOERR) then allocate(a(idmax)) call netcdf_io_check( nf90_get_var(self%id, self%a_varid, a, start=[1, tslot], count=[idmax,1]), "swiftest_io_netcdf_get_valid_masks nf90_getvar a_varid" ) - lvalid(:) = a(:) == a(:) + lvalid(:) = ieee_is_normal(a(:)) else lvalid(:) = .false. end if end if end if - plmask(:) = (Gmass(:) == Gmass(:)) + plmask(:) = ieee_is_normal(Gmass(:)) where(plmask(:)) plmask(:) = Gmass(:) > 0.0_DP tpmask(:) = .not. plmask(:) plmask(1) = .false. ! This is the central body @@ -1088,6 +1094,7 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) ! Select only active bodies plmask(:) = plmask(:) .and. lvalid(:) tpmask(:) = tpmask(:) .and. lvalid(:) + call ieee_set_halting_mode(IEEE_ALL,fpe_halting_modes) end associate From e10f09ec03a456eafd6ed2512f0fa227717437e0 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Wed, 24 May 2023 19:50:49 -0400 Subject: [PATCH 106/149] Fixed up the floating point flags for gfortran --- cmake/Modules/SetFortranFlags.cmake | 12 ++++++++++++ src/CMakeLists.txt | 6 +++--- src/swiftest/swiftest_io.f90 | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index ce7f3149b..0d06920de 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -75,6 +75,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-fno-underscoring" # GNU ) +# Compile code assuming that IEEE signaling NaNs may generate user-visible traps during floating-point operations. +# Setting this option disables optimizations that may change the number of exceptions visible with signaling NaNs. +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-fsignaling-nans " # GNU + ) + # Determines whether the current Fortran Standard behavior of the compiler is fully implemented. SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}" @@ -208,6 +214,7 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-init=snan,arrays" # Intel "/Qinit:snan,arrays" # Intel Windows + "-finit-real=snan" # GNU ) # Does not generate an interface block for each routine in a source file @@ -342,6 +349,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE}" SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" Fortran "-fp-model=precise" # Intel "/fp:precise" # Intel Windows + "-fno-unsafe-math-optimizations" # GNU + ) +# Disable transformations and optimizations that assume default floating-point rounding behavior. +SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" + Fortran "-frounding-math" ) SET_COMPILE_FLAG(STRICTMATH_FLAGS "${STRICTMATH_FLAGS}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef23fadc8..e241d3360 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,6 +14,8 @@ # Add the source files SET(STRICT_MATH_FILES ${SRC}/collision/collision_generate.f90 + ${SRC}/collision/collision_io.f90 + ${SRC}/collision/collision_util.f90 ${SRC}/fraggle/fraggle_generate.f90 ${SRC}/fraggle/fraggle_util.f90 ${SRC}/fraggle/fraggle_module.f90 @@ -31,6 +33,7 @@ SET(STRICT_MATH_FILES ${SRC}/rmvs/rmvs_step.f90 ${SRC}/swiftest/swiftest_drift.f90 ${SRC}/swiftest/swiftest_gr.f90 + ${SRC}/swiftest/swiftest_io.f90 ${SRC}/swiftest/swiftest_kick.f90 ${SRC}/swiftest/swiftest_user.f90 ${SRC}/swiftest/swiftest_obl.f90 @@ -60,10 +63,8 @@ SET(FAST_MATH_FILES ${SRC}/helio/helio_module.f90 ${SRC}/symba/symba_module.f90 ${SRC}/collision/collision_check.f90 - ${SRC}/collision/collision_io.f90 ${SRC}/collision/collision_regime.f90 ${SRC}/collision/collision_resolve.f90 - ${SRC}/collision/collision_util.f90 ${SRC}/encounter/encounter_check.f90 ${SRC}/encounter/encounter_io.f90 ${SRC}/encounter/encounter_util.f90 @@ -73,7 +74,6 @@ SET(FAST_MATH_FILES ${SRC}/rmvs/rmvs_encounter_check.f90 ${SRC}/rmvs/rmvs_util.f90 ${SRC}/swiftest/swiftest_discard.f90 - ${SRC}/swiftest/swiftest_io.f90 ${SRC}/swiftest/swiftest_util.f90 ${SRC}/symba/symba_discard.f90 ${SRC}/symba/symba_encounter_check.f90 diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index c3061858b..921ea8308 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -1067,7 +1067,7 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) if (status == NF90_NOERR) then allocate(body_status(idmax)) call netcdf_io_check( nf90_get_var(self%id, self%status_varid, body_status, start=[1, tslot], count=[idmax,1]), "swiftest_io_netcdf_get_valid_masks nf90_getvar status_varid" ) - lvalid(:) = body_status /= INACTIVE + lvalid(:) = body_status(:) /= INACTIVE else status = nf90_inq_varid(self%id, self%rh_varname, self%rh_varid) if (status == NF90_NOERR) then From 660eb485bbb4baf0d9774ed2ea73e5b11c27070a Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 11:31:48 -0400 Subject: [PATCH 107/149] Improved the way that semi-interacting bodies are identified from input --- .../Basic_Simulation/initial_conditions.py | 10 +-- src/swiftest/swiftest_io.f90 | 80 ++++++++----------- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/examples/Basic_Simulation/initial_conditions.py b/examples/Basic_Simulation/initial_conditions.py index 1e7cd5cd6..813bd8dfb 100755 --- a/examples/Basic_Simulation/initial_conditions.py +++ b/examples/Basic_Simulation/initial_conditions.py @@ -54,13 +54,13 @@ capom_pl = default_rng().uniform(0.0, 360.0, npl) omega_pl = default_rng().uniform(0.0, 360.0, npl) capm_pl = default_rng().uniform(0.0, 360.0, npl) -GM_pl = (np.array([6e23, 8e23, 1e24, 3e24, 5e24]) / sim.param['MU2KG']) * sim.GU -R_pl = np.full(npl, (3 * (GM_pl / sim.GU) / (4 * np.pi * density_pl)) ** (1.0 / 3.0)) -Rh_pl = a_pl * ((GM_pl) / (3 * sim.GU)) ** (1.0 / 3.0) +M_pl = np.array([6e23, 8e23, 1e24, 3e24, 5e24]) * sim.KG2MU +R_pl = np.full(npl, (3 * M_pl/ (4 * np.pi * density_pl)) ** (1.0 / 3.0)) Ip_pl = np.full((npl,3),0.4,) rot_pl = np.zeros((npl,3)) +mtiny = 1.01 * np.max(M_pl) -sim.add_body(name=name_pl, a=a_pl, e=e_pl, inc=inc_pl, capom=capom_pl, omega=omega_pl, capm=capm_pl, Gmass=GM_pl, radius=R_pl, rhill=Rh_pl, Ip=Ip_pl, rot=rot_pl) +sim.add_body(name=name_pl, a=a_pl, e=e_pl, inc=inc_pl, capom=capom_pl, omega=omega_pl, capm=capm_pl, mass=M_pl, radius=R_pl, Ip=Ip_pl, rot=rot_pl) # Add 10 user-defined test particles. ntp = 10 @@ -74,7 +74,7 @@ capm_tp = default_rng().uniform(0.0, 360.0, ntp) sim.add_body(name=name_tp, a=a_tp, e=e_tp, inc=inc_tp, capom=capom_tp, omega=omega_tp, capm=capm_tp) -sim.set_parameter(tstart=0.0, tstop=1.0e3, dt=0.01, istep_out=100, dump_cadence=0) +sim.set_parameter(tstart=0.0, tstop=1.0e3, dt=0.01, istep_out=100, dump_cadence=0, compute_conservation_values=True, mtiny=mtiny) # Display the run configuration parameters. sim.get_parameter() sim.save() diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 921ea8308..38b1735d7 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -123,8 +123,6 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) real(DP), dimension(NDIM) :: L_total_now, L_orbit_now, L_spin_now real(DP) :: ke_orbit_now, ke_spin_now, pe_now, E_orbit_now, be_now, be_cb_now, be_cb_orig, te_now real(DP) :: GMtot_now - character(len=STRMAX) :: errmsg - integer(I4B), parameter :: EGYIU = 72 character(len=*), parameter :: EGYTERMFMT = '(" DL/L0 = ", ES12.5, "; DE_orbit/|E0| = ", ES12.5, "; DE_total/|E0| = ", ES12.5, "; DM/M0 = ", ES12.5)' associate(nbody_system => self, pl => self%pl, cb => self%cb, npl => self%pl%nbody, display_unit => param%display_unit) @@ -204,10 +202,6 @@ module subroutine swiftest_io_conservation_report(self, param, lterminal) end associate return - - 667 continue - write(*,*) "Error writing energy and momentum tracking file: " // trim(adjustl(errmsg)) - call base_util_exit(FAILURE) end subroutine swiftest_io_conservation_report @@ -226,7 +220,6 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t real(DP) :: tfrac !! Fraction of total simulation time completed type(progress_bar), save :: pbar !! Object used to print out a progress bar character(len=64) :: pbarmessage - character(*), parameter :: symbacompactfmt = '(";NPLM",ES22.15,$)' #ifdef COARRAY character(*), parameter :: co_statusfmt = '("Image: ",I4, "; Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, tp = ", I6, ", ", I6)' @@ -465,7 +458,6 @@ module subroutine swiftest_io_get_args(integrator, param_file_name, display_styl character(len=STRMAX), dimension(:), allocatable :: arg integer(I4B), dimension(:), allocatable :: ierr integer :: i,narg - character(len=*),parameter :: linefmt = '(A)' narg = command_argument_count() if (narg > 0) then @@ -653,6 +645,7 @@ module subroutine swiftest_io_netcdf_get_t0_values_system(self, nc, param) ! Internals integer(I4B) :: itmax, idmax, tslot real(DP), dimension(:), allocatable :: vals + logical, dimension(:), allocatable :: plmask, tpmask real(DP), dimension(1) :: rtemp real(DP), dimension(NDIM) :: rot0, Ip0, L real(DP) :: mass0 @@ -689,7 +682,8 @@ module subroutine swiftest_io_netcdf_get_t0_values_system(self, nc, param) self%L_total_orig(:) = self%L_orbit_orig(:) + self%L_spin_orig(:) call netcdf_io_check( nf90_get_var(nc%id, nc%Gmass_varid, vals, start=[1,tslot], count=[idmax,1]), "netcdf_io_get_t0_values_system Gmass_varid" ) - self%GMtot_orig = vals(1) + sum(vals(2:idmax), vals(2:idmax) == vals(2:idmax)) + call nc%get_valid_masks(plmask,tpmask) + self%GMtot_orig = vals(1) + sum(vals(2:idmax), plmask(:)) cb%GM0 = vals(1) cb%dGM = cb%Gmass - cb%GM0 @@ -1030,7 +1024,7 @@ module subroutine swiftest_io_netcdf_open(self, param, readonly) end subroutine swiftest_io_netcdf_open - module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) + module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask, plmmask, Gmtiny) !! author: David A. Minton !! !! Given an open NetCDF, returns logical masks indicating which bodies in the body arrays are active pl and tp type at the current time. @@ -1039,15 +1033,17 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) use, intrinsic :: ieee_arithmetic implicit none ! Arguments - class(swiftest_netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset - logical, dimension(:), allocatable, intent(out) :: plmask !! Logical mask indicating which bodies are massive bodies - logical, dimension(:), allocatable, intent(out) :: tpmask !! Logical mask indicating which bodies are test particles + class(swiftest_netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset + logical, dimension(:), allocatable, intent(out) :: plmask !! Logical mask indicating which bodies are massive bodies + logical, dimension(:), allocatable, intent(out) :: tpmask !! Logical mask indicating which bodies are test particles + logical, dimension(:), allocatable, intent(out), optional :: plmmask !! Logical mask indicating which bodies are fully interacting massive bodies + real(DP), intent(in), optional :: Gmtiny !! The cutoff G*mass between semi-interacting and fully interacting massive bodies ! Internals real(DP), dimension(:), allocatable :: Gmass, a real(DP), dimension(:,:), allocatable :: rh integer(I4B), dimension(:), allocatable :: body_status logical, dimension(:), allocatable :: lvalid - integer(I4B) :: idmax, status, i + integer(I4B) :: idmax, status logical, dimension(size(IEEE_ALL)) :: fpe_halting_modes call ieee_get_halting_mode(IEEE_ALL,fpe_halting_modes) ! Save the current halting modes so we can turn them off temporarily @@ -1094,6 +1090,14 @@ module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) ! Select only active bodies plmask(:) = plmask(:) .and. lvalid(:) tpmask(:) = tpmask(:) .and. lvalid(:) + + if (present(plmmask) .and. present(Gmtiny)) then + allocate(plmmask, source=plmask) + where(plmask(:)) + plmmask = Gmass(:) > Gmtiny + endwhere + end if + call ieee_set_halting_mode(IEEE_ALL,fpe_halting_modes) end associate @@ -1316,10 +1320,6 @@ module function swiftest_io_netcdf_read_frame_system(self, nc, param) result(ier ierr = 0 return - - 667 continue - write(*,*) "Error reading nbody_system frame in netcdf_io_read_frame_system" - end function swiftest_io_netcdf_read_frame_system @@ -1335,21 +1335,15 @@ module subroutine swiftest_io_netcdf_read_hdr_system(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for reading a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: status, idmax + integer(I4B) :: status logical, dimension(:), allocatable :: plmask, tpmask, plmmask - real(DP), dimension(:), allocatable :: Gmtemp associate(tslot => nc%tslot) call netcdf_io_check( nf90_get_var(nc%id, nc%time_varid, self%t, start=[tslot]), "netcdf_io_read_hdr_system nf90_getvar time_varid" ) - call nc%get_valid_masks(plmask, tpmask) - if (param%lmtiny_pl) then - idmax = size(plmask) - allocate(plmmask(idmax)) - allocate(Gmtemp(idmax)) - call netcdf_io_check( nf90_get_var(nc%id, nc%Gmass_varid, Gmtemp, start=[1,tslot], count=[idmax,1]), "netcdf_io_read_hdr_system nf90_getvar Gmass_varid" ) - where(Gmtemp(:) /= Gmtemp(:)) Gmtemp(:) = 0.0_DP - plmmask(:) = plmask(:) .and. Gmtemp(:) > param%GMTINY + call nc%get_valid_masks(plmask, tpmask, plmmask, param%GMTINY) + else + call nc%get_valid_masks(plmask, tpmask) end if status = nf90_inq_varid(nc%id, nc%npl_varname, nc%npl_varid) @@ -1624,7 +1618,7 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, idslot, old_mode + integer(I4B) :: i, j, idslot, old_mode, tmp integer(I4B), dimension(:), allocatable :: ind real(DP), dimension(NDIM) :: vh !! Temporary variable to store heliocentric velocity values when converting from pseudovelocity in GR-enabled runs real(DP) :: a, e, inc, omega, capom, capm, varpi, lam, f, cape, capf @@ -1729,7 +1723,7 @@ module subroutine swiftest_io_netcdf_write_frame_body(self, nc, param) end select #endif - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode), "netcdf_io_write_frame_body nf90_set_fill old_mode" ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp), "netcdf_io_write_frame_body nf90_set_fill old_mode" ) return end subroutine swiftest_io_netcdf_write_frame_body @@ -1745,7 +1739,7 @@ module subroutine swiftest_io_netcdf_write_frame_cb(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to for writing a NetCDF dataset to file class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: idslot, old_mode + integer(I4B) :: idslot, old_mode, tmp associate(tslot => nc%tslot) call self%write_info(nc, param) @@ -1766,7 +1760,7 @@ module subroutine swiftest_io_netcdf_write_frame_cb(self, nc, param) call netcdf_io_check( nf90_put_var(nc%id, nc%rot_varid, self%rot(:) * RAD2DEG, start=[1, idslot, tslot], count=[NDIM,1,1]), "swiftest_io_netcdf_write_frame_cby nf90_put_var cb rot_varid" ) end if - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode), "swiftest_io_netcdf_write_frame_cb nf90_set_fill old_mode" ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp), "swiftest_io_netcdf_write_frame_cb nf90_set_fill old_mode" ) end associate return @@ -1848,7 +1842,7 @@ module subroutine swiftest_io_netcdf_write_info_body(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, idslot, old_mode + integer(I4B) :: i, j, idslot, old_mode, tmp integer(I4B), dimension(:), allocatable :: ind character(len=NAMELEN) :: charstring character(len=NAMELEN), dimension(self%nbody) :: origin_type @@ -1892,7 +1886,7 @@ module subroutine swiftest_io_netcdf_write_info_body(self, nc, param) end associate end select - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode) ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp) ) return end subroutine swiftest_io_netcdf_write_info_body @@ -1906,7 +1900,7 @@ module subroutine swiftest_io_netcdf_write_info_cb(self, nc, param) class(swiftest_netcdf_parameters), intent(inout) :: nc !! Parameters used to identify a particular NetCDF dataset class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: idslot, old_mode + integer(I4B) :: idslot, old_mode, tmp character(len=NAMELEN) :: charstring ! This string of spaces of length NAMELEN is used to clear out any old data left behind inside the string variables @@ -1935,7 +1929,7 @@ module subroutine swiftest_io_netcdf_write_info_cb(self, nc, param) call netcdf_io_check( nf90_put_var(nc%id, nc%discard_rh_varid, self%info%discard_rh(:), start=[1, idslot], count=[NDIM,1]), "netcdf_io_write_info_body nf90_put_var cb discard_rh_varid" ) call netcdf_io_check( nf90_put_var(nc%id, nc%discard_vh_varid, self%info%discard_vh(:), start=[1, idslot], count=[NDIM,1]), "netcdf_io_write_info_body nf90_put_var cb discard_vh_varid" ) end if - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode) ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp) ) return end subroutine swiftest_io_netcdf_write_info_cb @@ -1989,7 +1983,6 @@ module subroutine swiftest_io_param_reader(self, unit, iotype, v_list, iostat, i character(*),parameter :: linefmt = '(A)' !! Format code for simple text string integer(I4B) :: nseeds, nseeds_from_file logical :: seed_set = .false. !! Is the random seed set in the input file? - character(len=:), allocatable :: integrator real(DP) :: tratio, y #ifdef COARRAY type(swiftest_parameters), codimension[*], save :: coparam @@ -2517,9 +2510,6 @@ module subroutine swiftest_io_param_writer(self, unit, iotype, v_list, iostat, i integer, intent(out) :: iostat !! IO status code character(len=*), intent(inout) :: iomsg !! Message to pass if iostat /= 0 ! Internals - character(*),parameter :: Ifmt = '(I0)' !! Format label for integer values - character(*),parameter :: Rfmt = '(ES25.17)' !! Format label for real values - character(*),parameter :: Lfmt = '(L1)' !! Format label for logical values integer(I4B) :: nseeds associate(param => self) @@ -2588,7 +2578,6 @@ module subroutine swiftest_io_param_writer(self, unit, iotype, v_list, iostat, i iomsg = "UDIO not implemented" end associate - 667 continue return end subroutine swiftest_io_param_writer @@ -2935,7 +2924,7 @@ module subroutine swiftest_io_read_in_system(self, nc, param) if (ierr /=0) call base_util_exit(FAILURE) end if - param%loblatecb = ((self%cb%j2rp2 /= 0.0_DP) .or. (self%cb%j4rp4 /= 0.0_DP)) + param%loblatecb = ((abs(self%cb%j2rp2) > 0.0_DP) .or. (abs(self%cb%j4rp4) > 0.0_DP)) if (.not.param%loblatecb) then if (allocated(self%pl%aobl)) deallocate(self%pl%aobl) if (allocated(self%tp%aobl)) deallocate(self%tp%aobl) @@ -3083,7 +3072,6 @@ module subroutine swiftest_io_read_in_param(self, param_file_name) call self%reader(LUN, iotype= "none", v_list=[""], iostat = ierr, iomsg = errmsg) if (ierr == 0) return - 667 continue write(self%display_unit,*) "Error reading parameter file: " // trim(adjustl(errmsg)) call base_util_exit(FAILURE) end subroutine swiftest_io_read_in_param @@ -3177,7 +3165,7 @@ module subroutine swiftest_io_initialize_output_file_system(self, nc, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals logical, save :: lfirst = .true. !! Flag to determine if this is the first call of this method - character(len=STRMAX) :: errmsg + character(len=2*STRMAX) :: errmsg logical :: fileExists associate (pl => self%pl, tp => self%tp, npl => self%pl%nbody, ntp => self%tp%nbody) @@ -3191,13 +3179,13 @@ module subroutine swiftest_io_initialize_output_file_system(self, nc, param) select case(param%out_stat) case('APPEND') if (.not.fileExists) then - errmsg = param%outfile // " not found! You must specify OUT_STAT = NEW, REPLACE, or UNKNOWN" + errmsg = trim(adjustl(param%outfile)) // " not found! You must specify OUT_STAT = NEW, REPLACE, or UNKNOWN" goto 667 end if call nc%open(param) case('NEW') if (fileExists) then - errmsg = param%outfile // " Alread Exists! You must specify OUT_STAT = APPEND, REPLACE, or UNKNOWN" + errmsg = trim(adjustl(param%outfile)) // " Alread Exists! You must specify OUT_STAT = APPEND, REPLACE, or UNKNOWN" goto 667 end if call nc%initialize(param) From 8c9e0520d12ca5cb61cd58c1725c79f994172189 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 11:32:33 -0400 Subject: [PATCH 108/149] Removed the line in the executable that calls the shell default shell configuration file temporarily until I determine if it is necessary --- python/swiftest/swiftest/simulation_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 0ae7f540b..10a869ec1 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -2779,7 +2779,7 @@ def write_param(self, self.driver_script = os.path.join(self.simdir, "swiftest_driver.sh") with open(self.driver_script, 'w') as f: f.write(f"#{self._shell_full}\n") - f.write(f"source ~/.{self._shell}rc\n") + #f.write(f"source ~/.{self._shell}rc\n") f.write(f"cd {self.simdir}\n") f.write(f"{str(self.driver_executable)} {self.integrator} {str(self.param_file)} compact\n") From 54869bed837c9340fc6f6346904f44f1c66e1009 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 11:32:47 -0400 Subject: [PATCH 109/149] Improved GNU warning flags in debug mode --- cmake/Modules/SetFortranFlags.cmake | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 0d06920de..7b7752747 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -186,6 +186,20 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" "/warn:all" # Intel Windows "-Wall" # GNU ) +# This enables some extra warning flags that are not enabled by -Wall +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" + Fortran "-Wextra" # GNU + ) + +# Disable the warning that arrays may be uninitialized, which comes up due to a known bug in gfortran +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" + Fortran "-Wno-maybe-uninitialized" # GNU + ) +# Disable the warning about unused dummy arguments. These primarily occur due to interface rules for type-bound procedures used in extendable types. +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" + Fortran "-Wno-unused-dummy-argument" # GNU + ) + # Traceback SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" @@ -241,6 +255,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" "-ffpe-trap=zero,overflow,underflow" # GNU ) +# List of floating-point exceptions, whose flag status is printed to ERROR_UNIT when invoking STOP and ERROR STOP +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" + Fortran "-ffpe-summary=all" # GNU + ) + # Enables floating-point invalid, divide-by-zero, and overflow exceptions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-fpe0" # Intel From 511e425c3a3a24445eccba9714cbaf9c82058e45 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 11:49:56 -0400 Subject: [PATCH 110/149] Fixed inconsistent types in index flattener routines --- src/swiftest/swiftest_util.f90 | 60 +++++++++++++--------------------- 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index b063cb418..641fcc170 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1101,12 +1101,12 @@ module subroutine swiftest_util_flatten_eucl_plpl(self, param) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, j, err - integer(I8B) :: k, npl + integer(I4B) :: err, i, j + integer(I8B) :: k, npl8 - npl = int(self%nbody, kind=I8B) - associate(nplpl => self%nplpl) - nplpl = npl * (npl - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl + associate(npl => self%nbody, nplpl => self%nplpl) + npl8 = int(npl, kind=I8B) + nplpl = npl8 * (npl8 - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl if (param%lflatten_interactions) then if (allocated(self%k_plpl)) deallocate(self%k_plpl) ! Reset the index array if it's been set previously allocate(self%k_plpl(2, nplpl), stat=err) @@ -1118,7 +1118,7 @@ module subroutine swiftest_util_flatten_eucl_plpl(self, param) #else do concurrent (i=1:npl, j=1:npl, j>i) #endif - call swiftest_util_flatten_eucl_ij_to_k(self%nbody, i, j, k) + call swiftest_util_flatten_eucl_ij_to_k(npl, i, j, k) self%k_plpl(1, k) = i self%k_plpl(2, k) = j end do @@ -1145,17 +1145,18 @@ module subroutine swiftest_util_flatten_eucl_pltp(self, pl, param) class(swiftest_pl), intent(in) :: pl !! Swiftest massive body object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I8B) :: i, j, counter, npl, ntp + integer(I4B) :: i, j + integer(I8B) :: counter, npl8, ntp8 - ntp = int(self%nbody, kind=I8B) - npl = int(pl%nbody, kind=I8B) - associate(npltp => self%npltp) - npltp = npl * ntp + associate(ntp => self%nbody, npl => pl%nbody, npltp => self%npltp) + npl8 = int(npl, kind=I8B) + ntp8 = int(ntp, kind=I8B) + npltp = npl8 * ntp8 if (allocated(self%k_pltp)) deallocate(self%k_pltp) ! Reset the index array if it's been set previously allocate(self%k_pltp(2, npltp)) - do i = 1_I8B, npl - counter = (i - 1_I8B) * npl + 1_I8B - do j = 1_I8B, ntp + counter = 1_I8B + do i = 1, npl + do j = 1, ntp self%k_pltp(1, counter) = i self%k_pltp(2, counter) = j counter = counter + 1_I8B @@ -1643,7 +1644,8 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals class(swiftest_pl), allocatable :: tmp !! The discarded body list. - integer(I4B) :: i, k, npl, nadd, nencmin, nenc_old, idnew1, idnew2, idold1, idold2 + integer(I4B) :: i, npl, nadd, idnew1, idnew2, idold1, idold2 + integer(I8B) :: k, nenc_old, nencmin logical, dimension(:), allocatable :: lmask class(encounter_list), allocatable :: plplenc_old logical :: lencounter @@ -1708,7 +1710,7 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) if (allocated(nbody_system%plpl_encounter)) then ! Store the original plplenc list so we don't remove any of the original encounters nenc_old = nbody_system%plpl_encounter%nenc - if (nenc_old > 0) then + if (nenc_old > 0_I8B) then allocate(plplenc_old, source=nbody_system%plpl_encounter) call plplenc_old%copy(nbody_system%plpl_encounter) end if @@ -1730,10 +1732,10 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) end select ! Re-index the encounter list as the index values may have changed - if (nenc_old > 0) then + if (nenc_old > 0_I8B) then nencmin = min(nbody_system%plpl_encounter%nenc, plplenc_old%nenc) nbody_system%plpl_encounter%nenc = nencmin - do k = 1, nencmin + do k = 1_I8B, nencmin idnew1 = nbody_system%plpl_encounter%id1(k) idnew2 = nbody_system%plpl_encounter%id2(k) idold1 = plplenc_old%id1(k) @@ -1774,7 +1776,7 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) end if nencmin = count(lmask(:)) nbody_system%plpl_encounter%nenc = nencmin - if (nencmin > 0) then + if (nencmin > 0_I8B) then nbody_system%plpl_encounter%index1(1:nencmin) = pack(nbody_system%plpl_encounter%index1(1:nenc_old), lmask(1:nenc_old)) nbody_system%plpl_encounter%index2(1:nencmin) = pack(nbody_system%plpl_encounter%index2(1:nenc_old), lmask(1:nenc_old)) nbody_system%plpl_encounter%id1(1:nencmin) = pack(nbody_system%plpl_encounter%id1(1:nenc_old), lmask(1:nenc_old)) @@ -3325,9 +3327,7 @@ module subroutine swiftest_util_version() "Authors:", //, & " The Purdue University Swiftest Development team ", /, & " Lead by David A. Minton ", /, & - " Single loop blocking by Jacob R. Elliott", /, & - " Fragmentation by Carlisle A. Wishard and", //, & - " Jennifer L. L. Poutplin ", //, & + " Carlisle Wishard, Jennifer Pouplin, Jacob Elliott, Dana Singh." & "Please address comments and questions to:", //, & " David A. Minton", /, & " Department Earth, Atmospheric, & Planetary Sciences ",/, & @@ -3341,22 +3341,6 @@ module subroutine swiftest_util_version() "************************************************", /) - 100 FORMAT(/, "************* SWIFTER: Version ", F3.1, " *************", //, & - "Authors:", //, & - " Martin Duncan: Queen's University", /, & - " Hal Levison : Southwest Research Institute", //, & - "Please address comments and questions to:", //, & - " Hal Levison or David Kaufmann", /, & - " Department of Space Studies", /, & - " Southwest Research Institute", /, & - " 1050 Walnut Street, Suite 400", /, & - " Boulder, Colorado 80302", /, & - " 303-546-0290 (HFL), 720-240-0119 (DEK)", /, & - " 303-546-9687 (fax)", /, & - " hal@gort.boulder.swri.edu (HFL)", /, & - " kaufmann@boulder.swri.edu (DEK)", //, & - "************************************************", /) - return end subroutine swiftest_util_version From d854c21a1444866e7ee6b4a56582fc709fcf5be7 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 11:55:21 -0400 Subject: [PATCH 111/149] Improved the tslot finder using proper ieee arithmetic routine --- src/CMakeLists.txt | 2 +- src/netcdf_io/netcdf_io_implementations.f90 | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e241d3360..0e3decf3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,6 +25,7 @@ SET(STRICT_MATH_FILES ${SRC}/helio/helio_step.f90 ${SRC}/misc/lambda_function_module.f90 ${SRC}/misc/solver_module.f90 + ${SRC}/netcdf_io/netcdf_io_implementations.f90 ${SRC}/operator/operator_module.f90 ${SRC}/operator/operator_cross.f90 ${SRC}/operator/operator_mag.f90 @@ -69,7 +70,6 @@ SET(FAST_MATH_FILES ${SRC}/encounter/encounter_io.f90 ${SRC}/encounter/encounter_util.f90 ${SRC}/helio/helio_util.f90 - ${SRC}/netcdf_io/netcdf_io_implementations.f90 ${SRC}/rmvs/rmvs_discard.f90 ${SRC}/rmvs/rmvs_encounter_check.f90 ${SRC}/rmvs/rmvs_util.f90 diff --git a/src/netcdf_io/netcdf_io_implementations.f90 b/src/netcdf_io/netcdf_io_implementations.f90 index 831b17902..b4e7f0b12 100644 --- a/src/netcdf_io/netcdf_io_implementations.f90 +++ b/src/netcdf_io/netcdf_io_implementations.f90 @@ -59,6 +59,8 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) !! !! Given an open NetCDF file and a value of time t, finds the index of the time value (aka the time slot) to place a new set of data. !! The returned value of tslot will correspond to the first index value where the value of t is greater than or equal to the saved time value. + use, intrinsic :: ieee_exceptions + use, intrinsic :: ieee_arithmetic implicit none ! Arguments class(netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset @@ -67,7 +69,10 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) ! Internals real(DP), dimension(:), allocatable :: tvals integer(I4B) :: i + logical, dimension(size(IEEE_ALL)) :: fpe_halting_modes + call ieee_get_halting_mode(IEEE_ALL,fpe_halting_modes) ! Save the current halting modes so we can turn them off temporarily + call ieee_set_halting_mode(IEEE_ALL,.false.) if (.not.self%lfile_is_open) return tslot = 0 @@ -76,7 +81,7 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) if (self%max_tslot > 0) then allocate(tvals(self%max_tslot)) call netcdf_io_check( nf90_get_var(self%id, self%time_varid, tvals(:), start=[1]), "netcdf_io_find_tslot get_var" ) - where(tvals(:) /= tvals(:)) tvals(:) = huge(1.0_DP) + where(.not.ieee_is_normal(tvals(:))) tvals(:) = huge(1.0_DP) else allocate(tvals(1)) tvals(1) = huge(1.0_DP) @@ -91,6 +96,8 @@ module subroutine netcdf_io_find_tslot(self, t, tslot) self%max_tslot = max(self%max_tslot, tslot) self%tslot = tslot + call ieee_set_halting_mode(IEEE_ALL,fpe_halting_modes) + return end subroutine netcdf_io_find_tslot From 7b923bd97a30a32351468ac151e8db024abfbb00 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 12:00:56 -0400 Subject: [PATCH 112/149] Cleaned up unused variables, type conversion inconsistencies (except for unit conversions using quad precision) and other things flagged by gfortran warnings --- src/base/base_module.f90 | 2 +- src/collision/collision_check.f90 | 26 +++++++++++++----------- src/collision/collision_generate.f90 | 5 +++-- src/collision/collision_io.f90 | 4 ++-- src/collision/collision_module.f90 | 2 +- src/collision/collision_resolve.f90 | 20 ++++++++++--------- src/encounter/encounter_check.f90 | 8 ++++---- src/encounter/encounter_io.f90 | 4 ++-- src/fraggle/fraggle_generate.f90 | 2 -- src/fraggle/fraggle_util.f90 | 7 +++---- src/rmvs/rmvs_step.f90 | 5 ++--- src/swiftest/swiftest_module.f90 | 10 ++++++---- src/swiftest/swiftest_orbel.f90 | 5 ++--- src/symba/symba_encounter_check.f90 | 30 +++++++++++++++------------- src/symba/symba_kick.f90 | 15 +++++++------- src/symba/symba_step.f90 | 1 - src/symba/symba_util.f90 | 4 ---- 17 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 01c111661..0f91e3473 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -2078,7 +2078,7 @@ subroutine base_util_unique_DP(input_array, output_array, index_map) n = n + 1 lo = minval(input_array(:), mask=input_array(:) > lo) unique_array(n) = lo - where(input_array(:) == lo) index_map(:) = n + where(abs(input_array(:) - lo) < epsilon(1.0_DP) * lo) index_map(:) = n if (lo >= hi) exit enddo allocate(output_array(n), source=unique_array(1:n)) diff --git a/src/collision/collision_check.f90 b/src/collision/collision_check.f90 index 0e69032bc..ebb4a3671 100644 --- a/src/collision/collision_check.f90 +++ b/src/collision/collision_check.f90 @@ -77,7 +77,8 @@ module subroutine collision_check_plpl(self, nbody_system, param, t, dt, irec, l ! Internals logical, dimension(:), allocatable :: lcollision, lmask real(DP), dimension(NDIM) :: xr, vr - integer(I4B) :: i, j, k, nenc + integer(I4B) :: i, j + integer(I8B) :: k, nenc real(DP) :: rlim, Gmtot logical :: lany_closest @@ -102,9 +103,9 @@ module subroutine collision_check_plpl(self, nbody_system, param, t, dt, irec, l self%lclosest(:) = .false. #ifdef DOCONLOC - do concurrent(k = 1:nenc, lmask(k)) shared(self,pl,lmask, dt, lcollision) local(i,j,xr,vr,rlim,Gmtot) + do concurrent(k = 1_I8B:nenc, lmask(k)) shared(self,pl,lmask, dt, lcollision) local(i,j,xr,vr,rlim,Gmtot) #else - do concurrent(k = 1:nenc, lmask(k)) + do concurrent(k = 1_I8B:nenc, lmask(k)) #endif i = self%index1(k) j = self%index2(k) @@ -120,7 +121,7 @@ module subroutine collision_check_plpl(self, nbody_system, param, t, dt, irec, l if (lany_collision .or. lany_closest) then call pl%rh2rb(nbody_system%cb) ! Update the central body barycenteric position vector to get us out of DH and into bary - do k = 1, nenc + do k = 1_I8B, nenc if (.not.lcollision(k) .and. .not. self%lclosest(k)) cycle i = self%index1(k) j = self%index2(k) @@ -178,7 +179,8 @@ module subroutine collision_check_pltp(self, nbody_system, param, t, dt, irec, l ! Internals logical, dimension(:), allocatable :: lcollision, lmask real(DP), dimension(NDIM) :: xr, vr - integer(I4B) :: i, j, k, nenc + integer(I4B) :: i, j + integer(I8B) :: k, nenc logical :: lany_closest character(len=STRMAX) :: timestr, idstri, idstrj, message class(collision_list_pltp), allocatable :: tmp @@ -194,13 +196,13 @@ module subroutine collision_check_pltp(self, nbody_system, param, t, dt, irec, l nenc = self%nenc allocate(lmask(nenc)) - lmask(:) = (self%status(1:nenc) == ACTIVE) + lmask(:) = (self%status(1:nenc) == ACTIVE) select type(pl) class is (symba_pl) - select type(tp) - class is (symba_tp) - lmask(:) = lmask(:) .and. (tp%levelg(self%index2(1:nenc)) >= irec) - end select + select type(tp) + class is (symba_tp) + lmask(:) = lmask(:) .and. (tp%levelg(self%index2(1:nenc)) >= irec) + end select end select if (.not.any(lmask(:))) return @@ -209,9 +211,9 @@ module subroutine collision_check_pltp(self, nbody_system, param, t, dt, irec, l self%lclosest(:) = .false. #ifdef DOCONLOC - do concurrent(k = 1:nenc, lmask(k)) shared(self,pl,tp,lmask, dt, lcollision) local(i,j,xr,vr) + do concurrent(k = 1_I8B:nenc, lmask(k)) shared(self,pl,tp,lmask, dt, lcollision) local(i,j,xr,vr) #else - do concurrent(k = 1:nenc, lmask(k)) + do concurrent(k = 1_I8B:nenc, lmask(k)) #endif i = self%index1(k) j = self%index2(k) diff --git a/src/collision/collision_generate.f90 b/src/collision/collision_generate.f90 index 6162122cf..7f529ba02 100644 --- a/src/collision/collision_generate.f90 +++ b/src/collision/collision_generate.f90 @@ -170,7 +170,8 @@ module subroutine collision_generate_merge(self, nbody_system, param, t) class(base_parameters), intent(inout) :: param !! Current run configuration parameters real(DP), intent(in) :: t !! The time of the collision ! Internals - integer(I4B) :: i, j, k, ibiggest + integer(I4B) :: i, j, ibiggest + integer(I8B) :: k real(DP), dimension(NDIM) :: L_spin_new, L_residual real(DP) :: volume character(len=STRMAX) :: message @@ -231,7 +232,7 @@ module subroutine collision_generate_merge(self, nbody_system, param, t) call collision_util_velocity_torque(-L_residual(:), fragments%mass(1), fragments%rb(:,1), fragments%vb(:,1)) ! Update any encounter lists that have the removed bodies in them so that they instead point to the new body - do k = 1, nbody_system%plpl_encounter%nenc + do k = 1_I8B, nbody_system%plpl_encounter%nenc do j = 1, impactors%ncoll i = impactors%id(j) if (i == ibiggest) cycle diff --git a/src/collision/collision_io.f90 b/src/collision/collision_io.f90 index d0e8f30ae..cb4a544a0 100644 --- a/src/collision/collision_io.f90 +++ b/src/collision/collision_io.f90 @@ -365,7 +365,7 @@ module subroutine collision_io_netcdf_write_frame_snapshot(self, history, param) class(encounter_storage), intent(inout) :: history !! Collision history object class(base_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, idslot, old_mode, npl, stage + integer(I4B) :: i, idslot, old_mode, npl, stage, tmp character(len=NAMELEN) :: charstring class(swiftest_pl), allocatable :: pl @@ -427,7 +427,7 @@ module subroutine collision_io_netcdf_write_frame_snapshot(self, history, param) call netcdf_io_check( nf90_put_var(nc%id, nc%L_spin_varid, collider%L_spin(:,:), start=[1, 1, eslot], count=[NDIM, 2, 1]), "collision_io_netcdf_write_frame_snapshot nf90_put_var L_spin_varid before" ) end if - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode) ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp) ) end associate end select return diff --git a/src/collision/collision_module.f90 b/src/collision/collision_module.f90 index 82bec250b..a06d6d2c1 100644 --- a/src/collision/collision_module.f90 +++ b/src/collision/collision_module.f90 @@ -126,7 +126,7 @@ module collision real(DP), dimension(:), allocatable :: rmag !! Array of radial distance magnitudes of individual fragments in the collisional coordinate frame real(DP), dimension(:), allocatable :: vmag !! Array of radial distance magnitudes of individual fragments in the collisional coordinate frame real(DP), dimension(:), allocatable :: rotmag !! Array of rotation magnitudes of individual fragments - integer(I1B), dimension(:), allocatable :: origin_body !! Array of indices indicating which impactor body (1 or 2) the fragment originates from + integer(I4B), dimension(:), allocatable :: origin_body !! Array of indices indicating which impactor body (1 or 2) the fragment originates from real(DP), dimension(NDIM) :: L_orbit_tot !! Orbital angular momentum vector of all fragments real(DP), dimension(NDIM) :: L_spin_tot !! Spin angular momentum vector of all fragments real(DP), dimension(:,:), allocatable :: L_orbit !! Orbital angular momentum vector of each individual fragment diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index 7aa76166c..3fbbbee84 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -191,8 +191,9 @@ module subroutine collision_resolve_extract_plpl(self, nbody_system, param) logical, dimension(:), allocatable :: lplpl_collision logical, dimension(:), allocatable :: lplpl_unique_parent integer(I4B), dimension(:), pointer :: plparent - integer(I4B), dimension(:), allocatable :: collision_idx, unique_parent_idx - integer(I4B) :: i, index_coll, ncollisions, nunique_parent, nplplenc + integer(I4B) :: nunique_parent + integer(I8B) :: ncollisions, index_coll, k, nplplenc + integer(I8B), dimension(:), allocatable :: unique_parent_idx, collision_idx select type(nbody_system) class is (swiftest_nbody_system) @@ -201,14 +202,14 @@ module subroutine collision_resolve_extract_plpl(self, nbody_system, param) associate(idx1 => self%index1, idx2 => self%index2, plparent => pl%kin%parent) nplplenc = self%nenc allocate(lplpl_collision(nplplenc)) - lplpl_collision(:) = self%status(1:nplplenc) == COLLIDED + lplpl_collision(:) = self%status(1_I8B:nplplenc) == COLLIDED if (.not.any(lplpl_collision)) return ! Collisions have been detected in this step. So we need to determine which of them are between unique bodies. ! Get the subset of pl-pl encounters that lead to a collision ncollisions = count(lplpl_collision(:)) allocate(collision_idx(ncollisions)) - collision_idx = pack([(i, i=1, nplplenc)], lplpl_collision) + collision_idx = pack([(k, k=1_I8B, nplplenc)], lplpl_collision) ! Get the subset of collisions that involve a unique pair of parents allocate(lplpl_unique_parent(ncollisions)) @@ -223,7 +224,7 @@ module subroutine collision_resolve_extract_plpl(self, nbody_system, param) ! due to restructuring of parent/child relationships when there are large numbers of multi-body collisions in a single ! step lplpl_unique_parent(:) = .true. - do index_coll = 1, ncollisions + do index_coll = 1_I8B, ncollisions associate(ip1 => plparent(idx1(collision_idx(index_coll))), ip2 => plparent(idx2(collision_idx(index_coll)))) lplpl_unique_parent(:) = .not. ( any(plparent(idx1(unique_parent_idx(:))) == ip1) .or. & any(plparent(idx2(unique_parent_idx(:))) == ip1) .or. & @@ -537,7 +538,8 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) character(len=STRMAX) :: timestr, idstr integer(I4B), dimension(2) :: idx_parent !! Index of the two bodies considered the "parents" of the collision logical :: lgoodcollision - integer(I4B) :: i, j, nnew, loop, ncollisions + integer(I4B) :: i, j, nnew, loop + integer(I8B) :: k, ncollisions integer(I4B), dimension(:), allocatable :: idnew integer(I4B), parameter :: MAXCASCADE = 1000 @@ -576,9 +578,9 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) call swiftest_io_log_one_message(COLLISION_LOG_OUT, "***********************************************************" // & "***********************************************************") - do i = 1, ncollisions - idx_parent(1) = pl%kin(idx1(i))%parent - idx_parent(2) = pl%kin(idx2(i))%parent + do k = 1_I8B, ncollisions + idx_parent(1) = pl%kin(idx1(k))%parent + idx_parent(2) = pl%kin(idx2(k))%parent call impactors%consolidate(nbody_system, param, idx_parent, lgoodcollision) if ((.not. lgoodcollision) .or. any(pl%status(idx_parent(:)) /= COLLIDED)) cycle diff --git a/src/encounter/encounter_check.f90 b/src/encounter/encounter_check.f90 index b987f8799..2d934e7d8 100644 --- a/src/encounter/encounter_check.f90 +++ b/src/encounter/encounter_check.f90 @@ -636,7 +636,7 @@ module subroutine encounter_check_collapse_ragged_list(ragged_list, n1, nenc, in ! Internals integer(I4B) :: i integer(I8B) :: j1, j0, nenci - integer(I4B), dimension(n1) :: ibeg + integer(I8B), dimension(n1) :: ibeg associate(nenc_arr => ragged_list(:)%nenc) nenc = sum(nenc_arr(:)) @@ -865,7 +865,7 @@ module subroutine encounter_check_sweep_aabb_double_list(self, n1, n2, r1, v1, r if (loverlap(i)) then ibeg = self%aabb%ibeg(i) + 1_I8B iend = self%aabb%iend(i) - 1_I8B - nbox = iend - ibeg + 1 + nbox = int(iend - ibeg, kind=I4B) + 1 call encounter_check_all_sweep_one(i, nbox, r1(1,i), r1(2,i), r1(3,i), v1(1,i), v1(2,i), v1(3,i), & xind(ibeg:iend), yind(ibeg:iend), zind(ibeg:iend),& vxind(ibeg:iend), vyind(ibeg:iend), vzind(ibeg:iend), & @@ -881,7 +881,7 @@ module subroutine encounter_check_sweep_aabb_double_list(self, n1, n2, r1, v1, r if (loverlap(i)) then ibeg = self%aabb%ibeg(i) + 1_I8B iend = self%aabb%iend(i) - 1_I8B - nbox = iend - ibeg + 1 + nbox = int(iend - ibeg, kind=I4B) + 1 ii = i - n1 call encounter_check_all_sweep_one(ii, nbox, r2(1,ii), r2(2,ii), r2(3,ii), v2(1,ii), v2(2,ii), v2(3,ii), & xind(ibeg:iend), yind(ibeg:iend), zind(ibeg:iend),& @@ -958,7 +958,7 @@ module subroutine encounter_check_sweep_aabb_single_list(self, n, r, v, renc, dt if (loverlap(i)) then ibeg = self%aabb%ibeg(i) + 1_I8B iend = self%aabb%iend(i) - 1_I8B - nbox = int(iend - ibeg + 1, kind=I4B) + nbox = int(iend - ibeg, kind=I4B) + 1 lencounteri(ibeg:iend) = .true. call encounter_check_all_sweep_one(i, nbox, r(1,i), r(2,i), r(3,i), v(1,i), v(2,i), v(3,i), & xind(ibeg:iend), yind(ibeg:iend), zind(ibeg:iend),& diff --git a/src/encounter/encounter_io.f90 b/src/encounter/encounter_io.f90 index ceb3194fe..fb5de048f 100644 --- a/src/encounter/encounter_io.f90 +++ b/src/encounter/encounter_io.f90 @@ -231,7 +231,7 @@ module subroutine encounter_io_netcdf_write_frame_snapshot(self, history, param) class(base_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, idslot, old_mode, npl, ntp + integer(I4B) :: i, idslot, old_mode, npl, ntp, tmp character(len=STRMAX) :: charstring select type(param) @@ -284,7 +284,7 @@ module subroutine encounter_io_netcdf_write_frame_snapshot(self, history, param) call netcdf_io_check( nf90_put_var(nc%id, nc%ptype_varid, charstring, start=[1, idslot], count=[NAMELEN, 1]), "encounter_io_netcdf_write_frame_snapshot nf90_put_var tp particle_type_varid" ) end do - call netcdf_io_check( nf90_set_fill(nc%id, old_mode, old_mode) ) + call netcdf_io_check( nf90_set_fill(nc%id, old_mode, tmp) ) end associate end select end select diff --git a/src/fraggle/fraggle_generate.f90 b/src/fraggle/fraggle_generate.f90 index 0dec13fde..5ec993fba 100644 --- a/src/fraggle/fraggle_generate.f90 +++ b/src/fraggle/fraggle_generate.f90 @@ -361,8 +361,6 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu integer(I4B) :: i, j, loop, istart, nfrag, npl, ntp logical :: lsupercat, lhitandrun integer(I4B), parameter :: MAXLOOP = 10000 - real(DP), parameter :: cloud_size_scale_factor = 3.0_DP ! Scale factor to apply to the size of the cloud relative to the distance from the impact point. - ! A larger value puts more space between fragments initially real(DP), parameter :: rbuffer = 1.01_DP ! Body radii are inflated by this scale factor to prevent secondary collisions real(DP), parameter :: pack_density = 0.5236_DP ! packing density of loose spheres diff --git a/src/fraggle/fraggle_util.f90 b/src/fraggle/fraggle_util.f90 index 351bd05fa..19752442c 100644 --- a/src/fraggle/fraggle_util.f90 +++ b/src/fraggle/fraggle_util.f90 @@ -108,7 +108,6 @@ module subroutine fraggle_util_set_mass_dist(self, param) integer(I4B), parameter :: iMrem = 3 integer(I4B), parameter :: NFRAGMIN = iMrem + 2 !! Minimum number of fragments that can be generated integer(I4B), dimension(:), allocatable :: ind - integer(I4B), parameter :: MAXLOOP = 20 logical :: flipper logical, dimension(size(IEEE_ALL)) :: fpe_halting_modes @@ -174,7 +173,7 @@ module subroutine fraggle_util_set_mass_dist(self, param) mass(2) = impactors%mass_dist(iMslr) ! Recompute the slope parameter beta so that we span the complete size range - if (Mslr == min_mfrag) Mslr = Mslr + impactors%mass_dist(iMrem) / nfrag + if (abs(Mslr - min_mfrag) < epsilon(min_mfrag) * min_mfrag) Mslr = Mslr + impactors%mass_dist(iMrem) / nfrag mremaining = impactors%mass_dist(iMrem) ! The mass will be distributed in a power law where N>M=(M/Mslr)**(-beta/3) @@ -225,8 +224,8 @@ module subroutine fraggle_util_set_mass_dist(self, param) ! For catastrophic impacts, we will assign each of the n>2 fragments to one of the two original bodies so that the fragment cloud occupies ! roughly the same space as both original bodies. For all other disruption cases, we use body 2 as the center of the cloud. - fragments%origin_body(1) = 1 - fragments%origin_body(2) = 2 + fragments%origin_body(1) = 1_I1B + fragments%origin_body(2) = 2_I1B if (impactors%regime == COLLRESOLVE_REGIME_SUPERCATASTROPHIC) then mcumul = fragments%mass(1) flipper = .true. diff --git a/src/rmvs/rmvs_step.f90 b/src/rmvs/rmvs_step.f90 index 11f7d2756..4a5dcf3af 100644 --- a/src/rmvs/rmvs_step.f90 +++ b/src/rmvs/rmvs_step.f90 @@ -541,9 +541,8 @@ subroutine rmvs_peri_tp(tp, pl, t, dt, lfirst, inner_index, ipleP, param) integer(I4B), intent(in) :: ipleP !! index of RMVS planet being closely encountered class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: i, id1, id2 - real(DP) :: r2, mu, rhill2, vdotr, a, peri, capm, tperi, rpl - real(DP), dimension(NDIM) :: rh1, rh2, vh1, vh2 + integer(I4B) :: i + real(DP) :: r2, mu, rhill2, vdotr, a, peri, capm, tperi rhill2 = pl%rhill(ipleP)**2 mu = pl%Gmass(ipleP) diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 3d35749ee..8349c12c1 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -671,11 +671,13 @@ module subroutine swiftest_io_netcdf_get_t0_values_system(self, nc, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters end subroutine swiftest_io_netcdf_get_t0_values_system - module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask) + module subroutine swiftest_io_netcdf_get_valid_masks(self, plmask, tpmask, plmmask, Gmtiny) implicit none - class(swiftest_netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset - logical, dimension(:), allocatable, intent(out) :: plmask !! Logical mask indicating which bodies are massive bodies - logical, dimension(:), allocatable, intent(out) :: tpmask !! Logical mask indicating which bodies are test particles + class(swiftest_netcdf_parameters), intent(inout) :: self !! Parameters used to identify a particular NetCDF dataset + logical, dimension(:), allocatable, intent(out) :: plmask !! Logical mask indicating which bodies are massive bodies + logical, dimension(:), allocatable, intent(out) :: tpmask !! Logical mask indicating which bodies are test particles + logical, dimension(:), allocatable, intent(out), optional :: plmmask !! Logical mask indicating which bodies are fully interacting massive bodies + real(DP), intent(in), optional :: Gmtiny !! The cutoff G*mass between semi-interacting and fully interacting massive bodies end subroutine swiftest_io_netcdf_get_valid_masks module subroutine swiftest_io_netcdf_initialize_output(self, param) diff --git a/src/swiftest/swiftest_orbel.f90 b/src/swiftest/swiftest_orbel.f90 index dafb3e293..3827c1b59 100644 --- a/src/swiftest/swiftest_orbel.f90 +++ b/src/swiftest/swiftest_orbel.f90 @@ -236,7 +236,6 @@ real(DP) pure function swiftest_orbel_flon(e,icapn) real(DP), parameter :: a5 = 51891840._DP, a3 = 1037836800._DP real(DP), parameter :: b11 = 11 * a11, b9 = 9 * a9, b7 = 7 * a7 real(DP), parameter :: b5 = 5 * a5, b3 = 3 * a3 - real(DP), parameter :: THIRD = 1._DP / 3._DP ! Function to solve "Kepler's eqn" for F (here called ! x) for given e and CAPN. Only good for smallish CAPN @@ -725,7 +724,7 @@ pure elemental module subroutine swiftest_orbel_xv2aeq(mu, rx, ry, rz, vx, vy, v hz = rx*vy - ry*vx h2 = hx*hx + hy*hy + hz*hz - if (h2 == 0.0_DP) return + if (h2 < tiny(h2)) return energy = 0.5_DP * v2 - mu / r if (abs(energy * r / mu) < sqrt(TINYVALUE)) then iorbit_type = PARABOLA @@ -791,7 +790,7 @@ pure module subroutine swiftest_orbel_xv2aqt(mu, rx, ry, rz, vx, vy, vz, a, q, c hy = rz*vx - rx*vz hz = rx*vy - ry*vx h2 = hx*hx + hy*hy + hz*hz - if (h2 == 0.0_DP) return + if (h2 < tiny(h2)) return r = sqrt(rx*rx + ry*ry + rz*rz) v2 = vx*vx + vy*vy + vz*vz diff --git a/src/symba/symba_encounter_check.f90 b/src/symba/symba_encounter_check.f90 index 4bc92b79d..e66791409 100644 --- a/src/symba/symba_encounter_check.f90 +++ b/src/symba/symba_encounter_check.f90 @@ -96,11 +96,12 @@ module function symba_encounter_check_list_plpl(self, param, nbody_system, dt, i integer(I4B), intent(in) :: irec !! Current recursion level logical :: lany_encounter !! Returns true if there is at least one close encounter ! Internals - integer(I4B) :: i, j, k, lidx, nenc_enc + integer(I4B) :: i, j, nenc_enc + integer(I8B) :: k, lidx real(DP), dimension(NDIM) :: xr, vr real(DP) :: rlim2, rji2, rcrit12 logical, dimension(:), allocatable :: lencmask, lencounter - integer(I4B), dimension(:), allocatable :: eidx + integer(I8B), dimension(:), allocatable :: eidx lany_encounter = .false. if (self%nenc == 0) return @@ -116,13 +117,13 @@ module function symba_encounter_check_list_plpl(self, param, nbody_system, dt, i allocate(eidx(nenc_enc)) allocate(lencounter(nenc_enc)) - eidx(:) = pack([(k, k = 1, self%nenc)], lencmask(:)) + eidx(:) = pack([(k, k = 1_I8B, self%nenc)], lencmask(:)) lencounter(:) = .false. #ifdef DOCONLOC - do concurrent(lidx = 1:nenc_enc) shared(self,pl,eidx,lencounter,dt) local(i,j,k,xr,vr,rcrit12,rlim2,rji2) + do concurrent(lidx = 1_I8B:nenc_enc) shared(self,pl,eidx,lencounter,dt) local(i,j,k,xr,vr,rcrit12,rlim2,rji2) #else - do concurrent(lidx = 1:nenc_enc) + do concurrent(lidx = 1_I8B:nenc_enc) #endif k = eidx(lidx) i = self%index1(k) @@ -141,8 +142,8 @@ module function symba_encounter_check_list_plpl(self, param, nbody_system, dt, i lany_encounter = any(lencounter(:)) if (lany_encounter) then nenc_enc = count(lencounter(:)) - eidx(1:nenc_enc) = pack(eidx(:), lencounter(:)) - do lidx = 1, nenc_enc + eidx(1_I8B:nenc_enc) = pack(eidx(:), lencounter(:)) + do lidx = 1_I8B, nenc_enc k = eidx(lidx) i = self%index1(k) j = self%index2(k) @@ -168,11 +169,12 @@ module function symba_encounter_check_list_pltp(self, param, nbody_system, dt, i integer(I4B), intent(in) :: irec !! Current recursion level logical :: lany_encounter !! Returns true if there is at least one close encounter ! Internals - integer(I4B) :: i, j, k, lidx, nenc_enc + integer(I4B) :: i, j, nenc_enc + integer(I8B) :: k, lidx real(DP), dimension(NDIM) :: xr, vr real(DP) :: rlim2, rji2 logical, dimension(:), allocatable :: lencmask, lencounter - integer(I4B), dimension(:), allocatable :: eidx + integer(I8B), dimension(:), allocatable :: eidx lany_encounter = .false. if (self%nenc == 0) return @@ -190,12 +192,12 @@ module function symba_encounter_check_list_pltp(self, param, nbody_system, dt, i allocate(eidx(nenc_enc)) allocate(lencounter(nenc_enc)) - eidx(:) = pack([(k, k = 1, self%nenc)], lencmask(:)) + eidx(:) = pack([(k, k = 1_I8B, self%nenc)], lencmask(:)) lencounter(:) = .false. #ifdef DOCONLOC - do concurrent(lidx = 1:nenc_enc) shared(self,pl,tp,eidx,lencounter,dt) local(i,j,k,xr,vr,rlim2,rji2) + do concurrent(lidx = 1_I8B:nenc_enc) shared(self,pl,tp,eidx,lencounter,dt) local(i,j,k,xr,vr,rlim2,rji2) #else - do concurrent(lidx = 1:nenc_enc) + do concurrent(lidx = 1_I8B:nenc_enc) #endif k = eidx(lidx) i = self%index1(k) @@ -214,8 +216,8 @@ module function symba_encounter_check_list_pltp(self, param, nbody_system, dt, i lany_encounter = any(lencounter(:)) if (lany_encounter) then nenc_enc = count(lencounter(:)) - eidx(1:nenc_enc) = pack(eidx(:), lencounter(:)) - do lidx = 1, nenc_enc + eidx(1_I8B:nenc_enc) = pack(eidx(:), lencounter(:)) + do lidx = 1_I8B, nenc_enc k = eidx(lidx) i = self%index1(k) j = self%index2(k) diff --git a/src/symba/symba_kick.f90 b/src/symba/symba_kick.f90 index dc280ef1c..2db14456c 100644 --- a/src/symba/symba_kick.f90 +++ b/src/symba/symba_kick.f90 @@ -91,7 +91,8 @@ module subroutine symba_kick_getacch_tp(self, nbody_system, param, t, lbeg) real(DP), intent(in) :: t !! Current time logical, intent(in) :: lbeg !! Logical flag that determines whether or not this is the beginning or end of the step ! Internals - integer(I4B) :: i, j, k + integer(I4B) :: i, j + integer(I8B) :: k real(DP) :: rjj, fac real(DP), dimension(NDIM) :: dx @@ -142,7 +143,7 @@ module subroutine symba_kick_list_plpl(self, nbody_system, dt, irec, sgn) real(DP) :: r, rr, ri, ris, rim1, r2, ir3, fac, faci, facj real(DP), dimension(NDIM) :: dx logical, dimension(:), allocatable :: lgoodlevel - integer(I4B), dimension(:), allocatable :: good_idx + integer(I8B), dimension(:), allocatable :: good_idx if (self%nenc == 0) return @@ -170,7 +171,7 @@ module subroutine symba_kick_list_plpl(self, nbody_system, dt, irec, sgn) ngood = count(lgoodlevel(:)) if (ngood > 0_I8B) then allocate(good_idx(ngood)) - good_idx(:) = pack([(i, i = 1, nenc)], lgoodlevel(:)) + good_idx(:) = pack([(k, k = 1_I8B, nenc)], lgoodlevel(:)) #ifdef DOCONLOC do concurrent (k = 1:ngood) shared(self,pl,good_idx) local(i,j) @@ -212,7 +213,7 @@ module subroutine symba_kick_list_plpl(self, nbody_system, dt, irec, sgn) end do ngood = count(lgoodlevel(:)) if (ngood == 0_I8B) return - good_idx(1:ngood) = pack([(i, i = 1, nenc)], lgoodlevel(:)) + good_idx(1:ngood) = pack([(k, k = 1_I8B, nenc)], lgoodlevel(:)) do k = 1, ngood i = self%index1(good_idx(k)) @@ -251,7 +252,7 @@ module subroutine symba_kick_list_pltp(self, nbody_system, dt, irec, sgn) real(DP) :: r, rr, ri, ris, rim1, r2, ir3, fac, faci real(DP), dimension(NDIM) :: dx logical, dimension(:), allocatable :: lgoodlevel - integer(I4B), dimension(:), allocatable :: good_idx + integer(I8B), dimension(:), allocatable :: good_idx if (self%nenc == 0) return @@ -284,7 +285,7 @@ module subroutine symba_kick_list_pltp(self, nbody_system, dt, irec, sgn) if (ngood > 0_I8B) then allocate(good_idx(ngood)) - good_idx(:) = pack([(i, i = 1, nenc)], lgoodlevel(:)) + good_idx(:) = pack([(k, k = 1_I8B, nenc)], lgoodlevel(:)) #ifdef DOCONLOC do concurrent (k = 1_I8B:ngood) shared(self,tp,good_idx) local(j) @@ -323,7 +324,7 @@ module subroutine symba_kick_list_pltp(self, nbody_system, dt, irec, sgn) end do ngood = count(lgoodlevel(:)) if (ngood == 0_I8B) return - good_idx(1:ngood) = pack([(i, i = 1, nenc)], lgoodlevel(:)) + good_idx(1:ngood) = pack([(k, k = 1_I8B, nenc)], lgoodlevel(:)) do k = 1, ngood j = self%index2(good_idx(k)) diff --git a/src/symba/symba_step.f90 b/src/symba/symba_step.f90 index 6e72c3e95..22cdddeb6 100644 --- a/src/symba/symba_step.f90 +++ b/src/symba/symba_step.f90 @@ -27,7 +27,6 @@ module subroutine symba_step_system(self, param, t, dt) real(DP), intent(in) :: dt !! Current stepsize ! Internals logical :: lencounter - type(walltimer) :: timer1,timer2,timer3 !! Object used for computing elapsed wall time select type(pl => self%pl) class is (symba_pl) diff --git a/src/symba/symba_util.f90 b/src/symba/symba_util.f90 index 94b6aa9f5..c7abd79f3 100644 --- a/src/symba/symba_util.f90 +++ b/src/symba/symba_util.f90 @@ -71,8 +71,6 @@ module subroutine symba_util_dealloc_pl(self) implicit none ! Arguments class(symba_pl), intent(inout) :: self !! SyMBA massive body object - ! Internals - integer(I4B) :: i if (allocated(self%levelg)) deallocate(self%levelg) if (allocated(self%levelm)) deallocate(self%levelm) @@ -352,8 +350,6 @@ module subroutine symba_util_setup_pl(self, n, param) class(symba_pl), intent(inout) :: self !! SyMBA massive body object integer(I4B), intent(in) :: n !! Number of particles to allocate space for class(swiftest_parameters), intent(in) :: param !! Current run configuration parameter - ! Internals - integer(I4B) :: i !> Call allocation method for parent class. call self%helio_pl%setup(n, param) From 2ad3c7d84111128ea3aedc82611800946010f7fd Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 12:02:39 -0400 Subject: [PATCH 113/149] Fixed bad index resulting from refactor --- src/collision/collision_resolve.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collision/collision_resolve.f90 b/src/collision/collision_resolve.f90 index 3fbbbee84..0166711ab 100644 --- a/src/collision/collision_resolve.f90 +++ b/src/collision/collision_resolve.f90 @@ -601,7 +601,7 @@ module subroutine collision_resolve_plpl(self, nbody_system, param, t, dt, irec) call collision_history%take_snapshot(param,nbody_system, t, "after") - plpl_collision%status(i) = collider%status + plpl_collision%status(k) = collider%status call impactors%dealloc() end do From 7472c5cd63baccf6e583ba203de57767b5a9adb8 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Thu, 25 May 2023 14:03:11 -0400 Subject: [PATCH 114/149] Fixed problem in SyMBA where nplm was not being updated properly after massive body discards --- src/swiftest/swiftest_util.f90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 641fcc170..2c7b803bc 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1678,7 +1678,10 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) npl = pl%nbody end if - if (npl == 0) return + if (npl == 0) then + if (param%lmtiny_pl) pl%nplm = 0 + return + end if ! Reset all of the status flags for this body pl%status(1:npl) = ACTIVE @@ -1696,6 +1699,7 @@ module subroutine swiftest_util_rearray_pl(self, nbody_system, param) elsewhere pl%info(1:npl)%particle_type = PL_TYPE_NAME end where + pl%nplm = count(.not.pl%lmtiny(1:npl)) end if ! Reindex the new list of bodies From d46bec6186f90b241b1a71e991c83a73c1bade86 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 26 May 2023 12:41:37 -0400 Subject: [PATCH 115/149] Restructured Docker and Singularity scripts. Added new GNU version of the Docker container that is smaller than the Intel one (but also slower). --- docker/.gitignore | 13 ++++++-- docker/bin/.gitignore | 1 - docker/gnu/Dockerfile | 44 ++++++++++++++++++++++++++++ docker/gnu/bin/.gitignore | 0 docker/{ => gnu}/bin/swiftest_driver | 2 +- docker/install.sh | 5 ++++ docker/{ => intel}/Dockerfile | 0 docker/pull | 2 -- singularity/.gitignore | 5 ++-- singularity/bin/.gitignore | 1 - singularity/install.sh | 9 ++++++ singularity/pull | 2 -- singularity/setenv.sh | 7 +++-- singularity/swiftest.def | 2 -- 14 files changed, 76 insertions(+), 17 deletions(-) delete mode 100644 docker/bin/.gitignore create mode 100644 docker/gnu/Dockerfile create mode 100644 docker/gnu/bin/.gitignore rename docker/{ => gnu}/bin/swiftest_driver (62%) create mode 100755 docker/install.sh rename docker/{ => intel}/Dockerfile (100%) delete mode 100755 docker/pull delete mode 100644 singularity/bin/.gitignore create mode 100755 singularity/install.sh delete mode 100755 singularity/pull delete mode 100644 singularity/swiftest.def diff --git a/docker/.gitignore b/docker/.gitignore index 825a38873..215dd3f04 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,3 +1,10 @@ -!Dockerfile -!bin/ -!pull +* +!.gitignore +!install.sh +!gnu/ +!gnu/Dockerfile +!/gnu/bin/ +!/gnu/bin/swiftest_driver +!intel/ +!intel/Dockerfile +!/intel/bin/swiftest_driver diff --git a/docker/bin/.gitignore b/docker/bin/.gitignore deleted file mode 100644 index c1d7ed39d..000000000 --- a/docker/bin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!swiftest_driver diff --git a/docker/gnu/Dockerfile b/docker/gnu/Dockerfile new file mode 100644 index 000000000..ec64211bd --- /dev/null +++ b/docker/gnu/Dockerfile @@ -0,0 +1,44 @@ +FROM debian:stable-slim as build + +# kick everything off +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git build-essential gfortran && \ + apt-get update && apt-get upgrade -y && \ + rm -rf /var/lib/apt/lists/* + +# Get CMAKE and install it +RUN mkdir -p /opt/cmake/build && \ + cd /opt/cmake/build && \ + curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ + /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license + +# Get dependencies +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libnetcdf-dev libnetcdff-dev libcoarrays-dev libcoarrays-mpich-dev&& \ + rm -rf /var/lib/apt/lists/* + +ENV NETCDF_HOME="/usr" +ENV NETCDF_FORTRAN_HOME="/usr" +ENV INDIR="/opt/dist//usr/local" + +# Get Swiftest source code +RUN cd /opt/ && \ + git clone -b debug https://github.com/carlislewishard/swiftest.git && \ + cd swiftest && \ + mkdir build && \ + cd build && \ + cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release && \ + make && \ + make install + +#Production container +FROM debian:stable-slim +COPY --from=build /opt/dist / +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates gfortran libnetcdf-dev libnetcdff-dev && \ + rm -rf /var/lib/apt/lists/* + +ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file diff --git a/docker/gnu/bin/.gitignore b/docker/gnu/bin/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/docker/bin/swiftest_driver b/docker/gnu/bin/swiftest_driver similarity index 62% rename from docker/bin/swiftest_driver rename to docker/gnu/bin/swiftest_driver index d2cb42a90..f0a3638dd 100755 --- a/docker/bin/swiftest_driver +++ b/docker/gnu/bin/swiftest_driver @@ -1,2 +1,2 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:latest "$@" +docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest_driver:gnu "$@" diff --git a/docker/install.sh b/docker/install.sh new file mode 100755 index 000000000..82e97d58f --- /dev/null +++ b/docker/install.sh @@ -0,0 +1,5 @@ +#!/bin/sh -- +tag=${1:-intel} +echo "Installing swiftest_driver:${tag} container and executable script" +docker pull daminton/swiftest_driver:${tag} +cp -rf ${tag}/bin/swiftest_driver ../bin/ \ No newline at end of file diff --git a/docker/Dockerfile b/docker/intel/Dockerfile similarity index 100% rename from docker/Dockerfile rename to docker/intel/Dockerfile diff --git a/docker/pull b/docker/pull deleted file mode 100755 index 2367ba639..000000000 --- a/docker/pull +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -docker pull daminton/swiftest:latest \ No newline at end of file diff --git a/singularity/.gitignore b/singularity/.gitignore index abc0e7f03..2a259d270 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -1,3 +1,4 @@ -!swiftest.def !bin/ -!pull +!bin/swiftest_driver +!install.sh +!setenv.sh diff --git a/singularity/bin/.gitignore b/singularity/bin/.gitignore deleted file mode 100644 index c1d7ed39d..000000000 --- a/singularity/bin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!swiftest_driver diff --git a/singularity/install.sh b/singularity/install.sh new file mode 100755 index 000000000..ef3eb1b9e --- /dev/null +++ b/singularity/install.sh @@ -0,0 +1,9 @@ +#!/bin/sh -- +# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. +# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source: +# $ . ./install.sh +# +tag=${1:-intel} +singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag} +cp -rf bin/swiftest_driver ../bin/ +source ./setenv.sh \ No newline at end of file diff --git a/singularity/pull b/singularity/pull deleted file mode 100755 index a62acaf6e..000000000 --- a/singularity/pull +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -singularity pull --force swiftest.sif docker://daminton/swiftest:latest diff --git a/singularity/setenv.sh b/singularity/setenv.sh index 79931a298..308d49151 100755 --- a/singularity/setenv.sh +++ b/singularity/setenv.sh @@ -1,3 +1,4 @@ -#!/bin/sh -# source this file to set the path to the swiftest.sif file -export SWIFTEST_SIF="${PWD}/swiftest.sif" +#!/bin/sh -- +# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. +# $ . ./setenvl.sh +export SWIFTEST_SIF="${PWD}/swiftest_driver.sif" \ No newline at end of file diff --git a/singularity/swiftest.def b/singularity/swiftest.def deleted file mode 100644 index ce6c10f63..000000000 --- a/singularity/swiftest.def +++ /dev/null @@ -1,2 +0,0 @@ -Bootstrap: docker -From: daminton/swiftest:latest \ No newline at end of file From 64d6a304a481dde06a4281c5c76019811a902023 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 26 May 2023 13:13:36 -0400 Subject: [PATCH 116/149] Fixed typos and added some output --- docker/install.sh | 2 +- singularity/install.sh | 1 + singularity/setenv.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/install.sh b/docker/install.sh index 82e97d58f..f81e95e57 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -1,5 +1,5 @@ #!/bin/sh -- tag=${1:-intel} -echo "Installing swiftest_driver:${tag} container and executable script" +echo "Installing swiftest_driver:${tag} Docker container and executable script" docker pull daminton/swiftest_driver:${tag} cp -rf ${tag}/bin/swiftest_driver ../bin/ \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index ef3eb1b9e..50d6d56df 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -4,6 +4,7 @@ # $ . ./install.sh # tag=${1:-intel} +echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag} cp -rf bin/swiftest_driver ../bin/ source ./setenv.sh \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh index 308d49151..cc8905033 100755 --- a/singularity/setenv.sh +++ b/singularity/setenv.sh @@ -1,4 +1,4 @@ #!/bin/sh -- # This will set the SWIFTEST_SIF environment variable as long as it is executed by source. -# $ . ./setenvl.sh +# $ . ./setenv.sh export SWIFTEST_SIF="${PWD}/swiftest_driver.sif" \ No newline at end of file From 6cce4f023b86f795370b149c8f8d0c0b2427bf94 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 26 May 2023 13:17:17 -0400 Subject: [PATCH 117/149] Removed unneeeded "container" argument because now the install.sh scripts will replace the executable with a script that calls the container --- python/swiftest/swiftest/simulation_class.py | 31 ++------------------ 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 10a869ec1..8971b897a 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -91,8 +91,6 @@ def __init__(self,read_param: bool = False, integrator : {"symba","rmvs","whm","helio"}, default "symba" Name of the n-body integrator that will be used when executing a run. Parameter input file equivalent: None - container : {"docker", "singularity"}, default None - Specify whether the driver exectuable is run from a Docker or Singularity container. Setting to `None` read_param : bool, default False Read the parameter file given by `param_file`. param_file : str, path-like, or file-lke, default "param.in" @@ -340,7 +338,6 @@ def __init__(self,read_param: bool = False, self.init_cond = xr.Dataset() self.encounters = xr.Dataset() self.collisions = xr.Dataset() - self.container = None # Set the location of the parameter input file, choosing the default if it isn't specified. self.simdir = Path.cwd() / Path(simdir) @@ -408,8 +405,6 @@ def _run_swiftest_driver(self): # Get current environment variables env = os.environ.copy() - if self.container == "SINGULARITY" and "SWIFTEST_SIF" not in env: - env['SWIFTEST_SIF'] = Path(self.binary_source).parent.parent / "swiftest.sif" cmd = f"{env['SHELL']} -l {self.driver_script}" @@ -835,7 +830,6 @@ def set_parameter(self, verbose: bool = True, **kwargs): "encounter_save" : "NONE", "coarray" : False, "simdir" : self.simdir, - "container" : None, } param_file = kwargs.pop("param_file",None) @@ -899,7 +893,6 @@ def get_parameter(self, **kwargs): def set_integrator(self, codename: None | Literal["Swiftest", "Swifter", "Swift"] = "Swiftest", integrator: Literal["symba","rmvs","whm","helio"] | None = None, - container: Literal["docker", "singularity"] | None = None, mtiny: float | None = None, gmtiny: float | None = None, verbose: bool | None = None, @@ -912,10 +905,6 @@ def set_integrator(self, codename : {"swiftest", "swifter", "swift"}, optional integrator : {"symba","rmvs","whm","helio"}, optional Name of the n-body integrator that will be used when executing a run. - container : {"docker", "singularity"}, default None - Specify whether the driver exectuable is run from a Docker or Singularity container. - Setting to `None` uses the local executable. - *Note*: Only valid for Swiftest. mtiny : float, optional The minimum mass of fully interacting bodies. Bodies below this mass interact with the larger bodies, but not each other (SyMBA only). *Note.* Only mtiny or gmtiny is accepted, not both. @@ -938,14 +927,6 @@ def set_integrator(self, update_list = [] - if container is not None: - valid_container = ["DOCKER", "SINGULARITY"] - if container.upper() not in valid_container: - warnings.warn(f"{container} is not a valid container type. Valid options are None, ",",".join(valid_container),stacklevel=2) - self.container = None - else: - self.container = container.upper() - if codename is not None: valid_codename = ["Swiftest", "Swifter", "Swift"] if codename.title() not in valid_codename: @@ -960,13 +941,7 @@ def set_integrator(self, self.param['! VERSION'] = f"{self.codename} input file" update_list.append("codename") if self.codename == "Swiftest": - if self.container is None: - self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" - elif self.container == "DOCKER": - self.binary_source = Path(_pyfile).parent.parent.parent.parent / "docker" / "bin" / "swiftest_driver" - elif self.container == "SINGULARITY": - self.binary_source = Path(_pyfile).parent.parent.parent.parent / "singularity" / "bin" / "swiftest_driver" - + self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" self.binary_path = self.simdir.resolve() self.driver_executable = self.binary_path / "swiftest_driver" if not self.binary_source.exists(): @@ -1041,8 +1016,8 @@ def get_integrator(self,arg_list: str | List[str] | None = None, verbose: bool | valid_instance_vars = {"codename": self.codename, "integrator": self.integrator, "param_file": str(self.param_file), - "driver_executable": str(self.driver_executable), - "container": self.container} + "driver_executable": str(self.driver_executable) + } try: self.integrator From 0e130018452e8703e8f9c33b1efcf4a133c2c175 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 26 May 2023 13:46:52 -0400 Subject: [PATCH 118/149] Added documentation for container versions --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 033d86834..60893055f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ Swiftest also includes the collisional fragmentation algorithm **Fraggle**, an a #### Installation -**System Requirements** +In order to use Swiftest, you need to have a working `swiftest_driver` executable. Currently, this can be obtained by either compiling the source code on the system you plan to run simulations on (fastest), or by running it from a Docker/Singularity container compiled for an x86_64 CPU using the Intel Fortran compiler (slower) or compiled using the GNU/gfortran compiler (slowest). + +**Building the `swiftest_driver` executable** Swiftest is designed to be downloaded, compiled, and run on a Linux based system. It is untested on Windows systems. @@ -103,6 +105,29 @@ $ make The Swiftest executable, called ```swiftest_driver```, should now be created in the ```/swiftest/bin/``` directory. +**Download the `swiftest_driver` as a Docker or Singularity container. + +The Swiftest driver is available as a Docker container on DockerHub in two versions: Intel and GNU. The Intel version was compiled for the x86_64 CPU using the Intel classic Fortran compiler. The GNU version was compliled for the x86_64 CPU using gfortran. The Intel version is faster than the GNU version (though not as fast as a native compile to the target CPU that you wish to run it on due to vectorization optimizations that Swiftest takes advantage of), however it is much larger: The Intel version is ~2.7GB while the GNU version is ~300MB. The Singularity container pulls from the same DockerHub container. + +To facilitate installation of the container, we provide a set of shell scripts to help automate the process of installing container versions of the executable. To install the default Intel version of the docker container from within the `swiftest\` project directory + +``` +$ cd docker +$ . ./install.sh +``` + +To install the GNU version: + +``` +$ cd docker +$ . ./install.sh gnu +``` + +The Singularity versions are installed the same way, just replace `cd docker` with `cd singularity` above. + +Whether installing either the Docker or Singularity containers, the install script will copy an executable shell script `swiftest_driver` into `swiftest/bin/`. Not that when installing the Singularity container, the install script will set an environment variable called `SWIFTEST_SIF` that must point to the aboslute path of the container file called `swiftest_driver.sif`. To use the driver script in a future shell, rather than running the install script again, we suggest adding the environment variable definition to your shell startup script (e.g. add `export SWIFTEST_SIF="/path/to/swiftest/singularity/swiftest.sif"` to your `.zshrc`) + + **Swiftest Python Package** Included with Swiftest, in the ```/swiftest/python/swiftest/``` directory, is a Python package designed to facilitate seamless data processing and analysis. The Python package, also called Swiftest, can be used to generate input files, run Swiftest simulations, and process output files in the NetCDF file format. From b17b360c008246d393ea298a2b92115e56c6030b Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 26 May 2023 13:49:00 -0400 Subject: [PATCH 119/149] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60893055f..3671832cb 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ $ make The Swiftest executable, called ```swiftest_driver```, should now be created in the ```/swiftest/bin/``` directory. -**Download the `swiftest_driver` as a Docker or Singularity container. +**Download the `swiftest_driver` as a Docker or Singularity container.** The Swiftest driver is available as a Docker container on DockerHub in two versions: Intel and GNU. The Intel version was compiled for the x86_64 CPU using the Intel classic Fortran compiler. The GNU version was compliled for the x86_64 CPU using gfortran. The Intel version is faster than the GNU version (though not as fast as a native compile to the target CPU that you wish to run it on due to vectorization optimizations that Swiftest takes advantage of), however it is much larger: The Intel version is ~2.7GB while the GNU version is ~300MB. The Singularity container pulls from the same DockerHub container. From acf1bb8d5a58ee5e0d9d5b676cafc41e077d3d83 Mon Sep 17 00:00:00 2001 From: David Minton Date: Sun, 4 Jun 2023 16:20:50 -0400 Subject: [PATCH 120/149] Set up new Docker container to run the driver and Python packages. Still needs testing. --- CMakeLists.txt | 7 +- Dockerfile | 191 ++++++++++++++++++++++++++++ cmake/Modules/FindNETCDF.cmake | 11 +- cmake/Modules/SetFortranFlags.cmake | 21 ++- src/CMakeLists.txt | 15 +-- swiftest.sh | 3 + 6 files changed, 230 insertions(+), 18 deletions(-) create mode 100644 Dockerfile create mode 100755 swiftest.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 37cda5709..08ec7c9e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,10 +32,15 @@ OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" O OPTION(USE_OPENMP "Use OpenMP for parallelization" ON) OPTION(USE_SIMD "Use SIMD vectorization" ON) OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF) +OPTION(BUILD_SHARED_LIBS "Build using shared libraries" ON) -# Locate and set parallelization libraries. There are some CMake peculiarities +# Locate and set external libraries. There are some CMake peculiarities # taken care of here, such as the fact that the FindOpenMP routine doesn't know # about Fortran. +IF (NOT BUILD_SHARED_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so") +ENDIF () + INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake) INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake) INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..53a5ce831 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,191 @@ +FROM ubuntu:20.04 as build + +# kick everything off +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ + rm -rf /var/lib/apt/lists/* + +# Get CMAKE and install it +RUN mkdir -p cmake/build && \ + cd cmake/build && \ + curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ + /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license + +# Get the Intel compilers +# download the key to system keyring +RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ +| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null +# add signed entry to apt sources and configure the APT client to use Intel repository: +RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list +RUN apt-get -y update && apt-get upgrade -y +RUN apt-get install -y intel-hpckit + +# Set Intel compiler environment variables +ENV INTEL_DIR="/opt/intel/oneapi" +ENV LANG=C.UTF-8 +ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' +ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' +ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' +ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' +ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' +ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' +ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' +ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' +ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' +ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' +ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' +ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DAL_MAJOR_BINARY='1' +ENV DAL_MINOR_BINARY='1' +ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' +ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' +ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' +ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' +ENV FPGA_VARS_ARGS='' +ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' +ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' +ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' +ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' +ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' +ENV IPPCP_TARGET_ARCH='intel64' +ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' +ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' +ENV IPP_TARGET_ARCH='intel64' +ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' +ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' +ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' +ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' +ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' +ENV ONEAPI_ROOT='/opt/intel/oneapi' +ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' +ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' +ENV SETVARS_COMPLETED='1' +ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' +ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' +ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' +ENV VT_MPI='impi4' +ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' +ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' + +# Set HDF5 and NetCDF-specific Environment variables +ENV INSTALL_DIR="/usr/local" +ENV LIB_DIR="${INSTALL_DIR}/lib" +ENV LD_LIBRARY_PATH=${LIB_DIR}:${LD_LIBRARY_PATH} +RUN mkdir -p ${LIB_DIR} + +ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" +ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" +ENV CXX="${INTEL_DIR}/compiler/latest/linux/bin/icpx" +ENV LDFLAGS="-L${LIB_DIR}" +ENV NCDIR="${INSTALL_DIR}" +ENV NFDIR="${INSTALL_DIR}" +ENV HDF5_ROOT="${INSTALL_DIR}" +ENV HDF5_LIBDIR="${HDF5_ROOT}/lib" +ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" +ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" + +# Get the HDF5, NetCDF-C and NetCDF-Fortran libraries +RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz +RUN wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz +RUN wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz +RUN wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz + +# Install dependencies +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ + rm -rf /var/lib/apt/lists/* + +# Install HDF5 +RUN cd hdf && \ + ./HDF5-1.14.1-Linux.sh --skip-license && \ + cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ + cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ + +RUN cp zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ + +ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" +ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +RUN cd netcdf-c-4.9.2 && \ + cmake -S . -B build -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF && \ + cmake --build build && \ + cmake --install build + +# NetCDF-Fortran library +ENV F77=${FC} +ENV CFLAGS="-fPIC" +ENV FCFLAGS=${CFLAGS} +ENV FFLAGS=${CFLAGS} +ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl" +ENV LDFLAGS="-static-intel" +ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +RUN cd netcdf-fortran-4.6.1 && \ + ./configure --disable-shared --prefix=${NFDIR} && \ + make && \ + make install + +# # Swiftest +ENV NETCDF_HOME=${INSTALL_DIR} +ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} +ENV NETCDF_LIBRARY=${NETCDF_HOME} +ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" +COPY ./cmake/ /swiftest/cmake/ +COPY ./src/ /swiftest/src/ +COPY ./CMakeLists.txt /swiftest/ +RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)\n' \ + 'find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(HDF5_HL_LIBRARY NAMES libhdf5_hl.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(HDF5_LIBRARY NAMES libhdf5.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(Z_LIBRARY NAMES libz.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(ZSTD_LIBRARY NAMES libzstd.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(SZ_LIBRARY NAMES libsz.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(BZ2_LIBRARY NAMES libbz2.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(CURL_LIBRARY NAMES libcurl.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(XML2_LIBRARY NAMES libxml2.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'set(NETCDF_FOUND TRUE)\n' \ + 'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \ + 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ + 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake + +RUN cd swiftest && \ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release -DBUILD_SHARED_LIBS=OFF &&\ + cmake --build build --verbose && \ + cmake --install build + +# Production container +FROM continuumio/miniconda3 + +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev && \ + rm -rf /var/lib/apt/lists/* + +ENV LD_LIBRARY_PATH="/usr/local/lib" +COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ + +RUN conda update --all -y +RUN conda install conda-libmamba-solver -y +RUN conda config --set solver libmamba +RUN conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 -y +RUN conda update --all -y + +COPY ./python/ . +COPY --from=build /usr/local/bin/swiftest_driver /bin/ +RUN cd swiftest && conda develop . +ENV SHELL="/bin/bash" +ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/cmake/Modules/FindNETCDF.cmake b/cmake/Modules/FindNETCDF.cmake index 2355ad900..003d5b195 100644 --- a/cmake/Modules/FindNETCDF.cmake +++ b/cmake/Modules/FindNETCDF.cmake @@ -8,11 +8,14 @@ # If not, see: https://www.gnu.org/licenses. # - Finds the NetCDF libraries -find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME) -find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME) -find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME) + +find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME ENV CPATH) +find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH) +find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH) set(NETCDF_FOUND TRUE) set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR}) -set(NETCDF_LIBRARIES ${NETCDF_LIBRARY} ${NETCDF_FORTRAN_LIBRARY}) +# Note for posterity: When building static libraries, NETCDF_FORTRAN_LIBRARY must come *before* NETCDF_LIBRARY. Otherwise you get a bunch of "undefined reference to" errors +set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY}) + mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR) \ No newline at end of file diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 7b7752747..cb5a40768 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -50,9 +50,9 @@ ENDIF(BT STREQUAL "RELEASE") # If the compiler flags have already been set, return now ######################################################### -IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER) +IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE ) RETURN () -ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER) +ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE) ######################################################################## # Determine the appropriate flags for this compiler for each build type. @@ -118,6 +118,7 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" "/Qpad" # Intel Windows ) + IF (CONTAINERIZE) # There is some bug where -march=native doesn't work on Mac IF(APPLE) @@ -125,6 +126,18 @@ IF (CONTAINERIZE) ELSE() SET(GNUNATIVE "-march=generic") ENDIF() + + # Use static Intel libraries + SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" + Fortran "-static-intel" # Intel + ) + + IF (USE_OPENMP) + SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" + Fortran "-qopenmp-link=static" # Intel + ) + ENDIF (USE_OPENMP) + ELSE () # There is some bug where -march=native doesn't work on Mac IF(APPLE) @@ -148,7 +161,9 @@ IF (USE_SIMD) IF (CONTAINERIZE) # Optimize for an old enough processor that it should run on most computers SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran ${GNUNATIVE} # GNU + Fortran "-xSSE2" # Intel + "/QxSSE2" # Intel Windows + ${GNUNATIVE} # GNU ) ELSE () # Optimize for the host's architecture diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0e3decf3c..cd7567afc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -108,22 +108,17 @@ SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON) # Add the needed libraries and special compiler flags ##################################################### -# # Uncomment if you need to link to BLAS and LAPACK -TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${NETCDF_LIBRARIES} ${NETCDF_FORTRAN_LIBRARIES}) - - +TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES}) IF(USE_OPENMP) - SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES - COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}" - LINK_FLAGS "${OpenMP_Fortran_FLAGS}") + SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS "${OpenMP_Fortran_FLAGS} ") + SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY LINK_FLAGS "${OpenMP_Fortran_FLAGS} ") ENDIF(USE_OPENMP) IF(USE_COARRAY) TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DCOARRAY) - SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES - COMPILE_FLAGS "${Coarray_Fortran_FLAGS}" - LINK_FLAGS "${Coarray_Fortran_FLAGS}") + SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS "${Coarray_Fortran_FLAGS} ") + SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY LINK_FLAGS "${Coarray_Fortran_FLAGS} ") ENDIF(USE_COARRAY) diff --git a/swiftest.sh b/swiftest.sh new file mode 100755 index 000000000..91d53ff92 --- /dev/null +++ b/swiftest.sh @@ -0,0 +1,3 @@ +#!/bin/sh -- +#docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" +docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file From ae98dd89dc8ffd3226156690705d1ef56c620157 Mon Sep 17 00:00:00 2001 From: David Minton Date: Sun, 4 Jun 2023 19:38:22 -0400 Subject: [PATCH 121/149] Added a universal writable .astropy directory to be able to call the docker image as a regular user to that simdir will have the correct ownership --- Dockerfile | 2 + docker/.gitignore | 9 +-- docker/bin/swiftest | 2 + docker/gnu/Dockerfile | 44 ----------- docker/gnu/bin/.gitignore | 0 docker/gnu/bin/swiftest_driver | 2 - docker/install.sh | 8 +- docker/intel/Dockerfile | 129 --------------------------------- swiftest.sh | 3 - 9 files changed, 10 insertions(+), 189 deletions(-) create mode 100755 docker/bin/swiftest delete mode 100644 docker/gnu/Dockerfile delete mode 100644 docker/gnu/bin/.gitignore delete mode 100755 docker/gnu/bin/swiftest_driver delete mode 100644 docker/intel/Dockerfile delete mode 100755 swiftest.sh diff --git a/Dockerfile b/Dockerfile index 53a5ce831..99b0e97a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -187,5 +187,7 @@ RUN conda update --all -y COPY ./python/ . COPY --from=build /usr/local/bin/swiftest_driver /bin/ RUN cd swiftest && conda develop . +RUN mkdir -p /.astropy && \ + chmod -R 777 /.astropy ENV SHELL="/bin/bash" ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore index 215dd3f04..09c5585d1 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,10 +1,5 @@ * !.gitignore !install.sh -!gnu/ -!gnu/Dockerfile -!/gnu/bin/ -!/gnu/bin/swiftest_driver -!intel/ -!intel/Dockerfile -!/intel/bin/swiftest_driver +!bin +!bin/swiftest diff --git a/docker/bin/swiftest b/docker/bin/swiftest new file mode 100755 index 000000000..bc94b2cb0 --- /dev/null +++ b/docker/bin/swiftest @@ -0,0 +1,2 @@ +#!/bin/sh -- +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/docker/gnu/Dockerfile b/docker/gnu/Dockerfile deleted file mode 100644 index ec64211bd..000000000 --- a/docker/gnu/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM debian:stable-slim as build - -# kick everything off -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git build-essential gfortran && \ - apt-get update && apt-get upgrade -y && \ - rm -rf /var/lib/apt/lists/* - -# Get CMAKE and install it -RUN mkdir -p /opt/cmake/build && \ - cd /opt/cmake/build && \ - curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ - /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license - -# Get dependencies -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libnetcdf-dev libnetcdff-dev libcoarrays-dev libcoarrays-mpich-dev&& \ - rm -rf /var/lib/apt/lists/* - -ENV NETCDF_HOME="/usr" -ENV NETCDF_FORTRAN_HOME="/usr" -ENV INDIR="/opt/dist//usr/local" - -# Get Swiftest source code -RUN cd /opt/ && \ - git clone -b debug https://github.com/carlislewishard/swiftest.git && \ - cd swiftest && \ - mkdir build && \ - cd build && \ - cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release && \ - make && \ - make install - -#Production container -FROM debian:stable-slim -COPY --from=build /opt/dist / -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates gfortran libnetcdf-dev libnetcdff-dev && \ - rm -rf /var/lib/apt/lists/* - -ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file diff --git a/docker/gnu/bin/.gitignore b/docker/gnu/bin/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker/gnu/bin/swiftest_driver b/docker/gnu/bin/swiftest_driver deleted file mode 100755 index f0a3638dd..000000000 --- a/docker/gnu/bin/swiftest_driver +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest_driver:gnu "$@" diff --git a/docker/install.sh b/docker/install.sh index f81e95e57..1a0b2b20c 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -1,5 +1,5 @@ #!/bin/sh -- -tag=${1:-intel} -echo "Installing swiftest_driver:${tag} Docker container and executable script" -docker pull daminton/swiftest_driver:${tag} -cp -rf ${tag}/bin/swiftest_driver ../bin/ \ No newline at end of file +tag=${1:-latest} +echo "Installing swiftest:${tag} Docker container and executable script" +docker pull daminton/swiftest:${tag} +cp -rf bin/swiftest ../bin/ \ No newline at end of file diff --git a/docker/intel/Dockerfile b/docker/intel/Dockerfile deleted file mode 100644 index c6d728b2d..000000000 --- a/docker/intel/Dockerfile +++ /dev/null @@ -1,129 +0,0 @@ -FROM debian:stable-slim as build - -# kick everything off -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev procps && \ - rm -rf /var/lib/apt/lists/* - -# Get CMAKE and install it -RUN mkdir -p cmake/build && \ - cd cmake/build && \ - curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ - /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license - -# Get the Intel compilers -# download the key to system keyring -RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ -| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null -# add signed entry to apt sources and configure the APT client to use Intel repository: -RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list -RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y intel-hpckit - -# Build the NetCDF libraries -RUN mkdir -p /opt/build && mkdir -p /opt/dist -ENV INDIR="/opt/dist//usr/local" -ENV INTEL_DIR="/opt/intel/oneapi" -ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" -ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" - -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libhdf5-dev hdf5-tools zlib1g zlib1g-dev libxml2-dev libcurl4-gnutls-dev m4 && \ - rm -rf /var/lib/apt/lists/* - - -#NetCDF-c library -RUN git clone https://github.com/Unidata/netcdf-c.git -RUN cd netcdf-c && mkdir build && cd build && \ - cmake .. -DCMAKE_PREFIX_PATH="${LD_LIBRARY_PATH}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ - make && make install - -#NetCDF-Fortran library -RUN git clone https://github.com/Unidata/netcdf-fortran.git -RUN cd netcdf-fortran && mkdir build && cd build && \ - cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}" && \ - make && make install - -# #Swiftest -RUN git clone -b debug https://github.com/carlislewishard/swiftest.git -ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" -ENV NETCDF_HOME="${INDIR}" -ENV NETCDF_FORTRAN_HOME="${INDIR}" -ENV LANG=C.UTF-8 -ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' -ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' -ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' -ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' -ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' -ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' -ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' -ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' -ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' -ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' -ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' -ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DAL_MAJOR_BINARY='1' -ENV DAL_MINOR_BINARY='1' -ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' -ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' -ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' -ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' -ENV FPGA_VARS_ARGS='' -ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' -ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' -ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' -ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' -ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' -ENV IPPCP_TARGET_ARCH='intel64' -ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' -ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' -ENV IPP_TARGET_ARCH='intel64' -ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' -ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' -ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' -ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' -ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' -ENV ONEAPI_ROOT='/opt/intel/oneapi' -ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' -ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' -ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' -ENV SETVARS_COMPLETED='1' -ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' -ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' -ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' -ENV VT_MPI='impi4' -ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' -ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' - -RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release && make && make install - -#Production container -FROM debian:stable-slim -COPY --from=build /opt/dist / - -# Get the Intel runtime libraries -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl gpg-agent software-properties-common gnupg pkg-config procps && \ - rm -rf /var/lib/apt/lists/* - -RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add - -RUN echo "deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list -RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran -ENV NETCDF_HOME="/usr/local" -ENV LANG=C.UTF-8 -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/intel/oneapi/lib" -RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y libhdf5-dev libxml2-dev - -ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file diff --git a/swiftest.sh b/swiftest.sh deleted file mode 100755 index 91d53ff92..000000000 --- a/swiftest.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -- -#docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" -docker run -v $(pwd):$(pwd) -w $(pwd) -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file From da21136b8c8e72389553f9b6857fcc58d5989ba3 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Sun, 4 Jun 2023 19:39:46 -0400 Subject: [PATCH 122/149] Improved coarray outpt handling and preventing runs getting stuck on sync --- src/swiftest/swiftest_coarray.f90 | 19 +++++++++++++++---- src/swiftest/swiftest_io.f90 | 6 +++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index e97488a0f..7dbd1a816 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -24,20 +24,33 @@ module subroutine swiftest_coarray_balance_system(nbody_system, param) ! Internals integer(I4B), codimension[*], save :: ntp integer(I4B) :: img,ntp_min, ntp_max + character(len=NAMELEN) :: min_str, max_str, diff_str, ni_str ntp = nbody_system%tp%nbody sync all + write(param%display_unit,*) "Checking whether test particles need to be reblanced." ntp_min = huge(1) ntp_max = 0 do img = 1, num_images() if (ntp[img] < ntp_min) ntp_min = ntp[img] if (ntp[img] > ntp_max) ntp_max = ntp[img] end do + write(min_str,*) ntp_min + write(max_str,*) ntp_max + write(diff_str,*) ntp_max - ntp_min + write(ni_str,*) num_images() + write(param%display_unit,*) "ntp_min : " // trim(adjustl(min_str)) + write(param%display_unit,*) "ntp_max : " // trim(adjustl(max_str)) + write(param%display_unit,*) "difference: " // trim(adjustl(diff_str)) if (ntp_max - ntp_min >= num_images()) then + write(param%display_unit,*) trim(adjustl(diff_str)) // ">=" // trim(adjustl(ni_str)) // ": Rebalancing" call nbody_system%coarray_collect(param) call nbody_system%coarray_distribute(param) + write(param%display_unit,*) "Rebalancing complete" + else + write(param%display_unit,*) trim(adjustl(diff_str)) // "<" // trim(adjustl(ni_str)) // ": No rebalancing needed" end if - + call flush(param%display_unit) return end subroutine swiftest_coarray_balance_system @@ -700,10 +713,8 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) write(image_num_char,*) this_image() write(ntp_num_char,*) nbody_system%tp%nbody - if (this_image() /= 1) sync images(this_image() - 1) write(param%display_unit,*) "Image " // trim(adjustl(image_num_char)) // " ntp: " // trim(adjustl(ntp_num_char)) if (param%log_output) flush(param%display_unit) - if (this_image() < num_images()) sync images(this_image() + 1) deallocate(ntp, lspill_list, tmp, cotp) @@ -711,4 +722,4 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) end subroutine swiftest_coarray_distribute_system -end submodule s_swiftest_coarray \ No newline at end of file +end submodule s_swiftest_coarray diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 38b1735d7..eea186c48 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -416,9 +416,9 @@ module subroutine swiftest_io_dump_storage(self, param) associate(nc => self%nc) #ifdef COARRAY sync all - if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) write(param%display_unit,*) "File output started" call iotimer%start() + critical #endif call nc%open(param) do i = 1, self%iframe @@ -433,10 +433,10 @@ module subroutine swiftest_io_dump_storage(self, param) end do call nc%close() #ifdef COARRAY - if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) + end critical call iotimer%stop() - call iotimer%report(message="File output :", unit=param%display_unit) sync all + call iotimer%report(message="File output :", unit=param%display_unit) #endif end associate From 0d4935bd21e963fba635c53f50c831dba0ce698c Mon Sep 17 00:00:00 2001 From: David Minton Date: Mon, 5 Jun 2023 14:05:32 -0400 Subject: [PATCH 123/149] Minor changes to troubleshoot energy calc NaN in new container. --- Dockerfile | 43 +++++++++++++++++++++------------- singularity/install.sh | 2 +- src/swiftest/swiftest_util.f90 | 5 ++-- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99b0e97a3..5b0e942a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -136,10 +136,12 @@ RUN cd netcdf-fortran-4.6.1 && \ make && \ make install -# # Swiftest +# Swiftest ENV NETCDF_HOME=${INSTALL_DIR} ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} ENV NETCDF_LIBRARY=${NETCDF_HOME} +ENV FOR_COARRAY_NUM_IMAGES=1 +ENV OMP_NUM_THREADS=1 ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" COPY ./cmake/ /swiftest/cmake/ @@ -162,32 +164,41 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake RUN cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release -DBUILD_SHARED_LIBS=OFF &&\ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_SHARED_LIBS=OFF &&\ cmake --build build --verbose && \ cmake --install build -# Production container -FROM continuumio/miniconda3 - +# Driver container +FROM ubuntu:20.04 as Driver +COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ +COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev && \ + libsz2 libcurl3-gnutls libxml2 && \ rm -rf /var/lib/apt/lists/* +# Production container +FROM continuumio/miniconda3 + ENV LD_LIBRARY_PATH="/usr/local/lib" +ENV SHELL="/bin/bash" COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ - -RUN conda update --all -y -RUN conda install conda-libmamba-solver -y -RUN conda config --set solver libmamba -RUN conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 -y -RUN conda update --all -y - COPY ./python/ . COPY --from=build /usr/local/bin/swiftest_driver /bin/ -RUN cd swiftest && conda develop . -RUN mkdir -p /.astropy && \ + +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsz2 libcurl3-gnutls libxml2 && \ + rm -rf /var/lib/apt/lists/* && \ + conda update --all -y && \ + conda install conda-libmamba-solver -y && \ + conda config --set solver libmamba && \ + conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 dask -y && \ + conda update --all -y && \ + cd swiftest && conda develop . && \ + mkdir -p /.astropy && \ chmod -R 777 /.astropy -ENV SHELL="/bin/bash" + ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index 50d6d56df..ff2be8596 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -3,7 +3,7 @@ # In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source: # $ . ./install.sh # -tag=${1:-intel} +tag=${1:-latest} echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag} cp -rf bin/swiftest_driver ../bin/ diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index 2c7b803bc..c164a365a 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1101,10 +1101,11 @@ module subroutine swiftest_util_flatten_eucl_plpl(self, param) class(swiftest_pl), intent(inout) :: self !! Swiftest massive body object class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals - integer(I4B) :: err, i, j + integer(I4B) :: err, i, j, npl integer(I8B) :: k, npl8 - associate(npl => self%nbody, nplpl => self%nplpl) + associate(nplpl => self%nplpl) + npl = self%nbody npl8 = int(npl, kind=I8B) nplpl = npl8 * (npl8 - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl if (param%lflatten_interactions) then From 9bf02d01741a64aa7848138af2a629c54b972c3f Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 12:51:42 -0400 Subject: [PATCH 124/149] Fixed compiler flag typos that prevented the -standard-semantics flag from being set. Also moved the standards compliance warnings to the DEBUG build --- Dockerfile | 68 ++++++++++++----------------- cmake/Modules/SetFortranFlags.cmake | 14 +++--- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b0e942a1..cb81fb7f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,24 +2,18 @@ FROM ubuntu:20.04 as build # kick everything off RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ - rm -rf /var/lib/apt/lists/* - -# Get CMAKE and install it -RUN mkdir -p cmake/build && \ - cd cmake/build && \ - curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ - /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license - -# Get the Intel compilers -# download the key to system keyring -RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ -| gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null -# add signed entry to apt sources and configure the APT client to use Intel repository: -RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list -RUN apt-get -y update && apt-get upgrade -y -RUN apt-get install -y intel-hpckit + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ + /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt-get -y update && apt-get upgrade -y && \ + apt-get install -y intel-hpckit && \ # Set Intel compiler environment variables ENV INTEL_DIR="/opt/intel/oneapi" @@ -94,24 +88,19 @@ ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" # Get the HDF5, NetCDF-C and NetCDF-Fortran libraries -RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz -RUN wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz -RUN wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz -RUN wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz - -# Install dependencies -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ - rm -rf /var/lib/apt/lists/* - -# Install HDF5 -RUN cd hdf && \ - ./HDF5-1.14.1-Linux.sh --skip-license && \ - cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ - cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ - -RUN cp zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ +RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ + wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ + apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ + rm -rf /var/lib/apt/lists/* && \ + cd hdf && \ + ./HDF5-1.14.1-Linux.sh --skip-license && \ + cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ + cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ && \ + cp zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" @@ -126,7 +115,7 @@ RUN cd netcdf-c-4.9.2 && \ # NetCDF-Fortran library ENV F77=${FC} ENV CFLAGS="-fPIC" -ENV FCFLAGS=${CFLAGS} +ENV FCFLAGS="${CFLAGS} -standard-semantics" ENV FFLAGS=${CFLAGS} ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl" ENV LDFLAGS="-static-intel" @@ -161,9 +150,8 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'set(NETCDF_FOUND TRUE)\n' \ 'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \ 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ - 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake - -RUN cd swiftest && \ + 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ + cd swiftest && \ cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_SHARED_LIBS=OFF &&\ cmake --build build --verbose && \ cmake --install build diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index cb5a40768..e78782ef3 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -83,17 +83,11 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" # Determines whether the current Fortran Standard behavior of the compiler is fully implemented. -SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}" +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-standard-semantics" # Intel "/standard-semantics" # Intel Windows ) -# Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018). -SET_COMPILE_FLAG(CMAKE_Fortran_Flags "${CMAKE_Fortran_FLAGS}" - Fortran "-stand f18" # Intel - "/stand:f18" # Intel Windows - "-fstd=f2018" # GNU - ) # Allows for lines longer than 80 characters without truncation SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" @@ -215,6 +209,12 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-Wno-unused-dummy-argument" # GNU ) +# Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018). +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-stand f18" # Intel + "/stand:f18" # Intel Windows + "-fstd=f2018" # GNU + ) # Traceback SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" From 7f9cacbe8549d1305c057f0babce705df0d25b84 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 12:52:01 -0400 Subject: [PATCH 125/149] Set debug flags correctly for the standards check --- cmake/Modules/SetFortranFlags.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index e78782ef3..550738f13 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -210,7 +210,7 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" ) # Tells the compiler to issue compile-time messages for nonstandard language elements (Fortran 2018). -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" +SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG}" Fortran "-stand f18" # Intel "/stand:f18" # Intel Windows "-fstd=f2018" # GNU From ed4b2f0610b95fd893de3e81f6769cb532e441eb Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 12:54:11 -0400 Subject: [PATCH 126/149] Fixed issues that were causing do concurrent to give bad results when OMP_NUM_THREADS>1 when the F2018 locality-spec is used. --- src/helio/helio_gr.f90 | 16 ++++++++-------- src/swiftest/swiftest_gr.f90 | 26 +++++++++++++++----------- src/swiftest/swiftest_module.f90 | 10 +++++----- src/swiftest/swiftest_orbel.f90 | 32 ++++++++++++++++++-------------- src/swiftest/swiftest_util.f90 | 6 ++++-- src/whm/whm_gr.f90 | 16 ++++++++-------- 6 files changed, 58 insertions(+), 48 deletions(-) diff --git a/src/helio/helio_gr.f90 b/src/helio/helio_gr.f90 index 6de300cae..6b43714b7 100644 --- a/src/helio/helio_gr.f90 +++ b/src/helio/helio_gr.f90 @@ -75,14 +75,14 @@ pure module subroutine helio_gr_p4_pl(self, nbody_system, param, dt) if (self%nbody == 0) return - associate(pl => self) + associate(lmask => self%lmask, rh => self%rh, vb => self%vb, inv_c2 => param%inv_c2) npl = self%nbody #ifdef DOCONLOC - do concurrent(i = 1:npl, pl%lmask(i)) shared(param,pl,dt) + do concurrent(i = 1:npl, lmask(i)) shared(inv_c2, lmask, rh, vb, dt) #else - do concurrent(i = 1:npl, pl%lmask(i)) + do concurrent(i = 1:npl, lmask(i)) #endif - call swiftest_gr_p4_pos_kick(param, pl%rh(:, i), pl%vb(:, i), dt) + call swiftest_gr_p4_pos_kick(inv_c2, rh(1,i), rh(2,i), rh(3,i), vb(1,i), vb(2,i), vb(3,i), dt) end do end associate @@ -108,14 +108,14 @@ pure module subroutine helio_gr_p4_tp(self, nbody_system, param, dt) if (self%nbody == 0) return - associate(tp => self) + associate(rh => self%rh, vb => self%vb, lmask => self%lmask, inv_c2 => param%inv_c2) ntp = self%nbody #ifdef DOCONLOC - do concurrent(i = 1:ntp, tp%lmask(i)) shared(param,tp,dt) + do concurrent(i = 1:ntp, lmask(i)) shared(inv_c2, lmask, rh, vb, dt) #else - do concurrent(i = 1:ntp, tp%lmask(i)) + do concurrent(i = 1:ntp, lmask(i)) #endif - call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vb(:, i), dt) + call swiftest_gr_p4_pos_kick(inv_c2, rh(1,i), rh(2,i), rh(3,i), vb(1,i), vb(2,i), vb(3,i), dt) end do end associate diff --git a/src/swiftest/swiftest_gr.f90 b/src/swiftest/swiftest_gr.f90 index 1985f6dd1..083e5de1b 100644 --- a/src/swiftest/swiftest_gr.f90 +++ b/src/swiftest/swiftest_gr.f90 @@ -87,7 +87,7 @@ pure module subroutine swiftest_gr_kick_getacch(mu, x, lmask, n, inv_c2, agr) end subroutine swiftest_gr_kick_getacch - pure module subroutine swiftest_gr_p4_pos_kick(param, x, v, dt) + pure elemental module subroutine swiftest_gr_p4_pos_kick(inv_c2, rx, ry, rz, vx, vy, vz, dt) !! author: David A. Minton !! !! Position kick due to p**4 term in the post-Newtonian correction @@ -100,17 +100,21 @@ pure module subroutine swiftest_gr_p4_pos_kick(param, x, v, dt) !! Adapted from David A. Minton's Swifter routine gr_whm_p4.f90 implicit none ! Arguments - class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters - real(DP), dimension(:), intent(inout) :: x !! Position vector - real(DP), dimension(:), intent(in) :: v !! Velocity vector - real(DP), intent(in) :: dt !! Step size + real(DP), intent(in) :: inv_c2 !! One over speed of light squared (1/c**2) + real(DP), intent(inout) :: rx, ry, rz !! Position vector + real(DP), intent(in) :: vx, vy, vz !! Velocity vector + real(DP), intent(in) :: dt !! Step size ! Internals - real(DP), dimension(NDIM) :: dr - real(DP) :: vmag2 - - vmag2 = dot_product(v(:), v(:)) - dr(:) = -2 * param%inv_c2 * vmag2 * v(:) - x(:) = x(:) + dr(:) * dt + real(DP) :: drx, dry, drz + real(DP) :: vmag2 + + vmag2 = vx*vx + vy*vy + vz*vz + drx = -2 * inv_c2 * vmag2 * vx + dry = -2 * inv_c2 * vmag2 * vy + drz = -2 * inv_c2 * vmag2 * vz + rx = rx + drx * dt + ry = ry + dry * dt + rz = rz + drz * dt return end subroutine swiftest_gr_p4_pos_kick diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 8349c12c1..a54e2351b 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -550,12 +550,12 @@ pure module subroutine swiftest_gr_kick_getacch(mu, x, lmask, n, inv_c2, agr) real(DP), dimension(:,:), intent(out) :: agr !! Accelerations end subroutine swiftest_gr_kick_getacch - pure module subroutine swiftest_gr_p4_pos_kick(param, x, v, dt) + pure elemental module subroutine swiftest_gr_p4_pos_kick(inv_c2, rx, ry, rz, vx, vy, vz, dt) implicit none - class(swiftest_parameters), intent(in) :: param !! Current run configuration parameters - real(DP), dimension(:), intent(inout) :: x !! Position vector - real(DP), dimension(:), intent(in) :: v !! Velocity vector - real(DP), intent(in) :: dt !! Step size + real(DP), intent(in) :: inv_c2 !! One over speed of light squared (1/c**2) + real(DP), intent(inout) :: rx, ry, rz !! Position vector + real(DP), intent(in) :: vx, vy, vz !! Velocity vector + real(DP), intent(in) :: dt !! Step size end subroutine swiftest_gr_p4_pos_kick pure module subroutine swiftest_gr_pseudovel2vel(param, mu, rh, pv, vh) diff --git a/src/swiftest/swiftest_orbel.f90 b/src/swiftest/swiftest_orbel.f90 index 3827c1b59..431d182ab 100644 --- a/src/swiftest/swiftest_orbel.f90 +++ b/src/swiftest/swiftest_orbel.f90 @@ -21,24 +21,28 @@ module subroutine swiftest_orbel_el2xv_vec(self, cb) class(swiftest_body), intent(inout) :: self !! Swiftest body object class(swiftest_cb), intent(inout) :: cb !! Swiftest central body objec ! Internals - integer(I4B) :: i + integer(I4B) :: i, n if (self%nbody == 0) return + n = self%nbody call self%set_mu(cb) + associate(mu => self%mu, a => self%a, e => self%e, inc => self%inc, capom => self%capom, omega => self%omega, & + capm => self%capm, rh => self%rh, vh => self%vh) #ifdef DOCONLOC - do concurrent (i = 1:self%nbody) shared(self) + do concurrent (i = 1:n) shared(mu, a, e, inc, capom, omega, capm, rh, vh) #else - do concurrent (i = 1:self%nbody) + do concurrent (i = 1:n) #endif - call swiftest_orbel_el2xv(self%mu(i), self%a(i), self%e(i), self%inc(i), self%capom(i), & - self%omega(i), self%capm(i), self%rh(:, i), self%vh(:, i)) - end do + call swiftest_orbel_el2xv(mu(i), a(i), e(i), inc(i), capom(i), omega(i), capm(i), & + rh(1,i), rh(2,i), rh(3,i), vh(1,i), vh(2,i), vh(3,i)) + end do + end associate return end subroutine swiftest_orbel_el2xv_vec - pure subroutine swiftest_orbel_el2xv(mu, a, ie, inc, capom, omega, capm, x, v) + pure elemental subroutine swiftest_orbel_el2xv(mu, a, ie, inc, capom, omega, capm, rx, ry, rz, vx, vy, vz) !! author: David A. Minton !! !! Compute osculating orbital elements from relative C)rtesian position and velocity @@ -56,7 +60,7 @@ pure subroutine swiftest_orbel_el2xv(mu, a, ie, inc, capom, omega, capm, x, v) implicit none real(DP), intent(in) :: mu real(DP), intent(in) :: a, ie, inc, capom, omega, capm - real(DP), dimension(:), intent(out) :: x, v + real(DP), intent(out) :: rx, ry, rz, vx, vy, vz integer(I4B) :: iorbit_type real(DP) :: e, cape, capf, zpara, em1 @@ -129,12 +133,12 @@ pure subroutine swiftest_orbel_el2xv(mu, a, ie, inc, capom, omega, capm, x, v) vfac2 = ri * sqgma endif !-- - x(1) = d11 * xfac1 + d21 * xfac2 - x(2) = d12 * xfac1 + d22 * xfac2 - x(3) = d13 * xfac1 + d23 * xfac2 - v(1) = d11 * vfac1 + d21 * vfac2 - v(2) = d12 * vfac1 + d22 * vfac2 - v(3) = d13 * vfac1 + d23 * vfac2 + rx = d11 * xfac1 + d21 * xfac2 + ry = d12 * xfac1 + d22 * xfac2 + rz = d13 * xfac1 + d23 * xfac2 + vx = d11 * vfac1 + d21 * vfac2 + vy = d12 * vfac1 + d22 * vfac2 + vz = d13 * vfac1 + d23 * vfac2 return end subroutine swiftest_orbel_el2xv diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90 index c164a365a..f3ce3082f 100644 --- a/src/swiftest/swiftest_util.f90 +++ b/src/swiftest/swiftest_util.f90 @@ -1213,7 +1213,9 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) #else do concurrent (i = 1:npl, pl%lmask(i)) #endif - h(:) = pl%rb(:,i) .cross. pl%vb(:,i) + h(1) = pl%rb(2,i) * pl%vb(3,i) - pl%rb(3,i) * pl%vb(2,i) + h(2) = pl%rb(3,i) * pl%vb(1,i) - pl%rb(1,i) * pl%vb(3,i) + h(3) = pl%rb(1,i) * pl%vb(2,i) - pl%rb(2,i) * pl%vb(1,i) ! Angular momentum from orbit Lplorbit(:,i) = pl%mass(i) * h(:) @@ -1269,7 +1271,7 @@ module subroutine swiftest_util_get_energy_and_momentum_system(self, param) nbody_system%ke_orbit = 0.5_DP * (kecb + sum(kepl(1:npl), pl%lmask(1:npl))) #ifdef DOCONLOC - do concurrent (j = 1:NDIM) shared(nbody_system,pl,Lcborbit,Lplorbit) + do concurrent (j = 1:NDIM) shared(nbody_system,pl,Lcborbit,Lplorbit,npl) #else do concurrent (j = 1:NDIM) #endif diff --git a/src/whm/whm_gr.f90 b/src/whm/whm_gr.f90 index c46a5ce2d..6e23ca21b 100644 --- a/src/whm/whm_gr.f90 +++ b/src/whm/whm_gr.f90 @@ -84,14 +84,14 @@ pure module subroutine whm_gr_p4_pl(self, nbody_system, param, dt) if (self%nbody == 0) return - associate(pl => self) + associate(xj => self%xj, vj => self%vj, lmask => self%lmask, inv_c2 => param%inv_c2) npl = self%nbody #ifdef DOCONLOC - do concurrent(i = 1:npl, pl%lmask(i)) shared(pl,dt) + do concurrent(i = 1:npl, lmask(i)) shared(lmask, inv_c2, xj, vj,dt) #else - do concurrent(i = 1:npl, pl%lmask(i)) + do concurrent(i = 1:npl, lmask(i)) #endif - call swiftest_gr_p4_pos_kick(param, pl%xj(:, i), pl%vj(:, i), dt) + call swiftest_gr_p4_pos_kick(inv_c2, xj(1,i), xj(2,i), xj(3,i), vj(1,i), vj(2,i), vj(3,i), dt) end do end associate @@ -115,15 +115,15 @@ pure module subroutine whm_gr_p4_tp(self, nbody_system, param, dt) ! Internals integer(I4B) :: i, ntp - associate(tp => self) + associate(rh => self%rh, vh => self%vh, lmask => self%lmask, inv_c2 => param%inv_c2) ntp = self%nbody if (ntp == 0) return #ifdef DOCONLOC - do concurrent(i = 1:ntp, tp%lmask(i)) shared(tp,dt) + do concurrent(i = 1:ntp, lmask(i)) shared(lmask, rh, vh, inv_c2, dt) #else - do concurrent(i = 1:ntp, tp%lmask(i)) + do concurrent(i = 1:ntp, lmask(i)) #endif - call swiftest_gr_p4_pos_kick(param, tp%rh(:, i), tp%vh(:, i), dt) + call swiftest_gr_p4_pos_kick(inv_c2, rh(1,i), rh(2,i), rh(3,i), vh(1,i), vh(2,i), vh(3,i), dt) end do end associate From 2aacbe3f63a23730645552e70412efc1b68a7c16 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 12:54:37 -0400 Subject: [PATCH 127/149] Fixed line lengths based on standards compliance warning flags in debug mode --- src/base/base_module.f90 | 311 +++++++++++++++++++-------------- src/globals/globals_module.f90 | 18 +- 2 files changed, 189 insertions(+), 140 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 0f91e3473..502a42515 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -22,90 +22,100 @@ module base !> User defined parameters that are read in from the parameters input file. !> Each paramter is initialized to a default values. type, abstract :: base_parameters - character(STRMAX) :: integrator !! Name of the nbody integrator used - character(STRMAX) :: param_file_name !! The name of the parameter file - real(DP) :: t0 = 0.0_DP !! Integration reference time - real(DP) :: tstart = -1.0_DP !! Integration start time - real(DP) :: tstop = -1.0_DP !! Integration stop time - real(DP) :: dt = -1.0_DP !! Time step - integer(I8B) :: iloop = 0_I8B !! Main loop counter - integer(I8B) :: nloops = 0_I8B !! Total number of loops to execute - integer(I8B) :: istart = 0_I8B !! Starting index for loop counter - integer(I4B) :: iout = 0 !! Output cadence counter - integer(I4B) :: idump = 0 !! Dump cadence counter - integer(I4B) :: nout = 0 !! Current output step - integer(I4B) :: istep = 0 !! Current value of istep (used for time stretching) - character(STRMAX) :: incbfile = CB_INFILE !! Name of input file for the central body - character(STRMAX) :: inplfile = PL_INFILE !! Name of input file for massive bodies - character(STRMAX) :: intpfile = TP_INFILE !! Name of input file for test particles - character(STRMAX) :: nc_in = NC_INFILE !! Name of system input file for NetCDF input - character(STRMAX) :: in_type = "NETCDF_DOUBLE" !! Data representation type of input data files - character(STRMAX) :: in_form = "XV" !! Format of input data files ("EL" or ["XV"]) - integer(I4B) :: istep_out = -1 !! Number of time steps between saved outputs - integer(I4B) :: nstep_out = -1 !! Total number of saved outputs - real(DP) :: fstep_out = 1.0_DP !! The output step time stretching factor - logical :: ltstretch = .false. !! Whether to employ time stretching or not - character(STRMAX) :: outfile = BIN_OUTFILE !! Name of output binary file - character(STRMAX) :: out_type = "NETCDF_DOUBLE" !! Binary format of output file - character(STRMAX) :: out_form = "XVEL" !! Data to write to output file - character(STRMAX) :: out_stat = 'NEW' !! Open status for output binary file - integer(I4B) :: dump_cadence = 10 !! Number of output steps between dumping simulation data to file - real(DP) :: rmin = -1.0_DP !! Minimum heliocentric radius for test particle - real(DP) :: rmax = -1.0_DP !! Maximum heliocentric radius for test particle - real(DP) :: rmaxu = -1.0_DP !! Maximum unbound heliocentric radius for test particle - real(DP) :: qmin = -1.0_DP !! Minimum pericenter distance for test particle - character(STRMAX) :: qmin_coord = "HELIO" !! Coordinate frame to use for qmin (["HELIO"] or "BARY") - real(DP) :: qmin_alo = -1.0_DP !! Minimum semimajor axis for qmin - real(DP) :: qmin_ahi = -1.0_DP !! Maximum semimajor axis for qmin - real(QP) :: MU2KG = -1.0_QP !! Converts mass units to grams - real(QP) :: TU2S = -1.0_QP !! Converts time units to seconds - real(QP) :: DU2M = -1.0_QP !! Converts distance unit to centimeters - real(DP) :: GU = -1.0_DP !! Universal gravitational constant in the system units - real(DP) :: inv_c2 = -1.0_DP !! Inverse speed of light squared in the system units - real(DP) :: GMTINY = -1.0_DP !! Smallest G*mass that is fully gravitating - real(DP) :: min_GMfrag = -1.0_DP !! Smallest G*mass that can be produced in a fragmentation event - real(DP) :: nfrag_reduction = 30.0_DP !! Reduction factor for limiting the number of fragments in a collision - integer(I4B), dimension(:), allocatable :: seed !! Random seeds for fragmentation modeling - logical :: lmtiny_pl = .false. !! Include semi-interacting massive bodies - character(STRMAX) :: collision_model = "MERGE" !! The Coll - character(STRMAX) :: encounter_save = "NONE" !! Indicate if and how encounter data should be saved - logical :: lenc_save_trajectory = .false. !! Indicates that when encounters are saved, the full trajectory through recursion steps are saved - logical :: lenc_save_closest = .false. !! Indicates that when encounters are saved, the closest approach distance between pairs of bodies is saved - character(NAMELEN) :: interaction_loops = "ADAPTIVE" !! Method used to compute interaction loops. Options are "TRIANGULAR", "FLAT", or "ADAPTIVE" - character(NAMELEN) :: encounter_check_plpl = "ADAPTIVE" !! Method used to compute pl-pl encounter checks. Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" - character(NAMELEN) :: encounter_check_pltp = "ADAPTIVE" !! Method used to compute pl-tp encounter checks. Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" - logical :: lcoarray = .false. !! Use Coarrays for test particle parallelization. - - ! The following are used internally, and are not set by the user, but instead are determined by the input value of INTERACTION_LOOPS + character(STRMAX) :: integrator !! Name of the nbody integrator used + character(STRMAX) :: param_file_name !! The name of the parameter file + real(DP) :: t0 = 0.0_DP !! Integration reference time + real(DP) :: tstart = -1.0_DP !! Integration start time + real(DP) :: tstop = -1.0_DP !! Integration stop time + real(DP) :: dt = -1.0_DP !! Time step + integer(I8B) :: iloop = 0_I8B !! Main loop counter + integer(I8B) :: nloops = 0_I8B !! Total number of loops to execute + integer(I8B) :: istart = 0_I8B !! Starting index for loop counter + integer(I4B) :: iout = 0 !! Output cadence counter + integer(I4B) :: idump = 0 !! Dump cadence counter + integer(I4B) :: nout = 0 !! Current output step + integer(I4B) :: istep = 0 !! Current value of istep (used for time stretching) + character(STRMAX) :: incbfile = CB_INFILE !! Name of input file for the central body + character(STRMAX) :: inplfile = PL_INFILE !! Name of input file for massive bodies + character(STRMAX) :: intpfile = TP_INFILE !! Name of input file for test particles + character(STRMAX) :: nc_in = NC_INFILE !! Name of system input file for NetCDF input + character(STRMAX) :: in_type = "NETCDF_DOUBLE" !! Data representation type of input data files + character(STRMAX) :: in_form = "XV" !! Format of input data files ("EL" or ["XV"]) + integer(I4B) :: istep_out = -1 !! Number of time steps between saved outputs + integer(I4B) :: nstep_out = -1 !! Total number of saved outputs + real(DP) :: fstep_out = 1.0_DP !! The output step time stretching factor + logical :: ltstretch = .false. !! Whether to employ time stretching or not + character(STRMAX) :: outfile = BIN_OUTFILE !! Name of output binary file + character(STRMAX) :: out_type = "NETCDF_DOUBLE" !! Binary format of output file + character(STRMAX) :: out_form = "XVEL" !! Data to write to output file + character(STRMAX) :: out_stat = 'NEW' !! Open status for output binary file + integer(I4B) :: dump_cadence = 10 !! Number of output steps between dumping simulation data to file + real(DP) :: rmin = -1.0_DP !! Minimum heliocentric radius for test particle + real(DP) :: rmax = -1.0_DP !! Maximum heliocentric radius for test particle + real(DP) :: rmaxu = -1.0_DP !! Maximum unbound heliocentric radius for test particle + real(DP) :: qmin = -1.0_DP !! Minimum pericenter distance for test particle + character(STRMAX) :: qmin_coord = "HELIO" !! Coordinate frame to use for qmin (["HELIO"] or "BARY") + real(DP) :: qmin_alo = -1.0_DP !! Minimum semimajor axis for qmin + real(DP) :: qmin_ahi = -1.0_DP !! Maximum semimajor axis for qmin + real(QP) :: MU2KG = -1.0_QP !! Converts mass units to grams + real(QP) :: TU2S = -1.0_QP !! Converts time units to seconds + real(QP) :: DU2M = -1.0_QP !! Converts distance unit to centimeters + real(DP) :: GU = -1.0_DP !! Universal gravitational constant in the system units + real(DP) :: inv_c2 = -1.0_DP !! Inverse speed of light squared in the system units + real(DP) :: GMTINY = -1.0_DP !! Smallest G*mass that is fully gravitating + real(DP) :: min_GMfrag = -1.0_DP !! Smallest G*mass that can be produced in a fragmentation event + real(DP) :: nfrag_reduction = 30.0_DP !! Reduction factor for limiting the number of collision fragments + integer(I4B), dimension(:), allocatable :: seed !! Random seeds for fragmentation modeling + logical :: lmtiny_pl = .false. !! Include semi-interacting massive bodies + character(STRMAX) :: collision_model = "MERGE" !! The Coll + character(STRMAX) :: encounter_save = "NONE" !! Indicate if and how encounter data should be saved + logical :: lenc_save_trajectory = .false. !! Indicates that when encounters are saved, the full trajectory + !! through recursion steps are saved + logical :: lenc_save_closest = .false. !! Indicates that when encounters are saved, the closest approach + !! distance between pairs of bodies is saved + character(NAMELEN):: interaction_loops = "ADAPTIVE" !! Method used to compute interaction loops. + !! Options are "TRIANGULAR", "FLAT", or "ADAPTIVE" + character(NAMELEN):: encounter_check_plpl = "ADAPTIVE" !! Method used to compute pl-pl encounter checks. + !! Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" + character(NAMELEN):: encounter_check_pltp = "ADAPTIVE" !! Method used to compute pl-tp encounter checks. + !! Options are "TRIANGULAR", "SORTSWEEP", or "ADAPTIVE" + logical :: lcoarray = .false. !! Use Coarrays for test particle parallelization. + + ! The following are not set by the user, but instead are determined by the input value of INTERACTION_LOOPS logical :: lflatten_interactions = .false. !! Use the flattened upper triangular matrix for pl-pl interaction loops - logical :: lencounter_sas_plpl = .false. !! Use the Sort and Sweep algorithm to prune the encounter list before checking for close encounters - logical :: lencounter_sas_pltp = .false. !! Use the Sort and Sweep algorithm to prune the encounter list before checking for close encounters + logical :: lencounter_sas_plpl = .false. !! Use the Sort and Sweep algorithm to prune the encounter list before checking + !! for close encounters + logical :: lencounter_sas_pltp = .false. !! Use the Sort and Sweep algorithm to prune the encounter list before checking + !! for close encounters ! Logical flags to turn on or off various features of the code - logical :: lrhill_present = .false. !! Hill radii are given as an input rather than calculated by the code (can be used to inflate close encounter regions manually) + logical :: lrhill_present = .false. !! Hill radii are given as an input rather than calculated by the code (can be used to + !! inflate close encounter regions manually) logical :: lextra_force = .false. !! User defined force function turned on logical :: lbig_discard = .false. !! Save big bodies on every discard logical :: lclose = .false. !! Turn on close encounters logical :: lenergy = .false. !! Track the total energy of the system - logical :: loblatecb = .false. !! Calculate acceleration from oblate central body (automatically turns true if nonzero J2 is input) + logical :: loblatecb = .false. !! Calculate acceleration from oblate central body (automatically turns true if nonzero J2 + !! is input) logical :: lrotation = .false. !! Include rotation states of big bodies logical :: ltides = .false. !! Include tidal dissipation - ! Initial values to pass to the energy report subroutine (usually only used in the case of a restart, otherwise these will be updated with initial conditions values) - real(DP) :: E_orbit_orig = 0.0_DP !! Initial orbital energy + ! Initial values to pass to the energy report subroutine (usually only used in the case of a restart, otherwise these will be + ! updated with initial conditions values) + real(DP) :: E_orbit_orig = 0.0_DP !! Initial orbital energy real(DP) :: GMtot_orig = 0.0_DP !! Initial system mass - real(DP), dimension(NDIM) :: L_total_orig = 0.0_DP !! Initial total angular momentum vector - real(DP), dimension(NDIM) :: L_orbit_orig = 0.0_DP !! Initial orbital angular momentum - real(DP), dimension(NDIM) :: L_spin_orig = 0.0_DP !! Initial spin angular momentum vector - real(DP), dimension(NDIM) :: L_escape = 0.0_DP !! Angular momentum of bodies that escaped the system (used for bookeeping) + real(DP), dimension(NDIM) :: L_total_orig = 0.0_DP !! Initial total angular momentum vector + real(DP), dimension(NDIM) :: L_orbit_orig = 0.0_DP !! Initial orbital angular momentum + real(DP), dimension(NDIM) :: L_spin_orig = 0.0_DP !! Initial spin angular momentum vector + real(DP), dimension(NDIM) :: L_escape = 0.0_DP !! Angular momentum of escaped bodies (used for bookeeping) real(DP) :: GMescape = 0.0_DP !! Mass of bodies that escaped the system (used for bookeeping) - real(DP) :: E_collisions = 0.0_DP !! Energy lost from system due to collisions - real(DP) :: E_untracked = 0.0_DP !! Energy gained from system due to escaped bodies + real(DP) :: E_collisions = 0.0_DP !! Energy lost from system due to collisions + real(DP) :: E_untracked = 0.0_DP !! Energy gained from system due to escaped bodies logical :: lfirstenergy = .true. !! This is the first time computing energe logical :: lfirstkick = .true. !! Initiate the first kick in a symplectic step logical :: lrestart = .false. !! Indicates whether or not this is a restarted run - character(NAMELEN) :: display_style !! Style of the output display {"STANDARD", "COMPACT"}). Default is "STANDARD" + character(NAMELEN) :: display_style !! Style of the output display {["STANDARD"], "COMPACT"}). integer(I4B) :: display_unit !! File unit number for display (either to stdout or to a log file) logical :: log_output = .false. !! Logs the output to file instead of displaying it on the terminal @@ -137,8 +147,10 @@ subroutine abstract_io_param_reader(self, unit, iotype, v_list, iostat, iomsg) implicit none class(base_parameters), intent(inout) :: self !! Collection of parameters integer(I4B), intent(in) :: unit !! File unit number - character(len=*), intent(in) :: iotype !! Dummy argument passed to the input/output procedure contains the text from the char-literal-constant, prefixed with DT. - !! If you do not include a char-literal-constant, the iotype argument contains only DT. + character(len=*), intent(in) :: iotype !! Dummy argument passed to the input/output procedure contains the + !! text from the char-literal-constant, prefixed with DT. If you do + !! not include a char-literal-constant, the iotype argument contains + !! only DT. character(len=*), intent(in) :: v_list(:) !! The first element passes the integrator code to the reader integer(I4B), intent(out) :: iostat !! IO status code character(len=*), intent(inout) :: iomsg !! Message to pass if iostat /= 0 @@ -149,8 +161,10 @@ subroutine abstract_io_param_writer(self, unit, iotype, v_list, iostat, iomsg) implicit none class(base_parameters), intent(in) :: self !! Collection of parameters integer(I4B), intent(in) :: unit !! File unit number - character(len=*), intent(in) :: iotype !! Dummy argument passed to the input/output procedure contains the text from the char-literal-constant, prefixed with DT. - !! If you do not include a char-literal-constant, the iotype argument contains only DT. + character(len=*), intent(in) :: iotype !! Dummy argument passed to the input/output procedure contains the + !! text from the char-literal-constant, prefixed with DT. If you do + !! not include a char-literal-constant, the iotype argument contains + !! only DT. integer(I4B), intent(in) :: v_list(:) !! Not used in this procedure integer(I4B), intent(out) :: iostat !! IO status code character(len=*), intent(inout) :: iomsg !! Message to pass if iostat /= 0 @@ -168,7 +182,8 @@ end subroutine abstract_io_read_in_param type :: base_storage_frame class(*), allocatable :: item contains - procedure :: store => base_util_copy_store !! Stores a snapshot of the nbody system so that later it can be retrieved for saving to file. + procedure :: store => base_util_copy_store !! Stores a snapshot of the nbody system so that later it can be + !! retrieved for saving to file. generic :: assignment(=) => store final :: base_final_storage_frame end type @@ -179,25 +194,26 @@ end subroutine abstract_io_read_in_param integer(I4B) :: nframes !! Total number of frames that can be stored !! An class that establishes the pattern for various storage objects - type(base_storage_frame), dimension(:), allocatable :: frame !! Array of stored frames - integer(I4B) :: iframe = 0 !! Index of the last frame stored in the system - integer(I4B) :: nid !! Number of unique id values in all saved snapshots - integer(I4B), dimension(:), allocatable :: idvals !! The set of unique id values contained in the snapshots - integer(I4B), dimension(:), allocatable :: idmap !! The id value -> index map - integer(I4B) :: nt !! Number of unique time values in all saved snapshots - real(DP), dimension(:), allocatable :: tvals !! The set of unique time values contained in the snapshots - integer(I4B), dimension(:), allocatable :: tmap !! The t value -> index map + type(base_storage_frame), dimension(:), allocatable :: frame !! Array of stored frames + integer(I4B) :: iframe = 0 !! Index of the last frame stored in the system + integer(I4B) :: nid !! Number of unique id values in all saved snapshots + integer(I4B), dimension(:), allocatable :: idvals !! The set of unique id values contained in the snapshots + integer(I4B), dimension(:), allocatable :: idmap !! The id value -> index map + integer(I4B) :: nt !! Number of unique time values in all saved snapshots + real(DP), dimension(:), allocatable :: tvals !! The set of unique time values contained in the snapshots + integer(I4B), dimension(:), allocatable :: tmap !! The t value -> index map contains procedure :: dealloc => base_util_dealloc_storage !! Deallocates all allocatables - procedure :: reset => base_util_reset_storage !! Resets the storage object back to its original state by removing all of the saved items from the storage frames + procedure :: reset => base_util_reset_storage !! Resets the storage object back to its original state by removing all of + !! the saved items from the storage frames procedure :: resize => base_util_resize_storage !! Resizes storage if it is too small procedure :: setup => base_util_setup_storage !! Sets up a storage system with a set number of frames procedure :: save => base_util_snapshot_save !! Takes a snapshot of the current system end type base_storage - !> Class definition for the particle origin information object. This object is used to track time, location, and collisional regime - !> of fragments produced in collisional events. + !> Class definition for the particle origin information object. This object is used to track time, location, and collisional + !> regime of fragments produced in collisional events. type, abstract :: base_particle_info end type base_particle_info @@ -291,13 +307,15 @@ end subroutine abstract_util_dealloc_object subroutine base_util_append_arr_char_string(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of character string type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of character string type onto another. If the destination array is not allocated, or is not big + !! enough, this will allocate space for it. implicit none ! Arguments - character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array - character(len=STRMAX), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. - logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Destination array + character(len=STRMAX), dimension(:), allocatable, intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at + !! arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig @@ -336,13 +354,15 @@ end subroutine base_util_append_arr_char_string subroutine base_util_append_arr_DP(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of double precision type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of double precision type onto another. If the destination array is not allocated, or is not big + !! enough, this will allocate space for it. implicit none ! Arguments - real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. - logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + real(DP), dimension(:), allocatable, intent(inout) :: arr !! Destination array + real(DP), dimension(:), allocatable, intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at + !! arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig @@ -381,13 +401,15 @@ end subroutine base_util_append_arr_DP subroutine base_util_append_arr_DPvec(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of double precision vector type of size (NDIM, n) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of double precision vector type of size (NDIM, n) onto another. If the destination array is not + !! allocated, or is not big enough, this will allocate space for it. implicit none ! Arguments - real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array - real(DP), dimension(:,:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. - logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + real(DP), dimension(:,:), allocatable, intent(inout) :: arr !! Destination array + real(DP), dimension(:,:), allocatable, intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at + !! arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig @@ -428,13 +450,15 @@ end subroutine base_util_append_arr_DPvec subroutine base_util_append_arr_I4B(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of integer(I4B) onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of integer(I4B) onto another. If the destination array is not allocated, or is not big enough, + !! this will allocate space for it. implicit none ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array - integer(I4B), dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. - logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + integer(I4B), dimension(:), allocatable, intent(inout) :: arr !! Destination array + integer(I4B), dimension(:), allocatable, intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at + !! arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig @@ -473,13 +497,15 @@ end subroutine base_util_append_arr_I4B subroutine base_util_append_arr_logical(arr, source, nold, lsource_mask) !! author: David A. Minton !! - !! Append a single array of logical type onto another. If the destination array is not allocated, or is not big enough, this will allocate space for it. + !! Append a single array of logical type onto another. If the destination array is not allocated, or is not big enough, + !! this will allocate space for it. implicit none ! Arguments - logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array - logical, dimension(:), allocatable, intent(in) :: source !! Array to append - integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at arr(nold+1). Otherwise, the size of arr will be used. - logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to + logical, dimension(:), allocatable, intent(inout) :: arr !! Destination array + logical, dimension(:), allocatable, intent(in) :: source !! Array to append + integer(I4B), intent(in), optional :: nold !! Extent of original array. If passed, the source array will begin at + !! arr(nold+1). Otherwise, the size of arr will be used. + logical, dimension(:), intent(in), optional :: lsource_mask !! Logical mask indicating which elements to append to ! Internals integer(I4B) :: nnew, nsrc, nend_orig @@ -574,7 +600,8 @@ subroutine base_util_exit(code) character(*), parameter :: BAR = '("------------------------------------------------")' character(*), parameter :: SUCCESS_MSG = '(/, "Normal termination of Swiftest (version ", f3.1, ")")' character(*), parameter :: FAIL_MSG = '(/, "Terminating Swiftest (version ", f3.1, ") due to error!!")' - character(*), parameter :: USAGE_MSG = '("Usage: swiftest [bs|helio|ra15|rmvs|symba|tu4|whm] [standard|compact|progress|NONE]")' + character(*), parameter :: USAGE_MSG = '("Usage: swiftest [bs|helio|ra15|rmvs|symba|tu4|whm] ' // & + '[standard|compact|progress|NONE]")' character(*), parameter :: HELP_MSG = USAGE_MSG select case(code) @@ -605,7 +632,8 @@ subroutine base_util_fill_arr_char_string(keeps, inserts, lfill_list) ! Arguments character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep character(len=STRMAX), dimension(:), allocatable, intent(in) :: inserts !! Array of values to insert into keep - logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the keeps + logical, dimension(:), intent(in) :: lfill_list !! Logical array of bodies to merge into the + !! keeps if (.not.allocated(keeps) .or. .not.allocated(inserts)) return @@ -702,7 +730,8 @@ end subroutine base_util_fill_arr_logical subroutine base_util_reset_storage(self) !! author: David A. Minton !! - !! Resets the storage object back to its original state by removing all of the saved items from the storage frames, but does not deallocate the frames + !! Resets the storage object back to its original state by removing all of the saved items from the storage frames, but + !! does not deallocate the frames implicit none ! Arguments class(base_storage), intent(inout) :: self @@ -736,7 +765,7 @@ subroutine base_util_resize_arr_char_string(arr, nnew) character(len=STRMAX), dimension(:), allocatable, intent(inout) :: arr !! Array to resize integer(I4B), intent(in) :: nnew !! New size ! Internals - character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary storage array in case the input array is already allocated + character(len=STRMAX), dimension(:), allocatable :: tmp !! Temp. storage array in case the input array is already allocated integer(I4B) :: nold !! Old size if (nnew < 0) return @@ -1008,10 +1037,12 @@ end subroutine base_util_setup_storage subroutine base_util_snapshot_save(self, snapshot) !! author: David A. Minton !! - !! Checks the current size of the storage object against the required size and extends it by a factor of 2 more than requested if it is too small. - !! Note: The reason to extend it by a factor of 2 is for performance. When there are many enounters per step, resizing every time you want to add an - !! encounter takes significant computational effort. Resizing by a factor of 2 is a tradeoff between performance (fewer resize calls) and memory managment - !! Memory usage grows by a factor of 2 each time it fills up, but no more. + !! Checks the current size of the storage object against the required size and extends it by a factor of 2 more than + !! requested if it is too small. + !! Note: The reason to extend it by a factor of 2 is for performance. When there are many enounters per step, resizing + !! every time you want to add an encounter takes significant computational effort. Resizing by a factor of 2 is a tradeoff + !! between performance (fewer resize calls) and memory managment. Memory usage grows by a factor of 2 each time it fills + !! up, but no more. implicit none ! Arguments class(base_storage), intent(inout) :: self !! Storage encounter storage object @@ -1043,8 +1074,11 @@ subroutine base_util_spill_arr_char_string(keeps, discards, lspill_list, ldestru ! Arguments character(len=STRMAX), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep character(len=STRMAX), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into + !! the discards + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not + !! this operation should alter the keeps + !! array or not ! Internals integer(I4B) :: nspill, nkeep, nlist character(len=STRMAX), dimension(:), allocatable :: tmp !! Array of values to keep @@ -1087,7 +1121,8 @@ subroutine base_util_spill_arr_DP(keeps, discards, lspill_list, ldestructive) real(DP), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep real(DP), dimension(:), allocatable, intent(inout) :: discards !! Array of discards logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discardss - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation + !! should alter the keeps array or not ! Internals integer(I4B) :: nspill, nkeep, nlist real(DP), dimension(:), allocatable :: tmp !! Array of values to keep @@ -1130,7 +1165,8 @@ subroutine base_util_spill_arr_DPvec(keeps, discards, lspill_list, ldestructive) real(DP), dimension(:,:), allocatable, intent(inout) :: keeps !! Array of values to keep real(DP), dimension(:,:), allocatable, intent(inout) :: discards !! Array discards logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this + !! operation should alter the keeps array or not ! Internals integer(I4B) :: i, nspill, nkeep, nlist real(DP), dimension(:,:), allocatable :: tmp !! Array of values to keep @@ -1174,13 +1210,14 @@ subroutine base_util_spill_arr_I4B(keeps, discards, lspill_list, ldestructive) !! This is the inverse of a spill operation implicit none ! Arguments - integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I4B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + integer(I4B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + integer(I4B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive!! Logical flag indicating whether or not this + !! operation should alter the keeps array or not ! Internals integer(I4B) :: nspill, nkeep, nlist - integer(I4B), dimension(:), allocatable :: tmp !! Array of values to keep + integer(I4B), dimension(:), allocatable :: tmp !! Array of values to keep nkeep = count(.not.lspill_list(:)) nspill = count(lspill_list(:)) @@ -1217,10 +1254,11 @@ subroutine base_util_spill_arr_I8B(keeps, discards, lspill_list, ldestructive) !! This is the inverse of a spill operation implicit none ! Arguments - integer(I8B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep - integer(I8B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards - logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or not + integer(I8B), dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep + integer(I8B), dimension(:), allocatable, intent(inout) :: discards !! Array of discards + logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards + logical, intent(in) :: ldestructive!! Logical flag indicating whether or not this + !! operation should alter the keeps array or not ! Internals integer(I4B) :: nspill, nkeep, nlist integer(I8B), dimension(:), allocatable :: tmp !! Array of values to keep @@ -1263,7 +1301,8 @@ subroutine base_util_spill_arr_logical(keeps, discards, lspill_list, ldestructiv logical, dimension(:), allocatable, intent(inout) :: keeps !! Array of values to keep logical, dimension(:), allocatable, intent(inout) :: discards !! Array of discards logical, dimension(:), intent(in) :: lspill_list !! Logical array of bodies to spill into the discards - logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation should alter the keeps array or no + logical, intent(in) :: ldestructive !! Logical flag indicating whether or not this operation + !! should alter the keeps array or no ! Internals integer(I4B) :: nspill, nkeep, nlist logical, dimension(:), allocatable :: tmp !! Array of values to keep @@ -1918,7 +1957,7 @@ pure subroutine base_util_sort_rearrange_arr_char_string(arr, ind, n) integer(I4B), dimension(:), intent(in) :: ind !! Index to rearrange against integer(I4B), intent(in) :: n !! Number of elements in arr and ind to rearrange ! Internals - character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary copy of arry used during rearrange operation + character(len=STRMAX), dimension(:), allocatable :: tmp !! Temporary copy of arry used during rearrange operation if (.not. allocated(arr) .or. n <= 0) return allocate(tmp, mold=arr) @@ -2062,7 +2101,9 @@ subroutine base_util_unique_DP(input_array, output_array, index_map) ! Arguments real(DP), dimension(:), intent(in) :: input_array !! Unsorted input array real(DP), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) + integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such + !! that any for any index i, + !! output_array(index_map(i)) = input_array(i) ! Internals real(DP), dimension(:), allocatable :: unique_array integer(I4B) :: n @@ -2095,7 +2136,9 @@ subroutine base_util_unique_I4B(input_array, output_array, index_map) ! Arguments integer(I4B), dimension(:), intent(in) :: input_array !! Unsorted input array integer(I4B), dimension(:), allocatable, intent(out) :: output_array !! Sorted array of unique values - integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such that any for any index i, output_array(index_map(i)) = input_array(i) + integer(I4B), dimension(:), allocatable, intent(out) :: index_map !! An array of the same size as input_array that such + !! that any for any index i, + !! output_array(index_map(i)) = input_array(i) ! Internals integer(I4B), dimension(:), allocatable :: unique_array integer(I4B) :: n, lo, hi diff --git a/src/globals/globals_module.f90 b/src/globals/globals_module.f90 index fd26b3404..dd58d6dae 100644 --- a/src/globals/globals_module.f90 +++ b/src/globals/globals_module.f90 @@ -37,9 +37,12 @@ module globals real(DP), parameter :: GC = 6.6743E-11_DP !! Universal gravitational constant in SI units real(DP), parameter :: einsteinC = 299792458.0_DP !! Speed of light in SI units - integer(I4B), parameter :: LOWERCASE_BEGIN = iachar('a') !! ASCII character set parameter for lower to upper conversion - start of lowercase - integer(I4B), parameter :: LOWERCASE_END = iachar('z') !! ASCII character set parameter for lower to upper conversion - end of lowercase - integer(I4B), parameter :: UPPERCASE_OFFSET = iachar('A') - iachar('a') !! ASCII character set parameter for lower to upper conversion - offset between upper and lower + integer(I4B), parameter :: LOWERCASE_BEGIN = iachar('a') !! ASCII character set parameter for lower to upper conversion - start + !! of lowercase + integer(I4B), parameter :: LOWERCASE_END = iachar('z') !! ASCII character set parameter for lower to upper conversion - end of + !! lowercase + integer(I4B), parameter :: UPPERCASE_OFFSET = iachar('A') - iachar('a') !! ASCII character set parameter for lower to upper + !! conversion - offset between upper and lower real(SP), parameter :: VERSION_NUMBER = 1.0_SP !! Swiftest version @@ -103,9 +106,11 @@ module globals integer(I4B), parameter :: NDUMPFILES = 2 character(*), parameter :: PARAM_RESTART_FILE = "param.restart.in" #ifdef COARRAY - character(STRMAX) :: SWIFTEST_LOG_FILE !! Name of file to use to log output when using "COMPACT" or "PROGRESS" display style (each co-image gets its own log file) + character(STRMAX) :: SWIFTEST_LOG_FILE !! Name of file to use to log output when using "COMPACT" or + !! "PROGRESS" display style (each co-image gets its own log file) #else - character(*), parameter :: SWIFTEST_LOG_FILE = "swiftest.log" !! Name of file to use to log output when using "COMPACT" or "PROGRESS" display style + character(*), parameter :: SWIFTEST_LOG_FILE = "swiftest.log" !! Name of file to use to log output when using "COMPACT" or + !! "PROGRESS" display style #endif integer(I4B), parameter :: SWIFTEST_LOG_OUT = 33 !! File unit for log file when using "COMPACT" display style @@ -117,7 +122,8 @@ module globals character(*), parameter :: BIN_OUTFILE = 'data.nc' integer(I4B), parameter :: BINUNIT = 20 !! File unit number for the binary output file integer(I4B), parameter :: PARTICLEUNIT = 44 !! File unit number for the binary particle info output file - integer(I4B), parameter :: LUN = 42 !! File unit number for files that are opened and closed within a single subroutine call, and therefore should not collide + integer(I4B), parameter :: LUN = 42 !! File unit number for files that are opened and closed within a single + !! subroutine call, and therefore should not collide !> Miscellaneous constants: integer(I4B), parameter :: NDIM = 3 !! Number of dimensions in our reality From 6b474370c95e44ef50e98a5cba2eaaa14ef32bac Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 14:38:57 -0400 Subject: [PATCH 128/149] Fixed typo in Dockerfile and set the Swiftest build type to RELEASE --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb81fb7f6..ca322e71d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get upgrade -y && \ | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt-get -y update && apt-get upgrade -y && \ - apt-get install -y intel-hpckit && \ + apt-get install -y intel-hpckit # Set Intel compiler environment variables ENV INTEL_DIR="/opt/intel/oneapi" @@ -152,7 +152,7 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_SHARED_LIBS=OFF &&\ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ cmake --build build --verbose && \ cmake --install build From a621339a35db81095f68abbfd0d7b41177868de4 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 15:34:20 -0400 Subject: [PATCH 129/149] Fixed another Dockerfile typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ca322e71d..0ac00b25d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14 ./HDF5-1.14.1-Linux.sh --skip-license && \ cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ && \ - cp zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ + cp /zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" From 20865a6dc92a45353a6a05ca0a74be5281338999 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 15:35:08 -0400 Subject: [PATCH 130/149] Removed do concurrents from operators and switched to manual cross product when in a do concurrent --- src/collision/collision_util.f90 | 62 +++++++++----- src/encounter/encounter_check.f90 | 10 +-- src/fraggle/fraggle_generate.f90 | 130 ++++++++++++++++++++---------- src/operator/operator_cross.f90 | 42 ++-------- src/operator/operator_mag.f90 | 18 +---- src/operator/operator_unit.f90 | 18 +---- 6 files changed, 147 insertions(+), 133 deletions(-) diff --git a/src/collision/collision_util.f90 b/src/collision/collision_util.f90 index 179fb289c..75c836b4c 100644 --- a/src/collision/collision_util.f90 +++ b/src/collision/collision_util.f90 @@ -146,13 +146,15 @@ end subroutine collision_util_get_idvalues_snapshot module subroutine collision_util_get_energy_and_momentum(self, nbody_system, param, phase) !! Author: David A. Minton !! - !! Calculates total system energy in either the pre-collision outcome state (phase = "before") or the post-collision outcome state (lbefore = .false.) + !! Calculates total system energy in either the pre-collision outcome state (phase = "before") or the post-collision outcome + !! state (lbefore = .false.) implicit none ! Arguments class(collision_basic), intent(inout) :: self !! Encounter collision system object class(base_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object class(base_parameters), intent(inout) :: param !! Current Swiftest run configuration parameters - character(len=*), intent(in) :: phase !! One of "before" or "after", indicating which phase of the calculation this needs to be done + character(len=*), intent(in) :: phase !! One of "before" or "after", indicating which phase of the + !! calculation this needs to be done ! Internals integer(I4B) :: i, phase_val, nfrag @@ -179,9 +181,15 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par do concurrent(i = 1:2) #endif impactors%ke_orbit(i) = 0.5_DP * impactors%mass(i) * dot_product(impactors%vc(:,i), impactors%vc(:,i)) - impactors%ke_spin(i) = 0.5_DP * impactors%mass(i) * impactors%radius(i)**2 * impactors%Ip(3,i) * dot_product(impactors%rot(:,i), impactors%rot(:,i)) + impactors%ke_spin(i) = 0.5_DP * impactors%mass(i) * impactors%radius(i)**2 * impactors%Ip(3,i) & + * dot_product(impactors%rot(:,i), impactors%rot(:,i)) impactors%be(i) = -3 * impactors%Gmass(i) * impactors%mass(i) / (5 * impactors%radius(i)) - impactors%L_orbit(:,i) = impactors%mass(i) * impactors%rc(:,i) .cross. impactors%vc(:,i) + impactors%L_orbit(1,i) = impactors%mass(i) * (impactors%rc(2,i) * impactors%vc(3,i) & + - impactors%rc(3,i) * impactors%vc(2,i)) + impactors%L_orbit(2,i) = impactors%mass(i) * (impactors%rc(3,i) * impactors%vc(1,i) & + - impactors%rc(1,i) * impactors%vc(3,i)) + impactors%L_orbit(3,i) = impactors%mass(i) * (impactors%rc(1,i) * impactors%vc(2,i) & + - impactors%rc(2,i) * impactors%vc(1,i)) impactors%L_spin(:,i) = impactors%mass(i) * impactors%radius(i)**2 * impactors%Ip(3,i) * impactors%rot(:,i) end do self%L_orbit(:,phase_val) = sum(impactors%L_orbit(:,1:2),dim=2) @@ -190,7 +198,8 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par self%ke_orbit(phase_val) = sum(impactors%ke_orbit(1:2)) self%ke_spin(phase_val) = sum(impactors%ke_spin(1:2)) self%be(phase_val) = sum(impactors%be(1:2)) - call swiftest_util_get_potential_energy(2, [(.true., i = 1, 2)], 0.0_DP, impactors%Gmass, impactors%mass, impactors%rb, self%pe(phase_val)) + call swiftest_util_get_potential_energy(2, [(.true., i = 1, 2)], 0.0_DP, impactors%Gmass, impactors%mass, & + impactors%rb, self%pe(phase_val)) self%te(phase_val) = self%ke_orbit(phase_val) + self%ke_spin(phase_val) + self%be(phase_val) + self%pe(phase_val) else if (phase_val == 2) then #ifdef DOCONLOC @@ -199,11 +208,18 @@ module subroutine collision_util_get_energy_and_momentum(self, nbody_system, par do concurrent(i = 1:nfrag) #endif fragments%ke_orbit(i) = 0.5_DP * fragments%mass(i) * dot_product(fragments%vc(:,i), fragments%vc(:,i)) - fragments%ke_spin(i) = 0.5_DP * fragments%mass(i) * fragments%radius(i)**2 * fragments%Ip(3,i) * dot_product(fragments%rot(:,i), fragments%rot(:,i)) - fragments%L_orbit(:,i) = fragments%mass(i) * fragments%rc(:,i) .cross. fragments%vc(:,i) + fragments%ke_spin(i) = 0.5_DP * fragments%mass(i) * fragments%radius(i)**2 * fragments%Ip(3,i) & + * dot_product(fragments%rot(:,i), fragments%rot(:,i)) + fragments%L_orbit(1,i) = fragments%mass(i) * (fragments%rc(2,i) * fragments%vc(3,i) - & + fragments%rc(3,i) * fragments%vc(2,i)) + fragments%L_orbit(2,i) = fragments%mass(i) * (fragments%rc(3,i) * fragments%vc(1,i) - & + fragments%rc(1,i) * fragments%vc(3,i)) + fragments%L_orbit(3,i) = fragments%mass(i) * (fragments%rc(1,i) * fragments%vc(2,i) - & + fragments%rc(2,i) * fragments%vc(1,i)) fragments%L_spin(:,i) = fragments%mass(i) * fragments%radius(i)**2 * fragments%Ip(3,i) * fragments%rot(:,i) end do - call swiftest_util_get_potential_energy(nfrag, [(.true., i = 1, nfrag)], 0.0_DP, fragments%Gmass, fragments%mass, fragments%rb, fragments%pe) + call swiftest_util_get_potential_energy(nfrag, [(.true., i = 1, nfrag)], 0.0_DP, fragments%Gmass, fragments%mass, & + fragments%rb, fragments%pe) fragments%be = sum(-3*fragments%Gmass(1:nfrag)*fragments%mass(1:nfrag)/(5*fragments%radius(1:nfrag))) fragments%L_orbit_tot(:) = sum(fragments%L_orbit(:,1:nfrag),dim=2) fragments%L_spin_tot(:) = sum(fragments%L_spin(:,1:nfrag),dim=2) @@ -541,9 +557,11 @@ end subroutine collision_util_setup_fragments module subroutine collision_util_set_coordinate_collider(self) - + !! author: David A. Minton + !! !! - !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual fragments. + !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual + !! fragments. implicit none ! Arguments class(collision_basic), intent(inout) :: self !! Collisional nbody_system @@ -564,7 +582,8 @@ end subroutine collision_util_set_coordinate_collider module subroutine collision_util_set_coordinate_fragments(self) !! author: David A. Minton !! - !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual fragments. + !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual + !! fragments. implicit none ! Arguments class(collision_fragments), intent(inout) :: self !! Collisional nbody_system @@ -590,7 +609,8 @@ end subroutine collision_util_set_coordinate_fragments module subroutine collision_util_set_coordinate_impactors(self) !! author: David A. Minton !! - !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual fragments. + !! Defines the collisional coordinate nbody_system, including the unit vectors of both the nbody_system and individual + !! fragments. implicit none ! Arguments class(collision_impactors), intent(inout) :: self !! Collisional nbody_system @@ -602,8 +622,8 @@ module subroutine collision_util_set_coordinate_impactors(self) delta_v(:) = impactors%vb(:, 2) - impactors%vb(:, 1) delta_r(:) = impactors%rb(:, 2) - impactors%rb(:, 1) - ! We will initialize fragments on a plane defined by the pre-impact nbody_system, with the z-axis aligned with the angular momentum vector - ! and the y-axis aligned with the pre-impact distance vector. + ! We will initialize fragments on a plane defined by the pre-impact nbody_system, with the z-axis aligned with the angular + ! momentum vector and the y-axis aligned with the pre-impact distance vector. ! y-axis is the separation distance impactors%y_unit(:) = .unit.delta_r(:) @@ -632,14 +652,16 @@ module subroutine collision_util_set_coordinate_impactors(self) impactors%vc(:,1) = impactors%vb(:,1) - impactors%vbcom(:) impactors%vc(:,2) = impactors%vb(:,2) - impactors%vbcom(:) - ! Find the point of impact between the two bodies, defined as the location (in the collisional coordinate system) at the surface of body 1 along the line connecting the two bodies. + ! Find the point of impact between the two bodies, defined as the location (in the collisional coordinate system) at the + ! surface of body 1 along the line connecting the two bodies. impactors%rcimp(:) = impactors%rb(:,1) + impactors%radius(1) * impactors%y_unit(:) - impactors%rbcom(:) ! Set the velocity direction as the "bounce" direction" for disruptions, and body 2's direction for hit and runs if (impactors%regime == COLLRESOLVE_REGIME_HIT_AND_RUN) then impactors%bounce_unit(:) = .unit. impactors%vc(:,2) else - impactors%bounce_unit(:) = .unit. (impactors%vc(:,2) - 2 * dot_product(impactors%vc(:,2),impactors%y_unit(:)) * impactors%y_unit(:)) + impactors%bounce_unit(:) = .unit. (impactors%vc(:,2) - 2 * dot_product(impactors%vc(:,2),impactors%y_unit(:)) & + * impactors%y_unit(:)) end if end associate @@ -742,7 +764,8 @@ module subroutine collision_util_snapshot(self, param, nbody_system, t, arg) class(base_parameters), intent(inout) :: param !! Current run configuration parameters class(base_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system object to store real(DP), intent(in), optional :: t !! Time of snapshot if different from nbody_system time - character(*), intent(in), optional :: arg !! "before": takes a snapshot just before the collision. "after" takes the snapshot just after the collision. + character(*), intent(in), optional :: arg !! "before": takes a snapshot just before the collision. "after" + !! takes the snapshot just after the collision. ! Arguments class(collision_snapshot), allocatable, save :: snapshot character(len=:), allocatable :: stage @@ -816,8 +839,9 @@ module subroutine collision_util_snapshot(self, param, nbody_system, t, arg) write(message,*) trim(adjustl(plnew%info(i)%name)), " (", trim(adjustl(plnew%info(i)%particle_type)),")" call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) end do - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "***********************************************************" // & - "***********************************************************") + call swiftest_io_log_one_message(COLLISION_LOG_OUT, & + "***********************************************************" // & + "***********************************************************") allocate(after_snap%pl, source=plnew) end select deallocate(after_orig%pl) diff --git a/src/encounter/encounter_check.f90 b/src/encounter/encounter_check.f90 index 2d934e7d8..deef9bdda 100644 --- a/src/encounter/encounter_check.f90 +++ b/src/encounter/encounter_check.f90 @@ -179,7 +179,7 @@ subroutine encounter_check_all_sort_and_sweep_plpl(npl, r, v, renc, dt, nenc, in #else do concurrent (i = 1:npl) #endif - rmag = .mag.r(:,i) + rmag = norm2(r(:,i)) rmax(i) = rmag + RSWEEP_FACTOR * renc(i) rmin(i) = rmag - RSWEEP_FACTOR * renc(i) end do @@ -236,7 +236,7 @@ subroutine encounter_check_all_sort_and_sweep_plplm(nplm, nplt, rplm, vplm, rplt #else do concurrent (i = 1:nplm) #endif - rmag = .mag.rplm(:,i) + rmag = norm2(rplm(:,i)) rmax(i) = rmag + RSWEEP_FACTOR * rencm(i) rmin(i) = rmag - RSWEEP_FACTOR * rencm(i) end do @@ -245,7 +245,7 @@ subroutine encounter_check_all_sort_and_sweep_plplm(nplm, nplt, rplm, vplm, rplt #else do concurrent (i = 1:nplt) #endif - rmag = .mag.rplt(:,i) + rmag = norm2(rplt(:,i)) rmax(nplm+i) = rmag + RSWEEP_FACTOR * renct(i) rmin(nplm+i) = rmag - RSWEEP_FACTOR * renct(i) end do @@ -304,7 +304,7 @@ subroutine encounter_check_all_sort_and_sweep_pltp(npl, ntp, rpl, vpl, rtp, vtp, #else do concurrent (i = 1:npl) #endif - rmag = .mag.rpl(:,i) + rmag = norm2(rpl(:,i)) rmax(i) = rmag + RSWEEP_FACTOR * rencpl(i) rmin(i) = rmag - RSWEEP_FACTOR * rencpl(i) end do @@ -313,7 +313,7 @@ subroutine encounter_check_all_sort_and_sweep_pltp(npl, ntp, rpl, vpl, rtp, vtp, #else do concurrent (i = 1:ntp) #endif - rmag = .mag.rtp(:,i) + rmag = norm2(rtp(:,i)) rmax(npl+i) = rmag + RSWEEP_FACTOR * renctp(i) rmin(npl+i) = rmag - RSWEEP_FACTOR * renctp(i) end do diff --git a/src/fraggle/fraggle_generate.f90 b/src/fraggle/fraggle_generate.f90 index 5ec993fba..cd1df7033 100644 --- a/src/fraggle/fraggle_generate.f90 +++ b/src/fraggle/fraggle_generate.f90 @@ -59,7 +59,8 @@ module subroutine fraggle_generate(self, nbody_system, param, t) call self%set_mass_dist(param) call self%disrupt(nbody_system, param, t, lfailure) if (lfailure) then - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Fraggle failed to find a solution to match energy contraint. Treating this as a merge.") + call swiftest_io_log_one_message(COLLISION_LOG_OUT, & + "Fraggle failed to find a solution to match energy contraint. Treating this as a merge.") call self%merge(nbody_system, param, t) ! Use the default collision model, which is merge return end if @@ -131,8 +132,9 @@ module subroutine fraggle_generate_disrupt(self, nbody_system, param, t, lfailur real(DP), parameter :: fail_scale_initial = 1.0003_DP integer(I4B) :: nfrag_start - ! The minimization and linear solvers can sometimes lead to floating point exceptions. Rather than halting the code entirely if this occurs, we - ! can simply fail the attempt and try again. So we need to turn off any floating point exception halting modes temporarily + ! The minimization and linear solvers can sometimes lead to floating point exceptions. Rather than halting the code entirely + ! if this occurs, we can simply fail the attempt and try again. So we need to turn off any floating point exception halting + ! modes temporarily call ieee_get_halting_mode(IEEE_ALL,fpe_halting_modes) ! Save the current halting modes so we can turn them off temporarily fpe_quiet_modes(:) = .false. call ieee_set_halting_mode(IEEE_ALL,fpe_quiet_modes) @@ -168,7 +170,8 @@ module subroutine fraggle_generate_disrupt(self, nbody_system, param, t, lfailur if (.not.lfailure) then if (self%fragments%nbody /= nfrag_start) then write(message,*) self%fragments%nbody - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Fraggle found a solution with " // trim(adjustl(message)) // " fragments" ) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Fraggle found a solution with " // trim(adjustl(message)) & + // " fragments" ) end if call self%get_energy_and_momentum(nbody_system, param, phase="after") @@ -176,7 +179,8 @@ module subroutine fraggle_generate_disrupt(self, nbody_system, param, t, lfailur dE = self%te(2) - self%te(1) call swiftest_io_log_one_message(COLLISION_LOG_OUT, "All quantities in collision system natural units") - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "* Conversion factors (collision system units / nbody system units):") + call swiftest_io_log_one_message(COLLISION_LOG_OUT, & + "* Conversion factors (collision system units / nbody system units):") write(message,*) "* Mass: ", self%mscale call swiftest_io_log_one_message(COLLISION_LOG_OUT, message) write(message,*) "* Distance: ", self%dscale @@ -200,7 +204,8 @@ module subroutine fraggle_generate_disrupt(self, nbody_system, param, t, lfailur end if call self%set_original_scale() - self%max_rot = MAX_ROT_SI * param%TU2S ! Re-compute the spin limit from scratch so it doesn't drift due to floating point errors every time we convert + self%max_rot = MAX_ROT_SI * param%TU2S ! Re-compute the spin limit from scratch so it doesn't drift due to floating point + ! errors every time we convert ! Restore the big array if (lk_plpl) call pl%flatten(param) @@ -248,7 +253,8 @@ module subroutine fraggle_generate_hitandrun(self, nbody_system, param, t) end if ! The Fraggle disruption model (and its extended types allow for non-pure hit and run. - if (impactors%mass_dist(2) > 0.9_DP * impactors%mass(jproj)) then ! Pure hit and run, so we'll just keep the two bodies untouched + if (impactors%mass_dist(2) > 0.9_DP * impactors%mass(jproj)) then ! Pure hit and run, so we'll just keep the two bodies + ! untouched call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Pure hit and run. No new fragments generated.") call self%collision_basic%hitandrun(nbody_system, param, t) return @@ -327,7 +333,8 @@ module subroutine fraggle_generate_merge(self, nbody_system, param, t) if (rotmag < self%max_rot) then call self%collision_basic%merge(nbody_system, param, t) else - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Merger would break the spin barrier. Converting to pure hit and run" ) + call swiftest_io_log_one_message(COLLISION_LOG_OUT, & + "Merger would break the spin barrier. Converting to pure hit and run" ) impactors%mass_dist(1:2) = impactors%mass(1:2) call self%hitandrun(nbody_system, param, t) end if @@ -342,9 +349,9 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu !! Author: Jennifer L.L. Pouplin, Carlisle A. Wishard, and David A. Minton !! !! Initializes the position vectors of the fragments around the center of mass based on the collision style. - !! For hit and run with disruption, the fragments are generated in a random cloud around the smallest of the two colliders (body 2) - !! For disruptive collisions, the fragments are generated in a random cloud around the impact point. Bodies are checked for overlap and - !! regenerated if they overlap. + !! For hit and run with disruption, the fragments are generated in a random cloud around the smallest of the two colliders + !! (body 2). For disruptive collisions, the fragments are generated in a random cloud around the impact point. Bodies are + !! checked for overlap and regenerated if they overlap. implicit none ! Arguments class(collision_fraggle), intent(inout) :: collider !! Fraggle collision system object @@ -372,7 +379,8 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu lhitandrun = (impactors%regime == COLLRESOLVE_REGIME_HIT_AND_RUN) ! We will treat the first two fragments of the list as special cases. - ! Place the first two bodies at the centers of the two fragment clouds, but be sure they are sufficiently far apart to avoid overlap + ! Place the first two bodies at the centers of the two fragment clouds, but be sure they are sufficiently far apart to + ! avoid overlap if (lhitandrun) then rdistance = impactors%radius(2) istart = 2 @@ -385,11 +393,14 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu end if mass_rscale(1:istart-1) = 1.0_DP - ! Give the fragment positions a random value that is scaled with fragment mass so that the more massive bodies tend to be closer to the impact point - ! Later, velocities will be scaled such that the farther away a fragment is placed from the impact point, the higher will its velocity be. + ! Give the fragment positions a random value that is scaled with fragment mass so that the more massive bodies tend to be + ! closer to the impact point. Later, velocities will be scaled such that the farther away a fragment is placed from the + ! impact point, the higher will its velocity be. call random_number(mass_rscale(istart:nfrag)) mass_rscale(istart:nfrag) = (mass_rscale(istart:nfrag) + 1.0_DP) / 2 - mass_rscale(istart:nfrag) = mass_rscale(istart:nfrag) * (sum(fragments%mass(istart:nfrag)) / fragments%mass(istart:nfrag))**(0.125_DP) ! The power is arbitrary. It just gives the velocity a small mass dependence + ! The power of 0.125 in the scaling below is arbitrary. It just gives the velocity a small mass dependence + mass_rscale(istart:nfrag) = mass_rscale(istart:nfrag) * (sum(fragments%mass(istart:nfrag)) & + / fragments%mass(istart:nfrag))**(0.125_DP) mass_rscale(istart:nfrag) = mass_rscale(istart:nfrag) / minval(mass_rscale(istart:nfrag)) loverlap(:) = .true. @@ -402,8 +413,10 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu fragment_cloud_center(:,2) = impactors%rc(:,2) fragments%rc(:,1) = fragment_cloud_center(:,1) else ! Keep the first and second bodies at approximately their original location, but so as not to be overlapping - fragment_cloud_center(:,1) = impactors%rcimp(:) - rbuffer * max(fragments%radius(1),impactors%radius(1)) * impactors%y_unit(:) - fragment_cloud_center(:,2) = impactors%rcimp(:) + rbuffer * max(fragments%radius(2),impactors%radius(2)) * impactors%y_unit(:) + fragment_cloud_center(:,1) = impactors%rcimp(:) - rbuffer * max(fragments%radius(1),& + impactors%radius(1)) * impactors%y_unit(:) + fragment_cloud_center(:,2) = impactors%rcimp(:) + rbuffer * max(fragments%radius(2), & + impactors%radius(2)) * impactors%y_unit(:) fragment_cloud_radius(:) = rdistance / pack_density fragments%rc(:,1:2) = fragment_cloud_center(:,1:2) end if @@ -420,7 +433,8 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu ! Randomly place the n>2 fragments inside their cloud until none are overlapping #ifdef DOCONLOC - do concurrent(i = istart:nfrag, loverlap(i)) shared(fragments, impactors, fragment_cloud_radius, fragment_cloud_center, loverlap, mass_rscale, u, phi, theta, lhitandrun) local(j, direction) + do concurrent(i = istart:nfrag, loverlap(i)) shared(fragments, impactors, fragment_cloud_radius, fragment_cloud_center,& + loverlap, mass_rscale, u, phi, theta, lhitandrun) local(j, direction) #else do concurrent(i = istart:nfrag, loverlap(i)) #endif @@ -442,13 +456,15 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu ! Stretch out the hit and run cloud along the flight trajectory if (lhitandrun) then - fragments%rc(:,i) = fragments%rc(:,i) * (1.0_DP + 2 * fragment_cloud_radius(j) * mass_rscale(i) * impactors%bounce_unit(:)) + fragments%rc(:,i) = fragments%rc(:,i) * (1.0_DP + 2 * fragment_cloud_radius(j) * mass_rscale(i) & + * impactors%bounce_unit(:)) end if fragments%rc(:,i) = fragments%rc(:,i) + fragment_cloud_center(:,j) if (lhitandrun) then - fragments%rc(:,i) = fragments%rc(:,i) + 2 * fragment_cloud_radius(j) * mass_rscale(i) * impactors%bounce_unit(:) ! Shift the stretched cloud downrange + ! Shift the stretched cloud downrange + fragments%rc(:,i) = fragments%rc(:,i) + 2 * fragment_cloud_radius(j) * mass_rscale(i) * impactors%bounce_unit(:) else ! Make sure that the fragments are positioned away from the impact point direction = dot_product(fragments%rc(:,i) - impactors%rcimp(:), fragment_cloud_center(:,j) - impactors%rcimp(:)) @@ -460,7 +476,8 @@ module subroutine fraggle_generate_pos_vec(collider, nbody_system, param, lfailu end do ! Because body 1 and 2 are initialized near the original impactor positions, then if these bodies are still overlapping - ! when the rest are not, we will randomly walk their position in space so as not to move them too far from their starting position + ! when the rest are not, we will randomly walk their position in space so as not to move them too far from their + ! starting position if (all(.not.loverlap(istart:nfrag)) .and. any(loverlap(1:istart-1))) then #ifdef DOCONLOC do concurrent(i = 1:istart-1,loverlap(i)) shared(fragments,loverlap, u, theta, i) local(rwalk, dis) @@ -525,8 +542,10 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i, nfrag - real(DP), parameter :: FRAG_ROT_FAC = 0.1_DP ! Fraction of projectile rotation magnitude to add as random noise to fragment rotation - real(DP), parameter :: hitandrun_momentum_transfer = 0.01_DP ! Fraction of projectile momentum transfered to target in a hit and run + real(DP), parameter :: FRAG_ROT_FAC = 0.1_DP ! Fraction of projectile rotation magnitude to add as random noise to fragment + ! rotation + real(DP), parameter :: hitandrun_momentum_transfer = 0.01_DP ! Fraction of projectile momentum transfered to target in a hit + ! and run real(DP) :: mass_fac real(DP), dimension(NDIM) :: drot, dL integer(I4B), parameter :: MAXLOOP = 10 @@ -536,14 +555,15 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) nfrag = collider%fragments%nbody lhitandrun = (impactors%regime == COLLRESOLVE_REGIME_HIT_AND_RUN) - ! Initialize fragment rotations and velocities to be pre-impact rotation for body 1, and randomized for bodies >1 and scaled to the original rotation. - ! This will get updated later when conserving angular momentum + ! Initialize fragment rotations and velocities to be pre-impact rotation for body 1, and randomized for bodies >1 and + ! scaled to the original rotation. This will get updated later when conserving angular momentum mass_fac = fragments%mass(1) / impactors%mass(1) fragments%rot(:,1) = mass_fac**(5.0_DP/3.0_DP) * impactors%rot(:,1) ! If mass was added, also add spin angular momentum if (mass_fac > 1.0_DP) then - dL(:) = (fragments%mass(1) - impactors%mass(1)) * (impactors%rc(:,2) - impactors%rc(:,1)) .cross. (impactors%vc(:,2) - impactors%vc(:,1)) + dL(:) = (fragments%mass(1) - impactors%mass(1)) * (impactors%rc(:,2) - impactors%rc(:,1)) & + .cross. (impactors%vc(:,2) - impactors%vc(:,1)) drot(:) = dL(:) / (fragments%mass(1) * fragments%radius(1)**2 * fragments%Ip(3,1)) ! Check to make sure we haven't broken the spin barrier. Reduce the rotation change if so do i = 1, MAXLOOP @@ -559,7 +579,8 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) end if if (lhitandrun) then - dL(:) = hitandrun_momentum_transfer * impactors%mass(2) * (impactors%rc(:,2) - impactors%rc(:,1)) .cross. (impactors%vc(:,2) - impactors%vc(:,1)) + dL(:) = hitandrun_momentum_transfer * impactors%mass(2) * (impactors%rc(:,2) - impactors%rc(:,1)) & + .cross. (impactors%vc(:,2) - impactors%vc(:,1)) drot(:) = dL(:) / (fragments%mass(1) * fragments%radius(1)**2 * fragments%Ip(3,1)) do i = 1, MAXLOOP if (.mag.(fragments%rot(:,1) + drot(:)) < collider%max_rot) exit @@ -580,7 +601,8 @@ module subroutine fraggle_generate_rot_vec(collider, nbody_system, param) do concurrent (i = 2:nfrag) #endif mass_fac = fragments%mass(i) / impactors%mass(2) - fragments%rot(:,i) = mass_fac**(5.0_DP/3.0_DP) * impactors%rot(:,2) + 2 * (fragments%rot(:,i) - 1.0_DP) * FRAG_ROT_FAC * .mag.impactors%rot(:,2) + fragments%rot(:,i) = mass_fac**(5.0_DP/3.0_DP) * impactors%rot(:,2) + 2 * (fragments%rot(:,i) - 1.0_DP) * & + FRAG_ROT_FAC * norm2(impactors%rot(:,2)) end do fragments%rotmag(:) = .mag.fragments%rot(:,:) @@ -603,17 +625,22 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters logical, intent(out) :: lfailure !! Did the velocity computation fail? ! Internals - real(DP), parameter :: ENERGY_SUCCESS_METRIC = 0.1_DP !! Relative energy error to accept as a success (success also must be energy-losing in addition to being within the metric amount) + real(DP), parameter :: ENERGY_SUCCESS_METRIC = 0.1_DP !! Relative energy error to accept as a success (success also must be + !! energy-losing in addition to being within the metric amount) real(DP), parameter :: ENERGY_CONVERGENCE_TOL = 1e-3_DP !! Relative change in error before giving up on energy convergence - real(DP) :: MOMENTUM_SUCCESS_METRIC = 10*epsilon(1.0_DP) !! Relative angular momentum error to accept as a success (should be *much* stricter than energy) + real(DP) :: MOMENTUM_SUCCESS_METRIC = 10*epsilon(1.0_DP) !! Relative angular momentum error to accept as a success + !! (should be *much* stricter than energy) integer(I4B) :: i, j, loop, try, istart, nfrag, nsteps, nsteps_best, posloop logical :: lhitandrun, lsupercat - real(DP), dimension(NDIM) :: vimp_unit, rimp, vrot, vdisp, L_residual, L_residual_unit, L_residual_best, dL, drot, rot_new, dL_metric - real(DP) :: vimp, vmag, vesc, dE, E_residual, E_residual_best, E_residual_last, ke_avail, ke_remove, dE_best, fscale, dE_metric, mfrag, rn, dL1_mag, dE_conv + real(DP), dimension(NDIM) :: vimp_unit, rimp, vrot, vdisp, L_residual, L_residual_unit, L_residual_best, dL, drot, rot_new + real(DP), dimension(NDIM) :: dL_metric + real(DP) :: vimp, vmag, vesc, dE, E_residual, E_residual_best, E_residual_last, ke_avail, ke_remove, dE_best, fscale + real(DP) :: dE_metric, mfrag, rn, dL1_mag, dE_conv, vumag integer(I4B), dimension(:), allocatable :: vsign real(DP), dimension(:), allocatable :: vscale real(DP), dimension(:), allocatable :: dLi_mag - ! For the initial "guess" of fragment velocities, this is the minimum and maximum velocity relative to escape velocity that the fragments will have + ! For the initial "guess" of fragment velocities, this is the minimum and maximum velocity relative to escape velocity that + ! the fragments will have real(DP), parameter :: hitandrun_vscale = 0.25_DP real(DP) :: vmin_guess real(DP) :: vmax_guess @@ -688,25 +715,34 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu ! Set the velocities of all fragments using all of the scale factors determined above if (istart > 1) fragments%vc(:,1) = impactors%vc(:,1) * impactors%mass(1) / fragments%mass(1) #ifdef DOCONLOC - do concurrent(i = istart:fragments%nbody) shared(fragments,impactors,lhitandrun, vscale, vesc, vsign) local(j,vrot,vmag,vdisp,rimp,vimp_unit) + do concurrent(i = istart:fragments%nbody) shared(fragments,impactors,lhitandrun, vscale, vesc, vsign) & + local(j,vrot,vmag,vdisp,rimp,vimp_unit, vumag) #else do concurrent(i = istart:fragments%nbody) #endif j = fragments%origin_body(i) - vrot(:) = impactors%rot(:,j) .cross. (fragments%rc(:,i) - impactors%rc(:,j)) + vrot(1) = impactors%rot(2,j) * (fragments%rc(3,i) - impactors%rc(3,j)) & + - impactors%rot(3,j) * (fragments%rc(2,i) - impactors%rc(2,j)) + vrot(2) = impactors%rot(3,j) * (fragments%rc(1,i) - impactors%rc(1,j)) & + - impactors%rot(1,j) * (fragments%rc(3,i) - impactors%rc(3,j)) + vrot(3) = impactors%rot(1,j) * (fragments%rc(2,i) - impactors%rc(2,j)) & + - impactors%rot(2,j) * (fragments%rc(1,i) - impactors%rc(1,j)) if (lhitandrun) then - vdisp(:) = .unit.(fragments%rc(:,i) - impactors%rc(:,2)) * vesc + vumag = norm2(fragments%rc(:,i) - impactors%rc(:,2)) + vdisp(:) = (fragments%rc(:,i) - impactors%rc(:,2)) / vumag * vesc fragments%vc(:,i) = vsign(i) * impactors%bounce_unit(:) * vscale(i) + vrot(:) + vdisp(:) else vmag = vscale(i) rimp(:) = fragments%rc(:,i) - impactors%rcimp(:) - vimp_unit(:) = .unit. (rimp(:) + vsign(i) * impactors%bounce_unit(:)) + vumag = norm2(rimp(:) + vsign(i) * impactors%bounce_unit(:)) + vimp_unit(:) = (rimp(:) + vsign(i) * impactors%bounce_unit(:)) / vumag fragments%vc(:,i) = vmag * vimp_unit(:) + vrot(:) end if end do fragments%vmag(:) = .mag. fragments%vc(:,:) - ! Every time the collision-frame velocities are altered, we need to be sure to shift everything back to the center-of-mass frame + ! Every time the collision-frame velocities are altered, we need to be sure to shift everything back to the + ! center-of-mass frame call collision_util_shift_vector_to_origin(fragments%mass, fragments%vc) call fragments%set_coordinate_system() @@ -715,7 +751,8 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu nsteps = nsteps + 1 mfrag = sum(fragments%mass(istart:fragments%nbody)) - ! Try to put residual angular momentum into the spin, but if this would go past the spin barrier, then put it into velocity shear instead + ! Try to put residual angular momentum into the spin, but if this would go past the spin barrier, then put it into + ! velocity shear instead call collider_local%get_energy_and_momentum(nbody_system, param, phase="after") L_residual(:) = (collider_local%L_total(:,2) - collider_local%L_total(:,1)) L_residual_unit(:) = .unit. L_residual(:) @@ -740,7 +777,8 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu if (.mag.rot_new(:) < collider_local%max_rot) then fragments%rot(:,i) = rot_new(:) fragments%rotmag(i) = .mag.fragments%rot(:,i) - else ! We would break the spin barrier here. Add a random component of rotation that is less than what would break the limit. The rest will go in velocity shear + else ! We would break the spin barrier here. Add a random component of rotation that is less than what would + ! break the limit. The rest will go in velocity shear call random_number(drot) call random_number(rn) drot(:) = (rn * collider_local%max_rot - fragments%rotmag(i)) * 2 * (drot(:) - 0.5_DP) @@ -785,7 +823,8 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu ! Check if we've converged on our constraints if (all(dL_metric(:) <= 1.0_DP)) then - if ((abs(E_residual) < abs(E_residual_best)) .or. ((dE < 0.0_DP) .and. (dE_best >= 0.0_DP))) then ! This is our best case so far. Save it for posterity + if ((abs(E_residual) < abs(E_residual_best)) .or. ((dE < 0.0_DP) .and. (dE_best >= 0.0_DP))) then + ! This is our best case so far. Save it for posterity E_residual_best = E_residual L_residual_best(:) = L_residual(:) dE_best = dE @@ -799,7 +838,8 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu if (dE_conv < ENERGY_CONVERGENCE_TOL) exit inner end if - ! Remove a constant amount of velocity from the bodies so we don't shift the center of mass and screw up the momentum + ! Remove a constant amount of velocity from the bodies so we don't shift the center of mass and screw up the + ! momentum ke_avail = 0.0_DP do i = fragments%nbody, 1, -1 ke_avail = ke_avail + 0.5_DP * fragments%mass(i) * max(fragments%vmag(i) - vesc / try,0.0_DP)**2 @@ -830,9 +870,11 @@ module subroutine fraggle_generate_vel_vec(collider, nbody_system, param, lfailu write(message, *) nsteps if (lfailure) then - call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Fraggle velocity calculation failed to converge after " // trim(adjustl(message)) // " steps. The best solution found had:") + call swiftest_io_log_one_message(COLLISION_LOG_OUT, "Fraggle velocity calculation failed to converge after " & + // trim(adjustl(message)) // " steps. The best solution found had:") else - call swiftest_io_log_one_message(COLLISION_LOG_OUT,"Fraggle velocity calculation converged after " // trim(adjustl(message)) // " steps.") + call swiftest_io_log_one_message(COLLISION_LOG_OUT,"Fraggle velocity calculation converged after " & + // trim(adjustl(message)) // " steps.") call collider%get_energy_and_momentum(nbody_system, param, phase="after") L_residual(:) = (collider%L_total(:,2) - collider%L_total(:,1)) diff --git a/src/operator/operator_cross.f90 b/src/operator/operator_cross.f90 index ef80e1fb8..72c24a176 100644 --- a/src/operator/operator_cross.f90 +++ b/src/operator/operator_cross.f90 @@ -104,11 +104,7 @@ pure module function operator_cross_el_sp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_sp(A(:,i), B(:,i)) end do return @@ -122,11 +118,7 @@ pure module function operator_cross_el_dp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_dp(A(:,i), B(:,i)) end do return @@ -140,11 +132,7 @@ pure module function operator_cross_el_qp(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_qp(A(:,i), B(:,i)) end do return @@ -158,11 +146,7 @@ pure module function operator_cross_el_i1b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_i1b(A(:,i), B(:,i)) end do return @@ -176,11 +160,7 @@ pure module function operator_cross_el_i2b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_i2b(A(:,i), B(:,i)) end do return @@ -194,11 +174,7 @@ pure module function operator_cross_el_i4b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_i4b(A(:,i), B(:,i)) end do return @@ -212,11 +188,7 @@ pure module function operator_cross_el_i8b(A, B) result(C) n = size(A, 2) if (allocated(C)) deallocate(C) allocate(C, mold = A) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B,C) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n C(:,i) = operator_cross_i8b(A(:,i), B(:,i)) end do return diff --git a/src/operator/operator_mag.f90 b/src/operator/operator_mag.f90 index 55f653fb9..721e4a930 100644 --- a/src/operator/operator_mag.f90 +++ b/src/operator/operator_mag.f90 @@ -44,11 +44,7 @@ pure module function operator_mag_el_sp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n B(i) = norm2(A(:, i)) end do return @@ -63,11 +59,7 @@ pure module function operator_mag_el_dp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n B(i) = norm2(A(:, i)) end do return @@ -82,11 +74,7 @@ pure module function operator_mag_el_qp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(n)) call ieee_set_halting_mode(ieee_underflow, .false.) -#ifdef DOCONLOC - do concurrent (i = 1:n) shared(A,B) -#else - do concurrent (i = 1:n) -#endif + do i = 1,n B(i) = norm2(A(:, i)) end do return diff --git a/src/operator/operator_unit.f90 b/src/operator/operator_unit.f90 index a25ee1bb1..2a14f6645 100644 --- a/src/operator/operator_unit.f90 +++ b/src/operator/operator_unit.f90 @@ -89,11 +89,7 @@ pure module function operator_unit_el_sp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) -#ifdef DOCONLOC - do concurrent (i=1:n) shared(A,B) -#else - do concurrent (i=1:n) -#endif + do i=1,n B(:,i) = operator_unit_sp(A(:,i)) end do @@ -113,11 +109,7 @@ pure module function operator_unit_el_dp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) -#ifdef DOCONLOC - do concurrent (i=1:n) shared(A,B) -#else - do concurrent (i=1:n) -#endif + do i=1,n B(:,i) = operator_unit_dp(A(:,i)) end do @@ -136,11 +128,7 @@ pure module function operator_unit_el_qp(A) result(B) if (allocated(B)) deallocate(B) allocate(B(NDIM,n)) -#ifdef DOCONLOC - do concurrent (i=1:n) shared(A,B) -#else - do concurrent (i=1:n) -#endif + do i=1,n B(:,i) = operator_unit_qp(A(:,i)) end do From b8be3684a07da6424ad674a140bab8d411594890 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 16:01:35 -0400 Subject: [PATCH 131/149] Removed -standard-semantics from the default flag list, because it is not compatible with the pre-built NetCDF Fortran library. Kept it in the containerized version. --- Dockerfile | 1 + cmake/Modules/SetFortranFlags.cmake | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ac00b25d..d0f2dcb88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,6 +132,7 @@ ENV NETCDF_LIBRARY=${NETCDF_HOME} ENV FOR_COARRAY_NUM_IMAGES=1 ENV OMP_NUM_THREADS=1 ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV FFLAGS="-fPIC -standard-semantics" ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" COPY ./cmake/ /swiftest/cmake/ COPY ./src/ /swiftest/src/ diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 550738f13..d1edd9ae9 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -81,14 +81,6 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-fsignaling-nans " # GNU ) - -# Determines whether the current Fortran Standard behavior of the compiler is fully implemented. -SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-standard-semantics" # Intel - "/standard-semantics" # Intel Windows - ) - - # Allows for lines longer than 80 characters without truncation SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" Fortran "-ffree-line-length-none" # GNU (gfortran) From f7b8f46205f4c431627f4452506c54d195b3fbf9 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 18:21:47 -0400 Subject: [PATCH 132/149] Updated coarray code so it can compile in the container. --- Dockerfile | 14 +++++------ cmake/Modules/SetFortranFlags.cmake | 4 ++++ docker/bin/swiftest | 2 +- src/coarray/coarray_clone.f90 | 8 +++---- src/rmvs/rmvs_coarray.f90 | 10 +------- src/swiftest/swiftest_coarray.f90 | 36 ++++++++++++++++++----------- src/swiftest/swiftest_module.f90 | 8 +++++++ 7 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0f2dcb88..35e450d7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14 wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ rm -rf /var/lib/apt/lists/* && \ cd hdf && \ ./HDF5-1.14.1-Linux.sh --skip-license && \ @@ -153,14 +153,13 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=ON -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ cmake --build build --verbose && \ cmake --install build # Driver container FROM ubuntu:20.04 as Driver -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -172,9 +171,8 @@ FROM continuumio/miniconda3 ENV LD_LIBRARY_PATH="/usr/local/lib" ENV SHELL="/bin/bash" -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/ COPY ./python/ . +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /bin/ RUN apt-get update && apt-get upgrade -y && \ @@ -188,6 +186,8 @@ RUN apt-get update && apt-get upgrade -y && \ conda update --all -y && \ cd swiftest && conda develop . && \ mkdir -p /.astropy && \ - chmod -R 777 /.astropy + chmod -R 777 /.astropy && \ + mkdir -p /.config/matplotlib && \ + chmod -R 777 /.config/matplotlib ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index d1edd9ae9..47361de08 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -117,6 +117,10 @@ IF (CONTAINERIZE) SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" Fortran "-static-intel" # Intel ) + # Use static Intel MPI libraries + SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" + Fortran "-static_mpi" # Intel + ) IF (USE_OPENMP) SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" diff --git a/docker/bin/swiftest b/docker/bin/swiftest index bc94b2cb0..fead7f338 100755 --- a/docker/bin/swiftest +++ b/docker/bin/swiftest @@ -1,2 +1,2 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/src/coarray/coarray_clone.f90 b/src/coarray/coarray_clone.f90 index 9f7e1ea1a..893cff147 100644 --- a/src/coarray/coarray_clone.f90 +++ b/src/coarray/coarray_clone.f90 @@ -71,7 +71,7 @@ module subroutine coarray_component_clone_DP(var,src_img) sync all if (this_image() == si) then do img = 1, num_images() - tmp[img] = var + tmp[img] = var end do sync images(*) else @@ -117,7 +117,7 @@ module subroutine coarray_component_clone_DP_arr1D(var,src_img) allocate(tmp(n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:)[img] = var + tmp(:)[img] = var end do sync images(*) else @@ -167,7 +167,7 @@ module subroutine coarray_component_clone_DP_arr2D(var,src_img) allocate(tmp(n1[si],n2[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:,:)[img] = var(:,:) + tmp(:,:)[img] = var(:,:) end do sync images(*) else @@ -252,7 +252,7 @@ module subroutine coarray_component_clone_DP_vec2D(var,src_img) allocate(tmp(NDIM,n[si])[*]) if (this_image() == si) then do img = 1, num_images() - tmp(:,:)[img] = var(:,:) + tmp(:,:)[img] = var(:,:) end do sync images(*) else diff --git a/src/rmvs/rmvs_coarray.f90 b/src/rmvs/rmvs_coarray.f90 index 6f7467df8..3a7508159 100644 --- a/src/rmvs/rmvs_coarray.f90 +++ b/src/rmvs/rmvs_coarray.f90 @@ -68,7 +68,6 @@ module subroutine rmvs_coarray_coclone_pl(self) end subroutine rmvs_coarray_coclone_pl - module subroutine rmvs_coarray_coclone_system(self) !! author: David A. Minton !! @@ -141,14 +140,7 @@ module subroutine rmvs_coarray_component_clone_interp_arr1D(var,src_img) do i = 1, n[si] call tmp(i)%coclone() end do - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var(:) - end do - - sync images(*) - else - sync images(si) + if (this_image() /= si) then if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 7dbd1a816..8329856b9 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -54,6 +54,7 @@ module subroutine swiftest_coarray_balance_system(nbody_system, param) return end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_coclone_body(self) !! author: David A. Minton !! @@ -94,6 +95,21 @@ module subroutine swiftest_coarray_coclone_body(self) return end subroutine swiftest_coarray_coclone_body + module subroutine swiftest_coarray_coclone_kin(self) + !! author: David A. Minton + !! + !! Broadcasts the image 1 object to all other images in a coarray + implicit none + ! Arguments + class(swiftest_kinship),intent(inout),codimension[*] :: self !! Swiftest kinship object + + call coclone(self%parent) + call coclone(self%nchild) + call coclone(self%child) + + return + end subroutine swiftest_coarray_coclone_kin + module subroutine swiftest_coarray_coclone_nc(self) !! author: David A. Minton !! @@ -327,10 +343,6 @@ module subroutine swiftest_coarray_coclone_system(self) ! Internals integer(I4B) :: i - call self%cb%coclone() - call self%pl%coclone() - call self%tp%coclone() - call coclone(self%maxid) call coclone(self%t) call coclone(self%GMtot) @@ -474,7 +486,7 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) integer(I4B), intent(in),optional :: src_img ! Internals type(swiftest_kinship), dimension(:), codimension[:], allocatable :: tmp - integer(I4B) :: img, si + integer(I4B) :: i, img, si integer(I4B), allocatable :: n[:] logical, allocatable :: isalloc[:] @@ -484,21 +496,17 @@ module subroutine swiftest_coarray_component_clone_kin_arr1D(var,src_img) si = 1 end if - allocate(isalloc[*]) - allocate(n[*]) + sync all isalloc = allocated(var) if (isalloc) n = size(var) sync all if (.not. isalloc[si]) return allocate(tmp(n[si])[*]) - if (this_image() == si) then - do img = 1, num_images() - tmp(:)[img] = var - end do - sync images(*) - else - sync images(si) + do i = 1, n[si] + call tmp(i)%coclone() + end do + if (this_image() /= si) then if (allocated(var)) deallocate(var) allocate(var, source=tmp) end if diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index a54e2351b..b7bbd109c 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -93,6 +93,9 @@ module swiftest integer(I4B), dimension(:), allocatable :: child !! Index of children particles contains procedure :: dealloc => swiftest_util_dealloc_kin !! Deallocates all allocatable arrays +#ifdef COARRAY + procedure :: coclone => swiftest_coarray_coclone_kin !! Clones the image 1 body object to all other images in the coarray structure. +#endif final :: swiftest_final_kin !! Finalizes the Swiftest kinship object - deallocates all allocatables end type swiftest_kinship @@ -1776,6 +1779,11 @@ module subroutine swiftest_coarray_coclone_cb(self) class(swiftest_cb),intent(inout),codimension[*] :: self !! Swiftest cb object end subroutine swiftest_coarray_coclone_cb + module subroutine swiftest_coarray_coclone_kin(self) + implicit none + class(swiftest_kinship),intent(inout),codimension[*] :: self !! Swiftest kinship object + end subroutine swiftest_coarray_coclone_kin + module subroutine swiftest_coarray_coclone_nc(self) implicit none class(swiftest_netcdf_parameters),intent(inout),codimension[*] :: self !! Swiftest body object From c0546f649ee9ec7dd4e9b0cb87b23170f54bed4a Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 22:25:20 -0400 Subject: [PATCH 133/149] Improved Docker container --- Dockerfile | 51 ++++++++++++++++-------- docker/.gitignore | 3 +- docker/bin/swiftest_driver | 2 + docker/bin/{swiftest => swiftest_python} | 0 4 files changed, 38 insertions(+), 18 deletions(-) create mode 100755 docker/bin/swiftest_driver rename docker/bin/{swiftest => swiftest_python} (100%) diff --git a/Dockerfile b/Dockerfile index 35e450d7e..9fe400197 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,26 +154,19 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ cd swiftest && \ cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=ON -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ - cmake --build build --verbose && \ + cmake --build build && \ + cp bin/swiftest_driver /usr/local/bin/swiftest_driver_caf && \ + rm -rf build && \ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ + cmake --build build && \ cmake --install build # Driver container -FROM ubuntu:20.04 as Driver +FROM continuumio/miniconda3 as setup_conda COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libsz2 libcurl3-gnutls libxml2 && \ - rm -rf /var/lib/apt/lists/* - -# Production container -FROM continuumio/miniconda3 - -ENV LD_LIBRARY_PATH="/usr/local/lib" -ENV SHELL="/bin/bash" -COPY ./python/ . -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /usr/local/bin/swiftest_driver /bin/ +COPY ./python/. /opt/conda/pkgs/swiftest/python/ RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -184,10 +177,34 @@ RUN apt-get update && apt-get upgrade -y && \ conda config --set solver libmamba && \ conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 dask -y && \ conda update --all -y && \ - cd swiftest && conda develop . && \ - mkdir -p /.astropy && \ + cd /opt/conda/pkgs/swiftest/python/swiftest && conda develop . && \ + conda clean --all -y + +# Production container +FROM ubuntu:20.04 + +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsz2 libcurl3-gnutls libxml2 && \ + rm -rf /var/lib/apt/lists/* + +ENV LD_LIBRARY_PATH="/usr/local/lib" +ENV SHELL="/bin/bash" +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ +COPY --from=setup_conda /opt/conda/. /opt/conda/ +COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ +COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin +COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ + +RUN mkdir -p /.astropy && \ chmod -R 777 /.astropy && \ + mkdir -p /.cache/matplotlib && \ mkdir -p /.config/matplotlib && \ - chmod -R 777 /.config/matplotlib + chmod -R 777 /.cache/matplotlib && \ + chmod -R 777 /.config/matplotlib && \ + mkdir -p /opt/conda/pkgs/swiftest/bin && \ + ln -s /usr/local/bin/swiftest_driver /opt/conda/pkgs/swiftest/bin/swiftest_driver ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore index 09c5585d1..772f5a9d9 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -2,4 +2,5 @@ !.gitignore !install.sh !bin -!bin/swiftest +!bin/swiftest_python +!bin/swiftest_driver diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver new file mode 100755 index 000000000..0f7d8b5dd --- /dev/null +++ b/docker/bin/swiftest_driver @@ -0,0 +1,2 @@ +#!/bin/sh -- +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/docker/bin/swiftest b/docker/bin/swiftest_python similarity index 100% rename from docker/bin/swiftest rename to docker/bin/swiftest_python From 7152017d63c6e13f2c1be48d014329152499a9e7 Mon Sep 17 00:00:00 2001 From: David Minton Date: Tue, 6 Jun 2023 22:33:28 -0400 Subject: [PATCH 134/149] More improvements to containers --- Dockerfile | 16 ++-------------- singularity/bin/swiftest_driver | 2 +- singularity/install.sh | 2 +- singularity/setenv.sh | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9fe400197..2d7821992 100644 --- a/Dockerfile +++ b/Dockerfile @@ -161,11 +161,8 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN cmake --build build && \ cmake --install build -# Driver container -FROM continuumio/miniconda3 as setup_conda -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ -COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin -COPY --from=build /usr/local/bin/swiftest_driver /bin/ +# Production container +FROM continuumio/miniconda3 COPY ./python/. /opt/conda/pkgs/swiftest/python/ RUN apt-get update && apt-get upgrade -y && \ @@ -180,20 +177,11 @@ RUN apt-get update && apt-get upgrade -y && \ cd /opt/conda/pkgs/swiftest/python/swiftest && conda develop . && \ conda clean --all -y -# Production container -FROM ubuntu:20.04 - -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libsz2 libcurl3-gnutls libxml2 && \ - rm -rf /var/lib/apt/lists/* - ENV LD_LIBRARY_PATH="/usr/local/lib" ENV SHELL="/bin/bash" COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ -COPY --from=setup_conda /opt/conda/. /opt/conda/ COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver index dba0863a9..30fe6352b 100755 --- a/singularity/bin/swiftest_driver +++ b/singularity/bin/swiftest_driver @@ -1,2 +1,2 @@ #!/bin/sh -- -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file +singularity run --bind $(pwd):$(pwd) exec --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index ff2be8596..99cae29c1 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -5,6 +5,6 @@ # tag=${1:-latest} echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" -singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag} +singularity pull --force swiftest.sif docker://daminton/swiftest:${tag} cp -rf bin/swiftest_driver ../bin/ source ./setenv.sh \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh index cc8905033..82ca7d408 100755 --- a/singularity/setenv.sh +++ b/singularity/setenv.sh @@ -1,4 +1,4 @@ #!/bin/sh -- # This will set the SWIFTEST_SIF environment variable as long as it is executed by source. # $ . ./setenv.sh -export SWIFTEST_SIF="${PWD}/swiftest_driver.sif" \ No newline at end of file +export SWIFTEST="${PWD}/swiftest.sif" \ No newline at end of file From 35e0099e9934c18b07c5add1ca4cbf93207823f0 Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 08:52:57 -0400 Subject: [PATCH 135/149] Updates to singularity scripts and also added apptainer scripts --- apptainer/.gitignore | 5 +++++ apptainer/bin/swiftest_driver | 2 ++ apptainer/bin/swiftest_python | 2 ++ apptainer/install.sh | 10 ++++++++++ apptainer/setenv.sh | 4 ++++ singularity/.gitignore | 1 + singularity/bin/swiftest_driver | 2 +- singularity/bin/swiftest_python | 2 ++ 8 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 apptainer/.gitignore create mode 100755 apptainer/bin/swiftest_driver create mode 100755 apptainer/bin/swiftest_python create mode 100755 apptainer/install.sh create mode 100755 apptainer/setenv.sh create mode 100755 singularity/bin/swiftest_python diff --git a/apptainer/.gitignore b/apptainer/.gitignore new file mode 100644 index 000000000..6ddbdba9f --- /dev/null +++ b/apptainer/.gitignore @@ -0,0 +1,5 @@ +!bin/ +!bin/swiftest_driver +!bin/swiftest_python +!install.sh +!setenv.sh diff --git a/apptainer/bin/swiftest_driver b/apptainer/bin/swiftest_driver new file mode 100755 index 000000000..fa008737a --- /dev/null +++ b/apptainer/bin/swiftest_driver @@ -0,0 +1,2 @@ +#!/bin/sh -- +apptainer exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/apptainer/bin/swiftest_python b/apptainer/bin/swiftest_python new file mode 100755 index 000000000..b33012e4a --- /dev/null +++ b/apptainer/bin/swiftest_python @@ -0,0 +1,2 @@ +#!/bin/sh -- +apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/apptainer/install.sh b/apptainer/install.sh new file mode 100755 index 000000000..abf9f4f91 --- /dev/null +++ b/apptainer/install.sh @@ -0,0 +1,10 @@ +#!/bin/sh -- +# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. +# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source: +# $ . ./install.sh +# +tag=${1:-latest} +echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" +apptainer pull --force swiftest.sif docker://daminton/swiftest:${tag} +cp -rf bin/swiftest_driver ../bin/ +source ./setenv.sh \ No newline at end of file diff --git a/apptainer/setenv.sh b/apptainer/setenv.sh new file mode 100755 index 000000000..82ca7d408 --- /dev/null +++ b/apptainer/setenv.sh @@ -0,0 +1,4 @@ +#!/bin/sh -- +# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. +# $ . ./setenv.sh +export SWIFTEST="${PWD}/swiftest.sif" \ No newline at end of file diff --git a/singularity/.gitignore b/singularity/.gitignore index 2a259d270..6ddbdba9f 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -1,4 +1,5 @@ !bin/ !bin/swiftest_driver +!bin/swiftest_python !install.sh !setenv.sh diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver index 30fe6352b..03876195b 100755 --- a/singularity/bin/swiftest_driver +++ b/singularity/bin/swiftest_driver @@ -1,2 +1,2 @@ #!/bin/sh -- -singularity run --bind $(pwd):$(pwd) exec --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file +singularity exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/singularity/bin/swiftest_python b/singularity/bin/swiftest_python new file mode 100755 index 000000000..4061ff119 --- /dev/null +++ b/singularity/bin/swiftest_python @@ -0,0 +1,2 @@ +#!/bin/sh -- +singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file From aa0acbfacc44834a80d7d1c79248c8f10c1669fc Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 08:56:09 -0400 Subject: [PATCH 136/149] Fixed typos in Singularity/Apptainer scripts --- apptainer/bin/swiftest_python | 2 +- singularity/bin/swiftest_python | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apptainer/bin/swiftest_python b/apptainer/bin/swiftest_python index b33012e4a..6d562c962 100755 --- a/apptainer/bin/swiftest_python +++ b/apptainer/bin/swiftest_python @@ -1,2 +1,2 @@ #!/bin/sh -- -apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file +apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/singularity/bin/swiftest_python b/singularity/bin/swiftest_python index 4061ff119..2a7b13735 100755 --- a/singularity/bin/swiftest_python +++ b/singularity/bin/swiftest_python @@ -1,2 +1,2 @@ #!/bin/sh -- -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file +singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file From 5f2e9bfc6ce1df4f3bbc90dffb1fa1c7b0c762ef Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 08:58:03 -0400 Subject: [PATCH 137/149] Fixed typo in Singularity/Apptainer install scripts --- apptainer/setenv.sh | 2 +- singularity/setenv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apptainer/setenv.sh b/apptainer/setenv.sh index 82ca7d408..d9f215d7c 100755 --- a/apptainer/setenv.sh +++ b/apptainer/setenv.sh @@ -1,4 +1,4 @@ #!/bin/sh -- # This will set the SWIFTEST_SIF environment variable as long as it is executed by source. # $ . ./setenv.sh -export SWIFTEST="${PWD}/swiftest.sif" \ No newline at end of file +export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh index 82ca7d408..d9f215d7c 100755 --- a/singularity/setenv.sh +++ b/singularity/setenv.sh @@ -1,4 +1,4 @@ #!/bin/sh -- # This will set the SWIFTEST_SIF environment variable as long as it is executed by source. # $ . ./setenv.sh -export SWIFTEST="${PWD}/swiftest.sif" \ No newline at end of file +export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file From da9b30019a921bed44197cee321a0350f2e46ce5 Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 09:00:36 -0400 Subject: [PATCH 138/149] More typo fixes and cleanup --- apptainer/install.sh | 5 +++-- singularity/install.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apptainer/install.sh b/apptainer/install.sh index abf9f4f91..a36eaf3a1 100755 --- a/apptainer/install.sh +++ b/apptainer/install.sh @@ -1,10 +1,11 @@ #!/bin/sh -- # This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source: +# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: # $ . ./install.sh # tag=${1:-latest} -echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" +echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest:${tag} Docker container" apptainer pull --force swiftest.sif docker://daminton/swiftest:${tag} cp -rf bin/swiftest_driver ../bin/ +cp -rf bin/swiftest_python ../bin/ source ./setenv.sh \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index 99cae29c1..3c1f3445d 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -1,10 +1,11 @@ #!/bin/sh -- # This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source: +# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: # $ . ./install.sh # tag=${1:-latest} -echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container" +echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest:${tag} Docker container" singularity pull --force swiftest.sif docker://daminton/swiftest:${tag} cp -rf bin/swiftest_driver ../bin/ +cp -rf bin/swiftest_python ../bin/ source ./setenv.sh \ No newline at end of file From c8ba0905b5dce9d56c4f518c1b3031defc8fe00f Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 09:03:42 -0400 Subject: [PATCH 139/149] Typos in install scripts --- apptainer/install.sh | 2 +- singularity/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apptainer/install.sh b/apptainer/install.sh index a36eaf3a1..4250b5abb 100755 --- a/apptainer/install.sh +++ b/apptainer/install.sh @@ -4,7 +4,7 @@ # $ . ./install.sh # tag=${1:-latest} -echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest:${tag} Docker container" +echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" apptainer pull --force swiftest.sif docker://daminton/swiftest:${tag} cp -rf bin/swiftest_driver ../bin/ cp -rf bin/swiftest_python ../bin/ diff --git a/singularity/install.sh b/singularity/install.sh index 3c1f3445d..e7c47e2b3 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -4,7 +4,7 @@ # $ . ./install.sh # tag=${1:-latest} -echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest:${tag} Docker container" +echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" singularity pull --force swiftest.sif docker://daminton/swiftest:${tag} cp -rf bin/swiftest_driver ../bin/ cp -rf bin/swiftest_python ../bin/ From 10935fb97bc0dc1cc094de6d47ce8b3ba57174da Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 16:23:29 -0400 Subject: [PATCH 140/149] Made significant improvements to the Docker, Apptainer, and Singularity containers. --- Dockerfile | 40 +++++++++++--------- apptainer/.gitignore | 3 +- apptainer/bin/{swiftest_python => swiftest} | 2 + apptainer/bin/swiftest_driver | 2 + apptainer/install.sh | 26 +++++++++---- apptainer/setenv.sh | 4 -- docker/.gitignore | 2 +- docker/bin/swiftest | 4 ++ docker/bin/swiftest_driver | 4 +- docker/bin/swiftest_python | 2 - docker/install.sh | 5 ++- python/swiftest/swiftest/simulation_class.py | 23 +++++------ singularity/.gitignore | 1 - singularity/bin/swiftest_driver | 2 + singularity/bin/swiftest_python | 2 - singularity/install.sh | 26 +++++++++---- singularity/setenv.sh | 4 -- 17 files changed, 86 insertions(+), 66 deletions(-) rename apptainer/bin/{swiftest_python => swiftest} (60%) delete mode 100755 apptainer/setenv.sh create mode 100755 docker/bin/swiftest delete mode 100755 docker/bin/swiftest_python delete mode 100755 singularity/bin/swiftest_python delete mode 100755 singularity/setenv.sh diff --git a/Dockerfile b/Dockerfile index 2d7821992..b53f8dc8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -163,7 +163,12 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN # Production container FROM continuumio/miniconda3 -COPY ./python/. /opt/conda/pkgs/swiftest/python/ +SHELL ["/bin/bash", "--login", "-c"] +ENV SHELL="/bin/bash" +ENV PATH="/opt/conda/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/lib" + +COPY environment.yml . RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -172,27 +177,28 @@ RUN apt-get update && apt-get upgrade -y && \ conda update --all -y && \ conda install conda-libmamba-solver -y && \ conda config --set solver libmamba && \ - conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 dask -y && \ - conda update --all -y && \ - cd /opt/conda/pkgs/swiftest/python/swiftest && conda develop . && \ - conda clean --all -y + conda env create -f environment.yml && \ + conda init bash && \ + echo "conda activate swiftest-env" >> ~/.bashrc -ENV LD_LIBRARY_PATH="/usr/local/lib" -ENV SHELL="/bin/bash" -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ -COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin -COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ +COPY ./python/. /opt/conda/pkgs/ +COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/envs/swiftest-env/bin/ +COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/bin/ +COPY --from=build /usr/local/bin/swiftest_driver_caf /opt/conda/envs/swiftest-env/bin/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /opt/conda/envs/swiftest-env/bin/ -RUN mkdir -p /.astropy && \ +# Start new shell to activate the environment and install Swiftest +RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ + conda clean --all -y && \ + mkdir -p /.astropy && \ chmod -R 777 /.astropy && \ mkdir -p /.cache/matplotlib && \ mkdir -p /.config/matplotlib && \ chmod -R 777 /.cache/matplotlib && \ chmod -R 777 /.config/matplotlib && \ - mkdir -p /opt/conda/pkgs/swiftest/bin && \ - ln -s /usr/local/bin/swiftest_driver /opt/conda/pkgs/swiftest/bin/swiftest_driver + ln -s /opt/conda/bin/swiftest_driver /opt/conda/bin/driver -ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "swiftest-env"] \ No newline at end of file diff --git a/apptainer/.gitignore b/apptainer/.gitignore index 6ddbdba9f..baed94e88 100644 --- a/apptainer/.gitignore +++ b/apptainer/.gitignore @@ -1,5 +1,4 @@ !bin/ !bin/swiftest_driver -!bin/swiftest_python +!bin/swiftest !install.sh -!setenv.sh diff --git a/apptainer/bin/swiftest_python b/apptainer/bin/swiftest similarity index 60% rename from apptainer/bin/swiftest_python rename to apptainer/bin/swiftest index 6d562c962..ca0fcfadd 100755 --- a/apptainer/bin/swiftest_python +++ b/apptainer/bin/swiftest @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/apptainer/bin/swiftest_driver b/apptainer/bin/swiftest_driver index fa008737a..e88be2805 100755 --- a/apptainer/bin/swiftest_driver +++ b/apptainer/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} apptainer exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/apptainer/install.sh b/apptainer/install.sh index 4250b5abb..5d84ca34a 100755 --- a/apptainer/install.sh +++ b/apptainer/install.sh @@ -1,11 +1,21 @@ #!/bin/sh -- -# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: -# $ . ./install.sh +# This will install the Apptainer version of the swiftest_driver in place of the native compiled version into ../bin as +# well as the swiftest_python script that is used to execute a Python input file. +# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.apptainer. +# To change this, just set environment variable SIF_DIR prior to running this script. +# +# The script takes an optional argument "tag" if you want to pull a container other than "latest". # -tag=${1:-latest} -echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" -apptainer pull --force swiftest.sif docker://daminton/swiftest:${tag} +# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, +# which requires this script to be called via source: +# $ source ./install.sh +# or +# $ . ./install.sh +TAG=${1:-latest} + +SIF_DIR=${SIF_DIR:-${HOME}/.apptainer} +echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container" +apptainer pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG} +cp -rf bin/swiftest ../bin/ cp -rf bin/swiftest_driver ../bin/ -cp -rf bin/swiftest_python ../bin/ -source ./setenv.sh \ No newline at end of file +export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif \ No newline at end of file diff --git a/apptainer/setenv.sh b/apptainer/setenv.sh deleted file mode 100755 index d9f215d7c..000000000 --- a/apptainer/setenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -- -# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. -# $ . ./setenv.sh -export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore index 772f5a9d9..5c73deb60 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -2,5 +2,5 @@ !.gitignore !install.sh !bin -!bin/swiftest_python +!bin/swiftest !bin/swiftest_driver diff --git a/docker/bin/swiftest b/docker/bin/swiftest new file mode 100755 index 000000000..8c985e065 --- /dev/null +++ b/docker/bin/swiftest @@ -0,0 +1,4 @@ +#!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES mintongroup/swiftest "$@" \ No newline at end of file diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver index 0f7d8b5dd..146509c36 100755 --- a/docker/bin/swiftest_driver +++ b/docker/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES MintonGroup/swiftest "$@" \ No newline at end of file diff --git a/docker/bin/swiftest_python b/docker/bin/swiftest_python deleted file mode 100755 index fead7f338..000000000 --- a/docker/bin/swiftest_python +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/docker/install.sh b/docker/install.sh index 1a0b2b20c..296223149 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -1,5 +1,6 @@ #!/bin/sh -- tag=${1:-latest} echo "Installing swiftest:${tag} Docker container and executable script" -docker pull daminton/swiftest:${tag} -cp -rf bin/swiftest ../bin/ \ No newline at end of file +docker pull mintongroup/swiftest:${tag} +cp -rf bin/swiftest ../bin/ +cp -rf bin/swiftest_driver ../bin/ \ No newline at end of file diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 8971b897a..399ab0d38 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -470,6 +470,9 @@ def _type_scrub(output_data): sys.exit() except: warnings.warn(f"Error executing main swiftest_driver program", stacklevel=2) + res = p.communicate() + for line in res[1]: + print(line, end='') sys.exit() pbar.close() @@ -502,11 +505,9 @@ def run(self,dask: bool = False, **kwargs): if not self.binary_source.exists(): msg = "Path to swiftest_driver has not been set!" - msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}" + msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}" warnings.warn(msg,stacklevel=2) return - else: - shutil.copy2(self.binary_source, self.driver_executable) if not self.restart: self.clean() @@ -942,16 +943,15 @@ def set_integrator(self, update_list.append("codename") if self.codename == "Swiftest": self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" - self.binary_path = self.simdir.resolve() - self.driver_executable = self.binary_path / "swiftest_driver" + self.driver_executable = self.binary_source if not self.binary_source.exists(): - warnings.warn(f"Cannot find the Swiftest driver in {str(self.binary_path)}",stacklevel=2) + warnings.warn(f"Cannot find the Swiftest driver at {str(self.binary_source)}",stacklevel=2) self.driver_executable = None else: - if self.binary_path.exists(): + if self.binary_source.exists(): self.driver_executable.resolve() else: - self.binary_path = "NOT IMPLEMENTED FOR THIS CODE" + self.binary_source = "NOT IMPLEMENTED FOR THIS CODE" self.driver_executable = None update_list.append("driver_executable") @@ -1200,8 +1200,6 @@ def set_feature(self, msg = f"Cannot create the {self.simdir.resolve()} directory: File exists." msg += "\nDelete the file or change the location of param_file" raise NotADirectoryError(msg) - self.binary_path = self.simdir.resolve() - self.driver_executable = self.binary_path / "swiftest_driver" self.param_file = Path(kwargs.pop("param_file","param.in")) if self.codename == "Swiftest": @@ -2754,7 +2752,6 @@ def write_param(self, self.driver_script = os.path.join(self.simdir, "swiftest_driver.sh") with open(self.driver_script, 'w') as f: f.write(f"#{self._shell_full}\n") - #f.write(f"source ~/.{self._shell}rc\n") f.write(f"cd {self.simdir}\n") f.write(f"{str(self.driver_executable)} {self.integrator} {str(self.param_file)} compact\n") @@ -2991,11 +2988,9 @@ def save(self, self.write_param(param_file=param_file,**kwargs) if not self.binary_source.exists(): msg = "Path to swiftest_driver has not been set!" - msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}" + msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}" warnings.warn(msg,stacklevel=2) return - else: - shutil.copy2(self.binary_source, self.driver_executable) elif codename == "Swifter": swifter_param = io.swiftest2swifter_param(param) if "rhill" in self.data: diff --git a/singularity/.gitignore b/singularity/.gitignore index 6ddbdba9f..b34dd2df5 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -2,4 +2,3 @@ !bin/swiftest_driver !bin/swiftest_python !install.sh -!setenv.sh diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver index 03876195b..024e93115 100755 --- a/singularity/bin/swiftest_driver +++ b/singularity/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} singularity exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/singularity/bin/swiftest_python b/singularity/bin/swiftest_python deleted file mode 100755 index 2a7b13735..000000000 --- a/singularity/bin/swiftest_python +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index e7c47e2b3..233ee187f 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -1,11 +1,21 @@ #!/bin/sh -- -# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: -# $ . ./install.sh +# This will install the Singularity version of the swiftest_driver in place of the native compiled version into ../bin as +# well as the swiftest_python script that is used to execute a Python input file. +# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.singularity. +# To change this, just set environment variable SIF_DIR prior to running this script. +# +# The script takes an optional argument "tag" if you want to pull a container other than "latest". # -tag=${1:-latest} -echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" -singularity pull --force swiftest.sif docker://daminton/swiftest:${tag} +# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, +# which requires this script to be called via source: +# $ source ./install.sh +# or +# $ . ./install.sh +TAG=${1:-latest} + +SIF_DIR=${SIF_DIR:-${HOME}/.singularity} +echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container" +singularity pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG} +cp -rf bin/swiftest ../bin/ cp -rf bin/swiftest_driver ../bin/ -cp -rf bin/swiftest_python ../bin/ -source ./setenv.sh \ No newline at end of file +export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh deleted file mode 100755 index d9f215d7c..000000000 --- a/singularity/setenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -- -# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. -# $ . ./setenv.sh -export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file From 49e77a3cd714fe63c7ca67035c47b6b81f9924da Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 19:11:11 -0400 Subject: [PATCH 141/149] Added singularity swiftest script --- singularity/.gitignore | 2 +- singularity/bin/swiftest | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100755 singularity/bin/swiftest diff --git a/singularity/.gitignore b/singularity/.gitignore index b34dd2df5..baed94e88 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -1,4 +1,4 @@ !bin/ !bin/swiftest_driver -!bin/swiftest_python +!bin/swiftest !install.sh diff --git a/singularity/bin/swiftest b/singularity/bin/swiftest new file mode 100755 index 000000000..8a5849e51 --- /dev/null +++ b/singularity/bin/swiftest @@ -0,0 +1,4 @@ +#!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} +singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file From afb2c9f036439711a9cdf003755c7b7d4e1077a1 Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 19:29:32 -0400 Subject: [PATCH 142/149] Added flags to ensure that host environment variables don't get passed down to the container in Apptainer/Singularity (which wreaks havoc on the RCAC cluster) --- apptainer/bin/swiftest | 2 +- singularity/bin/swiftest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apptainer/bin/swiftest b/apptainer/bin/swiftest index ca0fcfadd..5d91b80d6 100755 --- a/apptainer/bin/swiftest +++ b/apptainer/bin/swiftest @@ -1,4 +1,4 @@ #!/bin/sh -- OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} -apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file +apptainer run --bind $(pwd):$(pwd) --cleanenv --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/singularity/bin/swiftest b/singularity/bin/swiftest index 8a5849e51..a384860fe 100755 --- a/singularity/bin/swiftest +++ b/singularity/bin/swiftest @@ -1,4 +1,4 @@ #!/bin/sh -- OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file +singularity run --bind $(pwd):$(pwd) --cleanenv --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file From 1bd3846742e67358a47c171c712b00eda9c0a72d Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 8 Jun 2023 09:06:13 -0400 Subject: [PATCH 143/149] Updated usage message to be more accurate. --- src/base/base_module.f90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/base/base_module.f90 b/src/base/base_module.f90 index 502a42515..51266238f 100644 --- a/src/base/base_module.f90 +++ b/src/base/base_module.f90 @@ -600,8 +600,8 @@ subroutine base_util_exit(code) character(*), parameter :: BAR = '("------------------------------------------------")' character(*), parameter :: SUCCESS_MSG = '(/, "Normal termination of Swiftest (version ", f3.1, ")")' character(*), parameter :: FAIL_MSG = '(/, "Terminating Swiftest (version ", f3.1, ") due to error!!")' - character(*), parameter :: USAGE_MSG = '("Usage: swiftest [bs|helio|ra15|rmvs|symba|tu4|whm] ' // & - '[standard|compact|progress|NONE]")' + character(*), parameter :: USAGE_MSG = '("Usage: swiftest ' // & + '[{standard}|compact|progress]")' character(*), parameter :: HELP_MSG = USAGE_MSG select case(code) From 5e6a200839487c0e53d3d65f16f0ea12b0ad8acf Mon Sep 17 00:00:00 2001 From: David Minton Date: Thu, 8 Jun 2023 09:20:47 -0400 Subject: [PATCH 144/149] Added a Dockerfile just for the driver --- docker/.gitignore | 1 + docker/Dockerfile.swiftest_driver | 176 ++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 docker/Dockerfile.swiftest_driver diff --git a/docker/.gitignore b/docker/.gitignore index 5c73deb60..c876f8c26 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -4,3 +4,4 @@ !bin !bin/swiftest !bin/swiftest_driver +!Dockerfile.swiftest_driver diff --git a/docker/Dockerfile.swiftest_driver b/docker/Dockerfile.swiftest_driver new file mode 100644 index 000000000..8414f9016 --- /dev/null +++ b/docker/Dockerfile.swiftest_driver @@ -0,0 +1,176 @@ +FROM ubuntu:20.04 as build + +# kick everything off +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ + /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt-get -y update && apt-get upgrade -y && \ + apt-get install -y intel-hpckit + +# Set Intel compiler environment variables +ENV INTEL_DIR="/opt/intel/oneapi" +ENV LANG=C.UTF-8 +ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' +ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' +ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' +ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' +ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' +ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' +ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' +ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' +ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' +ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' +ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' +ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' +ENV DAL_MAJOR_BINARY='1' +ENV DAL_MINOR_BINARY='1' +ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' +ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' +ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' +ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' +ENV FPGA_VARS_ARGS='' +ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' +ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' +ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' +ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' +ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' +ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' +ENV IPPCP_TARGET_ARCH='intel64' +ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' +ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' +ENV IPP_TARGET_ARCH='intel64' +ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' +ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' +ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' +ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' +ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' +ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' +ENV ONEAPI_ROOT='/opt/intel/oneapi' +ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' +ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' +ENV SETVARS_COMPLETED='1' +ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' +ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' +ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' +ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' +ENV VT_MPI='impi4' +ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' +ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' + +# Set HDF5 and NetCDF-specific Environment variables +ENV INSTALL_DIR="/usr/local" +ENV LIB_DIR="${INSTALL_DIR}/lib" +ENV LD_LIBRARY_PATH=${LIB_DIR}:${LD_LIBRARY_PATH} +RUN mkdir -p ${LIB_DIR} + +ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" +ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" +ENV CXX="${INTEL_DIR}/compiler/latest/linux/bin/icpx" +ENV LDFLAGS="-L${LIB_DIR}" +ENV NCDIR="${INSTALL_DIR}" +ENV NFDIR="${INSTALL_DIR}" +ENV HDF5_ROOT="${INSTALL_DIR}" +ENV HDF5_LIBDIR="${HDF5_ROOT}/lib" +ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" +ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" + +# Get the HDF5, NetCDF-C and NetCDF-Fortran libraries +RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ + wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ + apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ + rm -rf /var/lib/apt/lists/* && \ + cd hdf && \ + ./HDF5-1.14.1-Linux.sh --skip-license && \ + cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ + cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ && \ + cp /zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ + +ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" +ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +RUN cd netcdf-c-4.9.2 && \ + cmake -S . -B build -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF && \ + cmake --build build && \ + cmake --install build + +# NetCDF-Fortran library +ENV F77=${FC} +ENV CFLAGS="-fPIC" +ENV FCFLAGS="${CFLAGS} -standard-semantics" +ENV FFLAGS=${CFLAGS} +ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl" +ENV LDFLAGS="-static-intel" +ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +RUN cd netcdf-fortran-4.6.1 && \ + ./configure --disable-shared --prefix=${NFDIR} && \ + make && \ + make install + +# Swiftest +ENV NETCDF_HOME=${INSTALL_DIR} +ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} +ENV NETCDF_LIBRARY=${NETCDF_HOME} +ENV FOR_COARRAY_NUM_IMAGES=1 +ENV OMP_NUM_THREADS=1 +ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV FFLAGS="-fPIC -standard-semantics" +ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" +COPY ./cmake/ /swiftest/cmake/ +COPY ./src/ /swiftest/src/ +COPY ./CMakeLists.txt /swiftest/ +RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)\n' \ + 'find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(HDF5_HL_LIBRARY NAMES libhdf5_hl.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(HDF5_LIBRARY NAMES libhdf5.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(Z_LIBRARY NAMES libz.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(ZSTD_LIBRARY NAMES libzstd.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(SZ_LIBRARY NAMES libsz.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(BZ2_LIBRARY NAMES libbz2.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(CURL_LIBRARY NAMES libcurl.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'find_library(XML2_LIBRARY NAMES libxml2.a HINTS ENV LD_LIBRARY_PATH)\n' \ + 'set(NETCDF_FOUND TRUE)\n' \ + 'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \ + 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ + 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ + cd swiftest && \ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ + cmake --build build && \ + cmake --install build + +# Production container +FROM ubuntu:20.04 + +RUN apt-get update && apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsz2 libcurl3-gnutls libxml2 && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin/ +# COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ +COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ + + +ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file From bfe55af08d47bb1f602a71e01f1f2e01e9fab818 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 9 Jun 2023 17:16:06 -0400 Subject: [PATCH 145/149] Made a clean, streamlined Dockerfile that contains almost no dependencies in the end. It can also spit out the binary executables so it could be used as a build tool rather than running from inside the container --- CMakeLists.txt | 8 - Dockerfile | 241 +++++++++++----------------- cmake/Modules/SetFortranFlags.cmake | 52 +++--- docker/Dockerfile.swiftest_driver | 197 +++++++---------------- src/CMakeLists.txt | 2 +- 5 files changed, 173 insertions(+), 327 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08ec7c9e2..961683f3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,13 +34,6 @@ OPTION(USE_SIMD "Use SIMD vectorization" ON) OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF) OPTION(BUILD_SHARED_LIBS "Build using shared libraries" ON) -# Locate and set external libraries. There are some CMake peculiarities -# taken care of here, such as the fact that the FindOpenMP routine doesn't know -# about Fortran. -IF (NOT BUILD_SHARED_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so") -ENDIF () - INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake) INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake) INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake) @@ -51,7 +44,6 @@ INCLUDE(${CMAKE_MODULE_PATH}/SetFortranFlags.cmake) INCLUDE_DIRECTORIES($ENV{NETCDF_FORTRAN_HOME}/include;$ENV{NETCDF_HOME}/include) - # There is an error in CMAKE with this flag for pgf90. Unset it GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME) IF(FCNAME STREQUAL "pgf90") diff --git a/Dockerfile b/Dockerfile index b53f8dc8c..33c82f700 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,85 +1,73 @@ -FROM ubuntu:20.04 as build - -# kick everything off -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ - rm -rf /var/lib/apt/lists/* && \ - mkdir -p cmake/build && \ - cd cmake/build && \ - curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ - /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license && \ - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ - | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ - apt-get -y update && apt-get upgrade -y && \ - apt-get install -y intel-hpckit - -# Set Intel compiler environment variables -ENV INTEL_DIR="/opt/intel/oneapi" -ENV LANG=C.UTF-8 -ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' -ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' -ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' -ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' -ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' -ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' -ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' -ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' -ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' -ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' -ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' -ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DAL_MAJOR_BINARY='1' -ENV DAL_MINOR_BINARY='1' -ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' -ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' -ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' -ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' -ENV FPGA_VARS_ARGS='' -ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' -ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' -ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' -ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' -ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' -ENV IPPCP_TARGET_ARCH='intel64' -ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' -ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' -ENV IPP_TARGET_ARCH='intel64' -ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' -ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' -ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' -ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' -ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' -ENV ONEAPI_ROOT='/opt/intel/oneapi' -ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' -ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' -ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' -ENV SETVARS_COMPLETED='1' -ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' -ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' -ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' -ENV VT_MPI='impi4' -ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' -ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' - -# Set HDF5 and NetCDF-specific Environment variables +# Copyright 2023 - David Minton +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. +# +# This Dockerfile will build the Swiftest driver program with minimal external dependencies using the Intel Oneapi toolkit. +# This is done by building static versions of a minimal set of libraries that NetCDF-Fortran needs (Netcdf-C, HDF5, and Zlib). +# These, along with the Intel runtime libraries, are linked statically to the executable. Only the OS-specific libraries are linked +# dynamically. + +# This build target compiles all dependencies and the swiftest driver itself +FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu22.04 as build + +# The MACHINE_CODE_VALUE argument is a string that is used when compiling the swiftest_driver. It is appended to the "-x" compiler +# option: (-x${MACHINE_CODE_VALUE}). The default value is set to "sse2" which allows for certain SIMD instructions to be used while +# remaining # compatible with a wide range of CPUs. To get the highest performance, you can pass "host" as an argument, but the +# compiled binary # would only run on a CPU with an architecture compatible with the one that the build was performed on. +# For more details and other options, see: +# https://www.intel.com/content/www/us/en/docs/fortran-compiler/developer-guide-reference/2023-1/x-qx.html +ARG MACHINE_CODE_VALUE="sse2" + +# Build type options are DEBUG, RELEASE, PROFILE, or TESTING. +ARG BUILD_TYPE="RELEASE" + ENV INSTALL_DIR="/usr/local" -ENV LIB_DIR="${INSTALL_DIR}/lib" -ENV LD_LIBRARY_PATH=${LIB_DIR}:${LD_LIBRARY_PATH} -RUN mkdir -p ${LIB_DIR} - -ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" -ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" -ENV CXX="${INTEL_DIR}/compiler/latest/linux/bin/icpx" -ENV LDFLAGS="-L${LIB_DIR}" +ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx" +ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx" +ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx" +ENV F77="${FC}" + +# Get the HDF5, NetCDF-C, and NetCDF-Fortran libraries +RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ + wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + m4 && \ + rm -rf /var/lib/apt/lists/* + +RUN cd zlib-1.2.13 && \ + ./configure --prefix=${INSTALL_DIR} --static && \ + make && \ + make install + +RUN cd hdf5-1.14.1-2 && \ + ./configure --disable-shared \ + --enable-build-mode=production \ + --disable-fortran \ + --disable-java \ + --disable-cxx \ + --prefix=${INSTALL_DIR} \ + --with-zlib=${INSTALL_DIR} && \ + make && \ + make install + +RUN cd netcdf-c-4.9.2 && \ + ./configure --disable-shared \ + --disable-dap \ + --disable-libxml2 \ + --disable-byterange \ + --prefix=${INSTALL_DIR} && \ + make && \ + make install + ENV NCDIR="${INSTALL_DIR}" ENV NFDIR="${INSTALL_DIR}" ENV HDF5_ROOT="${INSTALL_DIR}" @@ -87,39 +75,12 @@ ENV HDF5_LIBDIR="${HDF5_ROOT}/lib" ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" -# Get the HDF5, NetCDF-C and NetCDF-Fortran libraries -RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ - wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ - apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ - rm -rf /var/lib/apt/lists/* && \ - cd hdf && \ - ./HDF5-1.14.1-Linux.sh --skip-license && \ - cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ - cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ && \ - cp /zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ - -ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" -ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" -RUN cd netcdf-c-4.9.2 && \ - cmake -S . -B build -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DBUILD_SHARED_LIBS=OFF && \ - cmake --build build && \ - cmake --install build - # NetCDF-Fortran library -ENV F77=${FC} ENV CFLAGS="-fPIC" ENV FCFLAGS="${CFLAGS} -standard-semantics" ENV FFLAGS=${CFLAGS} -ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl" -ENV LDFLAGS="-static-intel" -ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +ENV CPPFLAGS="-I${INSTALL_DIR}/include" +ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lm -lz" RUN cd netcdf-fortran-4.6.1 && \ ./configure --disable-shared --prefix=${NFDIR} && \ make && \ @@ -131,38 +92,34 @@ ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} ENV NETCDF_LIBRARY=${NETCDF_HOME} ENV FOR_COARRAY_NUM_IMAGES=1 ENV OMP_NUM_THREADS=1 -ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV FC="${ONEAPI_ROOT}/mpi/latest/bin/mpiifort" ENV FFLAGS="-fPIC -standard-semantics" -ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" +ENV LDFLAGS="-L/usr/local/lib" +ENV LIBS="-lhdf5_hl -lhdf5 -lz" COPY ./cmake/ /swiftest/cmake/ COPY ./src/ /swiftest/src/ COPY ./CMakeLists.txt /swiftest/ -RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)\n' \ - 'find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(HDF5_HL_LIBRARY NAMES libhdf5_hl.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(HDF5_LIBRARY NAMES libhdf5.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(Z_LIBRARY NAMES libz.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(ZSTD_LIBRARY NAMES libzstd.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(SZ_LIBRARY NAMES libsz.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(BZ2_LIBRARY NAMES libbz2.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(CURL_LIBRARY NAMES libcurl.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(XML2_LIBRARY NAMES libxml2.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'set(NETCDF_FOUND TRUE)\n' \ - 'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \ - 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ - 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ - cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=ON -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ - cmake --build build && \ - cp bin/swiftest_driver /usr/local/bin/swiftest_driver_caf && \ - rm -rf build && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ +RUN cd swiftest && \ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DMACHINE_CODE_VALUE=${MACHINE_CODE} \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DUSE_COARRAY=OFF \ + -DBUILD_SHARED_LIBS=OFF && \ cmake --build build && \ cmake --install build -# Production container -FROM continuumio/miniconda3 +# This build target creates a container that executes just the driver program +FROM ubuntu:22.04 as driver +COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin/ +ENTRYPOINT ["/usr/local/bin/swiftest_driver"] + +# This build target exports the binary to the host +FROM scratch AS export_driver +COPY --from=build /usr/local/bin/swiftest_driver / + +# This build target creates a container with a conda environment with all dependencies needed to run the Python front end and +# analysis tools +FROM continuumio/miniconda3 as python SHELL ["/bin/bash", "--login", "-c"] ENV SHELL="/bin/bash" ENV PATH="/opt/conda/bin:${PATH}" @@ -170,11 +127,7 @@ ENV LD_LIBRARY_PATH="/usr/local/lib" COPY environment.yml . -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libsz2 libcurl3-gnutls libxml2 && \ - rm -rf /var/lib/apt/lists/* && \ - conda update --all -y && \ +RUN conda update --all -y && \ conda install conda-libmamba-solver -y && \ conda config --set solver libmamba && \ conda env create -f environment.yml && \ @@ -183,12 +136,6 @@ RUN apt-get update && apt-get upgrade -y && \ COPY ./python/. /opt/conda/pkgs/ COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/envs/swiftest-env/bin/ -COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/bin/ -COPY --from=build /usr/local/bin/swiftest_driver_caf /opt/conda/envs/swiftest-env/bin/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /opt/conda/envs/swiftest-env/lib/ -COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /opt/conda/envs/swiftest-env/lib/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /opt/conda/envs/swiftest-env/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /opt/conda/envs/swiftest-env/bin/ # Start new shell to activate the environment and install Swiftest RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ @@ -199,6 +146,6 @@ RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ mkdir -p /.config/matplotlib && \ chmod -R 777 /.cache/matplotlib && \ chmod -R 777 /.config/matplotlib && \ - ln -s /opt/conda/bin/swiftest_driver /opt/conda/bin/driver + ln -s /opt/conda/envs/swiftest-env/bin/swiftest_driver /opt/conda/bin/driver ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "swiftest-env"] \ No newline at end of file diff --git a/cmake/Modules/SetFortranFlags.cmake b/cmake/Modules/SetFortranFlags.cmake index 47361de08..4c8cc9b85 100644 --- a/cmake/Modules/SetFortranFlags.cmake +++ b/cmake/Modules/SetFortranFlags.cmake @@ -46,6 +46,16 @@ ELSE() MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE not valid! ${BUILD_TYPE_MSG}") ENDIF(BT STREQUAL "RELEASE") +IF (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + IF (APPLE) + SET(MACHINE_CODE_VALUE "tune=native" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.") + ELSE () + SET(MACHINE_CODE_VALUE "arch=native" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.") + ENDIF () +ELSE () + SET(MACHINE_CODE_VALUE "host" CACHE STRING "Tells the compiler which processor features it may target, including which instruction sets and optimizations it may generate.") +ENDIF () + ######################################################### # If the compiler flags have already been set, return now ######################################################### @@ -105,14 +115,7 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" ) -IF (CONTAINERIZE) - # There is some bug where -march=native doesn't work on Mac - IF(APPLE) - SET(GNUNATIVE "-mtune=generic") - ELSE() - SET(GNUNATIVE "-march=generic") - ENDIF() - +IF (NOT BUILD_SHARED_LIBS) # Use static Intel libraries SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}" Fortran "-static-intel" # Intel @@ -127,17 +130,7 @@ IF (CONTAINERIZE) Fortran "-qopenmp-link=static" # Intel ) ENDIF (USE_OPENMP) - -ELSE () - # There is some bug where -march=native doesn't work on Mac - IF(APPLE) - SET(GNUNATIVE "-mtune=native") - ELSE() - SET(GNUNATIVE "-march=native") - ENDIF() -ENDIF (CONTAINERIZE) - - +ENDIF () IF (USE_SIMD) # Enables OpenMP SIMD compilation when OpenMP parallelization is disabled. @@ -148,21 +141,12 @@ IF (USE_SIMD) ) ENDIF (NOT USE_OPENMP) - IF (CONTAINERIZE) - # Optimize for an old enough processor that it should run on most computers - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-xSSE2" # Intel - "/QxSSE2" # Intel Windows - ${GNUNATIVE} # GNU - ) - ELSE () - # Optimize for the host's architecture - SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" - Fortran "-xhost" # Intel - "/QxHost" # Intel Windows - ${GNUNATIVE} # GNU - ) - ENDIF (CONTAINERIZE) + # Optimize for an old enough processor that it should run on most computers + SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" + Fortran "-x${MACHINE_CODE_VALUE}" # Intel + "/Qx${MACHINE_CODE_VALUE}" # Intel Windows + "-m${MACHINE_CODE_VALUE}" # GNU + ) # Generate an extended set of vector functions SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" diff --git a/docker/Dockerfile.swiftest_driver b/docker/Dockerfile.swiftest_driver index 8414f9016..08480c058 100644 --- a/docker/Dockerfile.swiftest_driver +++ b/docker/Dockerfile.swiftest_driver @@ -1,85 +1,60 @@ -FROM ubuntu:20.04 as build +# Copyright 2023 - David Minton +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. +# +# This Dockerfile will build the Swiftest driver program with minimal external dependencies using the Intel Oneapi toolkit. +# This is done by building static versions of a minimal set of libraries that NetCDF-Fortran needs (Netcdf-C, HDF5, and Zlib). +# These, along with the Intel runtime libraries, are linked statically to the executable. Only the OS-specific libraries are linked +# dynamically. +FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build -# kick everything off -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \ - rm -rf /var/lib/apt/lists/* && \ - mkdir -p cmake/build && \ - cd cmake/build && \ - curl -LO https://github.com/Kitware/CMake/releases/download/v3.26.2/cmake-3.26.2-linux-x86_64.sh && \ - /bin/bash cmake-3.26.2-linux-x86_64.sh --prefix=/usr/local --skip-license && \ - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ - | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ - apt-get -y update && apt-get upgrade -y && \ - apt-get install -y intel-hpckit +ENV INSTALL_DIR="/usr/local" +ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx" +ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx" +ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx" +ENV F77="${FC}" -# Set Intel compiler environment variables -ENV INTEL_DIR="/opt/intel/oneapi" -ENV LANG=C.UTF-8 -ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards' -ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0' -ENV APM='/opt/intel/oneapi/advisor/2023.1.0/perfmodels' -ENV CCL_CONFIGURATION='cpu_gpu_dpcpp' -ENV CCL_ROOT='/opt/intel/oneapi/ccl/2021.9.0' -ENV CLASSPATH='/opt/intel/oneapi/mpi/2021.9.0//lib/mpi.jar:/opt/intel/oneapi/dal/2023.1.0/lib/onedal.jar' -ENV CLCK_ROOT='/opt/intel/oneapi/clck/2021.7.3' -ENV CMAKE_PREFIX_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/..:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/cmake:/opt/intel/oneapi/dal/2023.1.0:/opt/intel/oneapi/compiler/2023.1.0/linux/IntelDPCPP:/opt/intel/oneapi/ccl/2021.9.0/lib/cmake/oneCCL' -ENV CMPLR_ROOT='/opt/intel/oneapi/compiler/2023.1.0' -ENV CPATH='/opt/intel/oneapi/tbb/2021.9.0/env/../include:/opt/intel/oneapi/mpi/2021.9.0//include:/opt/intel/oneapi/mkl/2023.1.0/include:/opt/intel/oneapi/ippcp/2021.7.0/include:/opt/intel/oneapi/ipp/2021.8.0/include:/opt/intel/oneapi/dpl/2022.1.0/linux/include:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/include:/opt/intel/oneapi/dev-utilities/2021.9.0/include:/opt/intel/oneapi/dal/2023.1.0/include:/opt/intel/oneapi/ccl/2021.9.0/include/cpu_gpu_dpcpp' -ENV CPLUS_INCLUDE_PATH='/opt/intel/oneapi/clck/2021.7.3/include' -ENV DAALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DALROOT='/opt/intel/oneapi/dal/2023.1.0' -ENV DAL_MAJOR_BINARY='1' -ENV DAL_MINOR_BINARY='1' -ENV DIAGUTIL_PATH='/opt/intel/oneapi/vtune/2023.1.0/sys_check/vtune_sys_check.py:/opt/intel/oneapi/debugger/2023.1.0/sys_check/debugger_sys_check.py:/opt/intel/oneapi/compiler/2023.1.0/sys_check/sys_check.sh:/opt/intel/oneapi/advisor/2023.1.0/sys_check/advisor_sys_check.py:' -ENV DNNLROOT='/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp' -ENV DPL_ROOT='/opt/intel/oneapi/dpl/2022.1.0' -ENV FI_PROVIDER_PATH='/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib/prov:/usr/lib64/libfabric' -ENV FPGA_VARS_ARGS='' -ENV FPGA_VARS_DIR='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV GDB_INFO='/opt/intel/oneapi/debugger/2023.1.0/documentation/info/' -ENV INFOPATH='/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib' -ENV INSPECTOR_2023_DIR='/opt/intel/oneapi/inspector/2023.1.0' -ENV INTELFPGAOCLSDKROOT='/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga' -ENV INTEL_LICENSE_FILE='/opt/intel/licenses:/root/intel/licenses:/opt/intel/oneapi/clck/2021.7.3/licensing:/opt/intel/licenses:/root/intel/licenses:/Users/Shared/Library/Application Support/Intel/Licenses' -ENV INTEL_PYTHONHOME='/opt/intel/oneapi/debugger/2023.1.0/dep' -ENV IPPCP_TARGET_ARCH='intel64' -ENV IPPCRYPTOROOT='/opt/intel/oneapi/ippcp/2021.7.0' -ENV IPPROOT='/opt/intel/oneapi/ipp/2021.8.0' -ENV IPP_TARGET_ARCH='intel64' -ENV I_MPI_ROOT='/opt/intel/oneapi/mpi/2021.9.0' -ENV LD_LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/itac/2021.9.0/slib:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/libipt/intel64/lib:/opt/intel/oneapi/debugger/2023.1.0/dep/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV LIBRARY_PATH='/opt/intel/oneapi/tbb/2021.9.0/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.9.0//libfabric/lib:/opt/intel/oneapi/mpi/2021.9.0//lib/release:/opt/intel/oneapi/mpi/2021.9.0//lib:/opt/intel/oneapi/mkl/2023.1.0/lib/intel64:/opt/intel/oneapi/ippcp/2021.7.0/lib/intel64:/opt/intel/oneapi/ipp/2021.8.0/lib/intel64:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/lib:/opt/intel/oneapi/dal/2023.1.0/lib/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/clck/2021.7.3/lib/intel64:/opt/intel/oneapi/ccl/2021.9.0/lib/cpu_gpu_dpcpp' -ENV MANPATH='/opt/intel/oneapi/mpi/2021.9.0/man:/opt/intel/oneapi/itac/2021.9.0/man:/opt/intel/oneapi/debugger/2023.1.0/documentation/man:/opt/intel/oneapi/compiler/2023.1.0/documentation/en/man/common:/opt/intel/oneapi/clck/2021.7.3/man::' -ENV MKLROOT='/opt/intel/oneapi/mkl/2023.1.0' -ENV NLSPATH='/opt/intel/oneapi/mkl/2023.1.0/lib/intel64/locale/%l_%t/%N:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/locale/%l_%t/%N' -ENV OCL_ICD_FILENAMES='libintelocl_emu.so:libalteracl.so:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64/libintelocl.so' -ENV ONEAPI_ROOT='/opt/intel/oneapi' -ENV PATH='/opt/intel/oneapi/vtune/2023.1.0/bin64:/opt/intel/oneapi/mpi/2021.9.0//libfabric/bin:/opt/intel/oneapi/mpi/2021.9.0//bin:/opt/intel/oneapi/mkl/2023.1.0/bin/intel64:/opt/intel/oneapi/itac/2021.9.0/bin:/opt/intel/oneapi/inspector/2023.1.0/bin64:/opt/intel/oneapi/dev-utilities/2021.9.0/bin:/opt/intel/oneapi/debugger/2023.1.0/gdb/intel64/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/clck/2021.7.3/bin/intel64:/opt/intel/oneapi/advisor/2023.1.0/bin64:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' -ENV PKG_CONFIG_PATH='/opt/intel/oneapi/vtune/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/tbb/2021.9.0/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/mkl/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ippcp/2021.7.0/lib/pkgconfig:/opt/intel/oneapi/inspector/2023.1.0/include/pkgconfig/lib64:/opt/intel/oneapi/dpl/2022.1.0/lib/pkgconfig:/opt/intel/oneapi/dnnl/2023.1.0/cpu_dpcpp_gpu_dpcpp/../lib/pkgconfig:/opt/intel/oneapi/dal/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/compiler/2023.1.0/lib/pkgconfig:/opt/intel/oneapi/ccl/2021.9.0/lib/pkgconfig:/opt/intel/oneapi/advisor/2023.1.0/include/pkgconfig/lib64:' -ENV PYTHONPATH='/opt/intel/oneapi/advisor/2023.1.0/pythonapi' -ENV SETVARS_COMPLETED='1' -ENV TBBROOT='/opt/intel/oneapi/tbb/2021.9.0/env/..' -ENV VTUNE_PROFILER_2023_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VTUNE_PROFILER_DIR='/opt/intel/oneapi/vtune/2023.1.0' -ENV VT_ADD_LIBS='-ldwarf -lelf -lvtunwind -lm -lpthread' -ENV VT_LIB_DIR='/opt/intel/oneapi/itac/2021.9.0/lib' -ENV VT_MPI='impi4' -ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0' -ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib' +# Get the HDF5, NetCDF-C, and NetCDF-Fortran libraries +RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ + wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ + wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz -# Set HDF5 and NetCDF-specific Environment variables -ENV INSTALL_DIR="/usr/local" -ENV LIB_DIR="${INSTALL_DIR}/lib" -ENV LD_LIBRARY_PATH=${LIB_DIR}:${LD_LIBRARY_PATH} -RUN mkdir -p ${LIB_DIR} +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + m4 && \ + rm -rf /var/lib/apt/lists/* + +RUN cd zlib-1.2.13 && \ + ./configure --prefix=${INSTALL_DIR} --static && \ + make && \ + make install + +RUN cd hdf5-1.14.1-2 && \ + ./configure --disable-shared \ + --enable-build-mode=production \ + --disable-fortran \ + --disable-java \ + --disable-cxx \ + --prefix=${INSTALL_DIR} \ + --with-zlib=${INSTALL_DIR} && \ + make && \ + make install + +RUN cd netcdf-c-4.9.2 && \ + ./configure --disable-shared \ + --disable-dap \ + --disable-libxml2 \ + --disable-byterange \ + --prefix=${INSTALL_DIR} && \ + make && \ + make install -ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc" -ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx" -ENV CXX="${INTEL_DIR}/compiler/latest/linux/bin/icpx" -ENV LDFLAGS="-L${LIB_DIR}" ENV NCDIR="${INSTALL_DIR}" ENV NFDIR="${INSTALL_DIR}" ENV HDF5_ROOT="${INSTALL_DIR}" @@ -87,39 +62,12 @@ ENV HDF5_LIBDIR="${HDF5_ROOT}/lib" ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" -# Get the HDF5, NetCDF-C and NetCDF-Fortran libraries -RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ - wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz && \ - apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \ - rm -rf /var/lib/apt/lists/* && \ - cd hdf && \ - ./HDF5-1.14.1-Linux.sh --skip-license && \ - cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \ - cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ && \ - cp /zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/ - -ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" -ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" -RUN cd netcdf-c-4.9.2 && \ - cmake -S . -B build -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DBUILD_SHARED_LIBS=OFF && \ - cmake --build build && \ - cmake --install build - # NetCDF-Fortran library -ENV F77=${FC} ENV CFLAGS="-fPIC" ENV FCFLAGS="${CFLAGS} -standard-semantics" ENV FFLAGS=${CFLAGS} -ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl" -ENV LDFLAGS="-static-intel" -ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" +ENV CPPFLAGS="-I${INSTALL_DIR}/include" +ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lm -lz" RUN cd netcdf-fortran-4.6.1 && \ ./configure --disable-shared --prefix=${NFDIR} && \ make && \ @@ -131,46 +79,21 @@ ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} ENV NETCDF_LIBRARY=${NETCDF_HOME} ENV FOR_COARRAY_NUM_IMAGES=1 ENV OMP_NUM_THREADS=1 -ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort" +ENV FC="${ONEAPI_ROOT}/mpi/latest/bin/mpiifort" ENV FFLAGS="-fPIC -standard-semantics" -ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2" +ENV LDFLAGS="-L/usr/local/lib" +ENV LIBS="-lhdf5_hl -lhdf5 -lz" COPY ./cmake/ /swiftest/cmake/ COPY ./src/ /swiftest/src/ COPY ./CMakeLists.txt /swiftest/ -RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)\n' \ - 'find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(HDF5_HL_LIBRARY NAMES libhdf5_hl.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(HDF5_LIBRARY NAMES libhdf5.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(Z_LIBRARY NAMES libz.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(ZSTD_LIBRARY NAMES libzstd.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(SZ_LIBRARY NAMES libsz.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(BZ2_LIBRARY NAMES libbz2.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(CURL_LIBRARY NAMES libcurl.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'find_library(XML2_LIBRARY NAMES libxml2.a HINTS ENV LD_LIBRARY_PATH)\n' \ - 'set(NETCDF_FOUND TRUE)\n' \ - 'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \ - 'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \ - 'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake && \ - cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF &&\ +RUN cd swiftest && \ + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DMACHINE_CODE_VALUE="sse2" -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF && \ cmake --build build && \ cmake --install build # Production container FROM ubuntu:20.04 -RUN apt-get update && apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libsz2 libcurl3-gnutls libxml2 && \ - rm -rf /var/lib/apt/lists/* - COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin/ -# COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ - ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cd7567afc..f360bcdbc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -108,7 +108,7 @@ SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON) # Add the needed libraries and special compiler flags ##################################################### -TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES}) +TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES} $ENV{LIBS}) IF(USE_OPENMP) SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS "${OpenMP_Fortran_FLAGS} ") From 14537731d598b6f6ff4d7a3474948f5208b30c1b Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 9 Jun 2023 18:05:04 -0400 Subject: [PATCH 146/149] Addedn environment.yml file to repo so that the conda environment can be created --- .gitignore | 1 + Dockerfile | 8 +-- docker/.gitignore | 1 - docker/Dockerfile.swiftest_driver | 99 ------------------------------- environment.yml | 22 +++++++ 5 files changed, 27 insertions(+), 104 deletions(-) delete mode 100644 docker/Dockerfile.swiftest_driver create mode 100644 environment.yml diff --git a/.gitignore b/.gitignore index e5c25df47..3ec1398e3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ dump* !singularity/ !Dockerfile !swiftest.def +!environment.yml bin/ build/* diff --git a/Dockerfile b/Dockerfile index 33c82f700..4ac07ec7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -125,12 +125,12 @@ ENV SHELL="/bin/bash" ENV PATH="/opt/conda/bin:${PATH}" ENV LD_LIBRARY_PATH="/usr/local/lib" -COPY environment.yml . - RUN conda update --all -y && \ conda install conda-libmamba-solver -y && \ - conda config --set solver libmamba && \ - conda env create -f environment.yml && \ + conda config --set solver libmamba + +COPY environment.yml . +RUN conda env create -f environment.yml && \ conda init bash && \ echo "conda activate swiftest-env" >> ~/.bashrc diff --git a/docker/.gitignore b/docker/.gitignore index c876f8c26..5c73deb60 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -4,4 +4,3 @@ !bin !bin/swiftest !bin/swiftest_driver -!Dockerfile.swiftest_driver diff --git a/docker/Dockerfile.swiftest_driver b/docker/Dockerfile.swiftest_driver deleted file mode 100644 index 08480c058..000000000 --- a/docker/Dockerfile.swiftest_driver +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2023 - David Minton -# This file is part of Swiftest. -# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty -# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# You should have received a copy of the GNU General Public License along with Swiftest. -# If not, see: https://www.gnu.org/licenses. -# -# This Dockerfile will build the Swiftest driver program with minimal external dependencies using the Intel Oneapi toolkit. -# This is done by building static versions of a minimal set of libraries that NetCDF-Fortran needs (Netcdf-C, HDF5, and Zlib). -# These, along with the Intel runtime libraries, are linked statically to the executable. Only the OS-specific libraries are linked -# dynamically. -FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build - -ENV INSTALL_DIR="/usr/local" -ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx" -ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx" -ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx" -ENV F77="${FC}" - -# Get the HDF5, NetCDF-C, and NetCDF-Fortran libraries -RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz && \ - wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz && \ - wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - m4 && \ - rm -rf /var/lib/apt/lists/* - -RUN cd zlib-1.2.13 && \ - ./configure --prefix=${INSTALL_DIR} --static && \ - make && \ - make install - -RUN cd hdf5-1.14.1-2 && \ - ./configure --disable-shared \ - --enable-build-mode=production \ - --disable-fortran \ - --disable-java \ - --disable-cxx \ - --prefix=${INSTALL_DIR} \ - --with-zlib=${INSTALL_DIR} && \ - make && \ - make install - -RUN cd netcdf-c-4.9.2 && \ - ./configure --disable-shared \ - --disable-dap \ - --disable-libxml2 \ - --disable-byterange \ - --prefix=${INSTALL_DIR} && \ - make && \ - make install - -ENV NCDIR="${INSTALL_DIR}" -ENV NFDIR="${INSTALL_DIR}" -ENV HDF5_ROOT="${INSTALL_DIR}" -ENV HDF5_LIBDIR="${HDF5_ROOT}/lib" -ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include" -ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin" - -# NetCDF-Fortran library -ENV CFLAGS="-fPIC" -ENV FCFLAGS="${CFLAGS} -standard-semantics" -ENV FFLAGS=${CFLAGS} -ENV CPPFLAGS="-I${INSTALL_DIR}/include" -ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lm -lz" -RUN cd netcdf-fortran-4.6.1 && \ - ./configure --disable-shared --prefix=${NFDIR} && \ - make && \ - make install - -# Swiftest -ENV NETCDF_HOME=${INSTALL_DIR} -ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} -ENV NETCDF_LIBRARY=${NETCDF_HOME} -ENV FOR_COARRAY_NUM_IMAGES=1 -ENV OMP_NUM_THREADS=1 -ENV FC="${ONEAPI_ROOT}/mpi/latest/bin/mpiifort" -ENV FFLAGS="-fPIC -standard-semantics" -ENV LDFLAGS="-L/usr/local/lib" -ENV LIBS="-lhdf5_hl -lhdf5 -lz" -COPY ./cmake/ /swiftest/cmake/ -COPY ./src/ /swiftest/src/ -COPY ./CMakeLists.txt /swiftest/ -RUN cd swiftest && \ - cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DMACHINE_CODE_VALUE="sse2" -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF && \ - cmake --build build && \ - cmake --install build - -# Production container -FROM ubuntu:20.04 - -COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin/ - -ENTRYPOINT ["/usr/local/bin/swiftest_driver"] \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 000000000..0e1dbb387 --- /dev/null +++ b/environment.yml @@ -0,0 +1,22 @@ +name: swiftest-env + +channels: + - conda-forge + - defaults + +dependencies: + - numpy + - scipy + - matplotlib + - pandas + - xarray + - h5netcdf + - netcdf4 + - dask + - bottleneck + - astropy + - astroquery + - tqdm + - x264 + - ffmpeg + - conda-build \ No newline at end of file From a0e5edc55ed7a781708ed09d06420f96ae677351 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 9 Jun 2023 19:03:52 -0400 Subject: [PATCH 147/149] Fixed issue that was leading to an incompatibility between glibc versions due to the miniconda3 container being based on an older Ubuntu. --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ac07ec7f..55c22e793 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # dynamically. # This build target compiles all dependencies and the swiftest driver itself -FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu22.04 as build +FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build # The MACHINE_CODE_VALUE argument is a string that is used when compiling the swiftest_driver. It is appended to the "-x" compiler # option: (-x${MACHINE_CODE_VALUE}). The default value is set to "sse2" which allows for certain SIMD instructions to be used while @@ -135,7 +135,7 @@ RUN conda env create -f environment.yml && \ echo "conda activate swiftest-env" >> ~/.bashrc COPY ./python/. /opt/conda/pkgs/ -COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/envs/swiftest-env/bin/ +COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/bin/swiftest_driver # Start new shell to activate the environment and install Swiftest RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ @@ -145,7 +145,6 @@ RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ mkdir -p /.cache/matplotlib && \ mkdir -p /.config/matplotlib && \ chmod -R 777 /.cache/matplotlib && \ - chmod -R 777 /.config/matplotlib && \ - ln -s /opt/conda/envs/swiftest-env/bin/swiftest_driver /opt/conda/bin/driver + chmod -R 777 /.config/matplotlib ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "swiftest-env"] \ No newline at end of file From 9a60cee064927b98ba53943872962eaa85a14dc2 Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 9 Jun 2023 20:06:24 -0400 Subject: [PATCH 148/149] Minor tweaks to the Dockerfile to isolate the dependency build from the driver build --- .gitignore | 1 - Dockerfile | 40 ++++++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 3ec1398e3..5056a889a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ dump* !docker/ !singularity/ !Dockerfile -!swiftest.def !environment.yml bin/ diff --git a/Dockerfile b/Dockerfile index 55c22e793..ce3e615c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,18 +13,7 @@ # dynamically. # This build target compiles all dependencies and the swiftest driver itself -FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build - -# The MACHINE_CODE_VALUE argument is a string that is used when compiling the swiftest_driver. It is appended to the "-x" compiler -# option: (-x${MACHINE_CODE_VALUE}). The default value is set to "sse2" which allows for certain SIMD instructions to be used while -# remaining # compatible with a wide range of CPUs. To get the highest performance, you can pass "host" as an argument, but the -# compiled binary # would only run on a CPU with an architecture compatible with the one that the build was performed on. -# For more details and other options, see: -# https://www.intel.com/content/www/us/en/docs/fortran-compiler/developer-guide-reference/2023-1/x-qx.html -ARG MACHINE_CODE_VALUE="sse2" - -# Build type options are DEBUG, RELEASE, PROFILE, or TESTING. -ARG BUILD_TYPE="RELEASE" +FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build_deps ENV INSTALL_DIR="/usr/local" ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx" @@ -86,6 +75,25 @@ RUN cd netcdf-fortran-4.6.1 && \ make && \ make install +FROM intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04 as build_driver +COPY --from=build_deps /usr/local/. /usr/local/ +ENV INSTALL_DIR="/usr/local" +ENV CC="${ONEAPI_ROOT}/compiler/latest/linux/bin/icx" +ENV FC="${ONEAPI_ROOT}/compiler/latest/linux/bin/ifx" +ENV CXX="${ONEAPI_ROOT}/compiler/latest/linux/bin/icpx" +ENV F77="${FC}" + +# The MACHINE_CODE_VALUE argument is a string that is used when compiling the swiftest_driver. It is appended to the "-x" compiler +# option: (-x${MACHINE_CODE_VALUE}). The default value is set to "sse2" which allows for certain SIMD instructions to be used while +# remaining # compatible with a wide range of CPUs. To get the highest performance, you can pass "host" as an argument, but the +# compiled binary # would only run on a CPU with an architecture compatible with the one that the build was performed on. +# For more details and other options, see: +# https://www.intel.com/content/www/us/en/docs/fortran-compiler/developer-guide-reference/2023-1/x-qx.html +ARG MACHINE_CODE_VALUE="sse2" + +# Build type options are DEBUG, RELEASE, PROFILE, or TESTING. +ARG BUILD_TYPE="RELEASE" + # Swiftest ENV NETCDF_HOME=${INSTALL_DIR} ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME} @@ -109,13 +117,13 @@ RUN cd swiftest && \ cmake --install build # This build target creates a container that executes just the driver program -FROM ubuntu:22.04 as driver -COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin/ +FROM ubuntu:20.04 as driver +COPY --from=build_driver /usr/local/bin/swiftest_driver /usr/local/bin/ ENTRYPOINT ["/usr/local/bin/swiftest_driver"] # This build target exports the binary to the host FROM scratch AS export_driver -COPY --from=build /usr/local/bin/swiftest_driver / +COPY --from=build_driver /usr/local/bin/swiftest_driver / # This build target creates a container with a conda environment with all dependencies needed to run the Python front end and # analysis tools @@ -135,7 +143,7 @@ RUN conda env create -f environment.yml && \ echo "conda activate swiftest-env" >> ~/.bashrc COPY ./python/. /opt/conda/pkgs/ -COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/bin/swiftest_driver +COPY --from=build_driver /usr/local/bin/swiftest_driver /opt/conda/bin/swiftest_driver # Start new shell to activate the environment and install Swiftest RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ From 2da4a93960ef2b0b501fb4b7fa76e2bc7b7ee7ce Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 12 Jun 2023 09:21:25 -0400 Subject: [PATCH 149/149] Fixed some synchronization issues with coarrays --- src/swiftest/swiftest_coarray.f90 | 5 ++++- src/swiftest/swiftest_io.f90 | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 7dbd1a816..62e44f5ac 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -42,15 +42,18 @@ module subroutine swiftest_coarray_balance_system(nbody_system, param) write(param%display_unit,*) "ntp_min : " // trim(adjustl(min_str)) write(param%display_unit,*) "ntp_max : " // trim(adjustl(max_str)) write(param%display_unit,*) "difference: " // trim(adjustl(diff_str)) + flush(param%display_unit) + sync all if (ntp_max - ntp_min >= num_images()) then write(param%display_unit,*) trim(adjustl(diff_str)) // ">=" // trim(adjustl(ni_str)) // ": Rebalancing" + flush(param%display_unit) call nbody_system%coarray_collect(param) call nbody_system%coarray_distribute(param) write(param%display_unit,*) "Rebalancing complete" else write(param%display_unit,*) trim(adjustl(diff_str)) // "<" // trim(adjustl(ni_str)) // ": No rebalancing needed" end if - call flush(param%display_unit) + flush(param%display_unit) return end subroutine swiftest_coarray_balance_system diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index eea186c48..70ddac5f7 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -437,6 +437,7 @@ module subroutine swiftest_io_dump_storage(self, param) call iotimer%stop() sync all call iotimer%report(message="File output :", unit=param%display_unit) + flush(param%display_unit) #endif end associate