From 9cf4bcb6c96eeec9a042adc60b4da1ba4197d478 Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 24 Apr 2023 16:49:27 -0400 Subject: [PATCH] Improved the efficiency of coarray test particles by allowing each image to write to file independently, and to only do a distribute/collect cycle if the system gets too far out of balance --- src/swiftest/swiftest_coarray.f90 | 38 +++++++- src/swiftest/swiftest_driver.f90 | 70 ++++++--------- src/swiftest/swiftest_io.f90 | 144 +++++++++++++++++++++++------- src/swiftest/swiftest_module.f90 | 12 +++ 4 files changed, 187 insertions(+), 77 deletions(-) diff --git a/src/swiftest/swiftest_coarray.f90 b/src/swiftest/swiftest_coarray.f90 index 9f255932d..84144ae77 100644 --- a/src/swiftest/swiftest_coarray.f90 +++ b/src/swiftest/swiftest_coarray.f90 @@ -11,6 +11,37 @@ use coarray contains + + module subroutine swiftest_coarray_balance_system(nbody_system, param) + !! author: David A. Minton + !! + !! Checks whether or not the system needs to be rebalance. Rebalancing occurs when the difference between the number of test particles between the + !! image with the smallest and largest number of test particles is larger than the number of images + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + integer(I4B), codimension[:], allocatable :: ntp + integer(I4B) :: img,ntp_min, ntp_max + + allocate(ntp[*]) + ntp = nbody_system%tp%nbody + sync all + ntp_min = huge(1) + ntp_max = 0 + do img = 1, num_images() + if (ntp[img] < ntp_min) ntp_min = ntp[img] + if (ntp[img] > ntp_max) ntp_max = ntp[img] + end do + if (ntp_max - ntp_min >= num_images()) then + call nbody_system%coarray_collect(param) + call nbody_system%coarray_distribute(param) + end if + + return + end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_coclone_body(self) !! author: David A. Minton !! @@ -451,13 +482,11 @@ module subroutine swiftest_coarray_collect_system(nbody_system, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i,j - integer(I4B), codimension[*], save :: ntp class(swiftest_tp), allocatable, codimension[:] :: cotp character(len=NAMELEN) :: image_num_char if (.not.param%lcoarray) return - sync all if (this_image() == 1) then write(image_num_char,*) num_images() write(param%display_unit,*) " Collecting test particles from " // trim(adjustl(image_num_char)) // " images." @@ -486,18 +515,18 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) integer(I4B) :: i, istart, iend, ntot, num_per_image, ncopy class(swiftest_tp), allocatable :: tp logical, dimension(:), allocatable :: lspill_list - integer(I4B), codimension[*], save :: ntp + integer(I4B), codimension[:], allocatable :: ntp character(len=NAMELEN) :: image_num_char class(swiftest_tp), allocatable, codimension[:] :: cotp class(swiftest_tp), allocatable :: tmp if (.not.param%lcoarray) return - sync all if (this_image() == 1) then write(image_num_char,*) num_images() write(param%display_unit,*) " Distributing test particles across " // trim(adjustl(image_num_char)) // " images." end if + allocate(ntp[*]) ntp = nbody_system%tp%nbody sync all ntot = ntp[1] @@ -529,4 +558,5 @@ module subroutine swiftest_coarray_distribute_system(nbody_system, param) return end subroutine swiftest_coarray_distribute_system + end submodule s_swiftest_coarray \ No newline at end of file diff --git a/src/swiftest/swiftest_driver.f90 b/src/swiftest/swiftest_driver.f90 index 81f1c8c17..69b820599 100644 --- a/src/swiftest/swiftest_driver.f90 +++ b/src/swiftest/swiftest_driver.f90 @@ -96,20 +96,20 @@ program swiftest_driver #endif #ifdef COARRAY - ! The following line lets us read in the input files one image at a time + ! The following line lets us read in the input files one image at a time. Letting each image read the input in is faster than broadcasting all of the data if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) #endif call nbody_system%initialize(system_history, param) #ifdef COARRAY + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - if (this_image() == 1) then + ! Distribute test particles to the various images + call nbody_system%coarray_distribute(param) #endif - ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. - call nbody_system%display_run_information(param, integration_timer, phase="first") -#ifdef COARRAY - end if ! this_image() == 1 -#endif + ! If this is a new run, compute energy initial conditions (if energy tracking is turned on) and write the initial conditions to file. + call nbody_system%display_run_information(param, integration_timer, phase="first") + if (param%lenergy) then if (param%lrestart) then call nbody_system%get_t0_values(system_history%nc, param) @@ -118,21 +118,10 @@ program swiftest_driver end if call nbody_system%conservation_report(param, lterminal=.true.) end if + call system_history%take_snapshot(param,nbody_system) - -#ifdef COARRAY - if (this_image() == 1) then -#endif - call nbody_system%dump(param, system_history) -#ifdef COARRAY - end if ! this_image() == 1 -#endif + call nbody_system%dump(param, system_history) -#ifdef COARRAY - if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) - ! Distribute test particles to the various images - call nbody_system%coarray_distribute(param) -#endif do iloop = istart, nloops !> Step the nbody_system forward in time call integration_timer%start() @@ -154,40 +143,37 @@ program swiftest_driver nout = nout + 1 istep = floor(istep_out * fstep_out**nout, kind=I4B) end if + + call system_history%take_snapshot(param,nbody_system) + + if (idump == dump_cadence) then + idump = 0 + call nbody_system%dump(param, system_history) + end if #ifdef COARRAY - call nbody_system%coarray_collect(param) if (this_image() == 1) then #endif - call system_history%take_snapshot(param,nbody_system) - - if (idump == dump_cadence) then - idump = 0 - call nbody_system%dump(param, system_history) - end if - call integration_timer%report(message="Integration steps:", unit=display_unit) - call nbody_system%display_run_information(param, integration_timer) - call integration_timer%reset() +#ifdef COARRAY + end if !(this_image() == 1) +#endif + call nbody_system%display_run_information(param, integration_timer) + call integration_timer%reset() +#ifdef COARRAY + if (this_image() == 1) then +#endif if (param%lenergy) call nbody_system%conservation_report(param, lterminal=.true.) #ifdef COARRAY - end if - call nbody_system%coarray_distribute(param) + end if ! (this_image() == 1) + call nbody_system%coarray_balance(param) #endif end if end if end do ! Dump any remaining history if it exists -#ifdef COARRAY - call nbody_system%coarray_collect(param) - if (this_image() == 1) then -#endif - call nbody_system%dump(param, system_history) - call nbody_system%display_run_information(param, integration_timer, phase="last") -#ifdef COARRAY - end if ! this_image() == 1 -#endif - + call nbody_system%dump(param, system_history) + call nbody_system%display_run_information(param, integration_timer, phase="last") end associate #ifdef COARRAY diff --git a/src/swiftest/swiftest_io.f90 b/src/swiftest/swiftest_io.f90 index 8b4e5ad26..a9763dc28 100644 --- a/src/swiftest/swiftest_io.f90 +++ b/src/swiftest/swiftest_io.f90 @@ -223,12 +223,26 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t type(progress_bar), save :: pbar !! Object used to print out a progress bar character(len=64) :: pbarmessage character(*), parameter :: symbacompactfmt = '(";NPLM",ES22.15,$)' +#ifdef COARRAY + character(*), parameter :: statusfmt = '("Image: ",I4, "; Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + '"; Number of active pl, tp = ", I6, ", ", I6)' + character(*), parameter :: symbastatfmt = '("Image: ",I4, "; Image: Time = ", ES12.5, "; fraction done = ", F6.3, ' // & + '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' +#else character(*), parameter :: statusfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, tp = ", I6, ", ", I6)' character(*), parameter :: symbastatfmt = '("Time = ", ES12.5, "; fraction done = ", F6.3, ' // & '"; Number of active pl, plm, tp = ", I6, ", ", I6, ", ", I6)' +#endif character(*), parameter :: pbarfmt = '("Time = ", ES12.5," of ",ES12.5)' +! The following will syncronize the images so that they report in order, and only write to file one at at ime + + +#ifdef COARRAY + ! The following line lets us read in the input files one image at a time + if (param%lcoarray .and. (this_image() /= 1)) sync images(this_image() - 1) +#endif phase_val = 1 if (present(phase)) then if (phase == "first") then @@ -240,36 +254,68 @@ module subroutine swiftest_io_display_run_information(self, param, integration_t tfrac = (self%t - param%t0) / (param%tstop - param%t0) - if (phase_val == 0) then - if (param%lrestart) then - write(param%display_unit, *) " *************** Swiftest restart " // trim(adjustl(param%integrator)) // " *************** " - else - write(param%display_unit, *) " *************** Swiftest start " // trim(adjustl(param%integrator)) // " *************** " - end if - if (param%display_style == "PROGRESS") then - call pbar%reset(param%nloops) - else if (param%display_style == "COMPACT") then - write(*,*) "SWIFTEST START " // trim(adjustl(param%integrator)) +#ifdef COARRAY + if (this_image() == 1) then +#endif + if (phase_val == 0) then + if (param%lrestart) then + write(param%display_unit, *) " *************** Swiftest restart " // trim(adjustl(param%integrator)) // " *************** " + else + write(param%display_unit, *) " *************** Swiftest start " // trim(adjustl(param%integrator)) // " *************** " + end if + if (param%display_style == "PROGRESS") then + call pbar%reset(param%nloops) + else if (param%display_style == "COMPACT") then + write(param%display_unit,*) "SWIFTEST START " // trim(adjustl(param%integrator)) + end if end if - end if +#ifdef COARRAY + end if !(this_image() == 1) +#endif if (param%display_style == "PROGRESS") then - write(pbarmessage,fmt=pbarfmt) self%t, param%tstop - call pbar%update(1_I8B,message=pbarmessage) +#ifdef COARRAY + if (this_image() == 1) then +#endif + write(pbarmessage,fmt=pbarfmt) self%t, param%tstop + call pbar%update(1_I8B,message=pbarmessage) +#ifdef COARRAY + end if !(this_image() == 1) +#endif else if (param%display_style == "COMPACT") then call self%compact_output(param,integration_timer) end if if (self%pl%nplm > 0) then +#ifdef COARRAY + write(param%display_unit, symbastatfmt) this_image(),self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody +#else write(param%display_unit, symbastatfmt) self%t, tfrac, self%pl%nbody, self%pl%nplm, self%tp%nbody +#endif else +#ifdef COARRAY + write(param%display_unit, statusfmt) this_image(),self%t, tfrac, self%pl%nbody, self%tp%nbody +#else write(param%display_unit, statusfmt) self%t, tfrac, self%pl%nbody, self%tp%nbody +#endif end if - if (phase_val == -1) then - write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " - if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) - end if +#ifdef COARRAY + if (this_image() == num_images()) then +#endif + if (phase_val == -1) then + write(param%display_unit, *)" *************** Swiftest stop " // trim(adjustl(param%integrator)) // " *************** " + if (param%display_style == "COMPACT") write(*,*) "SWIFTEST STOP" // trim(adjustl(param%integrator)) + end if +#ifdef COARRAY + end if ! this_image() == num_images() + + ! Allow the other images to report + if (param%lcoarray .and. (this_image() < num_images())) sync images(this_image() + 1) + + ! Wait for everyone to catch up + sync all +#endif return end subroutine swiftest_io_display_run_information @@ -330,19 +376,24 @@ module subroutine swiftest_io_dump_system(self, param, system_history) ! Dump the nbody_system history to file call system_history%dump(param) - allocate(param_restart, source=param) - param_restart%in_form = "XV" - param_restart%out_stat = 'APPEND' - param_restart%in_type = "NETCDF_DOUBLE" - param_restart%nc_in = param%outfile - param_restart%lrestart = .true. - param_restart%tstart = self%t - param_file_name = trim(adjustl(PARAM_RESTART_FILE)) - call param_restart%dump(param_file_name) - write(time_text,'(I0.20)') param%iloop - param_file_name = "param." // trim(adjustl(time_text)) // ".in" - call param_restart%dump(param_file_name) - +#ifdef COARRAY + if (this_image() == 1) then +#endif + allocate(param_restart, source=param) + param_restart%in_form = "XV" + param_restart%out_stat = 'APPEND' + param_restart%in_type = "NETCDF_DOUBLE" + param_restart%nc_in = param%outfile + param_restart%lrestart = .true. + param_restart%tstart = self%t + param_file_name = trim(adjustl(PARAM_RESTART_FILE)) + call param_restart%dump(param_file_name) + write(time_text,'(I0.20)') param%iloop + param_file_name = "param." // trim(adjustl(time_text)) // ".in" + call param_restart%dump(param_file_name) +#ifdef COARRAY + end if ! (this_image() == 1) +#endif return end subroutine swiftest_io_dump_system @@ -360,22 +411,53 @@ module subroutine swiftest_io_dump_storage(self, param) class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters ! Internals integer(I4B) :: i +#ifdef COARRAY + integer(I4B) :: img, tslot + integer(I4B), dimension(self%iframe) :: ntp_tot + integer(I4B), codimension[:], allocatable :: ntp +#endif if (self%iframe == 0) return call self%make_index_map() associate(nc => self%nc) - call nc%open(param) +#ifdef COARRAY + ! Get the sum of all test particles across snapshots from all images + allocate(ntp[*]) + ntp_tot(:) = 0 + do i = 1, self%iframe + if (allocated(self%frame(i)%item)) then + select type(nbody_system => self%frame(i)%item) + class is (swiftest_nbody_system) + ntp = nbody_system%tp%nbody + sync all + do img = 1, num_images() + ntp_tot(i) = ntp_tot(i) + ntp[img] + end do + end select + end if + end do + critical +#endif + call nc%open(param) do i = 1, self%iframe if (allocated(self%frame(i)%item)) then select type(nbody_system => self%frame(i)%item) class is (swiftest_nbody_system) call nbody_system%write_frame(nc, param) +#ifdef COARRAY + ! Record the correct number of test particles from all images + call nc%find_tslot(nbody_system%t, tslot) + call netcdf_io_check( nf90_put_var(nc%id, nc%ntp_varid, ntp_tot(i), start=[tslot]), "swiftest_io_dump_storage nf90_put_var ntp_varid" ) +#endif COARRAY end select deallocate(self%frame(i)%item) end if end do call nc%close() +#ifdef COARRAY + end critical +#endif end associate call self%reset() return diff --git a/src/swiftest/swiftest_module.f90 b/src/swiftest/swiftest_module.f90 index 5fac1d0ad..73fd6e2b4 100644 --- a/src/swiftest/swiftest_module.f90 +++ b/src/swiftest/swiftest_module.f90 @@ -421,6 +421,7 @@ module swiftest procedure :: coclone => swiftest_coarray_coclone_system !! Clones the image 1 body object to all other images in the coarray structure. procedure :: coarray_collect => swiftest_coarray_collect_system !! Collects all the test particles from other images into the image #1 test particle system procedure :: coarray_distribute => swiftest_coarray_distribute_system !! Distributes test particles from image #1 out to all images. + procedure :: coarray_balance => swiftest_coarray_balance_system !! Checks whether or not the test particle coarrays need to be rebalanced. #endif end type swiftest_nbody_system @@ -1700,6 +1701,17 @@ end subroutine swiftest_util_version #ifdef COARRAY interface + module subroutine swiftest_coarray_balance_system(nbody_system, param) + !! author: David A. Minton + !! + !! Checks whether or not the system needs to be rebalance. Rebalancing occurs when the image with the smallest number of test particles + !! has <90% of that of the image with the largest number of test particles. + implicit none + ! Arguments + class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + end subroutine swiftest_coarray_balance_system + module subroutine swiftest_coarray_collect_system(nbody_system, param) implicit none class(swiftest_nbody_system), intent(inout) :: nbody_system !! Swiftest nbody system