From a0a179b9aca7f7cb11d55657e55d35b3c2174d2b Mon Sep 17 00:00:00 2001 From: David A Minton Date: Mon, 20 Sep 2021 21:03:54 -0400 Subject: [PATCH] Moved logging into the walltime methods. Added a walltime submodule --- Makefile | 6 + examples/symba_mars_disk/param.in | 2 +- src/kick/kick.f90 | 38 +---- src/modules/swiftest_classes.f90 | 1 + src/modules/walltime_classes.f90 | 184 +----------------------- src/symba/symba_kick.f90 | 40 +----- src/walltime/walltime.f90 | 224 ++++++++++++++++++++++++++++++ 7 files changed, 238 insertions(+), 257 deletions(-) create mode 100644 src/walltime/walltime.f90 diff --git a/Makefile b/Makefile index 17ef3c006..56df61d5b 100644 --- a/Makefile +++ b/Makefile @@ -168,6 +168,11 @@ lib: ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \ ln -s $(SWIFTEST_HOME)/Makefile .; \ make libdir + cd $(SWIFTEST_HOME)/src/walltime; \ + rm -f Makefile.Defines Makefile; \ + ln -s $(SWIFTEST_HOME)/Makefile.Defines .; \ + ln -s $(SWIFTEST_HOME)/Makefile .; \ + make libdir fast: cd $(SWIFTEST_HOME)/src/fraggle; \ @@ -237,6 +242,7 @@ clean: cd $(SWIFTEST_HOME)/src/tides; rm -f Makefile.Defines Makefile *.gc* cd $(SWIFTEST_HOME)/src/user; rm -f Makefile.Defines Makefile *.gc* cd $(SWIFTEST_HOME)/src/util; rm -f Makefile.Defines Makefile *.gc* + cd $(SWIFTEST_HOME)/src/walltime; rm -f Makefile.Defines Makefile *.gc* cd $(SWIFTEST_HOME)/src/whm; rm -f Makefile.Defines Makefile *.gc* cd $(SWIFTEST_HOME)/bin; rm -f swiftest_* cd $(SWIFTEST_HOME)/bin; rm -f tool_* diff --git a/examples/symba_mars_disk/param.in b/examples/symba_mars_disk/param.in index fd0cc9134..023f31647 100644 --- a/examples/symba_mars_disk/param.in +++ b/examples/symba_mars_disk/param.in @@ -1,6 +1,6 @@ !Parameter file for the SyMBA-RINGMOONS test T0 0.0 -TSTOP 12000.0 +TSTOP 1200.0 DT 600.0 CB_IN cb.in PL_IN mars.in diff --git a/src/kick/kick.f90 b/src/kick/kick.f90 index 03b6a9493..b7b100e44 100644 --- a/src/kick/kick.f90 +++ b/src/kick/kick.f90 @@ -19,24 +19,17 @@ module subroutine kick_getacch_int_pl(self, param) character(len=STRMAX) :: tstr, nstr, cstr, mstr, lstyle character(len=1) :: schar + if (param%ladaptive_interactions) then if (lfirst) then call itimer%time_this_loop(param, self, self%nplpl) + write(itimer%loopname, *) "kick_getacch_int_pl" lfirst = .false. else if (itimer%check(param, self%nplpl)) call itimer%time_this_loop(param, self, self%nplpl) end if end if - if (itimer%is_on) then - write(tstr,*) param%t - write(schar,'(I1)') itimer%stage - if (itimer%stage == 1) then - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "kick_getacch_int_pl: loop timer turned on at t = " // trim(adjustl(tstr))) - end if - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "kick_getacch_int_pl: stage " // schar ) - end if - if (param%lflatten_interactions) then call kick_getacch_int_all_flat_pl(self%nbody, self%nplpl, self%k_plpl, self%xh, self%Gmass, self%radius, self%ah) else @@ -44,32 +37,7 @@ module subroutine kick_getacch_int_pl(self, param) end if if (param%ladaptive_interactions) then - if (itimer%is_on) then - if (param%lflatten_interactions) then - write(lstyle,*) "FLAT" - else - write(lstyle,*) "TRIANGULAR" - end if - call itimer%adapt(param, self, self%nplpl) - write(schar,'(I1)') itimer%stage - write(nstr,*) self%nplpl - write(cstr,*) itimer%count_finish_step - itimer%count_start_step - select case(itimer%stage) - case(1) - write(mstr,*) itimer%stage1_metric - case(2) - write(mstr,*) itimer%stage2_metric - end select - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, adjustl(lstyle) // " " // trim(adjustl(cstr)) // " " // trim(adjustl(nstr)) // " " // trim(adjustl(mstr))) - if (itimer%stage == 2) then - if (param%lflatten_interactions) then - write(lstyle,*) "FLAT " - else - write(lstyle,*) "TRIANGULAR" - end if - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "The fastest loop method tested is " // trim(adjustl(lstyle))) - end if - end if + if (itimer%is_on) call itimer%adapt(param, self, self%nplpl) end if return diff --git a/src/modules/swiftest_classes.f90 b/src/modules/swiftest_classes.f90 index 2c3b4721e..055eb4a79 100644 --- a/src/modules/swiftest_classes.f90 +++ b/src/modules/swiftest_classes.f90 @@ -126,6 +126,7 @@ module swiftest_classes character(NAMELEN) :: interaction_loops = "ADAPTIVE" !! Method used to compute interaction loops. Options are "TRIANGULAR", "FLAT", or "ADAPTIVE" ! The following are used internally, and are not set by the user, but instead are determined by the input value of INTERACTION_LOOPS logical :: lflatten_interactions = .false. !! Use the flattened upper triangular matrix for pl-pl interaction loops + logical :: lflatten_encounters = .false. !! Use the flattened upper triangular matrix for pl-pl encounter check loops logical :: ladaptive_interactions = .false. !! Adaptive interaction loop is turned on ! Logical flags to turn on or off various features of the code diff --git a/src/modules/walltime_classes.f90 b/src/modules/walltime_classes.f90 index a06f1103e..594931a1b 100644 --- a/src/modules/walltime_classes.f90 +++ b/src/modules/walltime_classes.f90 @@ -24,6 +24,7 @@ module walltime_classes end type walltimer type, extends(walltimer) :: interaction_timer + character(len=STRMAX) :: loopname !! Stores the name of the loop being timed for logging purposes integer(I8B) :: max_interactions = huge(1_I8B) !! Stores the number of pl-pl interactions that failed when attempting to flatten (e.g. out of memory). Adapting won't occur if ninteractions > max_interactions integer(I8B) :: last_interactions = 0 !! Number of interactions that were computed last time. The timer is only run if there has been a change to the number of interactions integer(I4B) :: step_counter = 0 !! Number of steps that have elapsed since the last timed loop @@ -103,189 +104,6 @@ end subroutine walltime_interaction_time_this_loop end interface - contains - - module subroutine walltime_finish(self, nsubsteps, message, param) - !! author: David A. Minton - !! - !! Ends the timer, setting step_finish to the current ticker value and printing the elapsed time information to the terminal - implicit none - ! Arguments - class(walltimer), intent(inout) :: self !! Walltimer object - integer(I4B), intent(in) :: nsubsteps !! Number of substeps used to compute the time per step - character(len=*), intent(in) :: message !! Message to prepend to the wall time terminal output - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - ! Internals - character(len=*), parameter :: walltimefmt = '" Wall time (s): ", es12.5, "; Wall time/step in this interval (s): ", es12.5' - character(len=STRMAX) :: fmt - integer(I8B) :: count_delta_step, count_delta_main - real(DP) :: wall_main !! Value of total elapsed time at the end of a timed step - real(DP) :: wall_step !! Value of elapsed time since the start of a timed step - real(DP) :: wall_per_substep !! Value of time per substep - - if (.not.self%lmain_is_started) then - write(*,*) "Wall timer error: The step finish time cannot be calculated because the timer is not started!" - return - end if - - call system_clock(self%count_finish_step) - - count_delta_step = self%count_finish_step - self%count_start_step - count_delta_main = self%count_finish_step - self%count_start_main - wall_step = count_delta_step / (self%count_rate * 1.0_DP) - wall_main = count_delta_main / (self%count_rate * 1.0_DP) - wall_per_substep = wall_step / nsubsteps - - fmt = '("' // adjustl(message) // '",' // walltimefmt // ')' - write(*,trim(adjustl(fmt))) wall_main, wall_per_substep - - call self%start(param) - - return - end subroutine walltime_finish - - - module subroutine walltime_reset(self, param) - !! author: David A. Minton - !! - !! Resets the clock ticker, settting main_start to the current ticker value - implicit none - ! Arguments - class(walltimer), intent(inout) :: self !! Walltimer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - - call system_clock(self%count_start_main, self%count_rate, self%count_max) - self%lmain_is_started = .true. - call self%start(param) - - return - end subroutine walltime_reset - - - module subroutine walltime_start(self, param) - !! author: David A. Minton - !! - !! Starts the timer, setting step_start to the current ticker value - implicit none - ! Arguments - class(walltimer), intent(inout) :: self !! Walltimer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - - if (.not.self%lmain_is_started) then - write(*,*) "Wall timer error: Cannot start the step time until reset is called at least once!" - return - end if - - call system_clock(self%count_start_step) - - return - end subroutine walltime_start - - - module subroutine walltime_interaction_adapt(self, param, pl, ninteractions) - !! author: David A. Minton - !! - !! Determines which of the two loop styles is fastest and keeps that one - implicit none - class(interaction_timer), intent(inout) :: self !! Walltimer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object - integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop and to determine if number of interactions has changed since the last timing - - ! Record the elapsed time - call system_clock(self%count_finish_step) - - select case(self%stage) - case(1) - self%stage1_metric = (self%count_finish_step - self%count_start_step) / real(ninteractions, kind=DP) - case(2) - self%stage2_metric = (self%count_finish_step - self%count_start_step) / real(ninteractions, kind=DP) - self%is_on = .false. - self%step_counter = 0 - if (self%stage1_metric < self%stage2_metric) call self%flip(param, pl) ! Go back to the original style, otherwise keep the stage2 style - end select - - return - end subroutine walltime_interaction_adapt - - - module function walltime_interaction_check(self, param, ninteractions) result(ltimeit) - !! author: David A. Minton - !! - !! Checks whether or not the loop should be timed and starts the timer if the conditions for starting are met - implicit none - ! Arguments - class(interaction_timer), intent(inout) :: self !! Walltimer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop and to determine if number of interactions has changed since the last timing - logical :: ltimeit !! Logical flag indicating whether this loop should be timed or not - ! Internals - character(len=STRMAX) :: tstring - - if (self%is_on) then ! Entering the second stage of the loop timing. Therefore we will swap the interaction style and time this loop - self%stage = self%stage + 1 - ltimeit = (self%stage == 2) - else - self%step_counter = min(self%step_counter + 1, INTERACTION_TIMER_CADENCE) - ltimeit = .false. - if (self%step_counter == INTERACTION_TIMER_CADENCE) then - ltimeit = (ninteractions /= self%last_interactions) - if (ltimeit) self%stage = 1 - end if - end if - self%is_on = ltimeit - - return - end function walltime_interaction_check - - - module subroutine walltime_interaction_flip_loop_style(self, param, pl) - !! author: David A. Minton - !! - !! Flips the interaction loop style from FLAT to TRIANGULAR or vice versa - implicit none - ! Arguments - class(interaction_timer), intent(inout) :: self !! Interaction loop timer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object - - param%lflatten_interactions = .not. param%lflatten_interactions - if (param%lflatten_interactions) then - call pl%flatten(param) - else - if (allocated(pl%k_plpl)) deallocate(pl%k_plpl) - end if - - return - end subroutine walltime_interaction_flip_loop_style - - - module subroutine walltime_interaction_time_this_loop(self, param, pl, ninteractions) - !! author: David A. Minton - !! - !! Resets the interaction loop timer, and saves the current value of the array flatten parameter - implicit none - ! Arguments - class(interaction_timer), intent(inout) :: self !! Interaction loop timer object - class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters - class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object - integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop) - - self%is_on = .true. - select case(self%stage) - case(1) - self%stage1_ninteractions = ninteractions - self%stage1_is_flattened = param%lflatten_interactions - case(2) - param%lflatten_interactions = self%stage1_is_flattened - call self%flip(param, pl) - case default - self%stage = 1 - end select - call self%reset(param) - - return - end subroutine walltime_interaction_time_this_loop end module walltime_classes \ No newline at end of file diff --git a/src/symba/symba_kick.f90 b/src/symba/symba_kick.f90 index f528c7dc5..4a11e532a 100644 --- a/src/symba/symba_kick.f90 +++ b/src/symba/symba_kick.f90 @@ -16,28 +16,17 @@ module subroutine symba_kick_getacch_int_pl(self, param) ! Internals type(interaction_timer), save :: itimer logical, save :: lfirst = .true. - character(len=STRMAX) :: tstr, nstr, cstr, mstr - character(len=11) :: lstyle - character(len=1) :: schar if (param%ladaptive_interactions) then if (lfirst) then call itimer%time_this_loop(param, self, self%nplpl) + write(itimer%loopname, *) "symba_kick_getacch_int_pl" lfirst = .false. else if (itimer%check(param, self%nplpl)) call itimer%time_this_loop(param, self, self%nplpl) end if end if - if (itimer%is_on) then - write(tstr,*) param%t - write(schar,'(I1)') itimer%stage - if (itimer%stage == 1) then - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "symba_kick_getacch_int_pl: loop timer turned on at t = " // trim(adjustl(tstr))) - end if - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "symba_kick_getacch_int_pl: stage " // schar ) - end if - if (param%lflatten_interactions) then call kick_getacch_int_all_flat_pl(self%nbody, self%nplplm, self%k_plpl, self%xh, self%Gmass, self%radius, self%ah) else @@ -45,32 +34,7 @@ module subroutine symba_kick_getacch_int_pl(self, param) end if if (param%ladaptive_interactions) then - if (itimer%is_on) then - if (param%lflatten_interactions) then - write(lstyle,*) "FLAT " - else - write(lstyle,*) "TRIANGULAR" - end if - call itimer%adapt(param, self, self%nplpl) - write(schar,'(I1)') itimer%stage - write(nstr,*) self%nplpl - write(cstr,*) itimer%count_finish_step - itimer%count_start_step - select case(itimer%stage) - case(1) - write(mstr,*) itimer%stage1_metric - case(2) - write(mstr,*) itimer%stage2_metric - end select - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, adjustl(lstyle) // " " // trim(adjustl(cstr)) // " " // trim(adjustl(nstr)) // " " // trim(adjustl(mstr))) - if (itimer%stage == 2) then - if (param%lflatten_interactions) then - write(lstyle,*) "FLAT " - else - write(lstyle,*) "TRIANGULAR" - end if - call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "The fastest loop method tested is " // trim(adjustl(lstyle))) - end if - end if + if (itimer%is_on) call itimer%adapt(param, self, self%nplpl) end if return diff --git a/src/walltime/walltime.f90 b/src/walltime/walltime.f90 new file mode 100644 index 000000000..325c9a2c4 --- /dev/null +++ b/src/walltime/walltime.f90 @@ -0,0 +1,224 @@ +submodule(walltime_classes) s_walltime + use swiftest +contains + + module subroutine walltime_finish(self, nsubsteps, message, param) + !! author: David A. Minton + !! + !! Ends the timer, setting step_finish to the current ticker value and printing the elapsed time information to the terminal + implicit none + ! Arguments + class(walltimer), intent(inout) :: self !! Walltimer object + integer(I4B), intent(in) :: nsubsteps !! Number of substeps used to compute the time per step + character(len=*), intent(in) :: message !! Message to prepend to the wall time terminal output + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + ! Internals + character(len=*), parameter :: walltimefmt = '" Wall time (s): ", es12.5, "; Wall time/step in this interval (s): ", es12.5' + character(len=STRMAX) :: fmt + integer(I8B) :: count_delta_step, count_delta_main + real(DP) :: wall_main !! Value of total elapsed time at the end of a timed step + real(DP) :: wall_step !! Value of elapsed time since the start of a timed step + real(DP) :: wall_per_substep !! Value of time per substep + + if (.not.self%lmain_is_started) then + write(*,*) "Wall timer error: The step finish time cannot be calculated because the timer is not started!" + return + end if + + call system_clock(self%count_finish_step) + + count_delta_step = self%count_finish_step - self%count_start_step + count_delta_main = self%count_finish_step - self%count_start_main + wall_step = count_delta_step / (self%count_rate * 1.0_DP) + wall_main = count_delta_main / (self%count_rate * 1.0_DP) + wall_per_substep = wall_step / nsubsteps + + fmt = '("' // adjustl(message) // '",' // walltimefmt // ')' + write(*,trim(adjustl(fmt))) wall_main, wall_per_substep + + call self%start(param) + + return + end subroutine walltime_finish + + + module subroutine walltime_reset(self, param) + !! author: David A. Minton + !! + !! Resets the clock ticker, settting main_start to the current ticker value + implicit none + ! Arguments + class(walltimer), intent(inout) :: self !! Walltimer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + + call system_clock(self%count_start_main, self%count_rate, self%count_max) + self%lmain_is_started = .true. + call self%start(param) + + return + end subroutine walltime_reset + + + module subroutine walltime_start(self, param) + !! author: David A. Minton + !! + !! Starts the timer, setting step_start to the current ticker value + implicit none + ! Arguments + class(walltimer), intent(inout) :: self !! Walltimer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + + if (.not.self%lmain_is_started) then + write(*,*) "Wall timer error: Cannot start the step time until reset is called at least once!" + return + end if + + call system_clock(self%count_start_step) + + return + end subroutine walltime_start + + + module subroutine walltime_interaction_adapt(self, param, pl, ninteractions) + !! author: David A. Minton + !! + !! Determines which of the two loop styles is fastest and keeps that one + implicit none + ! Arguments + class(interaction_timer), intent(inout) :: self !! Walltimer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object + integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop and to determine if number of interactions has changed since the last timing + ! Internals + character(len=STRMAX) :: tstr, nstr, cstr, mstr + character(len=11) :: lstyle + character(len=1) :: schar + + ! Record the elapsed time + call system_clock(self%count_finish_step) + + if (param%lflatten_interactions) then + write(lstyle,*) "FLAT " + else + write(lstyle,*) "TRIANGULAR" + end if + write(schar,'(I1)') self%stage + write(nstr,*) ninteractions + + select case(self%stage) + case(1) + self%stage1_metric = (self%count_finish_step - self%count_start_step) / real(ninteractions, kind=DP) + write(mstr,*) self%stage2_metric + case(2) + self%stage2_metric = (self%count_finish_step - self%count_start_step) / real(ninteractions, kind=DP) + self%is_on = .false. + self%step_counter = 0 + if (self%stage1_metric < self%stage2_metric) call self%flip(param, pl) ! Go back to the original style, otherwise keep the stage2 style + write(mstr,*) self%stage1_metric + end select + + write(cstr,*) self%count_finish_step - self%count_start_step + + call io_log_one_message(INTERACTION_TIMER_LOG_OUT, adjustl(lstyle) // " " // trim(adjustl(cstr)) // " " // trim(adjustl(nstr)) // " " // trim(adjustl(mstr))) + + if (self%stage == 2) then + if (param%lflatten_interactions) then + write(lstyle,*) "FLAT " + else + write(lstyle,*) "TRIANGULAR" + end if + call io_log_one_message(INTERACTION_TIMER_LOG_OUT, "The fastest loop method tested is " // trim(adjustl(lstyle))) + end if + + return + end subroutine walltime_interaction_adapt + + + module function walltime_interaction_check(self, param, ninteractions) result(ltimeit) + !! author: David A. Minton + !! + !! Checks whether or not the loop should be timed and starts the timer if the conditions for starting are met + implicit none + ! Arguments + class(interaction_timer), intent(inout) :: self !! Walltimer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop and to determine if number of interactions has changed since the last timing + logical :: ltimeit !! Logical flag indicating whether this loop should be timed or not + ! Internals + character(len=STRMAX) :: tstring + + if (self%is_on) then ! Entering the second stage of the loop timing. Therefore we will swap the interaction style and time this loop + self%stage = self%stage + 1 + ltimeit = (self%stage == 2) + else + self%step_counter = min(self%step_counter + 1, INTERACTION_TIMER_CADENCE) + ltimeit = .false. + if (self%step_counter == INTERACTION_TIMER_CADENCE) then + ltimeit = (ninteractions /= self%last_interactions) + if (ltimeit) self%stage = 1 + end if + end if + self%is_on = ltimeit + + return + end function walltime_interaction_check + + + module subroutine walltime_interaction_flip_loop_style(self, param, pl) + !! author: David A. Minton + !! + !! Flips the interaction loop style from FLAT to TRIANGULAR or vice versa + implicit none + ! Arguments + class(interaction_timer), intent(inout) :: self !! Interaction loop timer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object + + param%lflatten_interactions = .not. param%lflatten_interactions + if (param%lflatten_interactions) then + call pl%flatten(param) + else + if (allocated(pl%k_plpl)) deallocate(pl%k_plpl) + end if + + return + end subroutine walltime_interaction_flip_loop_style + + + module subroutine walltime_interaction_time_this_loop(self, param, pl, ninteractions) + !! author: David A. Minton + !! + !! Resets the interaction loop timer, and saves the current value of the array flatten parameter + implicit none + ! Arguments + class(interaction_timer), intent(inout) :: self !! Interaction loop timer object + class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters + class(swiftest_pl), intent(inout) :: pl !! Swiftest massive body object + integer(I8B), intent(in) :: ninteractions !! Current number of interactions (used to normalize the timed loop) + ! Internals + character(len=STRMAX) :: tstr + character(len=1) :: schar + + self%is_on = .true. + write(tstr,*) param%t + select case(self%stage) + case(1) + self%stage1_ninteractions = ninteractions + self%stage1_is_flattened = param%lflatten_interactions + call io_log_one_message(INTERACTION_TIMER_LOG_OUT, trim(adjustl(self%loopname)) // ": loop timer turned on at t = " // trim(adjustl(tstr))) + case(2) + param%lflatten_interactions = self%stage1_is_flattened + call self%flip(param, pl) + case default + self%stage = 1 + end select + + write(schar,'(I1)') self%stage + call io_log_one_message(INTERACTION_TIMER_LOG_OUT, trim(adjustl(self%loopname)) // ": stage " // schar ) + + call self%reset(param) + + return + end subroutine walltime_interaction_time_this_loop + +end submodule s_walltime \ No newline at end of file