From 0d4935bd21e963fba635c53f50c831dba0ce698c Mon Sep 17 00:00:00 2001
From: David Minton <daminton@purdue.edu>
Date: Mon, 5 Jun 2023 14:05:32 -0400
Subject: [PATCH] Minor changes to troubleshoot energy calc NaN in new
 container.

---
 Dockerfile                     | 43 +++++++++++++++++++++-------------
 singularity/install.sh         |  2 +-
 src/swiftest/swiftest_util.f90 |  5 ++--
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 99b0e97a3..5b0e942a1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -136,10 +136,12 @@ RUN cd netcdf-fortran-4.6.1 && \
   make && \
   make install  
 
-# # Swiftest
+# Swiftest
 ENV NETCDF_HOME=${INSTALL_DIR}
 ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME}
 ENV NETCDF_LIBRARY=${NETCDF_HOME}
+ENV FOR_COARRAY_NUM_IMAGES=1
+ENV OMP_NUM_THREADS=1
 ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort"
 ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2"
 COPY ./cmake/ /swiftest/cmake/
@@ -162,32 +164,41 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN
          'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake
 
 RUN cd swiftest && \
-  cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release -DBUILD_SHARED_LIBS=OFF &&\
+  cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DUSE_COARRAY=OFF -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_SHARED_LIBS=OFF &&\
   cmake --build build --verbose && \
   cmake --install build
 
-# Production container
-FROM continuumio/miniconda3
-
+# Driver container
+FROM ubuntu:20.04 as Driver
+COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/
+COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/
+COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin
 RUN apt-get update && apt-get upgrade -y && \
   DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-  libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev && \
+  libsz2 libcurl3-gnutls libxml2 && \
   rm -rf /var/lib/apt/lists/*
 
+# Production container
+FROM continuumio/miniconda3
+
 ENV LD_LIBRARY_PATH="/usr/local/lib"
+ENV SHELL="/bin/bash"
 COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/
 COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/
-
-RUN conda update --all -y
-RUN conda install conda-libmamba-solver -y
-RUN conda config --set solver libmamba
-RUN conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 -y
-RUN conda update --all -y
-
 COPY ./python/ .
 COPY --from=build /usr/local/bin/swiftest_driver /bin/
-RUN cd swiftest && conda develop .
-RUN mkdir -p /.astropy && \
+
+RUN apt-get update && apt-get upgrade -y && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  libsz2 libcurl3-gnutls libxml2 && \
+  rm -rf /var/lib/apt/lists/* && \
+  conda update --all -y && \
+  conda install conda-libmamba-solver -y && \
+  conda config --set solver libmamba && \
+  conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 dask -y && \
+  conda update --all -y && \
+  cd swiftest && conda develop . && \
+  mkdir -p /.astropy && \
   chmod -R 777 /.astropy
-ENV SHELL="/bin/bash"
+
 ENTRYPOINT ["/opt/conda/bin/python"]
\ No newline at end of file
diff --git a/singularity/install.sh b/singularity/install.sh
index 50d6d56df..ff2be8596 100755
--- a/singularity/install.sh
+++ b/singularity/install.sh
@@ -3,7 +3,7 @@
 # In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest_driver.sif, which requires this script to be called via source:
 # $ . ./install.sh
 # 
-tag=${1:-intel}
+tag=${1:-latest}
 echo "Installing swiftest_driver.sif Singularity container and executable script from swiftest_driver:${tag} Docker container"
 singularity pull --force swiftest_driver.sif docker://daminton/swiftest_driver:${tag}
 cp -rf bin/swiftest_driver ../bin/
diff --git a/src/swiftest/swiftest_util.f90 b/src/swiftest/swiftest_util.f90
index 2c7b803bc..c164a365a 100644
--- a/src/swiftest/swiftest_util.f90
+++ b/src/swiftest/swiftest_util.f90
@@ -1101,10 +1101,11 @@ module subroutine swiftest_util_flatten_eucl_plpl(self, param)
       class(swiftest_pl),         intent(inout) :: self  !! Swiftest massive body object
       class(swiftest_parameters), intent(inout) :: param !! Current run configuration parameters
       ! Internals
-      integer(I4B) :: err, i, j
+      integer(I4B) :: err, i, j, npl
       integer(I8B) :: k, npl8
 
-      associate(npl => self%nbody, nplpl => self%nplpl)
+      associate(nplpl => self%nplpl)
+         npl = self%nbody
          npl8 = int(npl, kind=I8B)
          nplpl = npl8 * (npl8 - 1_I8B) / 2_I8B ! number of entries in a strict lower triangle, npl x npl
          if (param%lflatten_interactions) then