From 10935fb97bc0dc1cc094de6d47ce8b3ba57174da Mon Sep 17 00:00:00 2001 From: David Minton Date: Wed, 7 Jun 2023 16:23:29 -0400 Subject: [PATCH] Made significant improvements to the Docker, Apptainer, and Singularity containers. --- Dockerfile | 40 +++++++++++--------- apptainer/.gitignore | 3 +- apptainer/bin/{swiftest_python => swiftest} | 2 + apptainer/bin/swiftest_driver | 2 + apptainer/install.sh | 26 +++++++++---- apptainer/setenv.sh | 4 -- docker/.gitignore | 2 +- docker/bin/swiftest | 4 ++ docker/bin/swiftest_driver | 4 +- docker/bin/swiftest_python | 2 - docker/install.sh | 5 ++- python/swiftest/swiftest/simulation_class.py | 23 +++++------ singularity/.gitignore | 1 - singularity/bin/swiftest_driver | 2 + singularity/bin/swiftest_python | 2 - singularity/install.sh | 26 +++++++++---- singularity/setenv.sh | 4 -- 17 files changed, 86 insertions(+), 66 deletions(-) rename apptainer/bin/{swiftest_python => swiftest} (60%) delete mode 100755 apptainer/setenv.sh create mode 100755 docker/bin/swiftest delete mode 100755 docker/bin/swiftest_python delete mode 100755 singularity/bin/swiftest_python delete mode 100755 singularity/setenv.sh diff --git a/Dockerfile b/Dockerfile index 2d7821992..b53f8dc8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -163,7 +163,12 @@ RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN # Production container FROM continuumio/miniconda3 -COPY ./python/. /opt/conda/pkgs/swiftest/python/ +SHELL ["/bin/bash", "--login", "-c"] +ENV SHELL="/bin/bash" +ENV PATH="/opt/conda/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/lib" + +COPY environment.yml . RUN apt-get update && apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -172,27 +177,28 @@ RUN apt-get update && apt-get upgrade -y && \ conda update --all -y && \ conda install conda-libmamba-solver -y && \ conda config --set solver libmamba && \ - conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 dask -y && \ - conda update --all -y && \ - cd /opt/conda/pkgs/swiftest/python/swiftest && conda develop . && \ - conda clean --all -y + conda env create -f environment.yml && \ + conda init bash && \ + echo "conda activate swiftest-env" >> ~/.bashrc -ENV LD_LIBRARY_PATH="/usr/local/lib" -ENV SHELL="/bin/bash" -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /usr/local/lib/ -COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /usr/local/bin/ -COPY --from=build /usr/local/bin/swiftest_driver /usr/local/bin -COPY --from=build /usr/local/bin/swiftest_driver_caf /usr/local/bin/ +COPY ./python/. /opt/conda/pkgs/ +COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/envs/swiftest-env/bin/ +COPY --from=build /usr/local/bin/swiftest_driver /opt/conda/bin/ +COPY --from=build /usr/local/bin/swiftest_driver_caf /opt/conda/envs/swiftest-env/bin/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libicaf.so /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/mpi/2021.9.0//lib/release/libmpi.so.12 /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin/libintlc.so.5 /opt/conda/envs/swiftest-env/lib/ +COPY --from=build /opt/intel/oneapi/mpi/latest/bin/mpiexec.hydra /opt/conda/envs/swiftest-env/bin/ -RUN mkdir -p /.astropy && \ +# Start new shell to activate the environment and install Swiftest +RUN cd /opt/conda/pkgs/swiftest && conda develop . && \ + conda clean --all -y && \ + mkdir -p /.astropy && \ chmod -R 777 /.astropy && \ mkdir -p /.cache/matplotlib && \ mkdir -p /.config/matplotlib && \ chmod -R 777 /.cache/matplotlib && \ chmod -R 777 /.config/matplotlib && \ - mkdir -p /opt/conda/pkgs/swiftest/bin && \ - ln -s /usr/local/bin/swiftest_driver /opt/conda/pkgs/swiftest/bin/swiftest_driver + ln -s /opt/conda/bin/swiftest_driver /opt/conda/bin/driver -ENTRYPOINT ["/opt/conda/bin/python"] \ No newline at end of file +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "swiftest-env"] \ No newline at end of file diff --git a/apptainer/.gitignore b/apptainer/.gitignore index 6ddbdba9f..baed94e88 100644 --- a/apptainer/.gitignore +++ b/apptainer/.gitignore @@ -1,5 +1,4 @@ !bin/ !bin/swiftest_driver -!bin/swiftest_python +!bin/swiftest !install.sh -!setenv.sh diff --git a/apptainer/bin/swiftest_python b/apptainer/bin/swiftest similarity index 60% rename from apptainer/bin/swiftest_python rename to apptainer/bin/swiftest index 6d562c962..ca0fcfadd 100755 --- a/apptainer/bin/swiftest_python +++ b/apptainer/bin/swiftest @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} apptainer run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/apptainer/bin/swiftest_driver b/apptainer/bin/swiftest_driver index fa008737a..e88be2805 100755 --- a/apptainer/bin/swiftest_driver +++ b/apptainer/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} apptainer exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/apptainer/install.sh b/apptainer/install.sh index 4250b5abb..5d84ca34a 100755 --- a/apptainer/install.sh +++ b/apptainer/install.sh @@ -1,11 +1,21 @@ #!/bin/sh -- -# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: -# $ . ./install.sh +# This will install the Apptainer version of the swiftest_driver in place of the native compiled version into ../bin as +# well as the swiftest_python script that is used to execute a Python input file. +# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.apptainer. +# To change this, just set environment variable SIF_DIR prior to running this script. +# +# The script takes an optional argument "tag" if you want to pull a container other than "latest". # -tag=${1:-latest} -echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" -apptainer pull --force swiftest.sif docker://daminton/swiftest:${tag} +# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, +# which requires this script to be called via source: +# $ source ./install.sh +# or +# $ . ./install.sh +TAG=${1:-latest} + +SIF_DIR=${SIF_DIR:-${HOME}/.apptainer} +echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container" +apptainer pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG} +cp -rf bin/swiftest ../bin/ cp -rf bin/swiftest_driver ../bin/ -cp -rf bin/swiftest_python ../bin/ -source ./setenv.sh \ No newline at end of file +export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif \ No newline at end of file diff --git a/apptainer/setenv.sh b/apptainer/setenv.sh deleted file mode 100755 index d9f215d7c..000000000 --- a/apptainer/setenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -- -# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. -# $ . ./setenv.sh -export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore index 772f5a9d9..5c73deb60 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -2,5 +2,5 @@ !.gitignore !install.sh !bin -!bin/swiftest_python +!bin/swiftest !bin/swiftest_driver diff --git a/docker/bin/swiftest b/docker/bin/swiftest new file mode 100755 index 000000000..8c985e065 --- /dev/null +++ b/docker/bin/swiftest @@ -0,0 +1,4 @@ +#!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES mintongroup/swiftest "$@" \ No newline at end of file diff --git a/docker/bin/swiftest_driver b/docker/bin/swiftest_driver index 0f7d8b5dd..146509c36 100755 --- a/docker/bin/swiftest_driver +++ b/docker/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} +docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti --entrypoint /usr/local/bin/swiftest_driver -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES MintonGroup/swiftest "$@" \ No newline at end of file diff --git a/docker/bin/swiftest_python b/docker/bin/swiftest_python deleted file mode 100755 index fead7f338..000000000 --- a/docker/bin/swiftest_python +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -ti -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0 "$@" \ No newline at end of file diff --git a/docker/install.sh b/docker/install.sh index 1a0b2b20c..296223149 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -1,5 +1,6 @@ #!/bin/sh -- tag=${1:-latest} echo "Installing swiftest:${tag} Docker container and executable script" -docker pull daminton/swiftest:${tag} -cp -rf bin/swiftest ../bin/ \ No newline at end of file +docker pull mintongroup/swiftest:${tag} +cp -rf bin/swiftest ../bin/ +cp -rf bin/swiftest_driver ../bin/ \ No newline at end of file diff --git a/python/swiftest/swiftest/simulation_class.py b/python/swiftest/swiftest/simulation_class.py index 8971b897a..399ab0d38 100644 --- a/python/swiftest/swiftest/simulation_class.py +++ b/python/swiftest/swiftest/simulation_class.py @@ -470,6 +470,9 @@ def _type_scrub(output_data): sys.exit() except: warnings.warn(f"Error executing main swiftest_driver program", stacklevel=2) + res = p.communicate() + for line in res[1]: + print(line, end='') sys.exit() pbar.close() @@ -502,11 +505,9 @@ def run(self,dask: bool = False, **kwargs): if not self.binary_source.exists(): msg = "Path to swiftest_driver has not been set!" - msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}" + msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}" warnings.warn(msg,stacklevel=2) return - else: - shutil.copy2(self.binary_source, self.driver_executable) if not self.restart: self.clean() @@ -942,16 +943,15 @@ def set_integrator(self, update_list.append("codename") if self.codename == "Swiftest": self.binary_source = Path(_pyfile).parent.parent.parent.parent / "bin" / "swiftest_driver" - self.binary_path = self.simdir.resolve() - self.driver_executable = self.binary_path / "swiftest_driver" + self.driver_executable = self.binary_source if not self.binary_source.exists(): - warnings.warn(f"Cannot find the Swiftest driver in {str(self.binary_path)}",stacklevel=2) + warnings.warn(f"Cannot find the Swiftest driver at {str(self.binary_source)}",stacklevel=2) self.driver_executable = None else: - if self.binary_path.exists(): + if self.binary_source.exists(): self.driver_executable.resolve() else: - self.binary_path = "NOT IMPLEMENTED FOR THIS CODE" + self.binary_source = "NOT IMPLEMENTED FOR THIS CODE" self.driver_executable = None update_list.append("driver_executable") @@ -1200,8 +1200,6 @@ def set_feature(self, msg = f"Cannot create the {self.simdir.resolve()} directory: File exists." msg += "\nDelete the file or change the location of param_file" raise NotADirectoryError(msg) - self.binary_path = self.simdir.resolve() - self.driver_executable = self.binary_path / "swiftest_driver" self.param_file = Path(kwargs.pop("param_file","param.in")) if self.codename == "Swiftest": @@ -2754,7 +2752,6 @@ def write_param(self, self.driver_script = os.path.join(self.simdir, "swiftest_driver.sh") with open(self.driver_script, 'w') as f: f.write(f"#{self._shell_full}\n") - #f.write(f"source ~/.{self._shell}rc\n") f.write(f"cd {self.simdir}\n") f.write(f"{str(self.driver_executable)} {self.integrator} {str(self.param_file)} compact\n") @@ -2991,11 +2988,9 @@ def save(self, self.write_param(param_file=param_file,**kwargs) if not self.binary_source.exists(): msg = "Path to swiftest_driver has not been set!" - msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_path)}" + msg += f"\nMake sure swiftest_driver is compiled and the executable is in {str(self.binary_source.parent)}" warnings.warn(msg,stacklevel=2) return - else: - shutil.copy2(self.binary_source, self.driver_executable) elif codename == "Swifter": swifter_param = io.swiftest2swifter_param(param) if "rhill" in self.data: diff --git a/singularity/.gitignore b/singularity/.gitignore index 6ddbdba9f..b34dd2df5 100644 --- a/singularity/.gitignore +++ b/singularity/.gitignore @@ -2,4 +2,3 @@ !bin/swiftest_driver !bin/swiftest_python !install.sh -!setenv.sh diff --git a/singularity/bin/swiftest_driver b/singularity/bin/swiftest_driver index 03876195b..024e93115 100755 --- a/singularity/bin/swiftest_driver +++ b/singularity/bin/swiftest_driver @@ -1,2 +1,4 @@ #!/bin/sh -- +OMP_NUM_THREADS=${OMP_NUM_THREADS:-`nproc --all`} +FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES:-1} singularity exec --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} swiftest_driver "$@" \ No newline at end of file diff --git a/singularity/bin/swiftest_python b/singularity/bin/swiftest_python deleted file mode 100755 index 2a7b13735..000000000 --- a/singularity/bin/swiftest_python +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -- -singularity run --bind $(pwd):$(pwd) --env OMP_NUM_THREADS=${OMP_NUM_THREADS},FOR_COARRAY_NUM_IMAGES=${FOR_COARRAY_NUM_IMAGES} ${SWIFTEST_SIF} "$@" \ No newline at end of file diff --git a/singularity/install.sh b/singularity/install.sh index e7c47e2b3..233ee187f 100755 --- a/singularity/install.sh +++ b/singularity/install.sh @@ -1,11 +1,21 @@ #!/bin/sh -- -# This will install the Singularity container version of the swiftest_driver in place of the native compiled version into ../bin. -# In order to use the executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, which requires this script to be called via source: -# $ . ./install.sh +# This will install the Singularity version of the swiftest_driver in place of the native compiled version into ../bin as +# well as the swiftest_python script that is used to execute a Python input file. +# The swiftest.sif file will be copied to the SIF_DIR directory. The default location is ${HOME}/.singularity. +# To change this, just set environment variable SIF_DIR prior to running this script. +# +# The script takes an optional argument "tag" if you want to pull a container other than "latest". # -tag=${1:-latest} -echo "Installing swiftest.sif Singularity container and executable script from swiftest:${tag} Docker container" -singularity pull --force swiftest.sif docker://daminton/swiftest:${tag} +# In order to use one executable script, the SWIFTEST_SIF environment variable must be set to point to the location of swiftest.sif, +# which requires this script to be called via source: +# $ source ./install.sh +# or +# $ . ./install.sh +TAG=${1:-latest} + +SIF_DIR=${SIF_DIR:-${HOME}/.singularity} +echo "Installing ${SIF_DIR}/swiftest.sif container from mintongroup/swiftest:${TAG} Docker container" +singularity pull --force ${SIF_DIR}/swiftest.sif docker://mintongroup/swiftest:${TAG} +cp -rf bin/swiftest ../bin/ cp -rf bin/swiftest_driver ../bin/ -cp -rf bin/swiftest_python ../bin/ -source ./setenv.sh \ No newline at end of file +export SWIFTEST_SIF=${SIF_DIR}/swiftest.sif \ No newline at end of file diff --git a/singularity/setenv.sh b/singularity/setenv.sh deleted file mode 100755 index d9f215d7c..000000000 --- a/singularity/setenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -- -# This will set the SWIFTEST_SIF environment variable as long as it is executed by source. -# $ . ./setenv.sh -export SWIFTEST_SIF="${PWD}/swiftest.sif" \ No newline at end of file