Merge branch 'debug' of https://github.com/carlislewishard/swiftest i…

…nto debug

CMakeLists.txt

@@ -32,10 +32,15 @@ OPTION(USE_COARRAY "Use Coarray Fortran for parallelization of test particles" O
 OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)
 OPTION(USE_SIMD "Use SIMD vectorization" ON)
 OPTION(CONTAINERIZE "Compiling for use in a Docker/Singularity container" OFF)
+OPTION(BUILD_SHARED_LIBS "Build using shared libraries" ON)
 
-# Locate and set parallelization libraries.  There are some CMake peculiarities
+# Locate and set external libraries.  There are some CMake peculiarities
 # taken care of here, such as the fact that the FindOpenMP routine doesn't know
 # about Fortran.
+IF (NOT BUILD_SHARED_LIBS)
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
+ENDIF ()
+
 INCLUDE(${CMAKE_MODULE_PATH}/SetParallelizationLibrary.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetUpNetCDF.cmake)
 INCLUDE(${CMAKE_MODULE_PATH}/SetMKL.cmake)

docker/intel/Dockerfile → Dockerfile

@@ -1,9 +1,9 @@
-FROM debian:stable-slim as build
+FROM ubuntu:20.04 as build
 
 # kick everything off
 RUN apt-get update && apt-get upgrade -y && \
   DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-  ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config libaec-dev procps && \
+  ca-certificates curl git wget gpg-agent software-properties-common build-essential gnupg pkg-config && \
   rm -rf /var/lib/apt/lists/*
 
 # Get CMAKE and install it
@@ -21,36 +21,8 @@ RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https:/
 RUN apt-get -y update && apt-get upgrade -y
 RUN apt-get install -y intel-hpckit
 
-# Build the NetCDF libraries
-RUN mkdir -p /opt/build && mkdir -p /opt/dist
-ENV INDIR="/opt/dist//usr/local"
+# Set Intel compiler environment variables
 ENV INTEL_DIR="/opt/intel/oneapi"
-ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc"
-ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx"
-
-RUN apt-get update && apt-get upgrade -y && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-  libhdf5-dev hdf5-tools zlib1g zlib1g-dev libxml2-dev libcurl4-gnutls-dev m4 && \
-  rm -rf /var/lib/apt/lists/*
-
-
-#NetCDF-c library
-RUN git clone https://github.com/Unidata/netcdf-c.git
-RUN cd netcdf-c && mkdir build && cd build && \
-   cmake ..  -DCMAKE_PREFIX_PATH="${LD_LIBRARY_PATH}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX="${INDIR}"  && \
-   make && make install
-
-#NetCDF-Fortran library
-RUN git clone https://github.com/Unidata/netcdf-fortran.git
-RUN cd netcdf-fortran && mkdir build && cd build && \
-   cmake .. -DCMAKE_INSTALL_PREFIX="${INDIR}"  && \
-   make && make install
-
-# #Swiftest
-RUN git clone -b debug https://github.com/carlislewishard/swiftest.git
-ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort"
-ENV NETCDF_HOME="${INDIR}"
-ENV NETCDF_FORTRAN_HOME="${INDIR}"
 ENV LANG=C.UTF-8
 ENV ACL_BOARD_VENDOR_PATH='/opt/Intel/OpenCLFPGA/oneAPI/Boards'
 ENV ADVISOR_2023_DIR='/opt/intel/oneapi/advisor/2023.1.0'
@@ -104,26 +76,118 @@ ENV VT_MPI='impi4'
 ENV VT_ROOT='/opt/intel/oneapi/itac/2021.9.0'
 ENV VT_SLIB_DIR='/opt/intel/oneapi/itac/2021.9.0/slib'
 
-RUN cd swiftest && cmake -P distclean.cmake && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH="${INDIR}" -DCMAKE_INSTALL_PREFIX="${INDIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release && make && make install
+# Set HDF5 and NetCDF-specific Environment variables
+ENV INSTALL_DIR="/usr/local"
+ENV LIB_DIR="${INSTALL_DIR}/lib"
+ENV LD_LIBRARY_PATH=${LIB_DIR}:${LD_LIBRARY_PATH}
+RUN mkdir -p ${LIB_DIR}
 
-#Production container
-FROM debian:stable-slim
-COPY --from=build /opt/dist /
+ENV CC="${INTEL_DIR}/compiler/latest/linux/bin/icx-cc"
+ENV FC="${INTEL_DIR}/compiler/latest/linux/bin/ifx"
+ENV CXX="${INTEL_DIR}/compiler/latest/linux/bin/icpx"
+ENV LDFLAGS="-L${LIB_DIR}"
+ENV NCDIR="${INSTALL_DIR}"
+ENV NFDIR="${INSTALL_DIR}"
+ENV HDF5_ROOT="${INSTALL_DIR}"
+ENV HDF5_LIBDIR="${HDF5_ROOT}/lib"
+ENV HDF5_INCLUDE_DIR="${HDF5_ROOT}/include"
+ENV HDF5_PLUGIN_PATH="${HDF5_LIBDIR}/plugin"
+
+# Get the HDF5, NetCDF-C and NetCDF-Fortran libraries
+RUN wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/bin/unix/hdf5-1.14.1-2-Std-ubuntu2004_64-Intel.tar.gz | tar xvz 
+RUN wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz
+RUN wget -qO- https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.1.tar.gz | tar xvz
+RUN wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz
 
-# Get the Intel runtime libraries
+# Install dependencies
 RUN apt-get update && apt-get upgrade -y && \
   DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-  ca-certificates curl gpg-agent software-properties-common gnupg pkg-config procps && \
+  libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev m4 && \
   rm -rf /var/lib/apt/lists/*
 
-RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | apt-key add -
-RUN echo "deb [trusted=yes] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list
-RUN apt-get -y update && apt-get upgrade -y
-RUN apt-get install -y intel-oneapi-runtime-openmp intel-oneapi-runtime-mkl intel-oneapi-runtime-mpi intel-oneapi-runtime-fortran 
-ENV NETCDF_HOME="/usr/local"
-ENV LANG=C.UTF-8
-ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/intel/oneapi/lib"
-RUN apt-get -y update && apt-get upgrade -y
-RUN apt-get install -y libhdf5-dev libxml2-dev 
+# Install HDF5
+RUN cd hdf && \
+  ./HDF5-1.14.1-Linux.sh --skip-license && \
+  cp -R HDF_Group/HDF5/1.14.1/lib/*.a ${HDF5_ROOT}/lib/ && \
+  cp -R HDF_Group/HDF5/1.14.1/include/* ${HDF5_ROOT}/include/ 
+
+RUN cp zlib-1.2.13/zlib.h ${HDF5_INCLUDE_DIR}/
+
+ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}"
+ENV LDFLAGS="-static-intel -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2"
+RUN cd netcdf-c-4.9.2 && \
+  cmake -S . -B build -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \
+                      -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+                      -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+                      -DBUILD_SHARED_LIBS=OFF && \
+  cmake --build build && \
+  cmake --install build
+
+# NetCDF-Fortran library
+ENV F77=${FC}
+ENV CFLAGS="-fPIC"
+ENV FCFLAGS=${CFLAGS}
+ENV FFLAGS=${CFLAGS}
+ENV CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include -I/usr/include/x86_64-linux-gnu/curl"
+ENV LDFLAGS="-static-intel"
+ENV LIBS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdf -lhdf5_hl -lhdf5 -lsz -lm -lz -lzstd -lbz2 -lcurl -lxml2" 
+RUN cd netcdf-fortran-4.6.1 && \
+  ./configure --disable-shared --prefix=${NFDIR} && \
+  make && \
+  make install  
+
+# # Swiftest
+ENV NETCDF_HOME=${INSTALL_DIR}
+ENV NETCDF_FORTRAN_HOME=${NETCDF_HOME}
+ENV NETCDF_LIBRARY=${NETCDF_HOME}
+ENV FC="${INTEL_DIR}/mpi/latest/bin/mpiifort"
+ENV LDFLAGS="-L/usr/local/lib -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lsz -lz -lzstd -lbz2 -lcurl -lxml2"
+COPY ./cmake/ /swiftest/cmake/
+COPY ./src/ /swiftest/src/
+COPY ./CMakeLists.txt /swiftest/
+RUN echo 'find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)\n' \
+         'find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(HDF5_HL_LIBRARY NAMES libhdf5_hl.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(HDF5_LIBRARY NAMES libhdf5.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(Z_LIBRARY NAMES libz.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(ZSTD_LIBRARY NAMES libzstd.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(SZ_LIBRARY NAMES libsz.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(BZ2_LIBRARY NAMES libbz2.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(CURL_LIBRARY NAMES libcurl.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'find_library(XML2_LIBRARY NAMES libxml2.a HINTS ENV LD_LIBRARY_PATH)\n' \
+         'set(NETCDF_FOUND TRUE)\n' \
+         'set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})\n' \
+         'set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY} ${HDF5_HL_LIBRARY} ${HDF5_LIBRARY} ${SZ_LIBRARY} ${Z_LIBRARY} ${ZSTD_LIBRARY} ${BZ2_LIBRARY} ${CURL_LIBRARY} ${XML2_LIBRARY} )\n' \
+         'mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)\n' > /swiftest/cmake/Modules/FindNETCDF.cmake
+
+RUN cd swiftest && \
+  cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCONTAINERIZE=ON -DCMAKE_BUILD_TYPE=release -DBUILD_SHARED_LIBS=OFF &&\
+  cmake --build build --verbose && \
+  cmake --install build
+
+# Production container
+FROM continuumio/miniconda3
+
+RUN apt-get update && apt-get upgrade -y && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  libxml2-dev libcurl4-gnutls-dev libzstd-dev libbz2-dev libaec-dev && \
+  rm -rf /var/lib/apt/lists/*
+
+ENV LD_LIBRARY_PATH="/usr/local/lib"
+COPY --from=build /opt/intel/oneapi/mpi/latest/lib/libmpifort.so.12 /usr/local/lib/
+COPY --from=build /opt/intel/oneapi/mpi/latest/lib/release/libmpi.so.12 /usr/local/lib/
+
+RUN conda update --all -y
+RUN conda install conda-libmamba-solver -y
+RUN conda config --set solver libmamba
+RUN conda install -c conda-forge conda-build numpy scipy matplotlib pandas xarray astropy astroquery tqdm x264 bottleneck ffmpeg h5netcdf netcdf4 -y
+RUN conda update --all -y
 
-ENTRYPOINT ["/usr/local/bin/swiftest_driver"]
+COPY ./python/ .
+COPY --from=build /usr/local/bin/swiftest_driver /bin/
+RUN cd swiftest && conda develop .
+RUN mkdir -p /.astropy && \
+  chmod -R 777 /.astropy
+ENV SHELL="/bin/bash"
+ENTRYPOINT ["/opt/conda/bin/python"]

cmake/Modules/FindNETCDF.cmake

@@ -8,11 +8,14 @@
 # If not, see: https://www.gnu.org/licenses. 
 
 # - Finds the NetCDF libraries 
-find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME)
-find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME)
-find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME)
+
+find_path(NETCDF_INCLUDE_DIR NAMES netcdf.mod HINTS ENV NETCDF_FORTRAN_HOME ENV CPATH)
+find_library(NETCDF_FORTRAN_LIBRARY NAMES netcdff HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH)
+find_library(NETCDF_LIBRARY NAMES netcdf HINTS ENV NETCDF_FORTRAN_HOME ENV LD_LIBRARY_PATH)
 
 set(NETCDF_FOUND TRUE)
 set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR})
-set(NETCDF_LIBRARIES ${NETCDF_LIBRARY} ${NETCDF_FORTRAN_LIBRARY})
+# Note for posterity: When building static libraries, NETCDF_FORTRAN_LIBRARY must come *before* NETCDF_LIBRARY. Otherwise you get a bunch of "undefined reference to" errors
+set(NETCDF_LIBRARIES ${NETCDF_FORTRAN_LIBRARY} ${NETCDF_LIBRARY})
+
 mark_as_advanced(NETCDF_LIBRARY NETCDF_FORTRAN_LIBRARY NETCDF_INCLUDE_DIR)

cmake/Modules/SetFortranFlags.cmake

@@ -50,9 +50,9 @@ ENDIF(BT STREQUAL "RELEASE")
 # If the compiler flags have already been set, return now
 #########################################################
 
-IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER)
+IF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE )
     RETURN ()
-ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE AND CMAKE_Fortran_FLAGS_CONTAINER)
+ENDIF(CMAKE_Fortran_FLAGS_RELEASE AND CMAKE_Fortran_FLAGS_TESTING AND CMAKE_Fortran_FLAGS_DEBUG AND CMAKE_Fortran_FLAGS_PROFILE)
 
 ########################################################################
 # Determine the appropriate flags for this compiler for each build type.
@@ -118,13 +118,26 @@ SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
                         "/Qpad" # Intel Windows
 
                 )
+
 IF (CONTAINERIZE)
         # There is some bug where -march=native doesn't work on Mac
         IF(APPLE)
                 SET(GNUNATIVE "-mtune=generic")
         ELSE()
                 SET(GNUNATIVE "-march=generic")
         ENDIF()
+
+        # Use static Intel libraries
+        SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}"
+                Fortran "-static-intel"  # Intel
+        )
+
+        IF (USE_OPENMP)
+                SET_COMPILE_FLAG(CMAKE_Fortran_LINK_FLAGS "${CMAKE_Fortran_LINK_FLAGS}"
+                        Fortran "-qopenmp-link=static"  # Intel
+                )
+        ENDIF (USE_OPENMP)
+
 ELSE ()
         # There is some bug where -march=native doesn't work on Mac
         IF(APPLE)
@@ -148,7 +161,9 @@ IF (USE_SIMD)
         IF (CONTAINERIZE)
                 # Optimize for an old enough processor that it should run on most computers
                 SET_COMPILE_FLAG(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}"
-                                Fortran ${GNUNATIVE}    # GNU
+                                Fortran "-xSSE2" # Intel
+                                        "/QxSSE2" # Intel Windows
+                                        ${GNUNATIVE}    # GNU
                                 )
         ELSE ()
                 # Optimize for the host's architecture

docker/.gitignore

@@ -1,10 +1,5 @@
 *
 !.gitignore
 !install.sh
-!gnu/
-!gnu/Dockerfile
-!/gnu/bin/
-!/gnu/bin/swiftest_driver
-!intel/
-!intel/Dockerfile
-!/intel/bin/swiftest_driver
+!bin
+!bin/swiftest

docker/bin/swiftest

@@ -0,0 +1,2 @@
+#!/bin/sh --
+docker run -v $(pwd):$(pwd) -w $(pwd) --user "$(id -u):$(id -g)" -t -e OMP_NUM_THREADS -e FOR_COARRAY_NUM_IMAGES swiftest:1.0.0  "$@"

docker/install.sh

@@ -1,5 +1,5 @@
 #!/bin/sh --
-tag=${1:-intel}
-echo "Installing swiftest_driver:${tag} Docker container and executable script"
-docker pull daminton/swiftest_driver:${tag}
-cp -rf ${tag}/bin/swiftest_driver ../bin/
+tag=${1:-latest}
+echo "Installing swiftest:${tag} Docker container and executable script"
+docker pull daminton/swiftest:${tag}
+cp -rf bin/swiftest ../bin/

src/CMakeLists.txt

@@ -108,22 +108,17 @@ SET_SOURCE_FILES_PROPERTIES(${SWIFTEST_src} PROPERTIES Fortran_PREPROCESS ON)
 # Add the needed libraries and special compiler flags
 #####################################################
 
-# # Uncomment if you need to link to BLAS and LAPACK
-TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} ${NETCDF_LIBRARIES} ${NETCDF_FORTRAN_LIBRARIES})
-
-
+TARGET_LINK_LIBRARIES(${SWIFTEST_DRIVER} PRIVATE ${NETCDF_FORTRAN_LIBRARIES} ${NETCDF_LIBRARIES})
 
 IF(USE_OPENMP)
-    SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
-                          COMPILE_FLAGS "${OpenMP_Fortran_FLAGS}"
-                          LINK_FLAGS "${OpenMP_Fortran_FLAGS}")
+    SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS  "${OpenMP_Fortran_FLAGS} ")
+    SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY LINK_FLAGS  "${OpenMP_Fortran_FLAGS} ")
 ENDIF(USE_OPENMP)
 
 IF(USE_COARRAY)
     TARGET_COMPILE_DEFINITIONS(${SWIFTEST_DRIVER} PRIVATE -DCOARRAY)
-    SET_TARGET_PROPERTIES(${SWIFTEST_DRIVER} PROPERTIES
-                          COMPILE_FLAGS "${Coarray_Fortran_FLAGS}"
-                          LINK_FLAGS "${Coarray_Fortran_FLAGS}")
+    SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY COMPILE_FLAGS  "${Coarray_Fortran_FLAGS} ")
+    SET_PROPERTY(TARGET ${SWIFTEST_DRIVER} APPEND_STRING PROPERTY LINK_FLAGS  "${Coarray_Fortran_FLAGS} ")
 ENDIF(USE_COARRAY)