From e438bdd1c16003dac65a9029afee0b84537494bc Mon Sep 17 00:00:00 2001 From: David Minton Date: Fri, 11 Aug 2023 11:38:51 -0400 Subject: [PATCH] Continuted improvements to automated build scripts --- .dockerignore | 3 ++- Dockerfile | 21 ++++++---------- buildscripts/.gitignore | 3 ++- buildscripts/build_all.sh | 3 +-- buildscripts/build_dependencies.sh | 11 ++++---- buildscripts/fetch_dependencies.sh | 7 ++++-- buildscripts/make_environment.sh | 39 +++++++++++++++++++++++++++++ buildscripts/swiftest-build-env.yml | 15 +++++++++++ 8 files changed, 77 insertions(+), 25 deletions(-) create mode 100755 buildscripts/make_environment.sh create mode 100644 buildscripts/swiftest-build-env.yml diff --git a/.dockerignore b/.dockerignore index a852af2f5..1d60b6d6e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ !swiftest/*.h !swiftest/tests/*.py !src/*/*.f90 -!buildscripts/*.sh \ No newline at end of file +!buildscripts/*.sh +!buildscripts/*.yml diff --git a/Dockerfile b/Dockerfile index 4d735d994..125417410 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,30 +15,23 @@ # This build target compiles all dependencies and the swiftest driver itself ARG BUILDIMAGE="intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04" FROM ${BUILDIMAGE} as build-deps +ENV SCRIPT_DIR="buildscripts" SHELL ["/bin/bash", "--login", "-c"] +ENV SHELL="/bin/bash" WORKDIR /swiftest COPY . ./ -RUN buildscripts/fetch_dependencies.sh +RUN ${SCRIPT_DIR}/fetch_dependencies.sh RUN if [ "$BUILDIMAGE" = "intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04" ]; then \ - buildscripts/build_dependencies.sh Intel; \ + ${SCRIPT_DIR}/build_dependencies.sh Intel; \ else \ conda init bash && \ - conda update --all -y && \ - conda env create --file environment.yml --name swiftest-build-env && \ + conda update --name base conda -y && \ + ${SCRIPT_DIR}/make_environment.sh && \ echo "conda activate swiftest-build-env" >> ~/.bashrc && \ source ~/.bashrc && \ - conda update all -y && \ - conda install -c conda-forge scikit-build cython cmake gfortran -y &&\ - buildscripts/build_dependencies.sh GNU; \ + ${SCRIPT_DIR}/build_dependencies.sh GNU; \ fi -# RUN buildscripts/make_environment.sh -# RUN echo "conda activate swiftest-build-env" >> ~/.bashrc -# RUN source ~/.bashrc && conda activate swiftest-build-env -# RUN conda update all -y -# RUN conda install -c conda-forge scikit-build cython cmake gfortran --name swiftest-build-env -y -# RUN buildscripts/build_dependencies.sh GNU - # ENV INSTALL_DIR="/usr/local" # ENV FC="${ONEAPI_ROOT}/mpi/latest/bin/mpiifort" # ENV CC="${ONEAPI_ROOT}/mpi/latest/bin/mpicc -cc=icx" diff --git a/buildscripts/.gitignore b/buildscripts/.gitignore index e2ee4346f..120c12345 100644 --- a/buildscripts/.gitignore +++ b/buildscripts/.gitignore @@ -1 +1,2 @@ -!*.sh \ No newline at end of file +!*.sh +!*.yml \ No newline at end of file diff --git a/buildscripts/build_all.sh b/buildscripts/build_all.sh index a3bd51563..e64d22c14 100755 --- a/buildscripts/build_all.sh +++ b/buildscripts/build_all.sh @@ -24,7 +24,6 @@ # Determine the platform and architecture SCRIPT_DIR=$(dirname "$0") -echo $SCRIPT_DIR read -r OS ARCH < <($SCRIPT_DIR/get_platform.sh) # Determine if we are in the correct directory (the script can either be run from the Swiftest project root directory or the @@ -49,7 +48,7 @@ case $OS in if [ "$ARCH" = "x86_64" ]; then BUILDIMAGE="intel/oneapi-hpckit:2023.1.0-devel-ubuntu20.04" else - BUILDIMAGE="condaforge/miniforge3:23.1.0-4" + BUILDIMAGE="condaforge/mambaforge:23.1.0-4" fi cmd="docker build --tag swiftest:latest --tag swiftest:${VERSION} --build-arg BUILDIMAGE=\"${BUILDIMAGE}\" ." echo "Executing Docker build:\n${cmd}" diff --git a/buildscripts/build_dependencies.sh b/buildscripts/build_dependencies.sh index ae40f81ea..066b9f180 100755 --- a/buildscripts/build_dependencies.sh +++ b/buildscripts/build_dependencies.sh @@ -11,11 +11,12 @@ # You should have received a copy of the GNU General Public License along with Swiftest. # If not, see: https://www.gnu.org/licenses. SCRIPT_DIR=$(dirname "$0") -cd $SCRIPT_DIR/.. +BUILD_DIR=${SCRIPT_DIR}/../build +mkdir -p ${BUILD_DIR} +cd $BUILD_DIR USTMT="Usage: $0 <{Intel}|GNU>" -if [[ ( $@ == "--help") || $@ == "-h" ]] -then +if [[ ( $@ == "--help") || $@ == "-h" ]]; then echo $USTMT exit 0 fi @@ -51,7 +52,7 @@ esac export F77=${FC} echo "Using $COMPILER compilers:\nFC: $FC\nCC: $CC\nCXX: $CXX\n" -export INSTALL_DIR=$(realpath .)/usr/local +export INSTALL_DIR=${BUILD_DIR} mkdir -p ${INSTALL_DIR} export NCDIR="${INSTALL_DIR}" export NFDIR="${INSTALL_DIR}" @@ -93,7 +94,7 @@ if [ $COMPILER = "Intel" ]; then export FFLAGS=${CFLAGS} else export FCFLAGS="${CFLAGS}" - export FFFLAGS="${CFLAGS}" + export FFLAGS="${CFLAGS}" fi export LIBS="$(${INSTALL_DIR}/bin/nc-config --libs)" diff --git a/buildscripts/fetch_dependencies.sh b/buildscripts/fetch_dependencies.sh index 469d0f147..2330d3683 100755 --- a/buildscripts/fetch_dependencies.sh +++ b/buildscripts/fetch_dependencies.sh @@ -10,9 +10,12 @@ # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. # You should have received a copy of the GNU General Public License along with Swiftest. # If not, see: https://www.gnu.org/licenses. +SCRIPT_DIR=$(realpath $(dirname $0)) +BUILD_DIR=$(realpath ${SCRIPT_DIR}/../build) + +mkdir -p ${BUILD_DIR} +cd $BUILD_DIR -SCRIPT_DIR=$(dirname "$0") -cd $SCRIPT_DIR/.. wget -qO- https://www.zlib.net/zlib-1.2.13.tar.gz | tar xvz wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz | tar xvz wget -qO- https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.9.2.tar.gz | tar xvz diff --git a/buildscripts/make_environment.sh b/buildscripts/make_environment.sh new file mode 100755 index 000000000..152c0ca0f --- /dev/null +++ b/buildscripts/make_environment.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# This script will create a miniforge3 conda environment in order to execute the build +# Zlib, hdf5, netcdf-c, netcdf-fortran +# +# Copyright 2023 - David Minton +# This file is part of Swiftest. +# Swiftest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +# Swiftest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with Swiftest. +# If not, see: https://www.gnu.org/licenses. + +SCRIPT_DIR=$(realpath $(dirname $0)) + +# Determine if mamba/conda is already installed on the system. If not, install Mambaforge +if command -v mamba; then + CONDABIN="mamba" +elif command -v conda; then + CONDABIN="conda" +else + BUILD_DIR=$(realpath ${SCRIPT_DIR}/../build) + mkdir -p ${BUILD_DIR} + read -r OS ARCH < <(${SCRIPT_DIR}/get_platform.sh) + unset PYTHONPATH + cd $BUILD_DIR + wget https://github.com/conda-forge/miniforge/releases/download/23.1.0-4/Mambaforge-23.1.0-4-${OS}-${ARCH}.sh + + MYSHELL=$(basename $SHELL) + INSTALL_DIR=${HOME}/mambaforge + ${SHELL} Mambaforge-23.1.0-4-${OS}-${ARCH}.sh -b -p ${INSTALL_DIR} + rm Mambaforge-23.1.0-4-${OS}-${ARCH}.sh + + CONDABIN="${INSTALL_DIR}/condabin/mamba" + ${CONDABIN} init $MYSHELL + ${CONDABIN} update --name base mamba -y +fi +cd $SCRIPT_DIR +${CONDABIN} env create --file swiftest-build-env.yml --name swiftest-build-env \ No newline at end of file diff --git a/buildscripts/swiftest-build-env.yml b/buildscripts/swiftest-build-env.yml new file mode 100644 index 000000000..0521c0639 --- /dev/null +++ b/buildscripts/swiftest-build-env.yml @@ -0,0 +1,15 @@ +name: swiftest-build-env + +channels: + - conda-forge + - defaults + +dependencies: + - python>=3.8 + - compilers>=1.6 + - setuptools>=42 + - scikit-build>=0.17 + - cmake>=3.5.0 + - cython>=3.0.0 + - make + - ninja \ No newline at end of file