diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 923fb7fb6f9cb51a0a404d190827ba798e90c7f0..c3d8f543eb8eb97a22300d9008a4fda35e16f483 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,5 +58,5 @@ test:bionic: cache: paths: - $CI_PROJECT_DIR/cache - key: "test_cache_0000" + key: "hysop_cache" diff --git a/ci/docker_images/ubuntu/focal/Dockerfile b/ci/docker_images/ubuntu/focal/Dockerfile index 292719cf80a7986fa83444b66371b5dd2fc43137..539ff1ef44dc5f54301ee204b51256ca491e3c78 100644 --- a/ci/docker_images/ubuntu/focal/Dockerfile +++ b/ci/docker_images/ubuntu/focal/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update RUN apt-get full-upgrade -y # get build tools and required libraries -RUN apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake git vim ssh curl wget ca-certificates gcc g++ gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgcc-9-dev libgfortran-9-dev libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python2.7-dev opencl-headers +RUN apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake git vim ssh curl wget ca-certificates gcc g++ gfortran lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libblas-dev liblapack-dev libgcc-9-dev libgfortran-9-dev libopenmpi-dev libhdf5-openmpi-dev libgmp-dev libmpfr-dev libmpc-dev libflint-dev libfftw-dev libfftw-mpi-dev python2.7-dev opencl-headers # python packages using pip RUN cd /tmp && \ @@ -20,8 +20,8 @@ RUN cd /tmp && \ python2.7 get-pip.py && \ pip2.7 install --upgrade pip && \ rm -f /tmp/get-pip.py -RUN pip2.7 install --upgrade numpy setuptools cffi wheel pytest pybind11 -RUN pip2.7 install --upgrade backports.weakref backports.tempfile scipy sympy matplotlib mpi4py gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac pycairo weave argparse_color_formatter networkx pyvis +RUN pip2.7 install --upgrade numpy setuptools cffi wheel pytest pybind11 cython +RUN pip2.7 install --upgrade backports.weakref backports.tempfile scipy sympy matplotlib mpi4py gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac weave argparse_color_formatter networkx pyvis RUN CC=mpicc HDF5_MPI="ON" pip2.7 install --upgrade --no-binary=h5py h5py # llvm + numba + llvmlite (llvmlite 0.32 has a bug with llvm8) @@ -134,7 +134,6 @@ RUN cd /tmp && \ git clone https://github.com/drwells/pyFFTW && \ cd pyFFTW && \ git checkout r2r-try-two && \ - sed -i 's/\(fftw3[fl]\?_\)threads/\1omp/g' setup.py && \ pip2.7 install . && \ cd - && \ rm -Rf /tmp/pyFFTW diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index f787eca15b76264ce04d180fa2a4347a01d6a471..22fb87ba690b58beb52cd0389b2297d5c4f304c0 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -feu -o pipefail if [ $# -ne 4 ]; then echo "Usage ./build build_folder CC CXX FC" diff --git a/ci/scripts/build_and_debug.sh b/ci/scripts/build_and_debug.sh index c5a1e150151fe3ffbe532118c41259c5f5669957..c90dcece1dba51862163f08fb0b5c6f4b2040037 100755 --- a/ci/scripts/build_and_debug.sh +++ b/ci/scripts/build_and_debug.sh @@ -4,7 +4,7 @@ set -euf -o pipefail # /hysop should be mounted as read only by run_tests_in_docker.sh if [[ ! -d '/hysop' ]]; then echo "This script should not be called from host, but from within a docker image." - echo " => /hysop has not been mounted (see hysop/ci/utils/run_tests_in_docker.sh)." + echo " => /hysop has not been mounted (see hysop/ci/utils/run_debug.sh)." exit 1 fi diff --git a/ci/scripts/build_and_test.sh b/ci/scripts/build_and_test.sh index 44264d82b27ee97760834ce8c6a8acc66325deb1..031bf449c749f600999172182369dcd3bc02f424 100755 --- a/ci/scripts/build_and_test.sh +++ b/ci/scripts/build_and_test.sh @@ -4,7 +4,7 @@ set -euf -o pipefail # /hysop should be mounted as read only by run_tests_in_docker.sh if [[ ! -d '/hysop' ]]; then echo "This script should not be called from host, but from within a docker image." - echo " => /hysop has not been mounted (see hysop/ci/utils/run_tests_in_docker.sh)." + echo " => /hysop has not been mounted (see hysop/ci/utils/run_ci.sh)." exit 1 fi @@ -25,4 +25,9 @@ ${SCRIPT_DIR}/version.sh ${SCRIPT_DIR}/config.sh "${HYSOP_BUILD_DIR}" "${HYSOP_INSTALL_DIR}" "${CC}" "${CXX}" "${FC}" ${SCRIPT_DIR}/build.sh "${HYSOP_BUILD_DIR}" "${CC}" "${CXX}" "${FC}" ${SCRIPT_DIR}/install.sh "${HYSOP_BUILD_DIR}" "${HYSOP_INSTALL_DIR}" -${SCRIPT_DIR}/test.sh "${HYSOP_INSTALL_DIR}" "${HYSOP_DIR}/hysop" "/cache" +time ${SCRIPT_DIR}/test.sh "${HYSOP_INSTALL_DIR}" "${HYSOP_DIR}/hysop" + +# clean everything because image may be commited to retain hysop cache +cd +rm -rf /tmp/hysop +pip2.7 uninstall hysop diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh index c2c8e6a22e62fe59508a351c7b6e0b3d0a4e28df..7343b5172a87542fabccf43bb4e689cf35f5436a 100755 --- a/ci/scripts/config.sh +++ b/ci/scripts/config.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -feu -o pipefail if [ $# -ne 5 ]; then echo "Usage ./config build_folder install_folder CC CXX FC" @@ -22,7 +22,7 @@ INSTALL_DIR="$2" mkdir -p $BUILD_DIR cd $BUILD_DIR -CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DHYSOP_INSTALL=$INSTALL_DIR $ROOT_DIR +CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DPYTHON_EXECUTABLE="$(which python2.7)" -DHYSOP_INSTALL=$INSTALL_DIR $ROOT_DIR if [ ! -f Makefile ]; then echo "The makefile has not been generated." diff --git a/ci/scripts/install.sh b/ci/scripts/install.sh index f37f559698dc82b7289fb63a9ae8ea84f20e202c..ca04e7e690d82d93abbb1eed440e8739411863a0 100755 --- a/ci/scripts/install.sh +++ b/ci/scripts/install.sh @@ -1,6 +1,5 @@ - #!/bin/bash -set -e +set -feu -o pipefail if [ $# -ne 2 ]; then echo "Usage ./install build_folder install_folder" diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 8051a5ccd50f7621d68a24b88470eae5d1ff07c3..60fff7147008e222e964b24052759556a479e6a5 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -1,15 +1,15 @@ #!/bin/bash -set -e +set -fe -o pipefail PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python2.7} if [ $# -lt 2 ]; then - echo "Usage ./test install_folder hysop_folder [cache_dir]" + echo "Usage ./test install_folder hysop_folder [cache_dir] [backup_cache_dir]" exit 1 fi -if [ $# -gt 3 ]; then - echo "Usage ./test install_folder hysop_folder [cache_dir]" +if [ $# -gt 4 ]; then + echo "Usage ./test install_folder hysop_folder [cache_dir] [backup_cache_dir]" exit 1 fi @@ -23,23 +23,39 @@ if [ ! -d "$2" ]; then exit 1 fi -INSTALL_DIR=$1 -HYSOP_DIR=$2 -if [ $# -eq 3 ]; then - CACHE_DIR=$3 +INSTALL_DIR="$1" +HYSOP_DIR="$2" + +if [ $# -gt 2 ]; then + CACHE_DIR="$3" HAS_CACHE_DIR=true else HAS_CACHE_DIR=false fi -if [ "$HAS_CACHE_DIR" = true ]; then - if [ -d "$CACHE_DIR" ]; then +if [ $# -gt 3 ]; then + BACKUP_CACHE_DIR="$4" +fi + +if [ "${HAS_CACHE_DIR}" = true ]; then + mkdir -p "${HOME}/cache" + if [ -d "${CACHE_DIR}" ]; then echo "Cache directory '$CACHE_DIR' was found." - mkdir -p /root/.cache - cp -r $CACHE_DIR/* /root/.cache + cp -r $CACHE_DIR/* "${HOME}/.cache" else + # Untill gitlab allows cache on failure we need + # to provide initial cache so that CI succeeds (< 1h tests) + # see https://gitlab.com/gitlab-org/gitlab/-/issues/18969 + # Initial cache will be builtin in the docker image. echo "Cache directory '$CACHE_DIR' was not found." - mkdir -p $CACHE_DIR + if [[ -d "${BACKUP_CACHE_DIR}" ]]; then + echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found." + cp -r ${BACKUP_CACHE_DIR}/* "${HOME}/.cache" + elif [[ -z "${BACKUP_CACHE_DIR}" ]]; then + echo "No backup cache directory has been specified." + else + echo "Backup directory '${BACKUP_CACHE_DIR}' was not found." + fi fi fi @@ -69,7 +85,7 @@ DO_LONG_TESTS=${DO_LONG_TESTS:-false} COMMON_TEST_OPTIONS='' TEST_DIR="$HYSOP_DIR" -COMMON_EXAMPLE_OPTIONS='-VNC -cp default -maxit 2' +COMMON_EXAMPLE_OPTIONS='-VNC -d16 -cp default -maxit 2' EXAMPLE_DIR="$HYSOP_DIR/../hysop_examples/examples" hysop_test() { diff --git a/ci/utils/build_docker_image.sh b/ci/utils/build_docker_image.sh index 79d908eb096d7c6f17349a527eb0c9c9b96ecba5..b6d5ce89ec10af63a955f63596818755ac4afe51 100755 --- a/ci/utils/build_docker_image.sh +++ b/ci/utils/build_docker_image.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -set -euf -o pipefail +set -feu -o pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" NTHREADS="$(python -c 'import psutil; print(psutil.cpu_count(logical=False))')" -UBUNTU_RELEASE='focal' +UBUNTU_RELEASE=${1:-bionic} -docker build --rm=true --build-arg "NTHREADS=$NTHREADS" "$@" -t "keckj/hysop:${UBUNTU_RELEASE}" -f "${SCRIPT_DIR}/../docker_images/ubuntu/${UBUNTU_RELEASE}/Dockerfile" "${SCRIPT_DIR}/.." +docker build --rm=true --build-arg "NTHREADS=$NTHREADS" -t "keckj/hysop:${UBUNTU_RELEASE}" -f "${SCRIPT_DIR}/../docker_images/ubuntu/${UBUNTU_RELEASE}/Dockerfile" "${SCRIPT_DIR}/../.." diff --git a/ci/utils/pull_docker_image.sh b/ci/utils/pull_docker_image.sh index b0efca8acb89d8dd4b4a119d87853562006907c0..7cf52350a8ee79807a51fd5e0e849fdf0fae3ecb 100755 --- a/ci/utils/pull_docker_image.sh +++ b/ci/utils/pull_docker_image.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash set -euf -o pipefail +UBUNTU_RELEASE=${1:-bionic} docker logout -docker pull keckj/hysop:bionic +docker pull "keckj/hysop:${UBUNTU_RELEASE}" diff --git a/ci/utils/push_docker_image.sh b/ci/utils/push_docker_image.sh index b19592e570d4a798249a1a202c425b3d05ee04d4..33aec3ccf331068848844267349b62e4e245854e 100755 --- a/ci/utils/push_docker_image.sh +++ b/ci/utils/push_docker_image.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euf -o pipefail +UBUNTU_RELEASE=${1:-bionic} docker login -docker push keckj/hysop:bionic +docker push "keckj/hysop:${UBUNTU_RELEASE}" docker logout diff --git a/ci/utils/run_ci.sh b/ci/utils/run_ci.sh index 927568a5984356827eb0d07da4a8c9c2100d0569..ce540c0a57274df43650fef5df9416b512455c17 100755 --- a/ci/utils/run_ci.sh +++ b/ci/utils/run_ci.sh @@ -1,13 +1,9 @@ #!/usr/bin/env bash set -feu -o pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -DOCKER_IMG='keckj/hysop:bionic' +UBUNTU_RELEASE=${1:-bionic} +DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}" CONTAINER_ID='hysop_build_and_test' -HYSOP_CACHE_DIR="${HOME}/.cache/hysop" - -if [[ ! -d "${HYSOP_CACHE_DIR}" ]]; then - mkdir -p "${HYSOP_CACHE_DIR}" -fi function remove_img() { docker stop "${CONTAINER_ID}" || true @@ -19,6 +15,10 @@ remove_img docker logout docker pull "${DOCKER_IMG}" -docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" -v "${HYSOP_CACHE_DIR}:/cache:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}" +docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}" docker start "${CONTAINER_ID}" + docker exec "${CONTAINER_ID}" /hysop/ci/scripts/build_and_test.sh + +# on test success, upload hysop cache to the docker images +docker commit "${CONTAINER_ID}" "${DOCKER_IMG}" diff --git a/ci/utils/run_debug.sh b/ci/utils/run_debug.sh index a2f85712c1db5128dbc3d635c4e277eca2d08a84..eb3bb7753820301c71112e837d0e637a5d520608 100755 --- a/ci/utils/run_debug.sh +++ b/ci/utils/run_debug.sh @@ -1,15 +1,9 @@ - - #!/usr/bin/env bash set -feu -o pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -DOCKER_IMG='keckj/hysop:bionic' +UBUNTU_RELEASE=${1:-bionic} +DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}" CONTAINER_ID='hysop_build_and_debug' -HYSOP_CACHE_DIR="${HOME}/.cache/hysop" - -if [[ ! -d "${HYSOP_CACHE_DIR}" ]]; then - mkdir -p "${HYSOP_CACHE_DIR}" -fi function remove_img() { docker stop "${CONTAINER_ID}" || true @@ -21,6 +15,6 @@ remove_img docker logout docker pull "${DOCKER_IMG}" -docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" -v "${HYSOP_CACHE_DIR}:/cache:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}" +docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}" docker start "${CONTAINER_ID}" docker exec -it "${CONTAINER_ID}" /hysop/ci/scripts/build_and_debug.sh diff --git a/ci/utils/run_docker_image.sh b/ci/utils/run_docker_image.sh index ac6e49cfc57af4a3d5d0db4dd936688b09183067..6690d09643ba5f4148bac539397fa82bac467f7d 100755 --- a/ci/utils/run_docker_image.sh +++ b/ci/utils/run_docker_image.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash set -euf -o pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -docker run -it -v "${SCRIPT_DIR}/../..:/hysop:ro" keckj/hysop:bionic +UBUNTU_RELEASE='bionic' +docker run -it -v "${SCRIPT_DIR}/../..:/hysop:ro" "keckj/hysop:${UBUNTU_RELEASE}" diff --git a/hysop/operator/tests/test_custom_symbolic.py b/hysop/operator/tests/test_custom_symbolic.py index 305c9a5c33c7d4c29eb9031a0888a26665ec7348..ff5930c3ac1d5a542d62068a17cfad512969ccab 100644 --- a/hysop/operator/tests/test_custom_symbolic.py +++ b/hysop/operator/tests/test_custom_symbolic.py @@ -792,7 +792,7 @@ class TestCustomSymbolic(object): if __name__ == '__main__': TestCustomSymbolic.setup_class(enable_extra_tests=False, - enable_debug_mode=False) + enable_debug_mode=False) enable_pretty_printing()