diff --git a/ci/docker_images/ubuntu/bionic/Dockerfile b/ci/docker_images/ubuntu/bionic/Dockerfile index 3e615c27def184c847b963f9ce65ca74bf2fec66..2609fefd9c5efb8885553be1a6ae129a6333f7a2 100644 --- a/ci/docker_images/ubuntu/bionic/Dockerfile +++ b/ci/docker_images/ubuntu/bionic/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get update RUN apt-get full-upgrade -y # get build tools and required libraries -RUN apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake git vim ssh clang gcc gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgfortran3 libgcc1 libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python python-dev python-pip python-tk opencl-headers +RUN apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake rsync git vim ssh clang gcc gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgfortran3 libgcc1 libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python python-dev python-pip python-tk opencl-headers # python packages using pip RUN pip install --upgrade pip diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 22fb87ba690b58beb52cd0389b2297d5c4f304c0..763cde2f49cfef652e6ac1436e6d2762a094cb90 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -feu -o pipefail +set -feux -o pipefail if [ $# -ne 4 ]; then echo "Usage ./build build_folder CC CXX FC" diff --git a/ci/scripts/build_and_debug.sh b/ci/scripts/build_and_debug.sh index c90dcece1dba51862163f08fb0b5c6f4b2040037..69f32d8e8dac5bb470c66a2b0be5da52374406c3 100755 --- a/ci/scripts/build_and_debug.sh +++ b/ci/scripts/build_and_debug.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euf -o pipefail +set -feux -o pipefail # /hysop should be mounted as read only by run_tests_in_docker.sh if [[ ! -d '/hysop' ]]; then diff --git a/ci/scripts/build_and_test.sh b/ci/scripts/build_and_test.sh index 031bf449c749f600999172182369dcd3bc02f424..112ad23b34d1689843a4c08fbd735dd447df3d13 100755 --- a/ci/scripts/build_and_test.sh +++ b/ci/scripts/build_and_test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euf -o pipefail +set -feux -o pipefail # /hysop should be mounted as read only by run_tests_in_docker.sh if [[ ! -d '/hysop' ]]; then diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh index 7343b5172a87542fabccf43bb4e689cf35f5436a..654212515aa4cabbfd7e5e7c00d6b49d5d0eb59e 100755 --- a/ci/scripts/config.sh +++ b/ci/scripts/config.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -feu -o pipefail +set -feux -o pipefail if [ $# -ne 5 ]; then echo "Usage ./config build_folder install_folder CC CXX FC" diff --git a/ci/scripts/install.sh b/ci/scripts/install.sh index ca04e7e690d82d93abbb1eed440e8739411863a0..076de1eeab6fc6c0ff0595d57f69c790816a2119 100755 --- a/ci/scripts/install.sh +++ b/ci/scripts/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -feu -o pipefail +set -feux -o pipefail if [ $# -ne 2 ]; then echo "Usage ./install build_folder install_folder" diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index df54161849d4f19c8b9830c6c6316dbc1c615cec..f7c0ec5c6718ab5da7837dfbff38942e140b95d3 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -fe -o pipefail +set -feux -o pipefail PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python2.7} @@ -25,6 +25,7 @@ fi INSTALL_DIR="$1" HYSOP_DIR="$2" +HYSOP_CACHE_DIR="${HOME}/.cache" if [ $# -gt 2 ]; then CACHE_DIR="$3" @@ -35,33 +36,37 @@ fi if [ $# -gt 3 ]; then BACKUP_CACHE_DIR="$4" + HAS_BACKUP_CACHE_DIR=true +else + HAS_BACKUP_CACHE_DIR=false fi if [ "${HAS_CACHE_DIR}" = true ]; then - mkdir -p "${HOME}/cache" if [ -d "${CACHE_DIR}" ]; then echo "Cache directory '$CACHE_DIR' was found." - cp -r $CACHE_DIR/* "${HOME}/.cache" + rsync -rtvu "${CACHE_DIR}/" "${HYSOP_CACHE_DIR}/" else # Untill gitlab allows cache on failure we need # to provide initial cache so that CI succeeds (< 1h tests) - # see https://gitlab.com/gitlab-org/gitlab/-/issues/18969 - # Initial cache will be builtin in the docker image. - echo "Cache directory '$CACHE_DIR' was not found." - if [[ -d "${BACKUP_CACHE_DIR}" ]]; then - echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found." - cp -r ${BACKUP_CACHE_DIR}/* "${HOME}/.cache" - elif [[ -z "${BACKUP_CACHE_DIR}" ]]; then - echo "No backup cache directory has been specified." + # See https://gitlab.com/gitlab-org/gitlab/-/issues/18969 + # Initial cache can be injected in the docker image, see hysop/ci/utils/run_ci.sh. + echo "Cache directory '$CACHE_DIR' does not exist, trying to use backup cache directory." + if [[ "${HAS_BACKUP_CACHE_DIR}" = true ]]; then + if [[ -d "${BACKUP_CACHE_DIR}" ]]; then + echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found." + rsync -rtvu "${BACKUP_CACHE_DIR}/" "${HYSOP_CACHE_DIR}/" + else + echo "Backup directory '${BACKUP_CACHE_DIR}' does not exist." + fi else - echo "Backup directory '${BACKUP_CACHE_DIR}' was not found." + echo "No backup cache directory has been specified." fi fi + mkdir -p "${HYSOP_CACHE_DIR}" + mkdir -p "${CACHE_DIR}" fi -mkdir -p "${HOME}/.cache" -mkdir -p "${CACHE_DIR}" -export PYTHONPATH="$INSTALL_DIR/lib/python2.7/site-packages:$INSTALL_DIR:$PYTHONPATH" +export PYTHONPATH="${INSTALL_DIR}/lib/python2.7/site-packages:${INSTALL_DIR}" export MPLBACKEND='cairo' export HYSOP_VERBOSE=0 export HYSOP_DEBUG=0 @@ -81,7 +86,7 @@ echo echo "Default testing OpenCL platform is:" ${PYTHON_EXECUTABLE} -c 'import hysop; from hysop.testsenv import iter_clenv; print next(iter(iter_clenv()));' -RUN_TESTS=${RUN_TESTS:-true} +RUN_TESTS=${RUN_TESTS:-false} RUN_EXAMPLES=${RUN_EXAMPLES:-false} RUN_LONG_TESTS=${RUN_LONG_TESTS:-false} @@ -129,15 +134,15 @@ if [ "$RUN_TESTS" = true ]; then hysop_test "operator/tests/test_poisson.py" hysop_test "operator/tests/test_solenoidal_projection.py" hysop_test "operator/tests/test_poisson_curl.py" - $HYSOP_DIR/fields/tests/test_cartesian.sh + ${HYSOP_DIR}/fields/tests/test_cartesian.sh fi -if [ "$RUN_LONG_TESTS" = true ]; then +if [ "${RUN_LONG_TESTS}" = true ]; then hysop_test "backend/device/codegen/kernels/tests/test_directional_advection.py" hysop_test "backend/device/codegen/kernels/tests/test_directional_remesh.py" fi -if [ "$RUN_EXAMPLES" = true ]; then +if [ "${RUN_EXAMPLES}" = true ]; then example_test "analytic/analytic.py" example_test "scalar_diffusion/scalar_diffusion.py" example_test "scalar_advection/scalar_advection.py" @@ -154,8 +159,14 @@ if [ "$RUN_EXAMPLES" = true ]; then example_test "particles_above_salt/particles_above_salt_symmetrized.py" fi -if [ "$HAS_CACHE_DIR" = true ]; then - cp -r ${HOME}/.cache/* $CACHE_DIR +if [ "${HAS_CACHE_DIR}" = true ]; then + echo + echo + ls "${HYSOP_CACHE_DIR}" + echo + ls "${CACHE_DIR}" + echo + rsync -rtvu "${HYSOP_CACHE_DIR}/" "${CACHE_DIR}/" find "${CACHE_DIR}" -name '*.lock' -delete fi diff --git a/ci/scripts/version.sh b/ci/scripts/version.sh index 6588201ce29c4ab1eb55fc99d70a1c1339ef647e..69b7fd9687bdee24eaeca08110b75c6f48626cf1 100755 --- a/ci/scripts/version.sh +++ b/ci/scripts/version.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -feux -o pipefail echo "HOST" uname -a echo