From 90baae87d640650342cdf8a9c43af3f84061d8a8 Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <Jean-Baptiste.Keck@imag.fr>
Date: Tue, 21 Apr 2020 09:18:51 +0200
Subject: [PATCH] this is a test to check what is happening for CI caching, do
not merge on success
---
ci/docker_images/ubuntu/bionic/Dockerfile | 2 +-
ci/scripts/build.sh | 2 +-
ci/scripts/build_and_debug.sh | 2 +-
ci/scripts/build_and_test.sh | 2 +-
ci/scripts/config.sh | 2 +-
ci/scripts/install.sh | 2 +-
ci/scripts/test.sh | 53 ++++++++++++++---------
ci/scripts/version.sh | 2 +-
8 files changed, 39 insertions(+), 28 deletions(-)
diff --git a/ci/docker_images/ubuntu/bionic/Dockerfile b/ci/docker_images/ubuntu/bionic/Dockerfile
index 3e615c27d..2609fefd9 100644
--- a/ci/docker_images/ubuntu/bionic/Dockerfile
+++ b/ci/docker_images/ubuntu/bionic/Dockerfile
@@ -12,7 +12,7 @@ RUN apt-get update
RUN apt-get full-upgrade -y
# get build tools and required libraries
-RUN apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake git vim ssh clang gcc gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgfortran3 libgcc1 libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python python-dev python-pip python-tk opencl-headers
+RUN apt-get install -y expat unzip xz-utils automake libtool pkg-config cmake rsync git vim ssh clang gcc gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgfortran3 libgcc1 libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python python-dev python-pip python-tk opencl-headers
# python packages using pip
RUN pip install --upgrade pip
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
index 22fb87ba6..763cde2f4 100755
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -feu -o pipefail
+set -feux -o pipefail
if [ $# -ne 4 ]; then
echo "Usage ./build build_folder CC CXX FC"
diff --git a/ci/scripts/build_and_debug.sh b/ci/scripts/build_and_debug.sh
index c90dcece1..69f32d8e8 100755
--- a/ci/scripts/build_and_debug.sh
+++ b/ci/scripts/build_and_debug.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-set -euf -o pipefail
+set -feux -o pipefail
# /hysop should be mounted as read only by run_tests_in_docker.sh
if [[ ! -d '/hysop' ]]; then
diff --git a/ci/scripts/build_and_test.sh b/ci/scripts/build_and_test.sh
index 031bf449c..112ad23b3 100755
--- a/ci/scripts/build_and_test.sh
+++ b/ci/scripts/build_and_test.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-set -euf -o pipefail
+set -feux -o pipefail
# /hysop should be mounted as read only by run_tests_in_docker.sh
if [[ ! -d '/hysop' ]]; then
diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh
index 7343b5172..654212515 100755
--- a/ci/scripts/config.sh
+++ b/ci/scripts/config.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -feu -o pipefail
+set -feux -o pipefail
if [ $# -ne 5 ]; then
echo "Usage ./config build_folder install_folder CC CXX FC"
diff --git a/ci/scripts/install.sh b/ci/scripts/install.sh
index ca04e7e69..076de1eea 100755
--- a/ci/scripts/install.sh
+++ b/ci/scripts/install.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -feu -o pipefail
+set -feux -o pipefail
if [ $# -ne 2 ]; then
echo "Usage ./install build_folder install_folder"
diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh
index df5416184..f7c0ec5c6 100755
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -fe -o pipefail
+set -feux -o pipefail
PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python2.7}
@@ -25,6 +25,7 @@ fi
INSTALL_DIR="$1"
HYSOP_DIR="$2"
+HYSOP_CACHE_DIR="${HOME}/.cache"
if [ $# -gt 2 ]; then
CACHE_DIR="$3"
@@ -35,33 +36,37 @@ fi
if [ $# -gt 3 ]; then
BACKUP_CACHE_DIR="$4"
+ HAS_BACKUP_CACHE_DIR=true
+else
+ HAS_BACKUP_CACHE_DIR=false
fi
if [ "${HAS_CACHE_DIR}" = true ]; then
- mkdir -p "${HOME}/cache"
if [ -d "${CACHE_DIR}" ]; then
echo "Cache directory '$CACHE_DIR' was found."
- cp -r $CACHE_DIR/* "${HOME}/.cache"
+ rsync -rtvu "${CACHE_DIR}/" "${HYSOP_CACHE_DIR}/"
else
# Untill gitlab allows cache on failure we need
# to provide initial cache so that CI succeeds (< 1h tests)
- # see https://gitlab.com/gitlab-org/gitlab/-/issues/18969
- # Initial cache will be builtin in the docker image.
- echo "Cache directory '$CACHE_DIR' was not found."
- if [[ -d "${BACKUP_CACHE_DIR}" ]]; then
- echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found."
- cp -r ${BACKUP_CACHE_DIR}/* "${HOME}/.cache"
- elif [[ -z "${BACKUP_CACHE_DIR}" ]]; then
- echo "No backup cache directory has been specified."
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/18969
+ # Initial cache can be injected in the docker image, see hysop/ci/utils/run_ci.sh.
+ echo "Cache directory '$CACHE_DIR' does not exist, trying to use backup cache directory."
+ if [[ "${HAS_BACKUP_CACHE_DIR}" = true ]]; then
+ if [[ -d "${BACKUP_CACHE_DIR}" ]]; then
+ echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found."
+ rsync -rtvu "${BACKUP_CACHE_DIR}/" "${HYSOP_CACHE_DIR}/"
+ else
+ echo "Backup directory '${BACKUP_CACHE_DIR}' does not exist."
+ fi
else
- echo "Backup directory '${BACKUP_CACHE_DIR}' was not found."
+ echo "No backup cache directory has been specified."
fi
fi
+ mkdir -p "${HYSOP_CACHE_DIR}"
+ mkdir -p "${CACHE_DIR}"
fi
-mkdir -p "${HOME}/.cache"
-mkdir -p "${CACHE_DIR}"
-export PYTHONPATH="$INSTALL_DIR/lib/python2.7/site-packages:$INSTALL_DIR:$PYTHONPATH"
+export PYTHONPATH="${INSTALL_DIR}/lib/python2.7/site-packages:${INSTALL_DIR}"
export MPLBACKEND='cairo'
export HYSOP_VERBOSE=0
export HYSOP_DEBUG=0
@@ -81,7 +86,7 @@ echo
echo "Default testing OpenCL platform is:"
${PYTHON_EXECUTABLE} -c 'import hysop; from hysop.testsenv import iter_clenv; print next(iter(iter_clenv()));'
-RUN_TESTS=${RUN_TESTS:-true}
+RUN_TESTS=${RUN_TESTS:-false}
RUN_EXAMPLES=${RUN_EXAMPLES:-false}
RUN_LONG_TESTS=${RUN_LONG_TESTS:-false}
@@ -129,15 +134,15 @@ if [ "$RUN_TESTS" = true ]; then
hysop_test "operator/tests/test_poisson.py"
hysop_test "operator/tests/test_solenoidal_projection.py"
hysop_test "operator/tests/test_poisson_curl.py"
- $HYSOP_DIR/fields/tests/test_cartesian.sh
+ ${HYSOP_DIR}/fields/tests/test_cartesian.sh
fi
-if [ "$RUN_LONG_TESTS" = true ]; then
+if [ "${RUN_LONG_TESTS}" = true ]; then
hysop_test "backend/device/codegen/kernels/tests/test_directional_advection.py"
hysop_test "backend/device/codegen/kernels/tests/test_directional_remesh.py"
fi
-if [ "$RUN_EXAMPLES" = true ]; then
+if [ "${RUN_EXAMPLES}" = true ]; then
example_test "analytic/analytic.py"
example_test "scalar_diffusion/scalar_diffusion.py"
example_test "scalar_advection/scalar_advection.py"
@@ -154,8 +159,14 @@ if [ "$RUN_EXAMPLES" = true ]; then
example_test "particles_above_salt/particles_above_salt_symmetrized.py"
fi
-if [ "$HAS_CACHE_DIR" = true ]; then
- cp -r ${HOME}/.cache/* $CACHE_DIR
+if [ "${HAS_CACHE_DIR}" = true ]; then
+ echo
+ echo
+ ls "${HYSOP_CACHE_DIR}"
+ echo
+ ls "${CACHE_DIR}"
+ echo
+ rsync -rtvu "${HYSOP_CACHE_DIR}/" "${CACHE_DIR}/"
find "${CACHE_DIR}" -name '*.lock' -delete
fi
diff --git a/ci/scripts/version.sh b/ci/scripts/version.sh
index 6588201ce..69b7fd968 100755
--- a/ci/scripts/version.sh
+++ b/ci/scripts/version.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -e
+set -feux -o pipefail
echo "HOST"
uname -a
echo
--
GitLab