From c8ee1be9c6dd9789f2c43f6eaa927a6843635fce Mon Sep 17 00:00:00 2001
From: Jean-Baptiste Keck <Jean-Baptiste.Keck@imag.fr>
Date: Mon, 20 Apr 2020 14:47:50 +0200
Subject: [PATCH] allow to inject local ci cache directly into docker images
for online ci
---
.gitlab-ci.yml | 2 +-
ci/docker_images/ubuntu/focal/Dockerfile | 7 ++--
ci/scripts/build.sh | 2 +-
ci/scripts/build_and_debug.sh | 2 +-
ci/scripts/build_and_test.sh | 9 +++-
ci/scripts/config.sh | 4 +-
ci/scripts/install.sh | 3 +-
ci/scripts/test.sh | 44 +++++++++++++-------
ci/utils/build_docker_image.sh | 6 +--
ci/utils/pull_docker_image.sh | 3 +-
ci/utils/push_docker_image.sh | 3 +-
ci/utils/run_ci.sh | 14 +++----
ci/utils/run_debug.sh | 12 ++----
ci/utils/run_docker_image.sh | 3 +-
hysop/operator/tests/test_custom_symbolic.py | 2 +-
15 files changed, 66 insertions(+), 50 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 923fb7fb6..c3d8f543e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -58,5 +58,5 @@ test:bionic:
cache:
paths:
- $CI_PROJECT_DIR/cache
- key: "test_cache_0000"
+ key: "hysop_cache"
diff --git a/ci/docker_images/ubuntu/focal/Dockerfile b/ci/docker_images/ubuntu/focal/Dockerfile
index 292719cf8..539ff1ef4 100644
--- a/ci/docker_images/ubuntu/focal/Dockerfile
+++ b/ci/docker_images/ubuntu/focal/Dockerfile
@@ -12,7 +12,7 @@ RUN apt-get update
RUN apt-get full-upgrade -y
# get build tools and required libraries
-RUN apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake git vim ssh curl wget ca-certificates gcc g++ gfortran cython swig lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libboost-all-dev libblas-dev liblapack-dev libcgal-dev libatlas-base-dev libopenblas-dev libgcc-9-dev libgfortran-9-dev libopenmpi-dev libhdf5-openmpi-dev libfftw3-dev libfftw3-mpi-dev libgmp-dev libmpfr-dev libmpc-dev libsparsehash-dev libcairo-dev libcairomm-1.0-dev libflint-dev python2.7-dev opencl-headers
+RUN apt-get install -y --no-install-recommends expat unzip xz-utils automake libtool pkg-config cmake git vim ssh curl wget ca-certificates gcc g++ gfortran lsb-core cpio libnuma1 libpciaccess0 libreadline-dev libblas-dev liblapack-dev libgcc-9-dev libgfortran-9-dev libopenmpi-dev libhdf5-openmpi-dev libgmp-dev libmpfr-dev libmpc-dev libflint-dev libfftw-dev libfftw-mpi-dev python2.7-dev opencl-headers
# python packages using pip
RUN cd /tmp && \
@@ -20,8 +20,8 @@ RUN cd /tmp && \
python2.7 get-pip.py && \
pip2.7 install --upgrade pip && \
rm -f /tmp/get-pip.py
-RUN pip2.7 install --upgrade numpy setuptools cffi wheel pytest pybind11
-RUN pip2.7 install --upgrade backports.weakref backports.tempfile scipy sympy matplotlib mpi4py gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac pycairo weave argparse_color_formatter networkx pyvis
+RUN pip2.7 install --upgrade numpy setuptools cffi wheel pytest pybind11 cython
+RUN pip2.7 install --upgrade backports.weakref backports.tempfile scipy sympy matplotlib mpi4py gmpy2 psutil py-cpuinfo Mako subprocess32 editdistance portalocker colors.py tee primefac weave argparse_color_formatter networkx pyvis
RUN CC=mpicc HDF5_MPI="ON" pip2.7 install --upgrade --no-binary=h5py h5py
# llvm + numba + llvmlite (llvmlite 0.32 has a bug with llvm8)
@@ -134,7 +134,6 @@ RUN cd /tmp && \
git clone https://github.com/drwells/pyFFTW && \
cd pyFFTW && \
git checkout r2r-try-two && \
- sed -i 's/\(fftw3[fl]\?_\)threads/\1omp/g' setup.py && \
pip2.7 install . && \
cd - && \
rm -Rf /tmp/pyFFTW
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
index f787eca15..22fb87ba6 100755
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -e
+set -feu -o pipefail
if [ $# -ne 4 ]; then
echo "Usage ./build build_folder CC CXX FC"
diff --git a/ci/scripts/build_and_debug.sh b/ci/scripts/build_and_debug.sh
index c5a1e1501..c90dcece1 100755
--- a/ci/scripts/build_and_debug.sh
+++ b/ci/scripts/build_and_debug.sh
@@ -4,7 +4,7 @@ set -euf -o pipefail
# /hysop should be mounted as read only by run_tests_in_docker.sh
if [[ ! -d '/hysop' ]]; then
echo "This script should not be called from host, but from within a docker image."
- echo " => /hysop has not been mounted (see hysop/ci/utils/run_tests_in_docker.sh)."
+ echo " => /hysop has not been mounted (see hysop/ci/utils/run_debug.sh)."
exit 1
fi
diff --git a/ci/scripts/build_and_test.sh b/ci/scripts/build_and_test.sh
index 44264d82b..031bf449c 100755
--- a/ci/scripts/build_and_test.sh
+++ b/ci/scripts/build_and_test.sh
@@ -4,7 +4,7 @@ set -euf -o pipefail
# /hysop should be mounted as read only by run_tests_in_docker.sh
if [[ ! -d '/hysop' ]]; then
echo "This script should not be called from host, but from within a docker image."
- echo " => /hysop has not been mounted (see hysop/ci/utils/run_tests_in_docker.sh)."
+ echo " => /hysop has not been mounted (see hysop/ci/utils/run_ci.sh)."
exit 1
fi
@@ -25,4 +25,9 @@ ${SCRIPT_DIR}/version.sh
${SCRIPT_DIR}/config.sh "${HYSOP_BUILD_DIR}" "${HYSOP_INSTALL_DIR}" "${CC}" "${CXX}" "${FC}"
${SCRIPT_DIR}/build.sh "${HYSOP_BUILD_DIR}" "${CC}" "${CXX}" "${FC}"
${SCRIPT_DIR}/install.sh "${HYSOP_BUILD_DIR}" "${HYSOP_INSTALL_DIR}"
-${SCRIPT_DIR}/test.sh "${HYSOP_INSTALL_DIR}" "${HYSOP_DIR}/hysop" "/cache"
+time ${SCRIPT_DIR}/test.sh "${HYSOP_INSTALL_DIR}" "${HYSOP_DIR}/hysop"
+
+# clean everything because image may be commited to retain hysop cache
+cd
+rm -rf /tmp/hysop
+pip2.7 uninstall hysop
diff --git a/ci/scripts/config.sh b/ci/scripts/config.sh
index c2c8e6a22..7343b5172 100755
--- a/ci/scripts/config.sh
+++ b/ci/scripts/config.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -e
+set -feu -o pipefail
if [ $# -ne 5 ]; then
echo "Usage ./config build_folder install_folder CC CXX FC"
@@ -22,7 +22,7 @@ INSTALL_DIR="$2"
mkdir -p $BUILD_DIR
cd $BUILD_DIR
-CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DHYSOP_INSTALL=$INSTALL_DIR $ROOT_DIR
+CC="$3" CXX="$4" FC="$5" cmake -DCMAKE_BUILD_TYPE=Release -DVERBOSE=OFF -DWITH_SCALES=ON -DPYTHON_EXECUTABLE="$(which python2.7)" -DHYSOP_INSTALL=$INSTALL_DIR $ROOT_DIR
if [ ! -f Makefile ]; then
echo "The makefile has not been generated."
diff --git a/ci/scripts/install.sh b/ci/scripts/install.sh
index f37f55969..ca04e7e69 100755
--- a/ci/scripts/install.sh
+++ b/ci/scripts/install.sh
@@ -1,6 +1,5 @@
-
#!/bin/bash
-set -e
+set -feu -o pipefail
if [ $# -ne 2 ]; then
echo "Usage ./install build_folder install_folder"
diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh
index 8051a5ccd..60fff7147 100755
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
@@ -1,15 +1,15 @@
#!/bin/bash
-set -e
+set -fe -o pipefail
PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python2.7}
if [ $# -lt 2 ]; then
- echo "Usage ./test install_folder hysop_folder [cache_dir]"
+ echo "Usage ./test install_folder hysop_folder [cache_dir] [backup_cache_dir]"
exit 1
fi
-if [ $# -gt 3 ]; then
- echo "Usage ./test install_folder hysop_folder [cache_dir]"
+if [ $# -gt 4 ]; then
+ echo "Usage ./test install_folder hysop_folder [cache_dir] [backup_cache_dir]"
exit 1
fi
@@ -23,23 +23,39 @@ if [ ! -d "$2" ]; then
exit 1
fi
-INSTALL_DIR=$1
-HYSOP_DIR=$2
-if [ $# -eq 3 ]; then
- CACHE_DIR=$3
+INSTALL_DIR="$1"
+HYSOP_DIR="$2"
+
+if [ $# -gt 2 ]; then
+ CACHE_DIR="$3"
HAS_CACHE_DIR=true
else
HAS_CACHE_DIR=false
fi
-if [ "$HAS_CACHE_DIR" = true ]; then
- if [ -d "$CACHE_DIR" ]; then
+if [ $# -gt 3 ]; then
+ BACKUP_CACHE_DIR="$4"
+fi
+
+if [ "${HAS_CACHE_DIR}" = true ]; then
+ mkdir -p "${HOME}/cache"
+ if [ -d "${CACHE_DIR}" ]; then
echo "Cache directory '$CACHE_DIR' was found."
- mkdir -p /root/.cache
- cp -r $CACHE_DIR/* /root/.cache
+ cp -r $CACHE_DIR/* "${HOME}/.cache"
else
+ # Untill gitlab allows cache on failure we need
+ # to provide initial cache so that CI succeeds (< 1h tests)
+ # see https://gitlab.com/gitlab-org/gitlab/-/issues/18969
+ # Initial cache will be builtin in the docker image.
echo "Cache directory '$CACHE_DIR' was not found."
- mkdir -p $CACHE_DIR
+ if [[ -d "${BACKUP_CACHE_DIR}" ]]; then
+ echo "Backup cache directory '${BACKUP_CACHE_DIR}' was found."
+ cp -r ${BACKUP_CACHE_DIR}/* "${HOME}/.cache"
+ elif [[ -z "${BACKUP_CACHE_DIR}" ]]; then
+ echo "No backup cache directory has been specified."
+ else
+ echo "Backup directory '${BACKUP_CACHE_DIR}' was not found."
+ fi
fi
fi
@@ -69,7 +85,7 @@ DO_LONG_TESTS=${DO_LONG_TESTS:-false}
COMMON_TEST_OPTIONS=''
TEST_DIR="$HYSOP_DIR"
-COMMON_EXAMPLE_OPTIONS='-VNC -cp default -maxit 2'
+COMMON_EXAMPLE_OPTIONS='-VNC -d16 -cp default -maxit 2'
EXAMPLE_DIR="$HYSOP_DIR/../hysop_examples/examples"
hysop_test() {
diff --git a/ci/utils/build_docker_image.sh b/ci/utils/build_docker_image.sh
index 79d908eb0..b6d5ce89e 100755
--- a/ci/utils/build_docker_image.sh
+++ b/ci/utils/build_docker_image.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
-set -euf -o pipefail
+set -feu -o pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
NTHREADS="$(python -c 'import psutil; print(psutil.cpu_count(logical=False))')"
-UBUNTU_RELEASE='focal'
+UBUNTU_RELEASE=${1:-bionic}
-docker build --rm=true --build-arg "NTHREADS=$NTHREADS" "$@" -t "keckj/hysop:${UBUNTU_RELEASE}" -f "${SCRIPT_DIR}/../docker_images/ubuntu/${UBUNTU_RELEASE}/Dockerfile" "${SCRIPT_DIR}/.."
+docker build --rm=true --build-arg "NTHREADS=$NTHREADS" -t "keckj/hysop:${UBUNTU_RELEASE}" -f "${SCRIPT_DIR}/../docker_images/ubuntu/${UBUNTU_RELEASE}/Dockerfile" "${SCRIPT_DIR}/../.."
diff --git a/ci/utils/pull_docker_image.sh b/ci/utils/pull_docker_image.sh
index b0efca8ac..7cf52350a 100755
--- a/ci/utils/pull_docker_image.sh
+++ b/ci/utils/pull_docker_image.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
set -euf -o pipefail
+UBUNTU_RELEASE=${1:-bionic}
docker logout
-docker pull keckj/hysop:bionic
+docker pull "keckj/hysop:${UBUNTU_RELEASE}"
diff --git a/ci/utils/push_docker_image.sh b/ci/utils/push_docker_image.sh
index b19592e57..33aec3ccf 100755
--- a/ci/utils/push_docker_image.sh
+++ b/ci/utils/push_docker_image.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
set -euf -o pipefail
+UBUNTU_RELEASE=${1:-bionic}
docker login
-docker push keckj/hysop:bionic
+docker push "keckj/hysop:${UBUNTU_RELEASE}"
docker logout
diff --git a/ci/utils/run_ci.sh b/ci/utils/run_ci.sh
index 927568a59..ce540c0a5 100755
--- a/ci/utils/run_ci.sh
+++ b/ci/utils/run_ci.sh
@@ -1,13 +1,9 @@
#!/usr/bin/env bash
set -feu -o pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-DOCKER_IMG='keckj/hysop:bionic'
+UBUNTU_RELEASE=${1:-bionic}
+DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}"
CONTAINER_ID='hysop_build_and_test'
-HYSOP_CACHE_DIR="${HOME}/.cache/hysop"
-
-if [[ ! -d "${HYSOP_CACHE_DIR}" ]]; then
- mkdir -p "${HYSOP_CACHE_DIR}"
-fi
function remove_img() {
docker stop "${CONTAINER_ID}" || true
@@ -19,6 +15,10 @@ remove_img
docker logout
docker pull "${DOCKER_IMG}"
-docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" -v "${HYSOP_CACHE_DIR}:/cache:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}"
+docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}"
docker start "${CONTAINER_ID}"
+
docker exec "${CONTAINER_ID}" /hysop/ci/scripts/build_and_test.sh
+
+# on test success, upload hysop cache to the docker images
+docker commit "${CONTAINER_ID}" "${DOCKER_IMG}"
diff --git a/ci/utils/run_debug.sh b/ci/utils/run_debug.sh
index a2f85712c..eb3bb7753 100755
--- a/ci/utils/run_debug.sh
+++ b/ci/utils/run_debug.sh
@@ -1,15 +1,9 @@
-
-
#!/usr/bin/env bash
set -feu -o pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-DOCKER_IMG='keckj/hysop:bionic'
+UBUNTU_RELEASE=${1:-bionic}
+DOCKER_IMG="keckj/hysop:${UBUNTU_RELEASE}"
CONTAINER_ID='hysop_build_and_debug'
-HYSOP_CACHE_DIR="${HOME}/.cache/hysop"
-
-if [[ ! -d "${HYSOP_CACHE_DIR}" ]]; then
- mkdir -p "${HYSOP_CACHE_DIR}"
-fi
function remove_img() {
docker stop "${CONTAINER_ID}" || true
@@ -21,6 +15,6 @@ remove_img
docker logout
docker pull "${DOCKER_IMG}"
-docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" -v "${HYSOP_CACHE_DIR}:/cache:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}"
+docker create -v "${SCRIPT_DIR}/../..:/hysop:ro" --name="${CONTAINER_ID}" -it "${DOCKER_IMG}"
docker start "${CONTAINER_ID}"
docker exec -it "${CONTAINER_ID}" /hysop/ci/scripts/build_and_debug.sh
diff --git a/ci/utils/run_docker_image.sh b/ci/utils/run_docker_image.sh
index ac6e49cfc..6690d0964 100755
--- a/ci/utils/run_docker_image.sh
+++ b/ci/utils/run_docker_image.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
set -euf -o pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-docker run -it -v "${SCRIPT_DIR}/../..:/hysop:ro" keckj/hysop:bionic
+UBUNTU_RELEASE='bionic'
+docker run -it -v "${SCRIPT_DIR}/../..:/hysop:ro" "keckj/hysop:${UBUNTU_RELEASE}"
diff --git a/hysop/operator/tests/test_custom_symbolic.py b/hysop/operator/tests/test_custom_symbolic.py
index 305c9a5c3..ff5930c3a 100644
--- a/hysop/operator/tests/test_custom_symbolic.py
+++ b/hysop/operator/tests/test_custom_symbolic.py
@@ -792,7 +792,7 @@ class TestCustomSymbolic(object):
if __name__ == '__main__':
TestCustomSymbolic.setup_class(enable_extra_tests=False,
- enable_debug_mode=False)
+ enable_debug_mode=False)
enable_pretty_printing()
--
GitLab