Commit 2eaece56 authored by Emmanuel Promayon's avatar Emmanuel Promayon

Merge branch 'feature/fix-ctest-failure' into 'develop'

Feature/fix ctest failure

See merge request !116
parents 9fc14383 9b9da131
......@@ -63,7 +63,9 @@ fi
QT_VERSION=$(qmake --version | grep Qt | cut -f4 -d" " )
# output all information to report file
echo "Docker container id....... $CONTAINER_ID" > /tmp/report.txt
TIME_STAMP=$(date --rfc-2822)
echo "Date ..................... $TIME_STAMP" > /tmp/report.txt
echo "Docker container id....... $CONTAINER_ID" >> /tmp/report.txt
echo "Job....................... $CI_JOB_NAME" >> /tmp/report.txt
echo "PROJECT_SOURCE_DIR........ $PROJECT_SOURCE_DIR" >> /tmp/report.txt
echo "CI_PROJECT_DIR............ $CI_PROJECT_DIR" >> /tmp/report.txt
......
......@@ -168,6 +168,7 @@ stages:
paths:
- ${PROJECT_LOG_DIR}/test.log
- ${PROJECT_LOG_DIR}/ci-test.log
- ${PROJECT_LOG_DIR}/rerun-test.log
when: always
expire_in: 7 days
......
......@@ -35,7 +35,7 @@ fi
echo "===== Running ctest ====="
ctest -VV \
ctest --extra-verbose \
-DCTEST_SITE="$CDASH_SITE" \
-DCI_MODE="$CAMITK_CI_MODE" \
-DCI_ID="P $CI_PIPELINE_ID - J $CI_BUILD_ID" \
......@@ -51,7 +51,7 @@ ctest -VV \
# see https://gitlab.kitware.com/cmake/cmake/issues/17767
# and https://gitlab.kitware.com/cmake/cmake/issues/16314
# echo "===== Re-running failed tests ====="
# ctest -VV \
# ctest --extra-verbose \
# --rerun-failed \
# -DCTEST_SITE="$CDASH_SITE" \
# -DCI_MODE="$CAMITK_CI_MODE" \
......@@ -62,7 +62,7 @@ ctest -VV \
# -DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \
# -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \
# -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1
#
# if [[ "$OS" != "win7" ]]; then
# # shutdown xvfb
# kill $xvfbPid
......@@ -81,7 +81,45 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then
echo "===== Failed tests ====="
grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log
echo
exit 1
echo "===== Re-running failed tests ====="
echo "Checking $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log for failed test" > ${PROJECT_LOG_DIR}/rerun-test.log
echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
# loop over failed
grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log |
while IFS= read -r line; do
testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g')
echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
echo "Re-running $testname..." >> ${PROJECT_LOG_DIR}/rerun-test.log
echo "Reason:" >> ${PROJECT_LOG_DIR}/rerun-test.log
# add ----> at the beginning of the line to distinguished real error log during rerunning
echo "----> $line" >> ${PROJECT_LOG_DIR}/rerun-test.log
echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
ctest --extra-verbose \
-DCTEST_SITE="$CDASH_SITE" \
-DCI_MODE="$CAMITK_CI_MODE" \
-DCI_ID="P $CI_PIPELINE_ID - J $CI_BUILD_ID" \
-DCI_BRANCH="$CI_COMMIT_REF_NAME" \
-DCI_BUILD_SETTINGS="$COMPILER_CONFIG" \
-DCI_PROJECT_LOG_DIRECTORY="$CI_PROJECT_DIR/$PROJECT_LOG_DIR" \
-DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \
-DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \
-DRERUN_TESTNAME=$testname \
-S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > >(tee --append ${PROJECT_LOG_DIR}/rerun-test.log | grep --line-buffered -e "Test \#") 2>&1
done
# check the rerun log
if grep --quiet "^[^-].*\*\*\*Failed" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log; then
echo
echo "===== Fatal errors in reran tests ====="
echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log"
echo
echo "===== Failed tests in reran tests ====="
grep -e "^[^-].*\*\*\*Failed" -e "^[^-].*SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log
echo
exit 1
else
echo "Re-ran tests OK"
fi
else
echo "Everything OK. No test needed to be re-ran." > ${PROJECT_LOG_DIR}/rerun-test.log
echo "OK"
fi
# Driver instructions for the reran a failed test during continuous integration
#
# 1. Define parameters
# CAMITK_SITE="[Gitlab Runner] debian stable" # the name of the current machine
# CAMITK_CI_MODE="Experimental" # ctest mode (Nightly, Continuous, Experimental...)
# CAMITK_CI_BRANCH="branch-name" # git branch name (check directly with git if not defined)
# CAMITK_CI_ID="Pipeline #$CI_PIPELINE_ID Job #$CI_BUILD_ID" # unique id
# CAMITK_SOURCE_DIR=~/Dev/CamiTK/src/camitk # path to CamiTK code source directory
# CAMITK_BUILD_DIR=~/Dev/CamiTK/build/camitk-exp # path to the intended build directory
# CAMITK_BUILD_SETTINGS="GCC-64bits-Debug" # compiler-arch-buildtype string
# RERUN_TESTNAME=..." # name of the test
#
# 2. run the command
# ctest -VV \
# -DCTEST_SITE="$CAMITK_SITE" \
# -DCI_MODE="$CAMITK_CI_MODE" \
# -DCI_ID="$CAMITK_CI_ID" \
# -DCI_BRANCH=$CAMITK_CI_BRANCH \
# -DCI_BUILD_SETTINGS="$COMPILER_CONFIG" \
# -DCTEST_SOURCE_DIRECTORY="$CAMITK_SOURCE_DIR" \
# -DCTEST_BINARY_DIRECTORY="$CAMITK_BUILD_DIR" \
# -DRERUN_TESTNAME=name-of-the-test-to-run \
# -S $CAMITK_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > test.log 2>&1
#
# It will submit a new report in the "configure" section of the dashboard
# identified as $SITE and $COMPILER_CONFIG.
#
# What this script does ?
# 1. INFORMATION STEP
# Configure SITE and BUILD information to be correctly display on the dashboard
# Loads information from the CTestConfig.cmake file.
# 2. RUN TEST
# Re-Run test RERUN_TESTNAME
set(CI_STAGE "Test")
# ------------------------ STEP 1: information step ------------------------
include("${CTEST_SOURCE_DIRECTORY}/sdk/cmake/ctest/ci-setup.cmake")
# ------------------------ STEP 2: test ------------------------
message(STATUS "Step 2. Rerun only test ${RERUN_TESTNAME}... Using ${NUMBER_OF_PROC} tests in parallel")
# set extra time for test to 30 min = 1800 s (default is 10 min = 600 s)
set(CTEST_TEST_TIMEOUT 1800)
# run test
message(STATUS "- Running test ${RERUN_TESTNAME}...")
# When testname is something-1, adding a "$" at the end, avoid rerunning the test names something-10 to something-19
set(TESTNAME_REGEX "${RERUN_TESTNAME}$")
ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND
PARALLEL_LEVEL ${NUMBER_OF_PROC} # Run test in parallel
RETURN_VALUE TEST_RES
INCLUDE ${TESTNAME_REGEX}
)
if (NOT TEST_RES EQUAL 0)
message(STATUS "- Error: Test ${RERUN_TESTNAME} failed with return value ${TEST_RES}")
else()
message(STATUS "- Test ${RERUN_TESTNAME} passed")
endif()
message(STATUS "- Submitting test ${RERUN_TESTNAME} results...")
ctest_submit(PARTS Test RETURN_VALUE TEST_RES)
if (NOT TEST_RES EQUAL 0)
message(STATUS "- Error: Submitting test ${RERUN_TESTNAME} results failed with return value ${TEST_RES}")
else()
message(STATUS "- Submitting test ${RERUN_TESTNAME} results passed")
endif()
message(STATUS "Test ${RERUN_TESTNAME} done")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment