diff --git a/.gitlab/test.sh b/.gitlab/test.sh index 71d3ec3473ed41d0a8a29b247be490253c115e7a..2cf181992a759fc452bd6d777b7c17a93fbcc3bb 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -2,6 +2,12 @@ # Uncomment next line to print each bash command before it is executed #set -x +# function to list failing test in ctest log file +# first argument = file to check +listFailedTests() { + echo "$(grep -e "\*\*\*Failed" -e "SegFault" -e "\*\*\****Exception" -e "\*\*\*Timeout" $1 )" +} + # Path needs to use forward slashes # This is ok on Linux but since gitlab-runner 11.7 on windows all path variables use backward slash instead of forward slash # → Replace all backslash to forward slash @@ -24,7 +30,7 @@ if [[ "$OS" != "win10" ]]; then # create a specific file for xauth export XAUTHORITY=$(mktemp) Xvfb $DISPLAY -screen 0 1280x1024x16 -ac -nolisten tcp -nolisten unix -auth $XAUTHORITY & > >(tee --append ${PROJECT_LOG_DIR}/test.log) 2>&1 - trap "kill $! || true" EXIT + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT # give some times to start the Xvfb sleep 10 # On linux, use QT_QPA_PLATFORM=offscreen @@ -46,10 +52,12 @@ ctest --extra-verbose \ -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \ -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1 -# Rerun only the failed tests -# There seems to be a bug in CMake, disabling for now -# see https://gitlab.kitware.com/cmake/cmake/issues/17767 -# and https://gitlab.kitware.com/cmake/cmake/issues/16314 +# Using ctest with its --rerun-failed flag in order to only rerun the failed tests +# seems to be buggy. +# See CMake bugs: +# - https://gitlab.kitware.com/cmake/cmake/issues/17767 +# - https://gitlab.kitware.com/cmake/cmake/issues/16314 +# Otherwise it would only requires to add the following lines: # echo "===== Re-running failed tests =====" # ctest --extra-verbose \ # --rerun-failed \ @@ -62,12 +70,8 @@ ctest --extra-verbose \ # -DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \ # -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \ # -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1 -# -# if [[ "$OS" != "win10" ]]; then -# # shutdown xvfb -# kill $xvfbPid -# fi +# So as for now, parse the log and do a manual rerun echo echo "===== Not run (disabled) tests =====" grep -e "Not Run" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log @@ -78,17 +82,20 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then echo "===== Fatal errors =====" echo "Found fatal error in $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log" echo - echo "===== Failed tests =====" - grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log + + testToReRun=$(listFailedTests $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log) + nrOfFailed=$(echo "$testToReRun" | wc -l) + echo "===== $nrOfFailed failed test(s) =====" + echo "$testToReRun" echo + echo "===== Re-running failed tests =====" echo "Checking $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log for failed test" > ${PROJECT_LOG_DIR}/rerun-test.log echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log - # loop over failed - grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log | - while IFS= read -r line; do - testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g') + # loop over failed tests + echo "$testToReRun" | while IFS= read -r line; do echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log + testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g') echo "Re-running $testname..." >> ${PROJECT_LOG_DIR}/rerun-test.log echo "Reason:" >> ${PROJECT_LOG_DIR}/rerun-test.log # add ----> at the beginning of the line to distinguished real error log during rerunning @@ -106,20 +113,28 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then -DRERUN_TESTNAME=$testname \ -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > >(tee --append ${PROJECT_LOG_DIR}/rerun-test.log | grep --line-buffered -e "Test \#") 2>&1 done - # check the rerun log - if grep --quiet "^[^-].*\*\*\*Failed" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log; then + + # check the rerun log (listFailedTests but remove the reason lines added in the lg) + reRunFailed=$(listFailedTests ${PROJECT_LOG_DIR}/rerun-test.log | grep "^[^-].*") + nrOfFailed=$(echo "$testToReRun" | wc -l) + if test -z "$reRunFailed"; then + echo "===== All re-ran tests passed =====" + echo "OK" + else echo - echo "===== Fatal errors in reran tests =====" - echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" + echo "===== Fatal errors in re-ran tests =====" + echo "Found fatal error in re-run test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" echo - echo "===== Failed tests in reran tests =====" - grep -e "^[^-].*\*\*\*Failed" -e "^[^-].*SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log + + + echo "===== $nrOfFailed failed test(s) during reran =====" + echo "$reRunFailed" echo + + # send error to the pipeline exit 1 - else - echo "Re-ran tests OK" fi else - echo "Everything OK. No test needed to be re-ran." > ${PROJECT_LOG_DIR}/rerun-test.log + echo "Everything OK. No test needed to be re-ran." >> ${PROJECT_LOG_DIR}/rerun-test.log echo "OK" fi