From 747e08e237b015e09cc655a75a1143d109dc72b8 Mon Sep 17 00:00:00 2001 From: Emmanuel Promayon Date: Sat, 21 Nov 2020 11:30:30 +0100 Subject: [PATCH 1/4] FIXED parse ctest log for more than just Failed and SegFault --- .gitlab/test.sh | 50 +++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/.gitlab/test.sh b/.gitlab/test.sh index 71d3ec34..46326917 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -2,6 +2,12 @@ # Uncomment next line to print each bash command before it is executed #set -x +# function to list failing test in ctest log file +# first argument = file to check +listFailedTests() { + echo "$(grep -e "\*\*\*Failed" -e "SegFault" -e "\*\*\****Exception" -e "\*\*\*Timeout" $1 )" +} + # Path needs to use forward slashes # This is ok on Linux but since gitlab-runner 11.7 on windows all path variables use backward slash instead of forward slash # → Replace all backslash to forward slash @@ -46,10 +52,12 @@ ctest --extra-verbose \ -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \ -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1 -# Rerun only the failed tests -# There seems to be a bug in CMake, disabling for now -# see https://gitlab.kitware.com/cmake/cmake/issues/17767 -# and https://gitlab.kitware.com/cmake/cmake/issues/16314 +# Using ctest with its --rerun-failed flag in order to only rerun the failed tests +# seems to be buggy. +# See CMake bugs: +# - https://gitlab.kitware.com/cmake/cmake/issues/17767 +# - https://gitlab.kitware.com/cmake/cmake/issues/16314 +# Otherwise it would only requires to add the following lines: # echo "===== Re-running failed tests =====" # ctest --extra-verbose \ # --rerun-failed \ @@ -62,12 +70,8 @@ ctest --extra-verbose \ # -DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \ # -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \ # -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1 -# -# if [[ "$OS" != "win10" ]]; then -# # shutdown xvfb -# kill $xvfbPid -# fi +# So as for now, parse the log and do a manual rerun echo echo "===== Not run (disabled) tests =====" grep -e "Not Run" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log @@ -79,16 +83,16 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then echo "Found fatal error in $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log" echo echo "===== Failed tests =====" - grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log + testToReRun=$(listFailedTests $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log) + echo "$testToReRun" echo echo "===== Re-running failed tests =====" echo "Checking $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log for failed test" > ${PROJECT_LOG_DIR}/rerun-test.log echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log - # loop over failed - grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log | - while IFS= read -r line; do - testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g') + # loop over failed tests + echo "$testToReRun" | while IFS= read -r line; do echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log + testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g') echo "Re-running $testname..." >> ${PROJECT_LOG_DIR}/rerun-test.log echo "Reason:" >> ${PROJECT_LOG_DIR}/rerun-test.log # add ----> at the beginning of the line to distinguished real error log during rerunning @@ -106,20 +110,26 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then -DRERUN_TESTNAME=$testname \ -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > >(tee --append ${PROJECT_LOG_DIR}/rerun-test.log | grep --line-buffered -e "Test \#") 2>&1 done - # check the rerun log - if grep --quiet "^[^-].*\*\*\*Failed" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log; then + # check the rerun log (listFailedTests but remove the reason lines added in the lg) + reRunFailed=$(listFailedTests ${PROJECT_LOG_DIR}/rerun-test.log | grep "^[^-].*") + if test -z "$reRunFailed"; then + echo "Re-ran tests OK" + else echo echo "===== Fatal errors in reran tests =====" echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" echo echo "===== Failed tests in reran tests =====" - grep -e "^[^-].*\*\*\*Failed" -e "^[^-].*SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log + echo "$reRunFailed" >> ${$PROJECT_LOG_DIR}/rerun-test.log echo exit 1 - else - echo "Re-ran tests OK" fi else - echo "Everything OK. No test needed to be re-ran." > ${PROJECT_LOG_DIR}/rerun-test.log + echo "Everything OK. No test needed to be re-ran." >> ${PROJECT_LOG_DIR}/rerun-test.log echo "OK" fi + +# shutdown xvfb +if [[ "$OS" != "win10" ]]; then + kill $xvfbPid +fi -- GitLab From 53c043a6948ec501fdebe0614d9c63a044071d4d Mon Sep 17 00:00:00 2001 From: Emmanuel Promayon Date: Sat, 21 Nov 2020 13:10:07 +0100 Subject: [PATCH 2/4] FIXED extra $ --- .gitlab/test.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.gitlab/test.sh b/.gitlab/test.sh index 46326917..6f09bb36 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -120,7 +120,7 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" echo echo "===== Failed tests in reran tests =====" - echo "$reRunFailed" >> ${$PROJECT_LOG_DIR}/rerun-test.log + echo "$reRunFailed" >> ${PROJECT_LOG_DIR}/rerun-test.log echo exit 1 fi @@ -128,8 +128,3 @@ else echo "Everything OK. No test needed to be re-ran." >> ${PROJECT_LOG_DIR}/rerun-test.log echo "OK" fi - -# shutdown xvfb -if [[ "$OS" != "win10" ]]; then - kill $xvfbPid -fi -- GitLab From 17d606a493d4ae4334670521632aa91141d79bad Mon Sep 17 00:00:00 2001 From: Emmanuel Promayon Date: Sat, 21 Nov 2020 16:12:38 +0100 Subject: [PATCH 3/4] FIXED more comprehensive log and pipeline output --- .gitlab/test.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitlab/test.sh b/.gitlab/test.sh index 6f09bb36..e73baa94 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -82,10 +82,13 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then echo "===== Fatal errors =====" echo "Found fatal error in $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log" echo - echo "===== Failed tests =====" + testToReRun=$(listFailedTests $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log) + nrOfFailed=$(echo "$testToReRun" | wc -l) + echo "===== $nrOfFailed failed test(s) =====" echo "$testToReRun" echo + echo "===== Re-running failed tests =====" echo "Checking $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log for failed test" > ${PROJECT_LOG_DIR}/rerun-test.log echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log @@ -110,18 +113,25 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then -DRERUN_TESTNAME=$testname \ -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > >(tee --append ${PROJECT_LOG_DIR}/rerun-test.log | grep --line-buffered -e "Test \#") 2>&1 done + # check the rerun log (listFailedTests but remove the reason lines added in the lg) reRunFailed=$(listFailedTests ${PROJECT_LOG_DIR}/rerun-test.log | grep "^[^-].*") + nrOfFailed=$(echo "$testToReRun" | wc -l) if test -z "$reRunFailed"; then - echo "Re-ran tests OK" + echo "===== All re-ran tests passed =====" + echo "OK" else echo - echo "===== Fatal errors in reran tests =====" - echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" + echo "===== Fatal errors in re-ran tests =====" + echo "Found fatal error in re-run test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log" echo - echo "===== Failed tests in reran tests =====" - echo "$reRunFailed" >> ${PROJECT_LOG_DIR}/rerun-test.log + + + echo "===== $nrOfFailed failed test(s) during reran =====" + echo "$reRunFailed" echo + + # send error to the pipeline exit 1 fi else -- GitLab From 54954abe603fc962613ae717ac465c0f6e34cc5a Mon Sep 17 00:00:00 2001 From: Emmanuel Promayon Date: Sat, 21 Nov 2020 16:18:35 +0100 Subject: [PATCH 4/4] FIXED (tentative) kill all background jobs at exit --- .gitlab/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/test.sh b/.gitlab/test.sh index e73baa94..2cf18199 100755 --- a/.gitlab/test.sh +++ b/.gitlab/test.sh @@ -30,7 +30,7 @@ if [[ "$OS" != "win10" ]]; then # create a specific file for xauth export XAUTHORITY=$(mktemp) Xvfb $DISPLAY -screen 0 1280x1024x16 -ac -nolisten tcp -nolisten unix -auth $XAUTHORITY & > >(tee --append ${PROJECT_LOG_DIR}/test.log) 2>&1 - trap "kill $! || true" EXIT + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT # give some times to start the Xvfb sleep 10 # On linux, use QT_QPA_PLATFORM=offscreen -- GitLab