Merge branch 'feature/fix-ctest-failure' into 'develop'

Feature/fix ctest failure

See merge request !116

.gitlab/before.sh

@@ -63,7 +63,9 @@ fi
 QT_VERSION=$(qmake --version | grep Qt | cut -f4 -d" " )
 
 # output all information to report file
-echo "Docker container id....... $CONTAINER_ID" > /tmp/report.txt
+TIME_STAMP=$(date --rfc-2822)
+echo "Date ..................... $TIME_STAMP" > /tmp/report.txt
+echo "Docker container id....... $CONTAINER_ID" >> /tmp/report.txt
 echo "Job....................... $CI_JOB_NAME" >> /tmp/report.txt
 echo "PROJECT_SOURCE_DIR........ $PROJECT_SOURCE_DIR" >> /tmp/report.txt
 echo "CI_PROJECT_DIR............ $CI_PROJECT_DIR" >> /tmp/report.txt

.gitlab/gitlab-ci.yml

@@ -168,6 +168,7 @@ stages:
     paths: 
     - ${PROJECT_LOG_DIR}/test.log
     - ${PROJECT_LOG_DIR}/ci-test.log
+    - ${PROJECT_LOG_DIR}/rerun-test.log
     when: always
     expire_in: 7 days 
 

.gitlab/test.sh

@@ -35,7 +35,7 @@ fi
 
 echo "===== Running ctest ====="
 
-ctest -VV \
+ctest --extra-verbose \
       -DCTEST_SITE="$CDASH_SITE" \
       -DCI_MODE="$CAMITK_CI_MODE" \
       -DCI_ID="P $CI_PIPELINE_ID - J $CI_BUILD_ID" \
@@ -51,7 +51,7 @@ ctest -VV \
 # see https://gitlab.kitware.com/cmake/cmake/issues/17767
 # and https://gitlab.kitware.com/cmake/cmake/issues/16314
 #     echo "===== Re-running failed tests ====="
-#     ctest -VV \
+#     ctest --extra-verbose \
 #         --rerun-failed \
 #         -DCTEST_SITE="$CDASH_SITE" \
 #         -DCI_MODE="$CAMITK_CI_MODE" \
@@ -62,7 +62,7 @@ ctest -VV \
 #         -DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \
 #         -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \
 #         -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-test.cmake > >(tee --append ${PROJECT_LOG_DIR}/test.log | grep --line-buffered -e "Test \#") 2>&1
-
+#
 # if [[ "$OS" != "win7" ]]; then
 #     # shutdown xvfb
 #     kill $xvfbPid
@@ -81,7 +81,45 @@ if grep --quiet "Fatal error" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/ci-test.log; then
     echo "===== Failed tests ====="
     grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log
     echo
-    exit 1
+    echo "===== Re-running failed tests ====="
+    echo "Checking $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log for failed test" > ${PROJECT_LOG_DIR}/rerun-test.log
+    echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
+    # loop over failed
+    grep -e "\*\*\*Failed" -e "SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/test.log | 
+    while IFS= read -r line; do 
+        testname=$(echo $line | sed -r 's/.+[0-9]+:\s(.*)\s\..*/\1/g')
+        echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
+        echo "Re-running $testname..." >> ${PROJECT_LOG_DIR}/rerun-test.log
+        echo "Reason:" >> ${PROJECT_LOG_DIR}/rerun-test.log
+        # add ----> at the beginning of the line to distinguished real error log during rerunning 
+        echo "----> $line" >> ${PROJECT_LOG_DIR}/rerun-test.log
+        echo "" >> ${PROJECT_LOG_DIR}/rerun-test.log
+        ctest --extra-verbose \
+              -DCTEST_SITE="$CDASH_SITE" \
+              -DCI_MODE="$CAMITK_CI_MODE" \
+              -DCI_ID="P $CI_PIPELINE_ID - J $CI_BUILD_ID" \
+              -DCI_BRANCH="$CI_COMMIT_REF_NAME" \
+              -DCI_BUILD_SETTINGS="$COMPILER_CONFIG" \
+              -DCI_PROJECT_LOG_DIRECTORY="$CI_PROJECT_DIR/$PROJECT_LOG_DIR" \
+              -DCTEST_SOURCE_DIRECTORY="$PROJECT_SOURCE_DIR" \
+              -DCTEST_BINARY_DIRECTORY="$PROJECT_BUILD_DIR" \
+              -DRERUN_TESTNAME=$testname \
+              -S $PROJECT_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > >(tee --append ${PROJECT_LOG_DIR}/rerun-test.log | grep --line-buffered -e "Test \#") 2>&1
+    done
+    # check the rerun log
+    if grep --quiet "^[^-].*\*\*\*Failed" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log; then
+        echo
+        echo "===== Fatal errors in reran tests ====="
+        echo "Found fatal error in rerun test log $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log"
+        echo 
+        echo "===== Failed tests in reran tests ====="
+        grep -e "^[^-].*\*\*\*Failed" -e "^[^-].*SegFault" $CI_PROJECT_DIR/$PROJECT_LOG_DIR/rerun-test.log
+        echo
+        exit 1
+    else
+        echo "Re-ran tests OK"
+    fi
 else
+    echo "Everything OK. No test needed to be re-ran." > ${PROJECT_LOG_DIR}/rerun-test.log
     echo "OK"
 fi

sdk/cmake/ctest/ci-rerun.cmake

+# Driver instructions for the reran a failed test during continuous integration
+#
+# 1. Define parameters
+# CAMITK_SITE="[Gitlab Runner] debian stable"                  # the name of the current machine 
+# CAMITK_CI_MODE="Experimental"                                # ctest mode (Nightly, Continuous, Experimental...)
+# CAMITK_CI_BRANCH="branch-name"                               # git branch name (check directly with git if not defined)
+# CAMITK_CI_ID="Pipeline #$CI_PIPELINE_ID Job #$CI_BUILD_ID"   # unique id 
+# CAMITK_SOURCE_DIR=~/Dev/CamiTK/src/camitk                    # path to CamiTK code source directory
+# CAMITK_BUILD_DIR=~/Dev/CamiTK/build/camitk-exp               # path to the intended build directory
+# CAMITK_BUILD_SETTINGS="GCC-64bits-Debug"                     # compiler-arch-buildtype string
+# RERUN_TESTNAME=..."                                          # name of the test
+#
+# 2. run the command
+# ctest -VV \
+#       -DCTEST_SITE="$CAMITK_SITE" \
+#       -DCI_MODE="$CAMITK_CI_MODE" \
+#       -DCI_ID="$CAMITK_CI_ID" \
+#       -DCI_BRANCH=$CAMITK_CI_BRANCH \
+#       -DCI_BUILD_SETTINGS="$COMPILER_CONFIG" \
+#       -DCTEST_SOURCE_DIRECTORY="$CAMITK_SOURCE_DIR" \
+#       -DCTEST_BINARY_DIRECTORY="$CAMITK_BUILD_DIR" \
+#       -DRERUN_TESTNAME=name-of-the-test-to-run \
+#       -S $CAMITK_SOURCE_DIR/sdk/cmake/ctest/ci-rerun.cmake > test.log 2>&1
+#
+# It will submit a new report in the "configure" section of the dashboard 
+# identified as $SITE and $COMPILER_CONFIG.
+#
+# What this script does ?
+# 1. INFORMATION STEP
+#       Configure SITE and BUILD information to be correctly display on the dashboard
+#       Loads information from the CTestConfig.cmake file.
+# 2. RUN TEST
+#       Re-Run test RERUN_TESTNAME
+
+set(CI_STAGE "Test")
+
+# ------------------------ STEP 1: information step ------------------------
+include("${CTEST_SOURCE_DIRECTORY}/sdk/cmake/ctest/ci-setup.cmake")
+
+# ------------------------ STEP 2: test ------------------------
+message(STATUS "Step 2. Rerun only test ${RERUN_TESTNAME}... Using ${NUMBER_OF_PROC} tests in parallel")
+
+# set extra time for test to 30 min = 1800 s (default is 10 min = 600 s)
+set(CTEST_TEST_TIMEOUT 1800)
+
+# run test
+message(STATUS "- Running test ${RERUN_TESTNAME}...")
+# When testname is something-1, adding a "$" at the end, avoid rerunning the test names something-10 to something-19
+set(TESTNAME_REGEX "${RERUN_TESTNAME}$") 
+
+ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND
+           PARALLEL_LEVEL ${NUMBER_OF_PROC}    # Run test in parallel
+           RETURN_VALUE TEST_RES
+           INCLUDE ${TESTNAME_REGEX}
+)
+       
+if (NOT TEST_RES EQUAL 0)
+    message(STATUS "- Error: Test ${RERUN_TESTNAME} failed with return value ${TEST_RES}")
+else()
+    message(STATUS "- Test ${RERUN_TESTNAME} passed")
+endif()
+        
+message(STATUS "- Submitting test ${RERUN_TESTNAME} results...")
+
+ctest_submit(PARTS Test RETURN_VALUE TEST_RES)
+        
+if (NOT TEST_RES EQUAL 0)
+    message(STATUS "- Error: Submitting test ${RERUN_TESTNAME} results failed with return value ${TEST_RES}")
+else()
+    message(STATUS "- Submitting test ${RERUN_TESTNAME} results passed")
+endif()
+
+message(STATUS "Test ${RERUN_TESTNAME} done")