Merge branch 'feature/asm-for-regression-test' into 'develop'

Feature/asm for regression test

See merge request !21

sdk/actions/image/reconstruction/CMakeLists.txt

@@ -8,3 +8,8 @@ camitk_extension(ACTION_EXTENSION
 
 # Recursively update the shiboken path variable containing the CamiTK SDK tree structure
 set(SHIBOKEN_CAMITK_SDK_PATH ${SHIBOKEN_CAMITK_SDK_PATH}:${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "") 
+
+# add a specific test to run the action, save the output and compare it to expected
+camitk_add_test_actionstatemachine(NAME action-reconstruction-regression-test
+                                   CAMITK_SCXML action-reconstruction-regression-test.xml
+                                   EXPECTED_OUTPUT_FILES action-reconstruction-output-test.vtk)

sdk/actions/image/reconstruction/testdata/action-reconstruction-regression-test.xml

+<scxml xmlns="http://camitk.imag.fr/3/smallScxml" initial="Initialize" xmlns:camitk="http://camitk.imag.fr/3/asm" xsi:schemaLocation="http://camitk.imag.fr/3/smallScxml/../resources/smallScxml.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <state id="Initialize">
+  <onentry>
+   <camitk:onState>
+    <camitk:description><![CDATA[This pipeline will process several actions on the input component(s): <br /> <ul><li>Open File</li><li>Reconstruction</li><li>Save</li>]]></camitk:description>
+   </camitk:onState>
+  </onentry>
+  <transition target="Action 1" event="Next"/>
+ </state>
+ <state id="Action 1">
+  <onentry>
+   <camitk:onState>
+    <camitk:description>Open data (component) from a given file</camitk:description>
+    <camitk:action>
+     <camitk:name>Open File</camitk:name>
+     <camitk:parameters>
+         <!--  
+         Bug? There are two lines for this parameter, the first is written with an empty name value:
+      <camitk:parameter name="File name" value="" type="QString"/> 
+         This should not appear.
+         -->
+      <camitk:parameter name="File Name" value="action-reconstruction-input.mha" type="QString"/>
+     </camitk:parameters>
+     <camitk:parameters/>
+     <camitk:outputs>
+      <camitk:component name="image" type="ImageComponent"/>
+     </camitk:outputs>
+    </camitk:action>
+   </camitk:onState>
+  </onentry>
+  <transition target="Action 2" event="Next"/>
+  <transition target="Initialize" event="Back"/>
+ </state>
+ <state id="Action 2">
+  <onentry>
+   <camitk:onState>
+    <camitk:description>&lt;br /> Marching cubes is a simple algorithm for creating a triangle mesh from an implicit function (one of the form f(x, y, z) = 0).&lt;br /> It works by iterating ("marching") over a uniform grid of cubes superimposed over a region of the function. If all 8 vertices of the cube are positive, or all 8 vertices are negative, the cube is entirely above or entirely below the surface and no triangles are emitted. Otherwise, the cube straddles the function and some triangles and vertices are generated. Since each vertex can either be positive or negative, there are technically 28 possible configurations, but many of these are equivalent to one another. &lt;br/> &lt;br/>This action uses this algorithm to build a 3D surfacic mesh of the input image.</camitk:description>
+    <camitk:action>
+     <camitk:name>Reconstruction</camitk:name>
+     <camitk:parameters>
+      <camitk:parameter name="Threshold" value="127.5" type="double"/>
+      <camitk:parameter name="Keep only largest component?" value="true" type="bool"/>
+      <camitk:parameter name="Subsample original image?" value="true" type="bool"/>
+      <camitk:parameter name="Subsampled image width (X)" value="32" type="int"/>
+      <camitk:parameter name="Subsampled image height (Y)" value="32" type="int"/>
+      <camitk:parameter name="Subsampled image depth (Z)" value="32" type="int"/>
+     </camitk:parameters>
+     <camitk:parameters/>
+     <camitk:inputs>
+      <camitk:component name="image" type="ImageComponent"/>
+     </camitk:inputs>
+     <camitk:outputs>
+      <camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
+     </camitk:outputs>
+    </camitk:action>
+   </camitk:onState>
+  </onentry>
+  <transition target="Action 3" event="Next"/>
+  <transition target="Action 1" event="Back">
+   <onTransition>
+    <camitk:close>
+     <camitk:component name="image" type="ImageComponent"/>
+    </camitk:close>
+   </onTransition>
+  </transition>
+ </state>
+ <state id="Action 3">
+  <onentry>
+   <camitk:onState>
+    <camitk:description>Save all the top-level of the selected components</camitk:description>
+    <camitk:action>
+     <camitk:name>Save</camitk:name>
+     <camitk:parameters/>
+     <camitk:inputs>
+      <camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
+     </camitk:inputs>
+    </camitk:action>
+   </camitk:onState>
+  </onentry>
+  <transition target="Bye" event="Next"/>
+  <transition target="Action 2" event="Back">
+   <onTransition>
+    <camitk:close>
+     <camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
+    </camitk:close>
+   </onTransition>
+  </transition>
+ </state>
+ <state id="Bye" final="true">
+  <onentry>
+   <camitk:onState>
+    <camitk:description><![CDATA[Thanks you for using the CamiTK Action State Machine !]]></camitk:description>
+   </camitk:onState>
+  </onentry>
+  <transition target="Initialize" event="Back to the beginning">
+   <onTransition>
+    <camitk:close>
+     <camitk:component name="image" type="ImageComponent"/>
+     <camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
+    </camitk:close>
+   </onTransition>
+  </transition>
+ </state>
+</scxml>

sdk/applications/actionstatemachine/ActionState.cpp

@@ -310,6 +310,13 @@ void ActionState::onExit(QEvent* event) {
     (*logStream) << "\t</state>" << endl;
 }
 
+// ------------- trigger -----------------
+void ActionState::trigger() {
+    onEntry(nullptr);
+    applyAction();
+    onExit(nullptr);
+}
+
 // ------------- getInputComponents -----------------
 const QMap< QString, QString >& ActionState::getInputComponents() {
     return inputComponentNames;

sdk/applications/actionstatemachine/ActionState.h

@@ -84,6 +84,9 @@ public:
     /// get the names and type of all output components
     const QMap<QString, QString>& getOutputComponents();
 
+    /// manually trigger the state
+    void trigger();
+    
 protected:
     /// Reimplemented from QState
     /// @{

sdk/applications/actionstatemachine/ActionStateMachine.cpp

@@ -127,6 +127,43 @@ ActionStateMachine::~ActionStateMachine() {
     logFile->close();
 }
 
+// ---------------------- autoNext ----------------------------
+void ActionStateMachine::autoNext() {
+    ActionState* currentState;
+    ActionState *nextState;
+    
+    std::cout << "Starting autonext..." << std::endl;
+    
+    // get the first initial state (there should only be one)
+    currentState = dynamic_cast<ActionState*> (machine.configuration().toList().first());
+    // loop until no more state is available
+    while (currentState) {
+        std::cout << "- Entering state \"" << currentState->getName().toStdString() << "\"" << std::endl;
+        nextState=nullptr;
+        // get the next state by looking for the "Next" transition
+        foreach (QAbstractTransition* currentStateTransition, currentState->transitions()) {
+            ActionTransition* currentActionTransition = dynamic_cast<ActionTransition*>(currentStateTransition);        
+            if (currentActionTransition->getButtonText() == "Next" || currentActionTransition->getButtonText() == "Quit") {
+                // "Next" or "Quit" is found, activate the transition
+                std::cout << "- Activating Transition \"" << currentActionTransition->getButtonText().toStdString()  << "\"" << std::endl;
+                currentActionTransition->onTransition(nullptr);
+                // update the next state to enter
+                nextState = dynamic_cast<ActionState*> (currentActionTransition->targetState());
+            }                
+        }
+        // activate currentState
+        if (currentState) {
+           std::cout << "- Triggering state" << std::endl; 
+           currentState->trigger();
+        }
+        // next state
+        currentState = nextState;
+    }
+
+    std::cout << "Quitting autonext and application" << std::endl;
+    quit();
+}
+
 // ---------------------- initMainWindow ----------------------------
 void ActionStateMachine::initMainWindow() {
 

sdk/applications/actionstatemachine/ActionStateMachine.h

@@ -68,6 +68,10 @@ public:
     /// where to save the files
     QString getSaveDirectory();
 
+public slots:
+    /// Automatically loop to apply to next state for all states
+    void autoNext();
+    
 protected:
 
     /// initialized main window

sdk/applications/actionstatemachine/CommandLineOptions.cli

@@ -5,7 +5,7 @@
 // cli CommandLineOptions.cli
 //
 // to update the manpage:
-// cli --stdout --man-prologue manpage-prologue.1.in --man-epilogue manpage-epilogue.1.in --generate-man CommandLineOptions.cli > camitk-actionstatemachine.1.in
+// cli --stdout --man-prologue resources/manpage-prologue.1.in --man-epilogue resources/manpage-epilogue.1.in --generate-man CommandLineOptions.cli > resources/camitk-actionstatemachine.1.in
 
 include <string>;
 
@@ -15,5 +15,6 @@ class options {
   std::string   --file | -f             {"<file>",
                                          "CamiTK SCXML document to play back in the state machine"};
   std::string   --outDir | -o           {"<dir>",
-                                         "Output directory (will contains logs and saved files)"};  
+                                         "Output directory (will contains logs and saved files)"};
+  bool          --autonext | -a         {"Automatically run the state and the \"Next\" transition until final state"};
 };
\ No newline at end of file

sdk/applications/actionstatemachine/CommandLineOptions.hxx

@@ -245,6 +245,9 @@ class options
   const std::string&
   outDir () const;
 
+  const bool&
+  autonext () const;
+
   // Print usage information.
   //
   static void
@@ -261,6 +264,7 @@ class options
   bool version_;
   std::string file_;
   std::string outDir_;
+  bool autonext_;
 };
 
 #include "CommandLineOptions.ixx"

sdk/applications/actionstatemachine/CommandLineOptions.ixx

@@ -132,3 +132,9 @@ outDir () const
   return this->outDir_;
 }
 
+inline const bool& options::
+autonext () const
+{
+  return this->autonext_;
+}
+

sdk/applications/actionstatemachine/main.cpp

@@ -32,6 +32,7 @@ using namespace camitk;
 #include <QString>
 #include <QMap>
 #include <QMessageBox>
+#include <QTimer>
 
 #include "CommandLineOptions.hxx"
 
@@ -62,6 +63,11 @@ int main(int argc, char** argv) {
             // create a camitk application
             ActionStateMachine am(argc, argv, QString(o.file().c_str()), QString(o.outDir().c_str()));
 
+            // automatically call autonext slot at application start up
+            if (o.autonext()) {
+                QTimer::singleShot(0, &am, SLOT(autoNext()));
+            }
+            
             return am.exec();
         }
     }
@@ -75,7 +81,7 @@ int main(int argc, char** argv) {
         return EXIT_FAILURE;
     }
     catch (const std::exception& e) {
-        std::cout << argv[0] << " aborted..." << std::endl << "std AbortException:" << std::endl << e.what() << std::endl;
+        std::cout << argv[0] << " aborted..." << std::endl << "std exception:" << std::endl << e.what() << std::endl;
         return EXIT_FAILURE;
     }
     catch (...) {

sdk/applications/actionstatemachine/resources/camitk-actionstatemachine.1.in

@@ -16,7 +16,7 @@ This application takes an CamiTK augmented version of a SCXML subset as input an
 interactively replays the described state-machine. The state-machine contains a list 
 of CamiTK actions and transitions.
 .PP
-The targeted users are R&D departments or laboratories.
+The targeted users are in R&D departments or laboratories.
 .PP
 .SH "OPTIONS"
 
@@ -36,9 +36,12 @@ CamiTK SCXML document to play back in the state machine
 .IP "\fB--outDir\fP|\fB-o\fP \fIdir\fP"
 Output directory (will contains logs and saved files)
 
+.IP "\fB--autonext\fP|\fB-a\fP"
+Automatically run the state and the "Next" transition until final state
+
 .SH "AUTHORS"
 CamiTK and camitk\-@APPLICATION_NAME@ is currently developed by the Univ. Grenoble Alpes,
-CNRS in the TIMC\-IMAG UMR 5525 laboratory (GMCAO).
+CNRS, in the TIMC\-IMAG UMR 5525 laboratory (GMCAO).
 .SH "SEE ALSO"
 The full documentation and tutorials for
 .B CamiTK

sdk/applications/imp/main.cpp

@@ -130,7 +130,7 @@ int main(int argc, char* argv[]) {
     }
     catch (const std::exception& e) {
         //QMessageBox::warning(NULL, QString(argv[0]) + " aborted...", "AbortException:" + QString(e.what()));
-        std::cout << argv[0] << " aborted..." << std::endl << "std AbortException:" << std::endl << e.what() << std::endl;
+        std::cout << argv[0] << " aborted..." << std::endl << "std exception:" << std::endl << e.what() << std::endl;
         return EXIT_FAILURE;
     }
     catch (...) {

sdk/applications/wizard/Main.cpp

@@ -82,7 +82,7 @@ int main(int argc, char* argv[]) {
         return EXIT_FAILURE;
     }
     catch (const std::exception& e) {
-        std::cout << argv[0] << " aborted..." << std::endl << "std AbortException:" << std::endl << e.what() << std::endl;
+        std::cout << argv[0] << " aborted..." << std::endl << "std exception:" << std::endl << e.what() << std::endl;
         return EXIT_FAILURE;
     }
     catch (...) {

sdk/cmake/modules/CamiTKMacros.cmake

@@ -28,6 +28,7 @@ include(camitk/cdash/projects/CamiTKSubProjectValidate)
 # -- CamiTK test macros (see also http://www.org/cmake/help/v2.8.8/cmake.html#section_PropertiesonTests)
 include(camitk/test/CamiTKInitTest)
 include(camitk/test/CamiTKAddTest)
+include(camitk/test/CamiTKAddTestActionStateMachine)
 include(camitk/test/CamiTKAdditionalActionTest)
 
 # -- packaging macros

sdk/cmake/modules/macros/camitk/test/CamiTKAddTest.cmake

@@ -3,7 +3,7 @@
 #!
 #! camitk_add_test is a macro to add a new test to the CTest infrastructure
 #! It encapsulates CMake add_test and adds useful way of testing programs.
-#! It cannot operate on its on, you need to call camitk_init_test first (and only once) before
+#! It cannot operate on its own, you need to call camitk_init_test first (and only once) before
 #! calling camitk_add_test
 #!
 #! Details on the runned test can be found in ${CMAKE_BINARY_DIR}/Testing/Temporary/target#

sdk/cmake/modules/macros/camitk/test/CamiTKAddTestActionStateMachine.cmake

+#!
+#! @ingroup group_sdk_cmake_camitk_test
+#!
+#! camitk_add_test_actionstatemachine is a macro to add a new test to the CTest infrastructure
+#! It encapsulates CMake add_test and use the action state machine to run a set of actions.
+#!
+#! Details on the runned test can be found in ${CMAKE_BINARY_DIR}/Testing/Temporary/target#
+#! where target is the executable name (see camitk_init_test() macro), and # is the test order number.
+#!
+#! Usage:
+#! \code
+#! camitk_add_test_actionstatemachine(NAME "testName"
+#!                                    CAMITK_SCXML "file.xml"
+#!                                    EXPECTED_OUTPUT_FILES file1 file2 file3...
+#! )
+#! \endcode
+#!
+#! It creates a target name 
+#! 
+#! \param NAME                                Name of the test, use "ctest -V -R testName" to run the test individually 
+#! \param CAMITK_SCXML                        The input xml file
+#! \param EXPECTED_OUTPUT_FILES               List of files that have to be compared with the asm output
+#!
+macro(camitk_add_test_actionstatemachine)
+    set(options "")
+    set(oneValueArgs NAME CAMITK_SCXML)
+    set(multiValueArgs EXPECTED_OUTPUT_FILES)
+    cmake_parse_arguments(CAMITK_ADD_TEST_ASM "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+    # determine cmake macro path
+    if (NOT EXISTS SDK_TOP_LEVEL_SOURCE_DIR)
+        # this macro is called outside the sdk 
+        set(CAMITK_CMAKE_MACRO_PATH ${CAMITK_CMAKE_DIR}/modules/macros/camitk/test)
+        if(NOT IS_DIRECTORY ${CAMITK_CMAKE_MACRO_PATH})
+            # inside opensource but not in sdk (modeling or imaging)
+            set(CAMITK_CMAKE_MACRO_PATH ${CMAKE_SOURCE_DIR}/sdk/cmake/modules/macros/camitk/test)
+        endif()
+    else()
+        # directly use the macro source dir
+        set(CAMITK_CMAKE_MACRO_PATH ${SDK_TOP_LEVEL_SOURCE_DIR}/cmake/modules/macros/camitk/test)
+    endif()   
+
+    #-- clean/create test output directory
+    set(CAMITK_TEST_OUTPUT_DIR "${CMAKE_BINARY_DIR}/Testing/Temporary/${CAMITK_ADD_TEST_ASM_NAME}")
+    file(REMOVE_RECURSE ${CAMITK_TEST_OUTPUT_DIR})
+    file(MAKE_DIRECTORY ${CAMITK_TEST_OUTPUT_DIR})
+
+    #-- replace the input file names
+    # Read the file in a variable
+    file(READ ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} ASM_INPUT_FILE_AS_STRING)
+    # get all the lines that have a non-empty "File Name" parameter
+    string(REGEX MATCHALL "<camitk:parameter name=\"File Name\" value=\".*\" type=\"QString\"/>" FILENAME_LINES "${ASM_INPUT_FILE_AS_STRING}")
+    # intialize ASM_INPUT_FILE_AS_STRING_OK
+    set(ASM_INPUT_FILE_AS_STRING_OK "${ASM_INPUT_FILE_AS_STRING}")
+    # for each lines that have a non-empty "File Name" parameter
+    foreach(CURRENT_LINE ${FILENAME_LINES})
+        # first replace the file name so that the file can be copied to the temp directory
+        string(REGEX REPLACE "value=\"(.*)\"" "value=\"${CAMITK_TEST_OUTPUT_DIR}/\\1\"" CURRENT_LINE_OK ${CURRENT_LINE})
+        # second replace the line in ASM_INPUT_FILE_AS_STRING_OK
+        string(REPLACE "${CURRENT_LINE}" "${CURRENT_LINE_OK}" ASM_INPUT_FILE_AS_STRING_OK "${ASM_INPUT_FILE_AS_STRING_OK}")
+        # extract the File Name
+        string(REGEX REPLACE ".*value=\"(.*)\" .*" "\\1" INPUT_FILE_NAME ${CURRENT_LINE})
+        # copy the corresponding file to the temp directory
+        # message("===== Input File = ${INPUT_FILE_NAME}")
+        file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${INPUT_FILE_NAME}
+             DESTINATION ${CAMITK_TEST_OUTPUT_DIR})
+    endforeach()
+
+    # Finally write the file in the temp directory
+    # Beware/Note: you need to add the quote around the ASM_INPUT_FILE_AS_STRING_OK value otherwise ";" will be
+    # interprated as list/argument separator and will be transformed to " " (for XML files that means that each 
+    # substitution will get damaged: &lt; -> &lt ... not good...)
+    file(WRITE ${CAMITK_TEST_OUTPUT_DIR}/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} "${ASM_INPUT_FILE_AS_STRING_OK}")
+
+    # And copy the expected output files
+    foreach(EXPECTED_OUTPUT_FILE ${CAMITK_ADD_TEST_ASM_EXPECTED_OUTPUT_FILES})
+        file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${EXPECTED_OUTPUT_FILE}
+             DESTINATION ${CAMITK_TEST_OUTPUT_DIR})
+    endforeach()
+
+    #-- run actionstatemachine
+    set(CAMITK_TEST_EXECUTABLE_ARG "-f ${CAMITK_TEST_OUTPUT_DIR}/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} -o ${CAMITK_TEST_OUTPUT_DIR} -a")
+
+    add_test(NAME ${CAMITK_ADD_TEST_ASM_NAME}
+             COMMAND ${CMAKE_COMMAND} 
+                            -DCAMITK_TEST_COMMAND=${CMAKE_BINARY_DIR}/bin/camitk-actionstatemachine
+                            -DCAMITK_TEST_COMMAND_ARG=${CAMITK_TEST_EXECUTABLE_ARG}
+                            -DCAMITK_TEST_EXPECTED_FILES=${CAMITK_ADD_TEST_ASM_EXPECTED_OUTPUT_FILES}
+                            -DCAMITK_TEST_OUTPUT_DIR=${CAMITK_TEST_OUTPUT_DIR}
+                            -DCAMITK_TEST_NAME=${CAMITK_ADD_TEST_ASM_NAME}
+                            -P ${CAMITK_CMAKE_MACRO_PATH}/CamiTKTestActionStateMachine.cmake
+    )
+
+    
+    # set the label for tests / associate tests to a project name in CDash 
+    if( CAMITK_ADD_TEST_PROJECT_NAME )
+        set_tests_properties( ${CAMITK_ADD_TEST_ASM_NAME} PROPERTIES LABELS ${CAMITK_ADD_TEST_PROJECT_NAME} )#
+    else()
+        set_tests_properties( ${CAMITK_ADD_TEST_ASM_NAME} PROPERTIES LABELS ${CAMITK_ADD_TEST_ASM_NAME} )
+    endif()
+
+endmacro()

sdk/cmake/modules/macros/camitk/test/CamiTKTestActionStateMachine.cmake

+#!
+#! @ingroup group_sdk_cmake_camitk_test
+#!
+#! This CMake file run the action state machine and compare the output files to the expected files
+#! The intended use is to execute this script from the camitk_add_test_actionstatemachine 
+#!
+#! Usage:
+#! \code
+#! add_test(NAME ${CAMITK_TEST_NAME}
+#!          COMMAND ${CMAKE_COMMAND}  
+#!                         -DCAMITK_TEST_COMMAND=${CMAKE_BINARY_DIR}/bin/camitk-actionstatemachine
+#!                         -DCAMITK_TEST_COMMAND_ARG=${CAMITK_TEST_EXECUTABLE_ARG}
+#!                         -DCAMITK_TEST_EXPECTED_FILES=${CAMITK_TEST_EXPECTED_FILES}
+#!                         -DCAMITK_TEST_OUTPUT_DIR=${CAMITK_TEST_OUTPUT_DIR}
+#!                         -DCAMITK_TEST_NAME=${CAMITK_TEST_NAME}
+#!                         -P ${CAMITK_CMAKE_MACRO_PATH}/CamiTKTestActionStateMachine.cmake
+#! )
+#! \endcode
+#!
+#! \param CAMITK_TEST_COMMAND should always be camitk-actionstatemachine
+#! \param CAMITK_TEST_EXECUTABLE_ARG should always contains -a and have the proper -f and -o options 
+#! \param CAMITK_TEST_EXPECTED_FILES is a list of files that are to be compared
+#! \param CAMITK_TEST_OUTPUT_DIR is the temporary directory where to run the test
+#!
+#! Considering the value of CAMITK_TEST_OUTPUT_DIR, running the test creates 
+#!    - file "command" which holds the command-line to execute for the test
+#!    - file "command-result" which holds the result status (exit value) of the command. The exit 
+#!      result can be 0=success, 1=failure or other value = failure
+#!    - file "command-output" which capture all the std output of the command
+#!    - file "test" which command is run to check the file (i.e., diff the files)
+#!    - file "test-output" the standard output of the diff cmake command
+#!    - file "test-result" contains the exit result (0=success, 1 or other=failure)
+
+#-- declare outputfiles
+set(CAMITK_TEST_COMMAND_FILE ${CAMITK_TEST_OUTPUT_DIR}/command) # which command is run to test the exectable
+set(CAMITK_TEST_COMMAND_RESULT_FILE ${CAMITK_TEST_OUTPUT_DIR}/command-result) # the exit result (0=success, 1=failure) of the tested command goes in this file
+set(CAMITK_TEST_COMMAND_OUTPUT_FILE ${CAMITK_TEST_OUTPUT_DIR}/command-output) # the output of the tested command goes in this files
+set(CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE ${CAMITK_TEST_OUTPUT_DIR}/test) # which command is run to diff the files
+set(CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE ${CAMITK_TEST_OUTPUT_DIR}/test-output) # output of the diff cmake command
+set(CAMITK_TEST_EXPECTED_FILE_RESULT_FILE ${CAMITK_TEST_OUTPUT_DIR}/test-result) # exit result (0=success, 1=failure)
+
+# clean/remove all output files and previous test attempt
+unset(PREVIOUS_ATTEMPTS)
+file(GLOB ALL_OUTPUT_FILES ${CAMITK_TEST_OUTPUT_DIR}/*)
+foreach(POTENTIAL_DIR ${ALL_OUTPUT_FILES})
+    # only remove subdirectories (as they are previous attempts of running the test and should be asm output directory)
+    if(IS_DIRECTORY ${POTENTIAL_DIR})
+        list(APPEND PREVIOUS_ATTEMPTS ${POTENTIAL_DIR})
+    endif()
+endforeach()
+file(REMOVE_RECURSE ${CAMITK_TEST_COMMAND_FILE} ${CAMITK_TEST_COMMAND_RESULT_FILE} ${CAMITK_TEST_COMMAND_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE} ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_RESULT_FILE} ${PREVIOUS_ATTEMPTS})
+
+#--  First run the executable
+# Build the command file (so the user can retrieve/review it later)
+file(WRITE ${CAMITK_TEST_COMMAND_FILE} "${CAMITK_TEST_COMMAND} ${CAMITK_TEST_COMMAND_ARG}")
+
+# expands all arguments
+string(REPLACE " " ";" CAMITK_TEST_COMMAND_ARG_LIST ${CAMITK_TEST_COMMAND_ARG})
+
+# execute the command and write the output to the command-output file, and the result to the command-result file
+execute_process(
+  COMMAND ${CAMITK_TEST_COMMAND} ${CAMITK_TEST_COMMAND_ARG_LIST}
+  RESULT_VARIABLE CAMITK_TEST_COMMAND_RESULT
+  OUTPUT_VARIABLE CAMITK_TEST_COMMAND_OUTPUT
+)
+file(WRITE ${CAMITK_TEST_COMMAND_RESULT_FILE} ${CAMITK_TEST_COMMAND_RESULT})
+file(WRITE ${CAMITK_TEST_COMMAND_OUTPUT_FILE} ${CAMITK_TEST_COMMAND_OUTPUT})
+
+set(CAMITK_TEST_EXPECTED_FILE_OUTPUT "Action state machine regression test called with:\n - CAMITK_TEST_NAME=${CAMITK_TEST_NAME}\n - CAMITK_TEST_COMMAND=${CMAKE_BINARY_DIR}/bin/camitk-actionstatemachine\n - CAMITK_TEST_COMMAND_ARG=${CAMITK_TEST_COMMAND_ARG}\n - CAMITK_TEST_EXPECTED_FILES=${CAMITK_TEST_EXPECTED_FILES}\n - CAMITK_TEST_OUTPUT_DIR=${CAMITK_TEST_OUTPUT_DIR}\n")
+set(CAMITK_TEST_EXPECTED_FILE_RESULT 0)
+set(CAMITK_TEST_EXPECTED_FILE_COMMAND)
+set(CAMITK_TEST_FAILED_FILE)
+
+#-- Then compare each expected output
+foreach(EXPECTED_OUTPUT_FILE ${CAMITK_TEST_EXPECTED_FILES})
+    # find the same file in the asm output directory (*T*)
+    file(GLOB_RECURSE OUTPUT_FILE "${CAMITK_TEST_OUTPUT_DIR}/*T*/${EXPECTED_OUTPUT_FILE}")
+    
+    # compare the files
+    execute_process(
+        COMMAND ${CMAKE_COMMAND} -E compare_files ${OUTPUT_FILE} ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}
+        RESULT_VARIABLE CAMITK_TEST_EXPECTED_FILE_RESULT_SINGLE
+        OUTPUT_VARIABLE CAMITK_TEST_EXPECTED_FILE_OUTPUT_SINGLE
+        OUTPUT_QUIET
+        ERROR_QUIET
+    )
+    
+    # update the output variables (command, output and result)
+    set(CAMITK_TEST_EXPECTED_FILE_OUTPUT
+        "${CAMITK_TEST_EXPECTED_FILE_OUTPUT}=================================================\nComparing ${OUTPUT_FILE} with ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}\nOutput:\n${CAMITK_TEST_EXPECTED_FILE_OUTPUT_SINGLE}\n")
+    set(CAMITK_TEST_EXPECTED_FILE_COMMAND 
+        "${CAMITK_TEST_EXPECTED_FILE_COMMAND}${CMAKE_COMMAND} -E compare_files ${OUTPUT_FILE} ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}\n")
+    if(CAMITK_TEST_EXPECTED_FILE_RESULT_SINGLE)
+        set(CAMITK_TEST_EXPECTED_FILE_OUTPUT "${CAMITK_TEST_EXPECTED_FILE_OUTPUT}Result:\n${CAMITK_TEST_NAME}: comparing ${EXPECTED_OUTPUT_FILE} failed:\nOutput file ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE} is not the same as ${OUTPUT_FILE}\n\n" )
+        set(CAMITK_TEST_EXPECTED_FILE_RESULT 1)
+        set(CAMITK_TEST_FAILED_FILE "${CAMITK_TEST_FAILED_FILE}${EXPECTED_OUTPUT_FILE}\n")
+    else()
+        set(CAMITK_TEST_EXPECTED_FILE_OUTPUT "${CAMITK_TEST_EXPECTED_FILE_OUTPUT}Result:\n${CAMITK_TEST_NAME}: comparing ${EXPECTED_OUTPUT_FILE} success!\n\n" )    
+    endif()
+endforeach()
+
+#-- write everything to the test-command, test-output and test-result files 
+#   These information detail the results for an easier diagnostic by a human
+file(WRITE ${CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE} ${CAMITK_TEST_EXPECTED_FILE_COMMAND})
+file(WRITE ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_OUTPUT})
+file(WRITE ${CAMITK_TEST_EXPECTED_FILE_RESULT_FILE} ${CAMITK_TEST_EXPECTED_FILE_RESULT})
+
+#-- check result
+if( CAMITK_TEST_EXPECTED_FILE_RESULT )
+    message("[FAIL]")
+    message(FATAL_ERROR "${CAMITK_TEST_NAME}: (one or more) output file do(es) not match the corresponding expected file\nSee ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} for more information" )
+else()
+    message("[OK]")
+endif()