Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit f77f04a5 authored by Emmanuel Promayon's avatar Emmanuel Promayon
Browse files

NEW camitk_add_test_actionstatemachine macro



This macro runs the actionstatemachine in automatic mode,
on a given CamiTK SCXML file, and compare the output files
to expected results.
This is a first step towards automatic/systematic regression test
Signed-off-by: Emmanuel Promayon's avatarEmmanuel Promayon <Emmanuel.Promayon@imag.fr>
parent c0f08a44
......@@ -7,4 +7,9 @@ camitk_extension( ACTION_EXTENSION
)
# Recursively update the shiboken path variable containing the CamiTK SDK tree structure
set(SHIBOKEN_CAMITK_SDK_PATH ${SHIBOKEN_CAMITK_SDK_PATH}:${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "")
\ No newline at end of file
set(SHIBOKEN_CAMITK_SDK_PATH ${SHIBOKEN_CAMITK_SDK_PATH}:${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "")
# add a specific test to run the action, save the output and compare it to expected
camitk_add_test_actionstatemachine(NAME "action-reconstruction-regression-test"
CAMITK_SCXML "action-reconstruction-regression-test.xml"
EXPECTED_OUTPUT_FILES "action-reconstruction-output-test.vtk")
\ No newline at end of file
<scxml xmlns="http://camitk.imag.fr/3/smallScxml" initial="Initialize" xmlns:camitk="http://camitk.imag.fr/3/asm" xsi:schemaLocation="http://camitk.imag.fr/3/smallScxml/../resources/smallScxml.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<state id="Initialize">
<onentry>
<camitk:onState>
<camitk:description><![CDATA[This pipeline will process several actions on the input component(s): <br /> <ul><li>Open File</li><li>Reconstruction</li><li>Save</li>]]></camitk:description>
</camitk:onState>
</onentry>
<transition target="Action 1" event="Next"/>
</state>
<state id="Action 1">
<onentry>
<camitk:onState>
<camitk:description>Open data (component) from a given file</camitk:description>
<camitk:action>
<camitk:name>Open File</camitk:name>
<camitk:parameters>
<!--
Bug? There are two lines for this parameter, the first is written with an empty name value:
<camitk:parameter name="File name" value="" type="QString"/>
This should not appear.
-->
<camitk:parameter name="File Name" value="action-reconstruction-input.mha" type="QString"/>
</camitk:parameters>
<camitk:parameters/>
<camitk:outputs>
<camitk:component name="image" type="ImageComponent"/>
</camitk:outputs>
</camitk:action>
</camitk:onState>
</onentry>
<transition target="Action 2" event="Next"/>
<transition target="Initialize" event="Back"/>
</state>
<state id="Action 2">
<onentry>
<camitk:onState>
<camitk:description>&lt;br /> Marching cubes is a simple algorithm for creating a triangle mesh from an implicit function (one of the form f(x, y, z) = 0).&lt;br /> It works by iterating ("marching") over a uniform grid of cubes superimposed over a region of the function. If all 8 vertices of the cube are positive, or all 8 vertices are negative, the cube is entirely above or entirely below the surface and no triangles are emitted. Otherwise, the cube straddles the function and some triangles and vertices are generated. Since each vertex can either be positive or negative, there are technically 28 possible configurations, but many of these are equivalent to one another. &lt;br/> &lt;br/>This action uses this algorithm to build a 3D surfacic mesh of the input image.</camitk:description>
<camitk:action>
<camitk:name>Reconstruction</camitk:name>
<camitk:parameters>
<camitk:parameter name="Threshold" value="127.5" type="double"/>
<camitk:parameter name="Keep only largest component?" value="true" type="bool"/>
<camitk:parameter name="Subsample original image?" value="true" type="bool"/>
<camitk:parameter name="Subsampled image width (X)" value="32" type="int"/>
<camitk:parameter name="Subsampled image height (Y)" value="32" type="int"/>
<camitk:parameter name="Subsampled image depth (Z)" value="32" type="int"/>
</camitk:parameters>
<camitk:parameters/>
<camitk:inputs>
<camitk:component name="image" type="ImageComponent"/>
</camitk:inputs>
<camitk:outputs>
<camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
</camitk:outputs>
</camitk:action>
</camitk:onState>
</onentry>
<transition target="Action 3" event="Next"/>
<transition target="Action 1" event="Back">
<onTransition>
<camitk:close>
<camitk:component name="image" type="ImageComponent"/>
</camitk:close>
</onTransition>
</transition>
</state>
<state id="Action 3">
<onentry>
<camitk:onState>
<camitk:description>Save all the top-level of the selected components</camitk:description>
<camitk:action>
<camitk:name>Save</camitk:name>
<camitk:parameters/>
<camitk:inputs>
<camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
</camitk:inputs>
</camitk:action>
</camitk:onState>
</onentry>
<transition target="Bye" event="Next"/>
<transition target="Action 2" event="Back">
<onTransition>
<camitk:close>
<camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
</camitk:close>
</onTransition>
</transition>
</state>
<state id="Bye" final="true">
<onentry>
<camitk:onState>
<camitk:description><![CDATA[Thanks you for using the CamiTK Action State Machine !]]></camitk:description>
</camitk:onState>
</onentry>
<transition target="Initialize" event="Back to the beginning">
<onTransition>
<camitk:close>
<camitk:component name="image" type="ImageComponent"/>
<camitk:component name="action-reconstruction-output-test.vtk" type="MeshComponent"/>
</camitk:close>
</onTransition>
</transition>
</state>
</scxml>
......@@ -28,6 +28,7 @@ include(camitk/cdash/projects/CamiTKSubProjectValidate)
# -- CamiTK test macros (see also http://www.org/cmake/help/v2.8.8/cmake.html#section_PropertiesonTests)
include(camitk/test/CamiTKInitTest)
include(camitk/test/CamiTKAddTest)
include(camitk/test/CamiTKAddTestActionStateMachine)
# -- packaging macros
include(camitk/packaging/CamiTKOpenSourcePackaging)
......
......@@ -3,7 +3,7 @@
#!
#! camitk_add_test is a macro to add a new test to the CTest infrastructure
#! It encapsulates CMake add_test and adds useful way of testing programs.
#! It cannot operate on its on, you need to call camitk_init_test first (and only once) before
#! It cannot operate on its own, you need to call camitk_init_test first (and only once) before
#! calling camitk_add_test
#!
#! Details on the runned test can be found in ${CMAKE_BINARY_DIR}/Testing/Temporary/target#
......
#!
#! @ingroup group_sdk_cmake_camitk_test
#!
#! camitk_add_test_actionstatemachine is a macro to add a new test to the CTest infrastructure
#! It encapsulates CMake add_test and use the action state machine to run a set of actions.
#!
#! Details on the runned test can be found in ${CMAKE_BINARY_DIR}/Testing/Temporary/target#
#! where target is the executable name (see camitk_init_test() macro), and # is the test order number.
#!
#! Usage:
#! \code
#! camitk_add_test_actionstatemachine(NAME "testName"
#! CAMITK_SCXML "file.xml"
#! EXPECTED_OUTPUT_FILES "file1;file2;file3;..."
#! )
#! \endcode
#!
#! It creates a target name
#!
#! \param NAME Name of the test, use "ctest -V -R testName" to run the test individually
#! \param CAMITK_SCXML The input xml file
#! \param EXPECTED_OUTPUT_FILES List of files that have to be compared with the asm output
#!
#TODO
# Questions:
# - do not use testdata/ (all files are copied to share/*/testdata, which is cumbersome) but use a specific "regressiontestdata" ? "asmtest" ? "testasm" ?
# We need to define specific names, always used for asm regression test
# - regressiontestdata/input.xml : the asm scxml file
# - regressiontestdata/somethingfilename.smth : input files to run the actions by asm and refered to in input.xml as "File Name" parameters
# - regressiontestdata/output-something2.smth2 : output files to check against (refer to in the component output of some actions and the save actions
#
# If the directory is detected in the source directory, then the test should be added automatically
macro(camitk_add_test_actionstatemachine)
parse_arguments(CAMITK_ADD_TEST_ASM
"NAME;CAMITK_SCXML;EXPECTED_OUTPUT_FILES;"
"" # possible options
${ARGN}
)
# determine cmake macro path
if (NOT EXISTS SDK_TOP_LEVEL_SOURCE_DIR)
# this macro is called outside the sdk
set(CAMITK_CMAKE_MACRO_PATH ${CAMITK_CMAKE_DIR}/modules/macros/camitk/test)
if(NOT IS_DIRECTORY ${CAMITK_CMAKE_MACRO_PATH})
# inside opensource but not in sdk (modeling or imaging)
set(CAMITK_CMAKE_MACRO_PATH ${CMAKE_SOURCE_DIR}/sdk/cmake/modules/macros/camitk/test)
endif()
else()
# directly use the macro source dir
set(CAMITK_CMAKE_MACRO_PATH ${SDK_TOP_LEVEL_SOURCE_DIR}/cmake/modules/macros/camitk/test)
endif()
#-- clean/create test output directory
set(CAMITK_TEST_OUTPUT_DIR "${CMAKE_BINARY_DIR}/Testing/Temporary/${CAMITK_ADD_TEST_ASM_NAME}")
file(REMOVE_RECURSE ${CAMITK_TEST_OUTPUT_DIR})
file(MAKE_DIRECTORY ${CAMITK_TEST_OUTPUT_DIR})
#-- replace the input file names
# Read the file in a variable
file(READ ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} ASM_INPUT_FILE_AS_STRING)
# get all the lines that have a non-empty "File Name" parameter
string(REGEX MATCHALL "<camitk:parameter name=\"File Name\" value=\".*\" type=\"QString\"/>" FILENAME_LINES "${ASM_INPUT_FILE_AS_STRING}")
# intialize ASM_INPUT_FILE_AS_STRING_OK
set(ASM_INPUT_FILE_AS_STRING_OK "${ASM_INPUT_FILE_AS_STRING}")
# for each lines that have a non-empty "File Name" parameter
foreach(CURRENT_LINE ${FILENAME_LINES})
# first replace the file name so that the file can be copied to the temp directory
string(REGEX REPLACE "value=\"(.*)\"" "value=\"${CAMITK_TEST_OUTPUT_DIR}/\\1\"" CURRENT_LINE_OK ${CURRENT_LINE})
# second replace the line in ASM_INPUT_FILE_AS_STRING_OK
string(REPLACE "${CURRENT_LINE}" "${CURRENT_LINE_OK}" ASM_INPUT_FILE_AS_STRING_OK "${ASM_INPUT_FILE_AS_STRING_OK}")
# extract the File Name
string(REGEX REPLACE ".*value=\"(.*)\" .*" "\\1" INPUT_FILE_NAME ${CURRENT_LINE})
# copy the corresponding file to the temp directory
# message("===== Input File = ${INPUT_FILE_NAME}")
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${INPUT_FILE_NAME}
DESTINATION ${CAMITK_TEST_OUTPUT_DIR})
endforeach()
# Finally write the file in the temp directory
# Beware/Note: you need to add the quote around the ASM_INPUT_FILE_AS_STRING_OK value otherwise ";" will be
# interprated as list/argument separator and will be transformed to " " (for XML files that means that each
# substitution will get damaged: &lt; -> &lt ... not good...)
file(WRITE ${CAMITK_TEST_OUTPUT_DIR}/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} "${ASM_INPUT_FILE_AS_STRING_OK}")
# And copy the expected output files
foreach(EXPECTED_OUTPUT_FILE ${CAMITK_ADD_TEST_ASM_EXPECTED_OUTPUT_FILES})
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/testdata/${EXPECTED_OUTPUT_FILE}
DESTINATION ${CAMITK_TEST_OUTPUT_DIR})
endforeach()
#-- run actionstatemachine
set(CAMITK_TEST_EXECUTABLE_ARG "-f ${CAMITK_TEST_OUTPUT_DIR}/${CAMITK_ADD_TEST_ASM_CAMITK_SCXML} -o ${CAMITK_TEST_OUTPUT_DIR} -a")
add_test(NAME ${CAMITK_ADD_TEST_ASM_NAME}
COMMAND ${CMAKE_COMMAND}
-DCAMITK_TEST_COMMAND=${CMAKE_BINARY_DIR}/bin/camitk-actionstatemachine
-DCAMITK_TEST_COMMAND_ARG=${CAMITK_TEST_EXECUTABLE_ARG}
-DCAMITK_TEST_EXPECTED_FILES=${CAMITK_ADD_TEST_ASM_EXPECTED_OUTPUT_FILES}
-DCAMITK_TEST_OUTPUT_DIR=${CAMITK_TEST_OUTPUT_DIR}
-DCAMITK_TEST_NAME=${CAMITK_ADD_TEST_ASM_NAME}
-P ${CAMITK_CMAKE_MACRO_PATH}/CamiTKTestActionStateMachine.cmake
)
# set the label for tests / associate tests to a project name in CDash
if( CAMITK_ADD_TEST_PROJECT_NAME )
set_tests_properties( ${CAMITK_ADD_TEST_ASM_NAME} PROPERTIES LABELS ${CAMITK_ADD_TEST_PROJECT_NAME} )#
else()
set_tests_properties( ${CAMITK_ADD_TEST_ASM_NAME} PROPERTIES LABELS ${CAMITK_ADD_TEST_ASM_NAME} )
endif()
endmacro()
#!
#! @ingroup group_sdk_cmake_camitk_test
#!
#! This CMake file run the action state machine and compare the output files to the expected files
#! The intended use is to execute this script from the camitk_add_test_actionstatemachine
#!
#! Usage:
#! \code
#! add_test(NAME ${CAMITK_TEST_NAME}
#! COMMAND ${CMAKE_COMMAND}
#! -DCAMITK_TEST_COMMAND=${CMAKE_BINARY_DIR}/bin/camitk-actionstatemachine
#! -DCAMITK_TEST_COMMAND_ARG=${CAMITK_TEST_EXECUTABLE_ARG}
#! -DCAMITK_TEST_EXPECTED_FILES=${CAMITK_TEST_EXPECTED_FILES}
#! -DCAMITK_TEST_OUTPUT_DIR=${CAMITK_TEST_OUTPUT_DIR}
#! -DCAMITK_TEST_NAME=${CAMITK_TEST_NAME}
#! -P ${CAMITK_CMAKE_MACRO_PATH}/CamiTKTestActionStateMachine.cmake
#! )
#! \endcode
#!
#! \param CAMITK_TEST_COMMAND should always be camitk-actionstatemachine
#! \param CAMITK_TEST_EXECUTABLE_ARG should always contains -a and have the proper -f and -o options
#! \param CAMITK_TEST_EXPECTED_FILES is a list of files that are to be compared
#! \param CAMITK_TEST_OUTPUT_DIR is the temporary directory where to run the test
#!
#! Considering the value of CAMITK_TEST_OUTPUT_DIR, running the test creates
#! - file "command" which holds the command-line to execute for the test
#! - file "command-result" which holds the result status (exit value) of the command. The exit
#! result can be 0=success, 1=failure or other value = failure
#! - file "command-output" which capture all the std output of the command
#! - file "test" which command is run to check the file (i.e., diff the files)
#! - file "test-output" the standard output of the diff cmake command
#! - file "test-result" contains the exit result (0=success, 1 or other=failure)
#-- declare outputfiles
set(CAMITK_TEST_COMMAND_FILE ${CAMITK_TEST_OUTPUT_DIR}/command) # which command is run to test the exectable
set(CAMITK_TEST_COMMAND_RESULT_FILE ${CAMITK_TEST_OUTPUT_DIR}/command-result) # the exit result (0=success, 1=failure) of the tested command goes in this file
set(CAMITK_TEST_COMMAND_OUTPUT_FILE ${CAMITK_TEST_OUTPUT_DIR}/command-output) # the output of the tested command goes in this files
set(CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE ${CAMITK_TEST_OUTPUT_DIR}/test) # which command is run to diff the files
set(CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE ${CAMITK_TEST_OUTPUT_DIR}/test-output) # output of the diff cmake command
set(CAMITK_TEST_EXPECTED_FILE_RESULT_FILE ${CAMITK_TEST_OUTPUT_DIR}/test-result) # exit result (0=success, 1=failure)
# clean/remove all output files
file(REMOVE_RECURSE ${CAMITK_TEST_COMMAND_FILE} ${CAMITK_TEST_COMMAND_RESULT_FILE} ${CAMITK_TEST_COMMAND_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE} ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_RESULT_FILE})
#-- First run the executable
# Build the command file (so the user can retrieve/review it later)
file(WRITE ${CAMITK_TEST_COMMAND_FILE} "${CAMITK_TEST_COMMAND} ${CAMITK_TEST_COMMAND_ARG}")
# expands all arguments
string(REPLACE " " ";" CAMITK_TEST_COMMAND_ARG_LIST ${CAMITK_TEST_COMMAND_ARG})
# execute the command and write the output to the command-output file, and the result to the command-result file
execute_process(
COMMAND ${CAMITK_TEST_COMMAND} ${CAMITK_TEST_COMMAND_ARG_LIST}
RESULT_VARIABLE CAMITK_TEST_COMMAND_RESULT
OUTPUT_VARIABLE CAMITK_TEST_COMMAND_OUTPUT
)
file(WRITE ${CAMITK_TEST_COMMAND_RESULT_FILE} ${CAMITK_TEST_COMMAND_RESULT})
file(WRITE ${CAMITK_TEST_COMMAND_OUTPUT_FILE} ${CAMITK_TEST_COMMAND_OUTPUT})
set(CAMITK_TEST_EXPECTED_FILE_OUTPUT)
set(CAMITK_TEST_EXPECTED_FILE_RESULT 0)
set(CAMITK_TEST_EXPECTED_FILE_COMMAND)
set(CAMITK_TEST_FAILED_FILE)
#-- Then compare each expected output
foreach(EXPECTED_OUTPUT_FILE ${CAMITK_TEST_EXPECTED_FILES})
# find the same file in the asm output directory (*T*)
file(GLOB_RECURSE OUTPUT_FILE "${CAMITK_TEST_OUTPUT_DIR}/*T*/${EXPECTED_OUTPUT_FILE}")
# compare the files
execute_process(
COMMAND ${CMAKE_COMMAND} -E compare_files ${OUTPUT_FILE} ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}
RESULT_VARIABLE CAMITK_TEST_EXPECTED_FILE_RESULT_SINGLE
OUTPUT_VARIABLE CAMITK_TEST_EXPECTED_FILE_OUTPUT_SINGLE
OUTPUT_QUIET
ERROR_QUIET
)
# update the output variables
set(CAMITK_TEST_EXPECTED_FILE_OUTPUT
"${CAMITK_TEST_EXPECTED_FILE_OUTPUT}=================================================\ncompare ${OUTPUT_FILE} with ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}\nOutput:\n${CAMITK_TEST_EXPECTED_FILE_OUTPUT_SINGLE}\n")
set(CAMITK_TEST_EXPECTED_FILE_COMMAND
"${CAMITK_TEST_EXPECTED_FILE_COMMAND}${CMAKE_COMMAND} -E compare_files ${OUTPUT_FILE} ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE}\n")
if(CAMITK_TEST_EXPECTED_FILE_RESULT_SINGLE)
set(CAMITK_TEST_EXPECTED_FILE_OUTPUT "${CAMITK_TEST_EXPECTED_FILE_OUTPUT}Result:\n${CAMITK_TEST_NAME}: comparing ${EXPECTED_OUTPUT_FILE} failed:\nOutput file ${CAMITK_TEST_OUTPUT_DIR}/${EXPECTED_OUTPUT_FILE} is not the same as ${OUTPUT_FILE}\n\n" )
set(CAMITK_TEST_EXPECTED_FILE_RESULT 1)
set(CAMITK_TEST_FAILED_FILE "${CAMITK_TEST_FAILED_FILE}${EXPECTED_OUTPUT_FILE}\n")
else()
set(CAMITK_TEST_EXPECTED_FILE_OUTPUT "${CAMITK_TEST_EXPECTED_FILE_OUTPUT}Result:\n${CAMITK_TEST_NAME}: comparing ${EXPECTED_OUTPUT_FILE} success!\n\n" )
endif()
endforeach()
# write everything to the test-command, test-output and test-result files
file(WRITE ${CAMITK_TEST_EXPECTED_FILE_COMMAND_FILE} ${CAMITK_TEST_EXPECTED_FILE_COMMAND})
file(WRITE ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} ${CAMITK_TEST_EXPECTED_FILE_OUTPUT})
file(WRITE ${CAMITK_TEST_EXPECTED_FILE_RESULT_FILE} ${CAMITK_TEST_EXPECTED_FILE_RESULT})
#-- check result
if( CAMITK_TEST_EXPECTED_FILE_RESULT )
message(SEND_ERROR "${CAMITK_TEST_NAME}: output file(s) do not match:\n${CAMITK_TEST_FAILED_FILE}See ${CAMITK_TEST_EXPECTED_FILE_OUTPUT_FILE} for more information" )
endif()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment