Commit ccdda396 authored by Jan Möbius's avatar Jan Möbius
Browse files

Fixed error in compare tool, not throwing an error on string mismatch. Moved...

Fixed error in compare tool, not throwing an error on string mismatch. Moved test binaries into extra folder. Fixed bad commit with wrong test generator refs #394

git-svn-id: http://www.openflipper.org/svnrepo/OpenFlipper/branches/Free@13670 383ad7c9-94d9-4d36-a494-682f7c89f535
parent 8933b5a3
...@@ -30,6 +30,7 @@ SET (CTEST_DROP_SITE_CDASH FALSE) ...@@ -30,6 +30,7 @@ SET (CTEST_DROP_SITE_CDASH FALSE)
make_directory(${OPENFLIPPER_TEST_RESULTS}) make_directory(${OPENFLIPPER_TEST_RESULTS})
make_directory(${OPENFLIPPER_TEST_BINARIES})
# ======================================================================== # ========================================================================
# ======================================================================== # ========================================================================
......
...@@ -5,20 +5,20 @@ if(BUILD_TESTING) ...@@ -5,20 +5,20 @@ if(BUILD_TESTING)
set_target_properties( minimalExecutionTest PROPERTIES set_target_properties( minimalExecutionTest PROPERTIES
RUNTIME_OUTPUT_NAME "minimalExecTest" RUNTIME_OUTPUT_NAME "minimalExecTest"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests" RUNTIME_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests" LIBRARY_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
) )
if (WIN32) if (WIN32)
add_custom_command (TARGET minimalExecutionTest POST_BUILD add_custom_command (TARGET minimalExecutionTest POST_BUILD
COMMAND ${CMAKE_COMMAND} -E COMMAND ${CMAKE_COMMAND} -E
copy_if_different copy_if_different
${CMAKE_BINARY_DIR}/tests/${CMAKE_CFG_INTDIR}/minimalExecTest.exe ${OPENFLIPPER_TEST_BINARIES}/${CMAKE_CFG_INTDIR}/minimalExecTest.exe
${CMAKE_BINARY_DIR}/tests/minimalExecTest.exe) ${OPENFLIPPER_TEST_BINARIES}/minimalExecTest.exe)
endif() endif()
ADD_TEST(NAME "Minimal_Execution_Test" ADD_TEST(NAME "Minimal_Execution_Test"
COMMAND ${CMAKE_BINARY_DIR}/tests/minimalExecTest ) COMMAND ${OPENFLIPPER_TEST_BINARIES}/minimalExecTest )
set_tests_properties( Minimal_Execution_Test PROPERTIES set_tests_properties( Minimal_Execution_Test PROPERTIES
TIMEOUT 10 TIMEOUT 10
......
...@@ -5,27 +5,23 @@ if(BUILD_TESTING) ...@@ -5,27 +5,23 @@ if(BUILD_TESTING)
find_package (Qt4 COMPONENTS QtCore ) find_package (Qt4 COMPONENTS QtCore )
endif() endif()
add_executable (comparatorTool comparator.cc ) add_executable (compareTool compareTool.cc )
target_link_libraries (comparatorTool target_link_libraries (compareTool
${QT_LIBRARIES} ${QT_LIBRARIES}
) )
set_target_properties ( compareTool PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
LIBRARY_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
set_target_properties ( comparatorTool PROPERTIES
RUNTIME_OUTPUT_NAME "compareTool"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
) )
if (WIN32) if (WIN32)
add_custom_command (TARGET comparatorTool POST_BUILD add_custom_command (TARGET compareTool POST_BUILD
COMMAND ${CMAKE_COMMAND} -E COMMAND ${CMAKE_COMMAND} -E
copy_if_different copy_if_different
${CMAKE_BINARY_DIR}/tests/${CMAKE_CFG_INTDIR}/compareTool.exe ${OPENFLIPPER_TEST_BINARIES}/${CMAKE_CFG_INTDIR}/compareTool.exe
${CMAKE_BINARY_DIR}/tests/compareTool.exe) ${OPENFLIPPER_TEST_BINARIES}/compareTool.exe)
endif() endif()
endif() endif()
...@@ -170,7 +170,7 @@ int main(int argv, char **args) ...@@ -170,7 +170,7 @@ int main(int argv, char **args)
if ( toplevelKeys[i].endsWith("_DOUBLE") ) { if ( toplevelKeys[i].endsWith("_DOUBLE") ) {
ok &= compareDouble(toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i])); ok &= compareDouble(toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i]));
} else } else
compareString( toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i])); ok &= compareString( toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i]));
} else { } else {
cerr << "Missing key in result file: " << toplevelKeys[i] << "\n"; cerr << "Missing key in result file: " << toplevelKeys[i] << "\n";
ok = false; ok = false;
......
...@@ -11,9 +11,14 @@ endif( NOT output_test ) ...@@ -11,9 +11,14 @@ endif( NOT output_test )
# output_test contains the info about the loaded file # output_test contains the info about the loaded file
if( NOT test_file_info ) if( NOT test_file_info )
message( FATAL_ERROR "Variable test_file_info= not defined" ) message( FATAL_ERROR "Variable test_file_info not defined" )
endif( NOT test_file_info ) endif( NOT test_file_info )
if( NOT result_checker )
message( FATAL_ERROR "Variable result_checker not defined" )
endif( NOT result_checker )
# convert the space-separated string to a list # convert the space-separated string to a list
separate_arguments( test_args ) separate_arguments( test_args )
message( ${test_args} ) message( ${test_args} )
...@@ -34,63 +39,29 @@ if ( NOT ${PROCESSRESULT} EQUAL 0 ) ...@@ -34,63 +39,29 @@ if ( NOT ${PROCESSRESULT} EQUAL 0 )
message(SEND_ERROR "Process execution failed!") message(SEND_ERROR "Process execution failed!")
endif() endif()
# ===================================================
# Collect list of what should be tested
# ===================================================
# Read the info file
file (STRINGS ${test_file_info} FILEINFO)
#Create a list of properties we want to test
foreach(loop ${FILEINFO})
string ( REGEX REPLACE "=.*$" "" TMP ${loop} )
list( APPEND CHECKS ${TMP})
endforeach()
# ===================================================
# Read the files
# ===================================================
# read the test output
file (READ ${output_test} TESTOUTPUT)
# read the test output
file (READ ${test_file_info} INFOFILE)
# =================================================== # ===================================================
# Compare # Compare
# =================================================== # ===================================================
# Now go through all checks:
foreach(loop ${CHECKS})
#Get value from results:
string (
REGEX REPLACE
"^.*${loop}=([^\n]*).*$" "\\1"
RESULT ${TESTOUTPUT}
)
# remove whitespace
string(STRIP ${RESULT} RESULT)
#Get value from Info: if ( WIN32 )
string ( set(result_checker "${result_checker}.exe")
REGEX REPLACE endif()
"^.*${loop}=([^\n]*).*$" "\\1"
EXPECTED ${INFOFILE}
)
# remove whitespace message( "Executing: ${result_checker} ${output_test} ${test_file_info} " )
string(STRIP ${EXPECTED} EXPECTED)
# Use Stringcompare here # Timeout after 2 minutes
if ( NOT ${EXPECTED} STREQUAL ${RESULT} ) execute_process(
message(WARNING "Mismatching values for ${loop}: EXPECTED ${EXPECTED} but got ${RESULT}!") COMMAND ${result_checker} ${output_test} ${test_file_info}
set(test_not_successful true) TIMEOUT 60
endif() RESULT_VARIABLE PROCESSRESULT
)
endforeach() if ( NOT ${PROCESSRESULT} EQUAL 0 )
message(SEND_ERROR "Compare Tool execution failed!")
set(test_not_successful TRUE)
endif()
if( test_not_successful ) if( test_not_successful )
message( SEND_ERROR "Test Failed! See messages above to see what went wrong!" ) message( SEND_ERROR "Test Failed! See messages above to see what went wrong!" )
......
# some argument checking:
# test_cmd is the command to run with all its arguments
if( NOT test_cmd )
message( FATAL_ERROR "Variable test_cmd not defined" )
endif( NOT test_cmd )
# output_blessed contains the name of the "blessed" output file
if( NOT md5 )
message( FATAL_ERROR "Variable md5 not defined" )
endif( NOT md5 )
# output_test contains the name of the output file the test_cmd will produce
if( NOT output_test )
message( FATAL_ERROR "Variable output_test not defined" )
endif( NOT output_test )
# convert the space-separated string to a list
separate_arguments( test_args )
message( ${test_args} )
execute_process(
COMMAND ${test_cmd} ${test_args}
)
execute_process(
COMMAND ${CMAKE_COMMAND} -E md5sum ${output_test}
RESULT_VARIABLE test_not_successful
OUTPUT_VARIABLE result_md5
)
#split into its two parts
string(REPLACE " " ";" result_md5 ${result_md5})
#Get the sum
list(GET result_md5 0 result_md5)
if ( NOT result_md5 STREQUAL md5 )
message(STATUS "MD5 missmatch")
message(STATUS "${result_md5}")
message(STATUS "${md5}")
set( test_not_successful "TRUE" )
endif()
if( test_not_successful )
message( SEND_ERROR "${output_test} md5 is ${result_md5} but should be ${md5}!" )
endif( test_not_successful )
...@@ -52,6 +52,7 @@ function( run_single_object_file_mesh_test FILEPLUGIN TEST_FILE TEST_SCRIPT ) ...@@ -52,6 +52,7 @@ function( run_single_object_file_mesh_test FILEPLUGIN TEST_FILE TEST_SCRIPT )
-D test_args:string=${args} -D test_args:string=${args}
-D output_test=${OUTPUT_TEST_DATA_FILE} -D output_test=${OUTPUT_TEST_DATA_FILE}
-D test_file_info=${TEST_FILE_INFO} -D test_file_info=${TEST_FILE_INFO}
-D result_checker=${OPENFLIPPER_TEST_BINARIES}/compareTool
-P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake -P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake
) )
...@@ -157,7 +158,7 @@ function( run_algorithm_test TEST_SCRIPT INPUT_FILE INPUT_REFERENCE ) ...@@ -157,7 +158,7 @@ function( run_algorithm_test TEST_SCRIPT INPUT_FILE INPUT_REFERENCE )
-D test_args:string=${args} -D test_args:string=${args}
-D output_test=${OUTPUT_TEST_DATA_FILE} -D output_test=${OUTPUT_TEST_DATA_FILE}
-D test_file_info=${TEST_FILE_INFO} -D test_file_info=${TEST_FILE_INFO}
-D result_checker=${CMAKE_BINARY_DIR}/tests/compareTool -D result_checker=${OPENFLIPPER_TEST_BINARIES}/compareTool
-P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake -P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake
) )
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment