Fixed error in compare tool, not throwing an error on string mismatch. Moved...

Fixed error in compare tool, not throwing an error on string mismatch. Moved test binaries into extra folder. Fixed bad commit with wrong test generator refs #394

git-svn-id: http://www.openflipper.org/svnrepo/OpenFlipper/branches/Free@13670 383ad7c9-94d9-4d36-a494-682f7c89f535

tests/CMakeLists.txt

@@ -30,6 +30,7 @@ SET (CTEST_DROP_SITE_CDASH FALSE)
 
 
   make_directory(${OPENFLIPPER_TEST_RESULTS})
+  make_directory(${OPENFLIPPER_TEST_BINARIES})
 
   # ========================================================================
   # ========================================================================

tests/MinimalExecutionTest/CMakeLists.txt

@@ -5,20 +5,20 @@ if(BUILD_TESTING)
   
   set_target_properties( minimalExecutionTest PROPERTIES
       RUNTIME_OUTPUT_NAME "minimalExecTest"
-      RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
-      LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
+      RUNTIME_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
+      LIBRARY_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
     )
 	
   if (WIN32)
     add_custom_command (TARGET minimalExecutionTest POST_BUILD
                         COMMAND ${CMAKE_COMMAND} -E
                         copy_if_different
-                          ${CMAKE_BINARY_DIR}/tests/${CMAKE_CFG_INTDIR}/minimalExecTest.exe
-                          ${CMAKE_BINARY_DIR}/tests/minimalExecTest.exe)
+                          ${OPENFLIPPER_TEST_BINARIES}/${CMAKE_CFG_INTDIR}/minimalExecTest.exe
+                          ${OPENFLIPPER_TEST_BINARIES}/minimalExecTest.exe)
   endif()
 
   ADD_TEST(NAME "Minimal_Execution_Test"
-           COMMAND ${CMAKE_BINARY_DIR}/tests/minimalExecTest )
+           COMMAND ${OPENFLIPPER_TEST_BINARIES}/minimalExecTest )
 
   set_tests_properties( Minimal_Execution_Test PROPERTIES
     TIMEOUT 10

tests/comparatorTool/CMakeLists.txt → tests/compareTool/CMakeLists.txt

@@ -5,27 +5,23 @@ if(BUILD_TESTING)
     find_package (Qt4 COMPONENTS QtCore )
   endif()
  
-  add_executable (comparatorTool comparator.cc )
+  add_executable (compareTool compareTool.cc )
 
-  target_link_libraries (comparatorTool
+  target_link_libraries (compareTool
     ${QT_LIBRARIES}
   ) 
-
-
-
  
-  set_target_properties ( comparatorTool PROPERTIES
-    RUNTIME_OUTPUT_NAME "compareTool"
-    RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
-    LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
+  set_target_properties ( compareTool PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
+    LIBRARY_OUTPUT_DIRECTORY "${OPENFLIPPER_TEST_BINARIES}"
   )
 	
   if (WIN32)
-    add_custom_command (TARGET comparatorTool POST_BUILD
+    add_custom_command (TARGET compareTool POST_BUILD
                         COMMAND ${CMAKE_COMMAND} -E
                         copy_if_different                               
-                          ${CMAKE_BINARY_DIR}/tests/${CMAKE_CFG_INTDIR}/compareTool.exe
-                          ${CMAKE_BINARY_DIR}/tests/compareTool.exe)
+                          ${OPENFLIPPER_TEST_BINARIES}/${CMAKE_CFG_INTDIR}/compareTool.exe
+                          ${OPENFLIPPER_TEST_BINARIES}/compareTool.exe)
   endif()
 
 endif()

tests/comparatorTool/comparator.cc → tests/compareTool/compareTool.cc

@@ -170,7 +170,7 @@ int main(int argv, char **args)
        if ( toplevelKeys[i].endsWith("_DOUBLE") ) {
          ok &= compareDouble(toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i]));
        } else
-         compareString( toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i]));
+         ok &= compareString( toplevelKeys[i],resultFile.value(toplevelKeys[i]), referenceFile.value(toplevelKeys[i]));
      } else {
        cerr << "Missing key in result file: " << toplevelKeys[i] << "\n";
        ok = false;

tests/run_file_test.cmake

@@ -11,9 +11,14 @@ endif( NOT output_test )
 
 # output_test contains the info about the loaded file
 if( NOT test_file_info )
-   message( FATAL_ERROR "Variable test_file_info= not defined" )
+   message( FATAL_ERROR "Variable test_file_info not defined" )
 endif( NOT test_file_info )
 
+if( NOT result_checker )
+   message( FATAL_ERROR "Variable result_checker not defined" )
+endif( NOT result_checker )
+
+
 # convert the space-separated string to a list
 separate_arguments( test_args )
 message( ${test_args} )
@@ -35,62 +40,28 @@ if ( NOT ${PROCESSRESULT} EQUAL 0 )
 endif()
 
 # ===================================================
-# Collect list of what should be tested
+# Compare
 # ===================================================
 
-# Read the info file
-file (STRINGS ${test_file_info} FILEINFO)
-
-#Create a list of properties we want to test
-foreach(loop ${FILEINFO})
-  string ( REGEX REPLACE "=.*$" "" TMP ${loop} )
-  list( APPEND CHECKS ${TMP})
-endforeach()
 
-# ===================================================
-# Read the files
-# ===================================================
 
-# read the test output
-file (READ ${output_test} TESTOUTPUT)
+if ( WIN32 )
+  set(result_checker "${result_checker}.exe")
+endif()
 
-# read the test output
-file (READ ${test_file_info} INFOFILE)
+message( "Executing: ${result_checker} ${output_test} ${test_file_info} " )
 
-# ===================================================
-# Compare
-# ===================================================
+# Timeout after 2 minutes
+execute_process(
+   COMMAND ${result_checker} ${output_test} ${test_file_info}
+   TIMEOUT 60
+   RESULT_VARIABLE PROCESSRESULT
+)
 
-# Now go through all checks:
-foreach(loop ${CHECKS})
-
-  #Get value from results:
-  string (
-      REGEX REPLACE
-      "^.*${loop}=([^\n]*).*$" "\\1"
-      RESULT  ${TESTOUTPUT}
-  )
-
-  # remove whitespace
-  string(STRIP ${RESULT} RESULT)
-
-  #Get value from Info:
-  string (
-      REGEX REPLACE
-      "^.*${loop}=([^\n]*).*$" "\\1"
-      EXPECTED  ${INFOFILE}
-  )
-  
-  # remove whitespace
-  string(STRIP ${EXPECTED} EXPECTED)
-
-  # Use Stringcompare here
-  if ( NOT ${EXPECTED} STREQUAL ${RESULT} )
-    message(WARNING "Mismatching values for ${loop}: EXPECTED ${EXPECTED} but got ${RESULT}!")
-    set(test_not_successful true)
-  endif()
-
-endforeach()
+if ( NOT ${PROCESSRESULT} EQUAL 0 )
+  message(SEND_ERROR "Compare Tool execution failed!")
+  set(test_not_successful TRUE)
+endif()
 
 if( test_not_successful )
    message( SEND_ERROR "Test Failed! See messages above to see what went wrong!" )

tests/run_test_md5.cmake

-# some argument checking:
-# test_cmd is the command to run with all its arguments
-if( NOT test_cmd )
-   message( FATAL_ERROR "Variable test_cmd not defined" )
-endif( NOT test_cmd )
-
-# output_blessed contains the name of the "blessed" output file
-if( NOT md5 )
-   message( FATAL_ERROR "Variable md5 not defined" )
-endif( NOT md5 )
-
-# output_test contains the name of the output file the test_cmd will produce
-if( NOT output_test )
-   message( FATAL_ERROR "Variable output_test not defined" )
-endif( NOT output_test )
-
-# convert the space-separated string to a list
-separate_arguments( test_args )
-message( ${test_args} )
-
-execute_process(
-   COMMAND ${test_cmd} ${test_args}
-)
-
-execute_process(
-   COMMAND ${CMAKE_COMMAND} -E md5sum ${output_test}
-   RESULT_VARIABLE test_not_successful
-   OUTPUT_VARIABLE result_md5
-)
-
-#split into its two parts
-string(REPLACE " " ";" result_md5 ${result_md5})
-
-#Get the sum
-list(GET result_md5 0 result_md5)
-
-if ( NOT result_md5 STREQUAL md5 )
-  message(STATUS "MD5 missmatch")
-  message(STATUS "${result_md5}")
-  message(STATUS "${md5}")
-  set( test_not_successful "TRUE" )
-endif()
-
-if( test_not_successful )
-   message( SEND_ERROR "${output_test} md5 is ${result_md5} but should be ${md5}!" )
-endif( test_not_successful )

tests/testGenerators.cmake

@@ -52,6 +52,7 @@ function( run_single_object_file_mesh_test FILEPLUGIN TEST_FILE TEST_SCRIPT  )
      -D test_args:string=${args}
      -D output_test=${OUTPUT_TEST_DATA_FILE}
      -D test_file_info=${TEST_FILE_INFO}
+     -D result_checker=${OPENFLIPPER_TEST_BINARIES}/compareTool
      -P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake
   )
 
@@ -157,7 +158,7 @@ function( run_algorithm_test TEST_SCRIPT INPUT_FILE INPUT_REFERENCE )
      -D test_args:string=${args}
      -D output_test=${OUTPUT_TEST_DATA_FILE}
      -D test_file_info=${TEST_FILE_INFO}
-     -D result_checker=${CMAKE_BINARY_DIR}/tests/compareTool
+     -D result_checker=${OPENFLIPPER_TEST_BINARIES}/compareTool
      -P ${CMAKE_SOURCE_DIR}/tests/run_file_test.cmake
   )