Commit e7cd7b16 authored by Martin Marinov's avatar Martin Marinov Committed by GitHub Enterprise
Browse files

REFORM-999 Redesign the test report and checksum compare SDK (#25)

* Remove report functionality wrappers and simplify.
* Add Checksum::Compare function definition and use this in the Test::make_report().
* Clean up code and docs.
* Remove the registry definition from TestChecksum.hh
* Add docs to Checksum::Object::compare_data()
parent e48c901e
......@@ -23,6 +23,7 @@
#include <xlocnum>
#include <xutility>
#include <xatomic.h>
#include <xhash>
#if _MSC_VER >= 1920 // Visual Studio 2019 or later
#include <atomic>
......
set(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/LongestCommonSubsequenceT.hh
${CMAKE_CURRENT_SOURCE_DIR}/ResultAnalysis.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksum.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumCompare.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumCompletion.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumCondition.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumDebugEvent.hh
......@@ -12,14 +12,15 @@ set(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumNumberT.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumTime.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestPaths.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestReport.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestResult.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestResultAnalysis.hh
${CMAKE_CURRENT_SOURCE_DIR}/types.hh
PARENT_SCOPE
)
set(my_sources
${CMAKE_CURRENT_SOURCE_DIR}/ResultAnalysis.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksum.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumCompletion.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumCondition.cc
......@@ -28,5 +29,7 @@ set(my_sources
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumFile.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestChecksumIssueNumber.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestPaths.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestReport.cc
${CMAKE_CURRENT_SOURCE_DIR}/TestResultAnalysis.cc
PARENT_SCOPE
)
// (C) Copyright 2021 by Autodesk, Inc.
#ifndef BASE_RESULTANALYSIS_HH_INCLUDE
#define BASE_RESULTANALYSIS_HH_INCLUDE
#ifdef TEST_ON
#include <Base/Config/BaseDefines.hh>
#include <Base/Security/Mandatory.hh>
#include <Base/Test/TestResult.hh>
#include <Base/Test/TestChecksum.hh>
#include <set>
#include <sstream>
#include <string>
namespace Test {
typedef Base::IOutputStream OStream;
typedef Checksum::Difference Difference;
template <typename TypeT>
struct BasicTypeWrap
{
TypeT val_ = 0;
operator TypeT&() { return val_; }
operator const TypeT&() const { return val_; }
};
typedef std::map<Checksum::Difference, BasicTypeWrap<size_t>> DifferenceDistribution;
/*!
Compares two test folders to detect differences in test execution and/or test results.
Returns a map with founded difference types and the number of occurrences.
Save in an output stream log a description
of the differences for any different checksums. Equal checksums are reported if
_short_frmt = false.
*/
DifferenceDistribution internal_compare(
const std::string& _test_dir0, /*< [in] Test directory. */
const std::string& _test_dir1, /*< [in] Test directory. */
OStream& _log, /*< [out] Description of the checksums and differences. */
const bool _short_frmt = false /*![in] Removes identical checksums from the log. */
);
/*!
Reads the single test report in the input test directory and copies them in
the output stream. Detects if the test has not ended normally and eventually
adds an Execution crash or hang checksum.
*/
Result internal_report(
const std::string& _test_dir, /*< [in] Test directory. */
OStream& _log /*< [out] Log of relevant information. */
);
/*!
Extracts from the test results a set of files that will be the tests baseline.
*/
void internal_make_baseline(
const std::string& _test_root_dir, /*<[in] test root directory */
const std::string& _bsln_root_dir /*<[in] directory where we want to save the baseline data*/
);
}//Test
#endif//TEST_ON
#endif//BASE_RESULTANALYSIS_HH_INCLUDE
......@@ -4,17 +4,25 @@
#include "TestChecksum.hh"
#include "Base/Debug/DebCallStack.hh"
#include <fstream>
#include <iostream>
#include <map>
#include <mutex>
namespace Test {
namespace Checksum {
namespace Test
{
namespace Checksum
{
Level run_lvl = L_NONE;
namespace
{
/*!
Definition of the checksums registry. It is a map from a string (that is the
checksum name to an Checksum::Object.
*/
typedef std::map<String, Object*> Registry;
Registry& registry_modify()
{
......@@ -29,6 +37,15 @@ const Registry& registry()
return registry_modify();
}
Difference compare_from_registry(
const String& _name, const Record& _old_rcrd, const Record& _new_rcrd)
{
auto reg_it = registry().find(_name);
return reg_it == registry().end()
? Difference(Difference::UNKNOWN, "Checksum not registered")
: reg_it->second->compare(_old_rcrd, _new_rcrd);
}
///////////////////////////////////////////////////////////////////////////////
// class Checksum implementation
......@@ -59,11 +76,7 @@ Difference Object::compare_data(const String& _old, const String& _new) const
}
Difference Object::compare(
const Path& /*_old_path*/,
const Record& _old_rcrd,
const Path& /*_new_path*/,
const Record& _new_rcrd
) const
const Record& _old_rcrd, const Record& _new_rcrd) const
{
const auto old_rslt_type = _old_rcrd.rslt.type();
const auto new_rslt_type = _new_rcrd.rslt.type();
......
// (C) Copyright 2021 by Autodesk, Inc.
#ifndef BASE_ICHECKSUM_HH_INCLUDE
#define BASE_ICHECKSUM_HH_INCLUDE
#ifndef BASE_TESTCHECKSUM_HH_INCLUDED
#define BASE_TESTCHECKSUM_HH_INCLUDED
#ifndef TEST_ON
#define TEST(CHKSM, RCRD)
......@@ -13,8 +12,6 @@
#include <Base/Test/TestResult.hh>
#include <Base/Test/TestChecksumLevel.hh>
#include <Base/Utils/OStringStream.hh>
#include <map>
#include <sstream>
namespace Test
{
......@@ -86,7 +83,7 @@ public:
Difference& operator+=(const Difference::Type& _type)
{
if (type_ < _type)
type_ = type_;
type_ = _type;
return *this;
}
......@@ -137,9 +134,7 @@ public:
results, but compares the the data simply as strings (no parsing).
*/
virtual Difference compare(
const Path& _old_path, //!<[in] Path to the left record
const Record& _old_rcrd, //!<[in] "Left" record
const Path& _new_path, //!<[in] Path to the right record
const Record& _new_rcrd //!<[in] "Right" record
) const;
......@@ -156,7 +151,12 @@ protected:
//! Add a record of the checksum (protected version)
void add(const Result& _rslt, const String& _data);
//! Compare the data, the default implementation does a string comparison
/*!
Compare the data, the default implementation does a string comparison.
This is called by \ref compare to compare the data only without taking into
account the type of the result. This can be overridden instead of \ref compare
if the result type is insignificant for the comparison.
*/
virtual Difference compare_data(const String& _old, const String& _new) const;
protected:
......@@ -170,17 +170,6 @@ private:
Object* operator=(const Object&);
};
/*!
Definition of the checksums registry. It is a map from a string (that is the
checksum name to an IChecksum.
*/
typedef std::map<String, Object*> Registry;
/*!
Function to get a static map with all the registered checksums.
*/
const Registry& registry();
} // namespace Checksum
} // namespace Test
......@@ -199,5 +188,4 @@ const Registry& registry();
}
#endif // TEST_ON
#endif // BASE_ICHECKSUM_HH_INCLUDE
#endif // BASE_TESTCHECKSUM_HH_INCLUDED
// (C) Copyright 2021 by Autodesk, Inc.
#ifndef BASE_TESTCHECKSUMCOMPARE_HH_INCLUDED
#define BASE_TESTCHECKSUMCOMPARE_HH_INCLUDED
#ifdef TEST_ON
#include <Base/Test/TestChecksum.hh>
#include <functional>
namespace Test
{
namespace Checksum
{
/*!
Function type to compare two records of the same checksum identified by name.
The compare function should return \ref Difference value to describe the
difference of the old vs the new value.
*/
typedef std::function<Difference(const String& /*!<[in] checksum name */,
const Record& /*!<[in] checksum old value */,
const Record& /*!<[in] checksum new value */)>
Compare;
/*!
Compare function that uses the compare method of registered checksums.
\note Make sure to wrap this appropriately for shared object export when using
in shared binary components in order to use the registry in the shared binary!
*/
Difference compare_from_registry(
const String& _name, const Record& _old_rcrd, const Record& _new_rcrd);
} // namespace Checksum
} // namespace Test
#endif // TEST_ON
#endif // BASE_TESTCHECKSUMCOMPARE_HH_INCLUDED
......@@ -16,10 +16,10 @@ Execution::Execution() : Object("Execution", L_STABLE) {}
void Execution::record(
const char* const _call, const Result& _rslt, const char* _err_msg)
{
std::stringstream mess;
Base::OStringStream mess;
mess << _err_msg << " : " << _call;
add(_rslt, mess.str());
add(_rslt, mess.str);
}
// Register the checksum to check test execution.
......
// (C) Copyright 2021 by Autodesk, Inc.
#ifndef TEST_ON
int main(int , const char* []) { return 0; }
#else
#ifdef TEST_ON
#include <Base/Security/Mandatory.hh>
#include <Base/Test/ResultAnalysis.hh>
#include <Base/Test/TestResult.hh>
#include <boost/utility/value_init.hpp>
#include "TestReport.hh"
#include "TestResult.hh"
#include "TestResultAnalysis.hh"
#include <array>
#include <cstring>
#include <fstream>
#include <iomanip>
#include <iostream>
......@@ -31,38 +27,17 @@ INSECURE_INCLUDE_SECTION_BEGIN
INSECURE_INCLUDE_SECTION_END
#endif // __APPLE__
// The following functions should be available in a wrapper in the parent
// project. This wrapper should provide an external interface to the following
// functions in ResultAnalysis.hh: internal_compare(), internal_report(),
// internal_make_basline()
namespace Test {
extern DifferenceDistribution compare(
const std::string& _test_dir0, /*< [in] Test directory. */
const std::string& _test_dir1, /*< [in] Test directory. */
OStream& _log, /*< [out] Description of the checksums and differences. */
const bool _shrt_frmt = false /*![in] Removes identical checksums from the log. */
);
extern Result report(
const std::string& _test_dir, /*< [in] Test directory. */
OStream& _log /*< [out] Log of relevant information. */
);
extern void make_baseline(
const std::string& _test_root_dir, /*<[in] test root directory */
const std::string& _bsln_root_dir /*<[in] directory where we want to save the baseline data*/
);
}
namespace {
namespace Test
{
namespace
{
enum ExitStatus {
enum ExitStatus
{
ES_OK = 0,
ES_DIFFERENCES = 1, // Test compare has detected differences
ES_WRONG_ARGUMENT,
ES_ROOT_NOT_FOUND,
ES_ROOT_NOT_FOUND,
ES_ROOT_NOT_DIR,
ES_NO_OUT_FILE,
ES_UPDATE_FAILED
......@@ -394,7 +369,7 @@ ExitStatus make_report(const char* const _root_dir)
// Analyze and sort test report information
std::cout << "Analyzing: " << test_list.tests().size() << " tests...\n";
std::map<Test::Result, boost::value_initialized<size_t>> test_stats;
std::map<Test::Result, size_t> test_stats;
size_t test_idx = 0;
for (const auto& test : test_list.tests())
{
......@@ -406,7 +381,7 @@ ExitStatus make_report(const char* const _root_dir)
const auto& test_name = test.name_;
const Test::Path test_dir = test_list.root_dir() / test_name;
auto out = Test::report(test_dir.string(), test_log);
auto out = Test::parse_report(test_dir.string(), test_log);
diff_log.emplace_back(out, test_name.string(), test_log.stream());
++test_stats[out];
}
......@@ -445,7 +420,8 @@ enum CompareOutputType
COT_MIRROR
};
namespace Update {
namespace Update
{
void copy(const fs::path& _root0, const fs::path& _root1, const fs::path& _name)
{
......@@ -505,8 +481,8 @@ void replace(const fs::path& _root0, const fs::path& _root1,
}//namespace Update
ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
const CompareOutputType _cot)
ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
const CompareOutputType _cot, const Checksum::Compare& _chks_cmpr)
{
// Finds differences in the executed tests and compares the common ones.
// Finds the tests executed in both suites.
......@@ -558,11 +534,11 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
std::cout << '\r' << ++test_idx;
Base::OutputStreamAdaptT<TestStream> test_log;
const Test::Path test_dir0 = test_lists[0].root_dir() / test.name_;
const Test::Path test_dir1 = test_lists[1].root_dir() / test.name_;
const auto test_dir0 = test_lists[0].root_dir() / test.name_;
const auto test_dir1 = test_lists[1].root_dir() / test.name_;
const auto diff = Test::compare(test_dir0.string(), test_dir1.string(),
test_log, _cot != COT_FULL_DIFF);
const auto diff = compare(test_dir0.string(), test_dir1.string(),
test_log, _chks_cmpr, _cot != COT_FULL_DIFF);
if (!diff.empty())
{
++diff_test_nmbr;
......@@ -635,24 +611,10 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
return ES_OK;
}
}
} // namespace
/*!
Process one or two test root directory arguments by extracting the list
of tests inside each. Produce as follows:
1. Test report (if one root directory has been specified)
2. Test comparison (it two root directories have been specified)
3. Update the first folder to match the second by copying and overriding test
output.
An optional third argument can be passed to specify behavior affecting options
(2) and (3) above. The values are as follow:
F : Produce the full baseline diff only
S : Produce the short baseline diff only
U : Update the first folder to match the second
Usage examples:
\todo
*/
int main(int _argc, const char* _argv[])
int report(const int _argc, const char* const _argv[],
const Checksum::Compare& _chks_cmpr)
{
if (_argc < 2 || _argc > 4)
{
......@@ -662,60 +624,60 @@ int main(int _argc, const char* _argv[])
if (_argc == 2)
return make_report(_argv[1]);
else
bool shrt = true, full = false; // by default, both the shrt and full formats
bool updt = false, mirr = false; // update or mirror the test results
if (_argc == 4) // the 4th argument is used now to select a single format
{
bool shrt = true, full = false; // by default, both the shrt and full formats
bool updt = false, mirr = false; // update or mirror the test results
if (_argc == 4) // the 4th argument is used now to select a single format
if (strcmp(_argv[3], "S") == 0) // short only
full = false;
if (strcmp(_argv[3], "F") == 0) // full only
shrt = false;
if (strcmp(_argv[3], "U") == 0) // update and (re)generate the short diff
updt = true;
if (strcmp(_argv[3], "M") == 0) // mirror and (re)generate the short diff
mirr = true;
}
auto rslt = ES_OK;
if (updt || mirr)
{
const char* const sync_type = updt ? "Update " : "Mirror ";
std::cout << sync_type << _argv[1] << " to match " << _argv[2] << " ..."
<< std::endl;
try
{
if (strcmp(_argv[3], "S") == 0) // short only
full = false;
if (strcmp(_argv[3], "F") == 0) // full only
shrt = false;
if (strcmp(_argv[3], "U") == 0) // update and (re)generate the short diff
updt = true;
if (strcmp(_argv[3], "M") == 0) // mirror and (re)generate the short diff
mirr = true;
rslt = make_compare(
_argv[1], _argv[2], updt ? COT_UPDATE : COT_MIRROR, _chks_cmpr);
}
ExitStatus rslt = ES_OK;
if (updt || mirr)
catch (const std::exception& excpt)
{
const char* const sync_type = updt ? "Update " : "Mirror ";
std::cout << sync_type << _argv[1] << " to match "
<< _argv[2] << " ..." << std::endl;
try
{
rslt = make_compare(_argv[1], _argv[2], updt ? COT_UPDATE : COT_MIRROR);
}
catch (const std::exception& excpt)
{
std::cout << "FAILED " << sync_type << " with " << excpt.what()
<< std::endl;
rslt = ES_UPDATE_FAILED;
}
catch (...)
{
std::cout << "FAILED " << sync_type << " with an unknown exception!"
<< std::endl;
rslt = ES_UPDATE_FAILED;
}
std::cout << "FAILED " << sync_type << " with " << excpt.what()
<< std::endl;
rslt = ES_UPDATE_FAILED;
}
else
{ //TODO: running make_compare() twice is slow, but is an useful compromise
// for now; we should revisit this once we have more tests.
if (shrt)
{
std::cout << "Producing the short diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_SHORT_DIFF);
}
if (full)
{
std::cout << "Producing the full diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_FULL_DIFF);
}
catch (...)
{
std::cout << "FAILED " << sync_type << " with an unknown exception!"
<< std::endl;
rslt = ES_UPDATE_FAILED;
}
}
else
{
if (shrt)
{
std::cout << "Producing the short diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_SHORT_DIFF, _chks_cmpr);
}
if (full)
{
std::cout << "Producing the full diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_FULL_DIFF, _chks_cmpr);
}
return rslt;
}
return rslt;
}
} // namespace Test
#endif//TEST_ON
// (C) Copyright 2021 by Autodesk, Inc.
#ifndef BASE_TESTREPORT_HH_INCLUDED
#define BASE_TESTREPORT_HH_INCLUDED
#ifdef TEST_ON
#include <Base/Test/TestChecksumCompare.hh>
#include <functional>
namespace Test
{
/*!
Process one or two test root directory arguments by extracting the list
of tests inside each. Produce as follows:
1. Test report (if one root directory has been specified)
2. Test comparison (it two root directories have been specified)
3. Update the first folder to match the second by copying and overriding test
output.
An optional third argument can be passed to specify behavior affecting options
(2) and (3) above. The values are as follow:
F: Produce the full baseline diff only
S: Produce the short baseline diff only
U: Update the first folder to match the second (don't remove unmatched reports)
M: Mirror the second folder onto the first (remove unmatched reports!)
Usage examples:
\todo
*/
int report(const int _argc, const char* const _argv[],
const Checksum::Compare& _chks_cmpr);
} // namespace Test
#endif // TEST_ON
#endif // BASE_TESTREPORT_HH_INCLUDED
......@@ -2,18 +2,19 @@
#ifdef TEST_ON
#include "ResultAnalysis.hh"
#include <Base/Security/Mandatory.hh>
#include <Base/Test/LongestCommonSubsequenceT.hh>
#include <Base/Test/TestChecksum.hh>
#include <Base/Test/TestChecksumCompletion.hh>
#include <Base/Test/TestChecksumFile.hh>
#include <Base/Debug/DebOut.hh>
#include "TestResultAnalysis.hh"
#include "LongestCommonSubsequenceT.hh"
#include "TestChecksum.hh"
#include "TestChecksumCompletion.hh"
#include "TestChecksumFile.hh"
#include <algorithm>
#include <cctype>
#include <iostream>
#include <fstream>
#include <iostream>
#include <sstream>
#ifdef __APPLE__
#include <boost/filesystem.hpp>
......@@ -24,12 +25,14 @@ INSECURE_INCLUDE_SECTION_BEGIN
INSECURE_INCLUDE_SECTION_END
#endif // __APPLE__
namespace Test {
namespace Test
{
typedef std::istringstream IStream;
typedef std::string String;
namespace {
namespace
{