Commit 0ecc9eeb authored by Imdad Sardharwalla's avatar Imdad Sardharwalla Committed by GitHub Enterprise
Browse files

REFORM-1015 Add parametrised constructor to Test::Paths class and expose...

REFORM-1015 Add parametrised constructor to Test::Paths class and expose functions in TestReport.hh (#28)

* Add parametrised constructor to Test::Paths class

* Update documentation and comments for Paths class

* Expose make_comparison() and make_summary_report() in TestReport.hh

* Update documentation and comments for TestReport
parent 3fcf1861
......@@ -15,17 +15,17 @@
#include <string>
// TODO: convert this to a debug option once we implement it
//#define TEST_WAIT_FOR_DEBUG_ATTACH
//#define TEST_WAIT_FOR_DEBUG_ATTACH
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
#include <thread>
#endif //TEST_WAIT_FOR_DEBUG_ATTACH
#endif // TEST_WAIT_FOR_DEBUG_ATTACH
namespace Test
{
#ifdef __APPLE__
namespace fs = boost::filesystem;
#else // __APPLE__
#else // __APPLE__
namespace fs = std::filesystem;
#endif // __APPLE__
......@@ -41,7 +41,9 @@ namespace fs = std::filesystem;
throw std::runtime_error(strm.str); \
}
#define TEST_THROW_if(COND, EXPR) if (COND) TEST_THROW(EXPR)
#define TEST_THROW_if(COND, EXPR) \
if (COND) \
TEST_THROW(EXPR)
Base::IOutputStream& operator<<(Base::IOutputStream& _os, const fs::path& _path)
{
......@@ -70,45 +72,90 @@ void make_directory(const fs::path& _dir)
}
}
}//namespace
} // namespace
Paths::Paths(
const int _argc, const char* const* _argv, const char* const _out_root)
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
void Paths::wait_for_debug_attach()
{
// allow to attached the debugger a test process executed by the test system
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
if (boost::filesystem::exists("c:/wait_to_debug_test.yes"))
// Wait for a debugger to be attached to the test process
if (fs::exists("c:/wait_to_debug_test.yes"))
{ // placing a breakpoint on the cycle below allows us to exit the sleep
std::chrono::seconds duration(10);
for (int i = 0; i < 100; ++i)
std::this_thread::sleep_for(duration);
}
}
#endif // TEST_WAIT_FOR_DEBUG_ATTACH
TEST_THROW_if(_argc < 2, "Test arguments are not supplied");
// parse the extract the input test folder & name
fs::path in_path(_argv[1]);
if (fs::is_directory(in_path))
{// we expect that the first argument is the test root directory
TEST_THROW_if(_argc < 3, // is the subpath argument provided?
"Subpath argument for test root folder " << in_path << " missing");
fs::path subpath(_argv[2]);
in_path /= subpath;
if (_out_root != nullptr)
{
const auto out_dir = fs::path(_out_root) / fs::path(subpath);
make_directory(out_dir);
fs::current_path(out_dir); // set output as current path
}
void Paths::init(const char* const _in_root, const char* const _in_rel_path,
const char* const _out_root)
{
fs::path in_root(_in_root);
fs::path in_rel_path(_in_rel_path);
fs::path in_path = in_root / in_rel_path;
// Verify arguments
TEST_THROW_if(!fs::is_directory(in_root), in_root << " is not a directory.");
TEST_THROW_if(!fs::is_regular_file(in_path), in_path << " is not a file.");
// If an output root directory has been supplied, determine and create the
// test output directory, and switch to it
if (_out_root != nullptr)
{
const auto out_dir = fs::path(_out_root) / in_rel_path;
make_directory(out_dir);
fs::current_path(out_dir); // set output as current path
}
// else: special case to run a test from the IDE as a single path
TEST_THROW_if(!fs::is_regular_file(in_path), in_path << " is not a file");
in_path_ = in_path.string();
in_dir_ = in_path.parent_path().string();
filename_ = in_path.filename().string();
}
Paths::Paths(const char* const _in_root, const char* const _in_rel_path,
const char* const _out_root)
{
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
wait_for_debug_attach();
#endif // TEST_WAIT_FOR_DEBUG_ATTACH
init(_in_root, _in_rel_path, _out_root);
}
Paths::Paths(
const int _argc, const char* const* _argv, const char* const _out_root)
{
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
wait_for_debug_attach();
#endif // TEST_WAIT_FOR_DEBUG_ATTACH
TEST_THROW_if(_argc < 2, "Test arguments have not been supplied.");
// Parse the first argument--this could be either the input root directory, or
// a direct path to a test file
fs::path in_path(_argv[1]);
if (fs::is_regular_file(in_path))
{
// Simple path to test file. This option is to facilitate debugging a test
// within an IDE.
in_path_ = in_path.string();
in_dir_ = in_path.parent_path().string();
filename_ = in_path.filename().string();
}
else if (_argc > 2)
{
// input root directory and input relative path are supplied, but no output
// root directory.
init(_argv[1], _argv[2], _out_root);
}
else
{
// Command-line arguments are not in the correct format.
TEST_THROW("Failed to parse command-line arguments.");
}
}
std::string Paths::input_filename_path(const char* const _flnm) const
{
fs::path inpt_flnm_path(input_directory());
......@@ -118,4 +165,4 @@ std::string Paths::input_filename_path(const char* const _flnm) const
} // namespace Test
#endif//TEST_ON
#endif // TEST_ON
......@@ -14,25 +14,43 @@ class Paths
{
public:
/*!
Set up the test environment from the test app command-line interface (CLI) and
the system environment.
Set up the test environment and determine the output directory structure. This
can be done in one of two ways:
We expect the following CLI argument format:
> test-app input_root_directory input_relative_file_path [test arguments]
1. By supplying the test root directory (in_root), the relative path to the
test file (in_rel_path), and an optional output root directory (out_root).
If an output root directory is supplied, a test-specific output directory is
created (including any parent directories) by appending the supplied output
root folder with the relative file. We then switch to this output folder.
The (absolute) path to the test file is in_path = <in_root>/<in_rel_path>.
To facilitate debugging a test within an IDE, we also support the format:
> test-app input_test_path [test arguments]
In this case the paths data is setup identically, but no test-specific output
folder is created, and the test output is stored within the current folder.
If the output root directory is supplied, the test-specific output
directory out_dir = <out_root>/<in_rel_path>/ is created (including any
parent directories). The current working directory is then switched to
out_dir.
2. By forwarding the command-line arguments (argc, argv) from the test binary
and optionally supplying an output root directory (out_root). We expect the
following format for the command-line arguments:
> test-bin in_root in_rel_path [test_arguments]
Having parsed the command-line arguments, we then proceed as in option 1.
To facilitate debugging a test within an IDE, we also support the format:
> test-bin in_path [test arguments]
In this case the Paths object is set up identically, but no test-specific
output folder is created, and the test output is stored within the current
working directory.
*/
Paths(const char* const _in_root, //!< input root directory
const char* const _in_rel_path, //!< input relative file path
const char* const _out_root = nullptr //!< output root directory
);
Paths(const int _argc, //!< number of executable arguments
const char* const* _argv, //!< executable arguments
const char* const _out_root = nullptr //!< output root folder
const char* const _out_root = nullptr //!< output root directory
);
//! The input test path (directory + filename)
......@@ -51,6 +69,16 @@ public:
std::string input_filename_path(const char* const _flnm) const;
private:
// Initialisation function for the class, called by the constructor
void init(const char* const _in_root, //!< input root directory
const char* const _in_rel_path, //!< input relative file path
const char* const _out_root = nullptr //!< output root directory
);
#ifdef TEST_WAIT_FOR_DEBUG_ATTACH
void wait_for_debug_attach();
#endif // TEST_WAIT_FOR_DEBUG_ATTACH
std::string in_path_;
std::string in_dir_;
std::string filename_;
......
......@@ -29,20 +29,9 @@ INSECURE_INCLUDE_SECTION_END
namespace Test
{
namespace
namespace
{
enum ExitStatus
{
ES_OK = 0,
ES_DIFFERENCES = 1, // Test compare has detected differences
ES_WRONG_ARGUMENT,
ES_ROOT_NOT_FOUND,
ES_ROOT_NOT_DIR,
ES_NO_OUT_FILE,
ES_UPDATE_FAILED
};
const char* const INDENT = " ";
#ifdef __APPLE__
......@@ -56,8 +45,8 @@ Contains the test name and eventually the error message that appear in CTest log
*/
struct TestNameAndResult
{
explicit TestNameAndResult(const Test::Path& _test_name)
: name_(_test_name)
explicit TestNameAndResult(const Test::Path& _test_name)
: name_(_test_name)
{}
Test::Path name_;
mutable std::string error_msg_;
......@@ -158,7 +147,7 @@ TestList::TestList(const char* _dir) : exp_test_nmbr_(0)
}
}
// Recursively visits sub-directories and store as tests all directories that
// Recursively visits sub-directories and store as tests all directories that
// does not have sub-directories.
void TestList::collect_tests(const Test::Path& _dir, const size_t _root_path_size)
{
......@@ -171,7 +160,7 @@ void TestList::collect_tests(const Test::Path& _dir, const size_t _root_path_siz
else if (it->path().filename() == Test::REPORT_FILENAME)
is_test = true;
}
if (!is_test || _dir == root_dir_)
return;
......@@ -218,17 +207,17 @@ struct TestInfo
// Store report information about any test that have been executed.
struct TestInfoReport : public TestInfo
{
TestInfoReport(Test::Result _rep_out, const std::string& _test_name,
TestInfoReport(Test::Result _rep_out, const std::string& _test_name,
const TestStream& _descr) : TestInfo(_test_name, _descr), rep_out_(_rep_out)
{}
TestInfoReport(TestInfoReport&& _oth) : TestInfo(std::move(_oth)), rep_out_(_oth.rep_out_)
{}
friend std::ofstream& operator<<(std::ofstream& _os,
friend std::ofstream& operator<<(std::ofstream& _os,
const TestInfoReport& _rprt)
{
_os << _rprt.name_ << INDENT << _rprt.rep_out_.message() << std::endl
_os << _rprt.name_ << INDENT << _rprt.rep_out_.message() << std::endl
<< _rprt.descr_ << std::endl;
return _os;
}
......@@ -249,7 +238,7 @@ struct TestInfoCompare : public TestInfo
: TestInfo(_test_name, _descr), diffs_(_diff)
{}
TestInfoCompare(TestInfoCompare&& _oth)
TestInfoCompare(TestInfoCompare&& _oth)
: TestInfo(std::move(_oth)), diffs_(_oth.diffs_)
{}
......@@ -296,7 +285,7 @@ struct PathLink
template <class StreamT>
friend StreamT& operator<<(StreamT& _os, const PathLink& _link)
{
_os << "\"file:////" <<
_os << "\"file:////" <<
fs::canonical(fs::absolute(_link.path_)).generic_string() << "\"";
return _os;
}
......@@ -307,7 +296,7 @@ void save_comparison(std::ofstream& log_file,
const std::list<TestInfoCompare>& diff_log, const TestList (&test_lists)[2],
const bool _short_frmt = false)
{
const char* test_flnm = _short_frmt ?
const char* test_flnm = _short_frmt ?
"baseline_short.diff" : "baseline_full.diff";
auto tests_root(test_lists[1].root_dir().native());
......@@ -327,7 +316,7 @@ void save_comparison(std::ofstream& log_file,
const auto diff_flnm = diff_drct / test_flnm;
std::ofstream rprt_file(diff_flnm.c_str());
rprt_file << PathLink(diff_drct) << std::endl; // Test report path link
for (const auto& test_list : test_lists)
for (const auto& test_list : test_lists)
{// Log the file folder(s)
auto test_dir = test_list.root_dir() / test_info.name_;
rprt_file << PathLink(test_dir) << std::endl;
......@@ -344,83 +333,15 @@ void save_comparison(std::ofstream& log_file,
}
}
std::ofstream& operator<<(std::ofstream& _os,
const std::list<TestInfoReport>& _rprts)
std::ofstream& operator<<(std::ofstream& _os,
const std::list<TestInfoReport>& _rprts)
{
for (const auto& rprt : _rprts)
_os << rprt << std::endl;
return _os;
}
ExitStatus make_report(const char* const _root_dir)
{
// Finds the list of executed tests.
std::list<TestInfoReport> diff_log;
TestList test_list(_root_dir);
std::string report_flnm((test_list.root_dir() / Test::REPORT_FILENAME).string());
std::ofstream log_file(report_flnm);
if (!log_file)
{
std::cout << "ERROR: cannot create report file.\n";
return ES_NO_OUT_FILE;
}
// Analyze and sort test report information
std::cout << "Analyzing: " << test_list.tests().size() << " tests...\n";
std::map<Test::Result, size_t> test_stats;
size_t test_idx = 0;
for (const auto& test : test_list.tests())
{
std::cout << '\r' << ++test_idx;
Base::OutputStreamAdaptT<TestStream> test_log;
if (!test.error_msg_.empty())
test_log.stream() << "Test Result: " << test.error_msg_ << std::endl;
const auto& test_name = test.name_;
const Test::Path test_dir = test_list.root_dir() / test_name;
auto out = Test::parse_report(test_dir.string(), test_log);
diff_log.emplace_back(out, test_name.string(), test_log.stream());
++test_stats[out];
}
std::cout << std::endl;
// Saves a test summary.
log_file << "Executed " << diff_log.size() << " tests" << std::endl;
if (diff_log.size() != test_list.expected_test_number())
log_file << "Error: expected " << test_list.expected_test_number() << " tests\n";
for (const auto& stat : test_stats)
{
log_file << INDENT << std::setw(5) << stat.second << " " <<
stat.first.message() << std::endl;
}
log_file << std::endl;
// Saves the tests results.
log_file << diff_log;
// Make the baseline
const Test::Path test_dir(_root_dir);
const Test::Path bsln_dir =
test_dir.parent_path() / (test_dir.filename().string() + "_base");
Test::make_baseline(test_dir.string(), bsln_dir.string());
return ES_OK;
}
enum CompareOutputType
{
COT_SHORT_DIFF,
COT_FULL_DIFF,
COT_UPDATE,
COT_MIRROR
};
namespace Update
namespace Update
{
void copy(const fs::path& _root0, const fs::path& _root1, const fs::path& _name)
......@@ -459,7 +380,7 @@ void add(const fs::path& _root0, const fs::path& _root1, const SetT& _diff)
copy(_root0, _root1, test.name_);
}
}
template <class SetT>
void remove(const fs::path& _root0, const SetT& _diff)
{
......@@ -471,7 +392,7 @@ void remove(const fs::path& _root0, const SetT& _diff)
}
void replace(const fs::path& _root0, const fs::path& _root1,
void replace(const fs::path& _root0, const fs::path& _root1,
const fs::path& _name)
{
std::cout << "Replace " << _name << std::endl;
......@@ -481,7 +402,68 @@ void replace(const fs::path& _root0, const fs::path& _root1,
}//namespace Update
ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
} // namespace
ExitStatus make_summary_report(const char* const _root_dir)
{
// Finds the list of executed tests.
std::list<TestInfoReport> diff_log;
TestList test_list(_root_dir);
std::string report_flnm((test_list.root_dir() / Test::REPORT_FILENAME).string());
std::ofstream log_file(report_flnm);
if (!log_file)
{
std::cout << "ERROR: cannot create report file.\n";
return ES_NO_OUT_FILE;
}
// Analyze and sort test report information
std::cout << "Analyzing: " << test_list.tests().size() << " tests...\n";
std::map<Test::Result, size_t> test_stats;
size_t test_idx = 0;
for (const auto& test : test_list.tests())
{
std::cout << '\r' << ++test_idx;
Base::OutputStreamAdaptT<TestStream> test_log;
if (!test.error_msg_.empty())
test_log.stream() << "Test Result: " << test.error_msg_ << std::endl;
const auto& test_name = test.name_;
const Test::Path test_dir = test_list.root_dir() / test_name;
auto out = Test::parse_report(test_dir.string(), test_log);
diff_log.emplace_back(out, test_name.string(), test_log.stream());
++test_stats[out];
}
std::cout << std::endl;
// Saves a test summary.
log_file << "Executed " << diff_log.size() << " tests" << std::endl;
if (diff_log.size() != test_list.expected_test_number())
log_file << "Error: expected " << test_list.expected_test_number() << " tests\n";
for (const auto& stat : test_stats)
{
log_file << INDENT << std::setw(5) << stat.second << " " <<
stat.first.message() << std::endl;
}
log_file << std::endl;
// Saves the tests results.
log_file << diff_log;
// Make the baseline
const Test::Path test_dir(_root_dir);
const Test::Path bsln_dir =
test_dir.parent_path() / (test_dir.filename().string() + "_base");
Test::make_baseline(test_dir.string(), bsln_dir.string());
return ES_OK;
}
ExitStatus make_comparison(const char* const _dir0, const char* const _dir1,
const CompareOutputType _cot, const Checksum::Compare& _chks_cmpr)
{
// Finds differences in the executed tests and compares the common ones.
......@@ -522,10 +504,10 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
// Finds the common set of tests.
std::vector<TestNameAndResult> cmmn_tests;
std::set_intersection(
test_lists[0].tests().cbegin(), test_lists[0].tests().cend(),
test_lists[0].tests().cbegin(), test_lists[0].tests().cend(),
test_lists[1].tests().cbegin(), test_lists[1].tests().cend(),
std::inserter(cmmn_tests, cmmn_tests.end()));
// Compares the results of the common set of tests.
std::cout << "Comparing " << cmmn_tests.size() << " tests\n";
size_t diff_test_nmbr = 0, test_idx = 0;
......@@ -550,12 +532,12 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
}
}
// same results are in the full difference report
if (!diff.empty() || _cot == COT_FULL_DIFF)
if (!diff.empty() || _cot == COT_FULL_DIFF)
diff_log.emplace_back(diff, test.name_.string(), test_log.stream());
}
// Test compare summary.
std::cout << "\n\nCompare test has detected differences in " <<
std::cout << "\n\nCompare test has detected differences in " <<
diff_test_nmbr << " tests.\n";
if (_cot != COT_UPDATE && _cot != COT_MIRROR)
......@@ -576,14 +558,14 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
{
std::vector<TestNameAndResult> diff_tests;
std::set_difference(
test_lists[i].tests().cbegin(), test_lists[i].tests().cend(),
test_lists[i].tests().cbegin(), test_lists[i].tests().cend(),
test_lists[1 - i].tests().cbegin(), test_lists[1 - i].tests().cend(),
std::inserter(diff_tests, diff_tests.end()));
if (diff_tests.empty())
continue;
if (_cot == COT_UPDATE || _cot == COT_MIRROR)
{// delete test baseline on the left only if the MIRROR operation is used
if (i == 0 && _cot == COT_MIRROR) // delete tests on the left
if (i == 0 && _cot == COT_MIRROR) // delete tests on the left
Update::remove(test_lists[0].root_dir(), diff_tests);
else if (i == 1) // add the test baseline to the left set
{
......@@ -611,8 +593,6 @@ ExitStatus make_compare(const char* const _dir0, const char* const _dir1,
return ES_OK;
}
} // namespace
int report(const int _argc, const char* const _argv[],
const Checksum::Compare& _chks_cmpr)
{
......@@ -623,7 +603,7 @@ int report(const int _argc, const char* const _argv[],
}
if (_argc == 2)
return make_report(_argv[1]);
return make_summary_report(_argv[1]);
bool shrt = true, full = false; // by default, produce the short format
bool updt = false, mirr = false; // update or mirror the test results
......@@ -648,7 +628,7 @@ int report(const int _argc, const char* const _argv[],
<< std::endl;
try
{
rslt = make_compare(
rslt = make_comparison(
_argv[1], _argv[2], updt ? COT_UPDATE : COT_MIRROR, _chks_cmpr);
}
catch (const std::exception& excpt)
......@@ -669,12 +649,12 @@ int report(const int _argc, const char* const _argv[],
if (shrt)
{
std::cout << "Producing the short diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_SHORT_DIFF, _chks_cmpr);
rslt = make_comparison(_argv[1], _argv[2], COT_SHORT_DIFF, _chks_cmpr);
}
if (full)
{
std::cout << "Producing the full diff..." << std::endl;
rslt = make_compare(_argv[1], _argv[2], COT_FULL_DIFF, _chks_cmpr);
rslt = make_comparison(_argv[1], _argv[2], COT_FULL_DIFF, _chks_cmpr);
}
}
return rslt;
......
......@@ -10,24 +10,76 @@
namespace Test
{
enum ExitStatus
{
ES_OK = 0,
ES_DIFFERENCES = 1, // Test compare has detected differences
ES_WRONG_ARGUMENT,
ES_ROOT_NOT_FOUND,
ES_ROOT_NOT_DIR,
ES_NO_OUT_FILE,
ES_UPDATE_FAILED
};
/*!
Determines the action of the make_comparison() function:
* COT_SHORT_DIFF: Produce a comparison report that only includes checksums that
are different between <root_dir_0> and <root_dir_1>.
* COT_FULL_DIFF: Produce a comparison report that includes all checksums, even
those that are identical between <root_dir_0> and <root_dir_1>.
* COT_UPDATE: Copy all summary reports from <root_dir_1> to <root_dir_0>
(potentially overwriting summary reports in <root_dir_0>). Any summary reports
that exist in <root_dir_0> but not in <root_dir_1> are *unaffected*.
* COT_MIRROR: Copy all summary reports from <root_dir_1> to <root_dir_0>
(potentially overwriting summary reports in <root_dir_0>). Any summary reports
that exist in <root_dir_0> but not in <root_dir_1> are *deleted*.
*/
enum CompareOutputType
{
COT_SHORT_DIFF,
COT_FULL_DIFF,
COT_UPDATE,
COT_MIRROR
};