Commit 442e1843 authored by Martin Marinov's avatar Martin Marinov
Browse files

Test::Checksum::Difference now differentiates between complete failure/success...

Test::Checksum::Difference now differentiates between complete failure/success and the less important regression/improvement.
parent ca776b5a
......@@ -73,22 +73,65 @@ Difference Object::compare(
#define DIFFERENCE(TYPE) data_diff += Difference(Difference::TYPE)
switch (old_rslt_type)
{
case Result::OK :
case Result::OK:
switch (new_rslt_type)
{
return new_rslt_type == Result::WARNING ?
DIFFERENCE(SUSPICIOUS) : DIFFERENCE(REGRESSED);
}
case Result::WARNING: DIFFERENCE(SUSPICIOUS); break;
case Result::ERROR: DIFFERENCE(REGRESSED); break;
default: DIFFERENCE(FAILED); break; // FAILURE, CRASH, HANG
};
break;
case Result::WARNING :
switch (new_rslt_type)
{
return new_rslt_type == Result::OK ?
DIFFERENCE(IMPROVED) : DIFFERENCE(REGRESSED);
}
default : // ERROR, CRASH, HANG, etc
case Result::OK: DIFFERENCE(IMPROVED); break;
case Result::ERROR: DIFFERENCE(REGRESSED); break;
default: DIFFERENCE(FAILED); break; // FAILURE, CRASH, HANG
};
break;
case Result::ERROR :
switch (new_rslt_type)
{
return new_rslt_type == Result::OK || new_rslt_type == Result::WARNING ?
DIFFERENCE(IMPROVED) : DIFFERENCE(UNKNOWN);
}
case Result::OK:
case Result::WARNING: DIFFERENCE(IMPROVED); break;
default: DIFFERENCE(FAILED); break; // FAILURE, CRASH, HANG
};
break;
case Result::FAILURE:
switch (new_rslt_type)
{
// worked with or w/o issues, now fails
case Result::OK:
case Result::WARNING:
case Result::ERROR: DIFFERENCE(WORKED); break;
// gracious failure replaced by HANG or CRASH?!
default: DIFFERENCE(FAILED); break; // CRASH, HANG
};
break;
case Result::CRASH:
switch (new_rslt_type)
{
case Result::OK:
case Result::WARNING:
case Result::ERROR: DIFFERENCE(WORKED); break;
// CRASH replaced by gracious failure!
case Result::FAILURE: DIFFERENCE(IMPROVED); break;
default: DIFFERENCE(FAILED); break; // CRASH replaced by HANG?!
};
break;
case Result::HANG:
switch (new_rslt_type)
{
case Result::OK:
case Result::WARNING:
case Result::ERROR: DIFFERENCE(WORKED); break;
// HANG replaced by gracious failure!
case Result::FAILURE: DIFFERENCE(IMPROVED); break;
case Result::CRASH: DIFFERENCE(SUSPICIOUS); break; // HANG is now CRASH!
};
break;
}
return data_diff;
}
void Object::add(const Result& _rslt, const String& _data)
......
......@@ -41,12 +41,14 @@ class Difference
public:
enum Type
{
EQUAL,
UNKNOWN,
IMPROVED,
NEGLEGIBLE,
SUSPICIOUS,
REGRESSED
EQUAL, // result is bitwise identical
UNKNOWN, // non-negligible difference, but of unknown quality
IMPROVED, // result is better
NEGLEGIBLE, // result is negligibly different
SUSPICIOUS, // result is different, and the new result might be worse
REGRESSED, // result is worse
WORKED, // result works now, but used to fail
FAILED // result fails now, but used to work
};
static const char* const type_text(const Type _type)
......@@ -58,7 +60,9 @@ public:
"IMPROVED",
"NEGLEGIBLE",
"SUSPICIOUS",
"REGRESSED"
"REGRESSED",
"WORKED",
"FAILED"
};
return dscr[_type];
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment