third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ts: count skipped tests via SkipTestException

Browse Source 
- apply tag 'skip_other'
This commit is contained in:
Alexander Alekhin 2019-11-26 14:01:18 +03:00
parent 16ad53f354
commit 5639f5a296
7 changed files with 64 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
modules/ts/include/opencv2/ts.hpp
View File

@ -180,12 +180,21 @@ using testing::tuple_size;
using testing::tuple_element;
class SkipTestException: public cv::Exception
namespace details {
class SkipTestExceptionBase: public cv::Exception
{
public:
SkipTestExceptionBase(bool handlingTags);
SkipTestExceptionBase(const cv::String& message, bool handlingTags);
};
}
class SkipTestException: public details::SkipTestExceptionBase
{
public:
int dummy; // workaround for MacOSX Xcode 7.3 bug (don't make class "empty")
SkipTestException() : dummy(0) {}
SkipTestException(const cv::String& message) : dummy(0) { this->msg = message; }
SkipTestException() : details::SkipTestExceptionBase(false), dummy(0) {}
SkipTestException(const cv::String& message) : details::SkipTestExceptionBase(message, false), dummy(0) { }
};
/** Apply tag to the current test

10
modules/ts/include/opencv2/ts/ts_ext.hpp
View File

@ -16,6 +16,9 @@ extern int testThreads;
void testSetUp();
void testTearDown();
bool checkBigDataTests();
}
// check for required "opencv_test" namespace
@ -37,7 +40,7 @@ void testTearDown();
Body(); \
CV__TEST_CLEANUP \
} \
catch (const cvtest::SkipTestException& e) \
catch (const cvtest::details::SkipTestExceptionBase& e) \
{ \
printf("[ SKIP ] %s\n", e.what()); \
} \
@ -74,9 +77,8 @@ void testTearDown();
#define CV__TEST_BIGDATA_BODY_IMPL(name) \
{ \
if (!cvtest::runBigDataTests) \
if (!cvtest::checkBigDataTests()) \
{ \
printf("[ SKIP ] BigData tests are disabled\n"); \
return; \
} \
CV__TRACE_APP_FUNCTION_NAME(name); \
@ -85,7 +87,7 @@ void testTearDown();
Body(); \
CV__TEST_CLEANUP \
} \
catch (const cvtest::SkipTestException& e) \
catch (const cvtest::details::SkipTestExceptionBase& e) \
{ \
printf("[ SKIP ] %s\n", e.what()); \
} \

4
modules/ts/include/opencv2/ts/ts_perf.hpp
View File

@ -386,7 +386,7 @@ public:
static enum PERF_STRATEGY getCurrentModulePerformanceStrategy();
static enum PERF_STRATEGY setModulePerformanceStrategy(enum PERF_STRATEGY strategy);
class PerfSkipTestException: public cv::Exception
class PerfSkipTestException: public cvtest::SkipTestException
{
public:
int dummy; // workaround for MacOSX Xcode 7.3 bug (don't make class "empty")
@ -531,7 +531,7 @@ void PrintTo(const Size& sz, ::std::ostream* os);
::cvtest::testSetUp(); \
RunPerfTestBody(); \
} \
catch (cvtest::SkipTestException& e) \
catch (cvtest::details::SkipTestExceptionBase& e) \
{ \
printf("[ SKIP ] %s\n", e.what()); \
} \

25
modules/ts/src/ts.cpp
View File

@ -125,6 +125,20 @@ bool required_opencv_test_namespace = false; // compilation check for non-refac
namespace cvtest
{
details::SkipTestExceptionBase::SkipTestExceptionBase(bool handlingTags)
{
if (!handlingTags)
{
testTagIncreaseSkipCount("skip_other", true, true);
}
}
details::SkipTestExceptionBase::SkipTestExceptionBase(const cv::String& message, bool handlingTags)
{
if (!handlingTags)
testTagIncreaseSkipCount("skip_other", true, true);
this->msg = message;
}
uint64 param_seed = 0x12345678; // real value is passed via parseCustomOptions function
static std::string path_join(const std::string& prefix, const std::string& subpath)
@ -850,6 +864,17 @@ void testTearDown()
}
}
bool checkBigDataTests()
{
if (!runBigDataTests)
{
testTagIncreaseSkipCount("skip_bigdata", true, true);
printf("[ SKIP ] BigData tests are disabled\n");
return false;
}
return true;
}
void parseCustomOptions(int argc, char **argv)
{
const string command_line_keys = string(

10
modules/ts/src/ts_perf.cpp
View File

@ -2003,16 +2003,16 @@ void TestBase::RunPerfTestBody()
implConf.GetImpl();
#endif
}
catch(const SkipTestException&)
{
metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
throw;
}
catch(const PerfSkipTestException&)
{
metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
return;
}
catch(const cvtest::details::SkipTestExceptionBase&)
{
metrics.terminationReason = performance_metrics::TERM_SKIP_TEST;
throw;
}
catch(const PerfEarlyExitException&)
{
metrics.terminationReason = performance_metrics::TERM_INTERRUPT;

19
modules/ts/src/ts_tags.cpp
View File

@ -23,8 +23,10 @@ static std::map<std::string, int>& getTestTagsSkipExtraCounts()
static std::map<std::string, int> testTagsSkipExtraCounts;
return testTagsSkipExtraCounts;
}
static void increaseTagsSkipCount(const std::string& tag, bool isMain)
void testTagIncreaseSkipCount(const std::string& tag, bool isMain, bool appendSkipTests)
{
if (appendSkipTests)
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
std::map<std::string, int>& counts = isMain ? getTestTagsSkipCounts() : getTestTagsSkipExtraCounts();
std::map<std::string, int>::iterator i = counts.find(tag);
if (i == counts.end())
@ -192,11 +194,14 @@ public:
{
if (!skipped_tests.empty())
{
std::cout << "[ SKIPSTAT ] " << skipped_tests.size() << " tests via tags" << std::endl;
std::cout << "[ SKIPSTAT ] " << skipped_tests.size() << " tests skipped" << std::endl;
const std::vector<std::string>& skipTags = getTestTagsSkipList();
const std::map<std::string, int>& counts = getTestTagsSkipCounts();
const std::map<std::string, int>& countsExtra = getTestTagsSkipExtraCounts();
for (std::vector<std::string>::const_iterator i = skipTags.begin(); i != skipTags.end(); ++i)
std::vector<std::string> skipTags_all = skipTags;
skipTags_all.push_back("skip_bigdata");
skipTags_all.push_back("skip_other");
for (std::vector<std::string>::const_iterator i = skipTags_all.begin(); i != skipTags_all.end(); ++i)
{
int c1 = 0;
std::map<std::string, int>::const_iterator i1 = counts.find(*i);
@ -301,7 +306,7 @@ void checkTestTags()
if (found != tags.size())
{
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
throw details::SkipTestExceptionBase("Test tags don't pass required tags list (--test_tag parameter)", true);
}
}
}
@ -317,7 +322,7 @@ void checkTestTags()
const std::string& testTag = testTags[i];
if (isTestTagSkipped(testTag, skipTag))
{
increaseTagsSkipCount(skipTag, skip_message.empty());
testTagIncreaseSkipCount(skipTag, skip_message.empty());
if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)";
}
}
@ -327,7 +332,7 @@ void checkTestTags()
const std::string& testTag = testTagsImplied[i];
if (isTestTagSkipped(testTag, skipTag))
{
increaseTagsSkipCount(skipTag, skip_message.empty());
testTagIncreaseSkipCount(skipTag, skip_message.empty());
if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped (implied '" + skipTag + "' is in skip list)";
}
}
@ -335,7 +340,7 @@ void checkTestTags()
if (!skip_message.empty())
{
skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
throw SkipTestException(skip_message);
throw details::SkipTestExceptionBase(skip_message, true);
}
}

2
modules/ts/src/ts_tags.hpp
View File

@ -21,6 +21,8 @@ namespace cvtest {
void activateTestTags(const cv::CommandLineParser& parser);
void testTagIncreaseSkipCount(const std::string& tag, bool isMain = true, bool appendSkipTests = false);
} // namespace
#endif // OPENCV_TS_SRC_TAGS_HPP