Merge pull request #18287 from mpashchenkov:mp/ocv-gapi-blue-branch

[G-API]: Add four kernels to parse NN outputs & provide information in Streaming scenarios

* Kernels from GL "blue" branch, acc and perf tests

* Code cleanup

* Output fix

* Comment fix

* Added new file for parsers, stylistic corrections

* Added end line

* Namespace fix

* Code cleanup

* nnparsers.hpp moved to gapi/infer/, nnparsers -> parsers

* Removed cv:: from parsers.hpp
This commit is contained in:
Maxim Pashchenkov
2020-09-18 16:31:16 +03:00
committed by GitHub
parent 830d8d6b75
commit a63cee2139
16 changed files with 1423 additions and 1 deletions
@@ -11,6 +11,7 @@
#include <iostream>
#include "gapi_tests_common.hpp"
#include "gapi_parsers_tests_common.hpp"
namespace opencv_test
{
@@ -149,6 +150,15 @@ GAPI_TEST_FIXTURE(WarpPerspectiveTest, initMatrixRandU,
GAPI_TEST_FIXTURE(WarpAffineTest, initMatrixRandU,
FIXTURE_API(CompareMats, double , double, int, int, cv::Scalar),
6, cmpF, angle, scale, flags, border_mode, border_value)
GAPI_TEST_EXT_BASE_FIXTURE(ParseSSDBLTest, ParserSSDTest, initNothing,
FIXTURE_API(float, int), 2, confidence_threshold, filter_label)
GAPI_TEST_EXT_BASE_FIXTURE(ParseSSDTest, ParserSSDTest, initNothing,
FIXTURE_API(float, bool, bool), 3, confidence_threshold, alignment_to_square, filter_out_of_bounds)
GAPI_TEST_EXT_BASE_FIXTURE(ParseYoloTest, ParserYoloTest, initNothing,
FIXTURE_API(float, float, int), 3, confidence_threshold, nms_threshold, num_classes)
GAPI_TEST_FIXTURE(SizeTest, initMatrixRandU, <>, 0)
GAPI_TEST_FIXTURE(SizeRTest, initNothing, <>, 0)
} // opencv_test
#endif //OPENCV_GAPI_CORE_TESTS_HPP
@@ -9,6 +9,7 @@
#define OPENCV_GAPI_CORE_TESTS_INL_HPP
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/infer/parsers.hpp>
#include "gapi_core_tests.hpp"
namespace opencv_test
@@ -1578,6 +1579,95 @@ TEST_P(ReInitOutTest, TestWithAdd)
run_and_compare();
}
TEST_P(ParseSSDBLTest, ParseTest)
{
cv::Mat in_mat = generateSSDoutput(sz);
std::vector<cv::Rect> boxes_gapi, boxes_ref;
std::vector<int> labels_gapi, labels_ref;
// G-API code //////////////////////////////////////////////////////////////
cv::GMat in;
cv::GOpaque<cv::Size> op_sz;
auto out = cv::gapi::parseSSD(in, op_sz, confidence_threshold, filter_label);
cv::GComputation c(cv::GIn(in, op_sz), cv::GOut(std::get<0>(out), std::get<1>(out)));
c.apply(cv::gin(in_mat, sz), cv::gout(boxes_gapi, labels_gapi), getCompileArgs());
// Reference code //////////////////////////////////////////////////////////
parseSSDBLref(in_mat, sz, confidence_threshold, filter_label, boxes_ref, labels_ref);
// Comparison //////////////////////////////////////////////////////////////
EXPECT_TRUE(boxes_gapi == boxes_ref);
EXPECT_TRUE(labels_gapi == labels_ref);
}
TEST_P(ParseSSDTest, ParseTest)
{
cv::Mat in_mat = generateSSDoutput(sz);
std::vector<cv::Rect> boxes_gapi, boxes_ref;
// G-API code //////////////////////////////////////////////////////////////
cv::GMat in;
cv::GOpaque<cv::Size> op_sz;
auto out = cv::gapi::parseSSD(in, op_sz, confidence_threshold,
alignment_to_square, filter_out_of_bounds);
cv::GComputation c(cv::GIn(in, op_sz), cv::GOut(out));
c.apply(cv::gin(in_mat, sz), cv::gout(boxes_gapi), getCompileArgs());
// Reference code //////////////////////////////////////////////////////////
parseSSDref(in_mat, sz, confidence_threshold, alignment_to_square,
filter_out_of_bounds, boxes_ref);
// Comparison //////////////////////////////////////////////////////////////
EXPECT_TRUE(boxes_gapi == boxes_ref);
}
TEST_P(ParseYoloTest, ParseTest)
{
cv::Mat in_mat = generateYoloOutput(num_classes);
auto anchors = cv::gapi::nn::parsers::GParseYolo::defaultAnchors();
std::vector<cv::Rect> boxes_gapi, boxes_ref;
std::vector<int> labels_gapi, labels_ref;
// G-API code //////////////////////////////////////////////////////////////
cv::GMat in;
cv::GOpaque<cv::Size> op_sz;
auto out = cv::gapi::parseYolo(in, op_sz, confidence_threshold, nms_threshold, anchors);
cv::GComputation c(cv::GIn(in, op_sz), cv::GOut(std::get<0>(out), std::get<1>(out)));
c.apply(cv::gin(in_mat, sz), cv::gout(boxes_gapi, labels_gapi), getCompileArgs());
// Reference code //////////////////////////////////////////////////////////
parseYoloRef(in_mat, sz, confidence_threshold, nms_threshold, num_classes, anchors, boxes_ref, labels_ref);
// Comparison //////////////////////////////////////////////////////////////
EXPECT_TRUE(boxes_gapi == boxes_ref);
EXPECT_TRUE(labels_gapi == labels_ref);
}
TEST_P(SizeTest, ParseTest)
{
cv::GMat in;
cv::Size out_sz;
auto out = cv::gapi::size(in);
cv::GComputation c(cv::GIn(in), cv::GOut(out));
c.apply(cv::gin(in_mat1), cv::gout(out_sz), getCompileArgs());
EXPECT_EQ(out_sz, sz);
}
TEST_P(SizeRTest, ParseTest)
{
cv::Rect rect(cv::Point(0,0), sz);
cv::Size out_sz;
cv::GOpaque<cv::Rect> op_rect;
auto out = cv::gapi::size(op_rect);
cv::GComputation c(cv::GIn(op_rect), cv::GOut(out));
c.apply(cv::gin(rect), cv::gout(out_sz), getCompileArgs());
EXPECT_EQ(out_sz, sz);
}
} // opencv_test
#endif //OPENCV_GAPI_CORE_TESTS_INL_HPP
@@ -0,0 +1,397 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#ifndef OPENCV_GAPI_PARSERS_TESTS_COMMON_HPP
#define OPENCV_GAPI_PARSERS_TESTS_COMMON_HPP
#include "gapi_tests_common.hpp"
#include "../../include/opencv2/gapi/infer/parsers.hpp"
namespace opencv_test
{
class ParserSSDTest
{
public:
cv::Mat generateSSDoutput(const cv::Size& in_sz)
{
constexpr int maxN = 200;
constexpr int objSize = 7;
std::vector<int> dims{ 1, 1, maxN, objSize };
cv::Mat mat(dims, CV_32FC1);
auto data = mat.ptr<float>();
for (int i = 0; i < maxN; ++i)
{
float* it = data + i * objSize;
auto ssdIt = generateItem(i, in_sz);
it[0] = ssdIt.image_id;
it[1] = ssdIt.label;
it[2] = ssdIt.confidence;
it[3] = ssdIt.rc_left;
it[4] = ssdIt.rc_top;
it[5] = ssdIt.rc_right;
it[6] = ssdIt.rc_bottom;
}
return mat;
}
void parseSSDref(const cv::Mat& in_ssd_result,
const cv::Size& in_size,
const float confidence_threshold,
const bool alignment_to_square,
const bool filter_out_of_bounds,
std::vector<cv::Rect>& out_boxes)
{
out_boxes.clear();
const auto &in_ssd_dims = in_ssd_result.size;
CV_Assert(in_ssd_dims.dims() == 4u);
const int MAX_PROPOSALS = in_ssd_dims[2];
const int OBJECT_SIZE = in_ssd_dims[3];
CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size
const float *data = in_ssd_result.ptr<float>();
cv::Rect surface({0,0}, in_size), rc;
float image_id, confidence;
int label;
for (int i = 0; i < MAX_PROPOSALS; ++i)
{
std::tie(rc, image_id, confidence, label)
= extract(data + i*OBJECT_SIZE, in_size);
if (image_id < 0.f)
{
break; // marks end-of-detections
}
if (confidence < confidence_threshold)
{
continue; // skip objects with low confidence
}
if (alignment_to_square)
{
adjustBoundingBox(rc);
}
const auto clipped_rc = rc & surface;
if (filter_out_of_bounds)
{
if (clipped_rc.area() != rc.area())
{
continue;
}
}
out_boxes.emplace_back(clipped_rc);
}
}
void parseSSDBLref(const cv::Mat& in_ssd_result,
const cv::Size& in_size,
const float confidence_threshold,
const int filter_label,
std::vector<cv::Rect>& out_boxes,
std::vector<int>& out_labels)
{
out_boxes.clear();
out_labels.clear();
const auto &in_ssd_dims = in_ssd_result.size;
CV_Assert(in_ssd_dims.dims() == 4u);
const int MAX_PROPOSALS = in_ssd_dims[2];
const int OBJECT_SIZE = in_ssd_dims[3];
CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size
cv::Rect surface({0,0}, in_size), rc;
float image_id, confidence;
int label;
const float *data = in_ssd_result.ptr<float>();
for (int i = 0; i < MAX_PROPOSALS; i++)
{
std::tie(rc, image_id, confidence, label)
= extract(data + i*OBJECT_SIZE, in_size);
if (image_id < 0.f)
{
break; // marks end-of-detections
}
if (confidence < confidence_threshold ||
(filter_label != -1 && label != filter_label))
{
continue; // filter out object classes if filter is specified
}
out_boxes.emplace_back(rc & surface);
out_labels.emplace_back(label);
}
}
private:
void adjustBoundingBox(cv::Rect& boundingBox)
{
auto w = boundingBox.width;
auto h = boundingBox.height;
boundingBox.x -= static_cast<int>(0.067 * w);
boundingBox.y -= static_cast<int>(0.028 * h);
boundingBox.width += static_cast<int>(0.15 * w);
boundingBox.height += static_cast<int>(0.13 * h);
if (boundingBox.width < boundingBox.height)
{
auto dx = (boundingBox.height - boundingBox.width);
boundingBox.x -= dx / 2;
boundingBox.width += dx;
}
else
{
auto dy = (boundingBox.width - boundingBox.height);
boundingBox.y -= dy / 2;
boundingBox.height += dy;
}
}
std::tuple<cv::Rect, float, float, int> extract(const float* it,
const cv::Size& in_size)
{
float image_id = it[0];
int label = static_cast<int>(it[1]);
float confidence = it[2];
float rc_left = it[3];
float rc_top = it[4];
float rc_right = it[5];
float rc_bottom = it[6];
cv::Rect rc; // map relative coordinates to the original image scale
rc.x = static_cast<int>(rc_left * in_size.width);
rc.y = static_cast<int>(rc_top * in_size.height);
rc.width = static_cast<int>(rc_right * in_size.width) - rc.x;
rc.height = static_cast<int>(rc_bottom * in_size.height) - rc.y;
return std::make_tuple(rc, image_id, confidence, label);
}
int randInRange(const int start, const int end)
{
GAPI_Assert(start <= end);
return start + std::rand() % (end - start + 1);
}
cv::Rect generateBox(const cv::Size& in_sz)
{
// Generated rectangle can reside outside of the initial image by border pixels
constexpr int border = 10;
constexpr int minW = 16;
constexpr int minH = 16;
cv::Rect box;
box.width = randInRange(minW, in_sz.width + 2*border);
box.height = randInRange(minH, in_sz.height + 2*border);
box.x = randInRange(-border, in_sz.width + border - box.width);
box.y = randInRange(-border, in_sz.height + border - box.height);
return box;
}
struct SSDitem
{
float image_id = 0.0f;
float label = 0.0f;
float confidence = 0.0f;
float rc_left = 0.0f;
float rc_top = 0.0f;
float rc_right = 0.0f;
float rc_bottom = 0.0f;
};
SSDitem generateItem(const int i, const cv::Size& in_sz)
{
const auto normalize = [](int v, int range) { return static_cast<float>(v) / range; };
SSDitem it;
it.image_id = static_cast<float>(i);
it.label = static_cast<float>(randInRange(0, 9));
it.confidence = static_cast<float>(std::rand()) / RAND_MAX;
auto box = generateBox(in_sz);
it.rc_left = normalize(box.x, in_sz.width);
it.rc_right = normalize(box.x + box.width, in_sz.width);
it.rc_top = normalize(box.y, in_sz.height);
it.rc_bottom = normalize(box.y + box.height, in_sz.height);
return it;
}
};
class ParserYoloTest
{
public:
cv::Mat generateYoloOutput(const int num_classes)
{
std::vector<int> dims = { 1, 13, 13, (num_classes + 5) * 5 };
cv::Mat mat(dims, CV_32FC1);
auto data = mat.ptr<float>();
const size_t range = dims[0] * dims[1] * dims[2] * dims[3];
for (size_t i = 0; i < range; ++i)
{
data[i] = static_cast<float>(std::rand()) / RAND_MAX;
}
return mat;
}
void parseYoloRef(const cv::Mat& in_yolo_result,
const cv::Size& in_size,
const float confidence_threshold,
const float nms_threshold,
const int num_classes,
const std::vector<float>& anchors,
std::vector<cv::Rect>& out_boxes,
std::vector<int>& out_labels)
{
YoloParams params;
constexpr auto side_square = 13 * 13;
this->m_out = in_yolo_result.ptr<float>();
this->m_side = 13;
this->m_lcoords = params.coords;
this->m_lclasses = num_classes;
std::vector<Detection> detections;
for (int i = 0; i < side_square; ++i)
{
for (int b = 0; b < params.num; ++b)
{
float scale = this->scale(i, b);
if (scale < confidence_threshold)
{
continue;
}
double x = this->x(i, b);
double y = this->y(i, b);
double height = this->height(i, b, anchors[2 * b + 1]);
double width = this->width(i, b, anchors[2 * b]);
for (int label = 0; label < num_classes; ++label)
{
float prob = scale * classConf(i,b,label);
if (prob < confidence_threshold)
{
continue;
}
auto box = toBox(x, y, height, width, in_size);
detections.emplace_back(Detection(box, prob, label));
}
}
}
std::stable_sort(std::begin(detections), std::end(detections),
[](const Detection& a, const Detection& b)
{
return a.conf > b.conf;
});
if (nms_threshold < 1.0f)
{
for (const auto& d : detections)
{
if (std::end(out_boxes) ==
std::find_if(std::begin(out_boxes), std::end(out_boxes),
[&d, nms_threshold](const cv::Rect& r)
{
float rectOverlap = 1.f - static_cast<float>(jaccardDistance(r, d.rect));
return rectOverlap > nms_threshold;
}))
{
out_boxes. emplace_back(d.rect);
out_labels.emplace_back(d.label);
}
}
}
else
{
for (const auto& d: detections)
{
out_boxes. emplace_back(d.rect);
out_labels.emplace_back(d.label);
}
}
}
private:
struct Detection
{
Detection(const cv::Rect& in_rect, const float in_conf, const int in_label)
: rect(in_rect), conf(in_conf), label(in_label)
{}
cv::Rect rect;
float conf = 0.0f;
int label = 0;
};
struct YoloParams
{
int num = 5;
int coords = 4;
};
float scale(const int i, const int b)
{
int obj_index = index(i, b, m_lcoords);
return m_out[obj_index];
}
double x(const int i, const int b)
{
int box_index = index(i, b, 0);
int col = i % m_side;
return (col + m_out[box_index]) / m_side;
}
double y(const int i, const int b)
{
int box_index = index(i, b, 0);
int row = i / m_side;
return (row + m_out[box_index + m_side * m_side]) / m_side;
}
double width(const int i, const int b, const float anchor)
{
int box_index = index(i, b, 0);
return std::exp(m_out[box_index + 2 * m_side * m_side]) * anchor / m_side;
}
double height(const int i, const int b, const float anchor)
{
int box_index = index(i, b, 0);
return std::exp(m_out[box_index + 3 * m_side * m_side]) * anchor / m_side;
}
float classConf(const int i, const int b, const int label)
{
int class_index = index(i, b, m_lcoords + 1 + label);
return m_out[class_index];
}
cv::Rect toBox(const double x, const double y, const double h, const double w, const cv::Size& in_sz)
{
auto h_scale = in_sz.height;
auto w_scale = in_sz.width;
cv::Rect r;
r.x = static_cast<int>((x - w / 2) * w_scale);
r.y = static_cast<int>((y - h / 2) * h_scale);
r.width = static_cast<int>(w * w_scale);
r.height = static_cast<int>(h * h_scale);
return r;
}
int index(const int i, const int b, const int entry)
{
return b * m_side * m_side * (m_lcoords + m_lclasses + 1) + entry * m_side * m_side + i;
}
const float* m_out = nullptr;
int m_side = 0, m_lcoords = 0, m_lclasses = 0;
};
} // namespace opencv_test
#endif // OPENCV_GAPI_PARSERS_TESTS_COMMON_HPP
@@ -351,6 +351,27 @@ struct TestWithParamsSpecific : public TestWithParamsBase<ParamsSpecific<Specifi
Fixture() { InitF(type, sz, dtype); } \
};
/**
* @private
* @brief Create G-API test fixture with TestWithParams base class and additional base class.
* @param Fixture test fixture name.
@param ExtBase additional base class.
* @param InitF callable that will initialize default available members (from TestFunctional)
* @param API base class API. Specifies types of user-defined parameters. If there are no such
* parameters, empty angle brackets ("<>") must be specified.
* @param Number number of user-defined parameters (corresponds to the number of types in API).
* if there are no such parameters, 0 must be specified.
* @param ... list of names of user-defined parameters. if there are no parameters, the list
* must be empty.
*/
#define GAPI_TEST_EXT_BASE_FIXTURE(Fixture, ExtBase, InitF, API, Number, ...) \
struct Fixture : public TestWithParams API, public ExtBase { \
static_assert(Number == AllParams::specific_params_size, \
"Number of user-defined parameters doesn't match size of __VA_ARGS__"); \
__WRAP_VAARGS(DEFINE_SPECIFIC_PARAMS_##Number(__VA_ARGS__)) \
Fixture() { InitF(type, sz, dtype); } \
};
/**
* @private
* @brief Create G-API test fixture with TestWithParamsSpecific base class
@@ -496,4 +496,43 @@ INSTANTIATE_TEST_CASE_P(ReInitOutTestCPU, ReInitOutTest,
Values(cv::Size(640, 400),
cv::Size(10, 480))));
INSTANTIATE_TEST_CASE_P(ParseTestCPU, ParseSSDBLTest,
Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1),
Values(cv::Size(1920, 1080)),
Values(-1),
Values(CORE_CPU),
Values(0.3f, 0.5f, 0.7f),
Values(-1, 0, 1)));
INSTANTIATE_TEST_CASE_P(ParseTestCPU, ParseSSDTest,
Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1),
Values(cv::Size(1920, 1080)),
Values(-1),
Values(CORE_CPU),
Values(0.3f, 0.5f, 0.7f),
testing::Bool(),
testing::Bool()));
INSTANTIATE_TEST_CASE_P(ParseTestCPU, ParseYoloTest,
Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1),
Values(cv::Size(1920, 1080)),
Values(-1),
Values(CORE_CPU),
Values(0.3f, 0.5f, 0.7f),
Values(0.5f, 1.0f),
Values(80, 7)));
INSTANTIATE_TEST_CASE_P(SizeTestCPU, SizeTest,
Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1),
Values(cv::Size(32, 32),
cv::Size(640, 320)),
Values(-1),
Values(CORE_CPU)));
INSTANTIATE_TEST_CASE_P(SizeRTestCPU, SizeRTest,
Combine(Values(CV_8UC1, CV_8UC3, CV_32FC1),
Values(cv::Size(32, 32),
cv::Size(640, 320)),
Values(-1),
Values(CORE_CPU)));
}