third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #18240 from mpashchenkov:mp/ocv-gapi-input-cnn-reshape

Browse Source 
[G-API]: Adding reshape for CNN input.

* Added CNN input IE reshape

* rbs

* Added unordered_set instead vector

* Alignment
This commit is contained in:
Maxim Pashchenkov 2021-03-10 19:06:46 +03:00 committed by GitHub
parent ddd2447192
commit 12fa8d8444
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 398 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
modules/gapi/include/opencv2/gapi/infer/ie.hpp
View File

@ -8,6 +8,7 @@
#define OPENCV_GAPI_INFER_IE_HPP
#include <unordered_map>
#include <unordered_set>
#include <string>
#include <array>
#include <tuple> // tuple, tuple_size
@ -68,6 +69,9 @@ namespace detail {
bool is_generic;
IEConfig config;
std::map<std::string, std::vector<std::size_t>> reshape_table;
std::unordered_set<std::string> layer_names_to_reshape;
// NB: Number of asyncrhonious infer requests
size_t nireq;
};
@ -95,6 +99,8 @@ public:
, detail::ParamDesc::Kind::Load
, false
, {}
, {}
, {}
, 1u} {
};
@ -106,6 +112,8 @@ public:
, detail::ParamDesc::Kind::Import
, false
, {}
, {}
, {}
, 1u} {
};
@ -148,6 +156,36 @@ public:
return *this;
}
Params<Net>& cfgInputReshape(std::map<std::string, std::vector<std::size_t>>&& reshape_table) {
desc.reshape_table = std::move(reshape_table);
return *this;
}
Params<Net>& cfgInputReshape(const std::map<std::string, std::vector<std::size_t>>& reshape_table) {
desc.reshape_table = reshape_table;
return *this;
}
Params<Net>& cfgInputReshape(std::string&& layer_name, std::vector<size_t>&& layer_dims) {
desc.reshape_table.emplace(layer_name, layer_dims);
return *this;
}
Params<Net>& cfgInputReshape(const std::string& layer_name, const std::vector<size_t>& layer_dims) {
desc.reshape_table.emplace(layer_name, layer_dims);
return *this;
}
Params<Net>& cfgInputReshape(std::unordered_set<std::string>&& layer_names) {
desc.layer_names_to_reshape = std::move(layer_names);
return *this;
}
Params<Net>& cfgInputReshape(const std::unordered_set<std::string>& layer_names) {
desc.layer_names_to_reshape = layer_names;
return *this;
}
// BEGIN(G-API's network parametrization API)
GBackend backend() const { return cv::gapi::ie::backend(); }
std::string tag() const { return Net::tag(); }
@ -165,13 +203,13 @@ public:
const std::string &model,
const std::string &weights,
const std::string &device)
: desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, 1u}, m_tag(tag) {
: desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) {
};
Params(const std::string &tag,
const std::string &model,
const std::string &device)
: desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, 1u}, m_tag(tag) {
: desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) {
};
Params& pluginConfig(IEConfig&& cfg) {

106
modules/gapi/src/backends/ie/giebackend.cpp
View File

@ -45,6 +45,7 @@
#include "backends/ie/giebackend/giewrapper.hpp"
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
#include "logger.hpp"
#if INF_ENGINE_RELEASE < 2021010000
#include "ie_compound_blob.h"
@ -224,6 +225,9 @@ struct IEUnit {
// but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap
inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo());
outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo());
if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) {
GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network");
}
} else {
cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind"));
}
@ -249,6 +253,11 @@ struct IEUnit {
if (params.num_out == 1u && params.output_names.empty()) {
params.output_names = { outputs.begin()->first };
}
if (!params.reshape_table.empty()) {
GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <=
params.num_in &&
"Number of layers to reshape must be less than or equal to number of inputs");
}
}
// This method is [supposed to be] called at Island compilation stage
@ -669,6 +678,46 @@ void cv::gimpl::ie::GIEExecutable::run(cv::gimpl::GIslandExecutable::IInput &in
namespace cv {
namespace gimpl {
namespace ie {
static void configureInputReshapeByImage(const IE::InputInfo::Ptr& ii,
const cv::GMetaArg mm,
IE::ICNNNetwork::InputShapes& input_reshape_table) {
const auto& layer_name = ii->name();
// Finding name in reshape table
const auto name_pos_in_table = input_reshape_table.find(layer_name);
// If contains then reshape for this layer already configured by shapes
// otherwise create a new element of reshape table with name and dimension
// which based on input image size.
if (name_pos_in_table != input_reshape_table.end()) {
GAPI_Assert(false &&
"Names of layers for reshape with specified dimensions shouldn't intersect with names for reshape by image");
}
cv::Size image_sz;
switch (mm.index()) {
case cv::GMetaArg::index_of<cv::GMatDesc>():
{
const auto &meta = util::get<cv::GMatDesc>(mm);
image_sz = meta.size;
break;
}
case cv::GMetaArg::index_of<cv::GFrameDesc>():
{
const auto &meta = util::get<cv::GFrameDesc>(mm);
image_sz = meta.size;
break;
}
default:
util::throw_error(std::runtime_error("Unsupported input meta for IE backend"));
}
auto input_dims = ii->getTensorDesc().getDims();
const auto size = input_dims.size();
if (size <= 1) {
GAPI_Assert(false && "Unsupported number of dimensions for reshape by image");
}
input_dims.at(size - 2) = static_cast<size_t>(image_sz.height);
input_dims.at(size - 1) = static_cast<size_t>(image_sz.width);
// Adding new element to reshape table
input_reshape_table.emplace(layer_name, input_dims);
}
static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) {
switch (mm.index()) {
@ -732,22 +781,34 @@ struct Infer: public cv::detail::KernelTag {
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
// meta order.
GAPI_Assert(uu.params.input_names.size() == in_metas.size()
&& "Known input layers count doesn't match input meta count");
for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names),
ade::util::toRange(in_metas))) {
auto &&ii = uu.inputs.at(std::get<0>(it));
const auto & mm = std::get<1>(it);
const auto &input_name = std::get<0>(it);
auto &&ii = uu.inputs.at(input_name);
const auto & mm = std::get<1>(it);
configureInputInfo(ii, mm);
if (uu.params.layer_names_to_reshape.find(input_name) !=
uu.params.layer_names_to_reshape.end()) {
configureInputReshapeByImage(ii, mm, input_reshape_table);
}
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
// FIXME: This isn't the best place to call reshape function.
// Сorrect solution would be to do this in compile() method of network,
// but now input meta isn't passed to compile() method.
if (!input_reshape_table.empty()) {
const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
}
// FIXME: It would be nice here to have an exact number of network's
// input/output parameters. Probably GCall should store it here for us.
// It doesn't, as far as I know..
@ -802,6 +863,7 @@ struct InferROI: public cv::detail::KernelTag {
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// FIXME: So far it is pretty limited
@ -809,11 +871,23 @@ struct InferROI: public cv::detail::KernelTag {
GAPI_Assert(2u == in_metas.size());
// 0th is ROI, 1st is input image
auto &&ii = uu.inputs.at(uu.params.input_names.at(0));
const auto &input_name = uu.params.input_names.at(0);
auto &&ii = uu.inputs.at(input_name);
auto &&mm = in_metas.at(1u);
configureInputInfo(ii, mm);
if (uu.params.layer_names_to_reshape.find(input_name) !=
uu.params.layer_names_to_reshape.end()) {
configureInputReshapeByImage(ii, mm, input_reshape_table);
}
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
// FIXME: This isn't the best place to call reshape function.
// Сorrect solution would be to do this in compile() method of network,
// but now input meta isn't passed to compile() method.
if (!input_reshape_table.empty()) {
const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
}
// FIXME: It would be nice here to have an exact number of network's
// input/output parameters. Probably GCall should store it here for us.
// It doesn't, as far as I know..
@ -870,6 +944,7 @@ struct InferList: public cv::detail::KernelTag {
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
@ -882,9 +957,20 @@ struct InferList: public cv::detail::KernelTag {
auto &&ii = uu.inputs.at(input_name);
const auto & mm = in_metas[idx++];
configureInputInfo(ii, mm);
if (uu.params.layer_names_to_reshape.find(input_name) !=
uu.params.layer_names_to_reshape.end()) {
configureInputReshapeByImage(ii, mm, input_reshape_table);
}
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
// FIXME: This isn't the best place to call reshape function.
// Сorrect solution would be to do this in compile() method of network,
// but now input meta isn't passed to compile() method.
if (!input_reshape_table.empty()) {
const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
}
// roi-list version is much easier at the moment.
// All our outputs are vectors which don't have
// metadata at the moment - so just create a vector of
@ -973,6 +1059,7 @@ struct InferList2: public cv::detail::KernelTag {
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
@ -1023,7 +1110,18 @@ struct InferList2: public cv::detail::KernelTag {
if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) {
// This is a cv::Rect -- configure the IE preprocessing
configureInputInfo(ii, mm_0);
if (uu.params.layer_names_to_reshape.find(input_name) !=
uu.params.layer_names_to_reshape.end()) {
configureInputReshapeByImage(ii, mm_0, input_reshape_table);
}
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
// FIXME: This isn't the best place to call reshape function.
// Сorrect solution would be to do this in compile() method of network,
// but now input meta isn't passed to compile() method.
if (!input_reshape_table.empty()) {
const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
}
} else {
// This is a cv::GMat (equals to: cv::Mat)
// Just validate that it is really the type

256
modules/gapi/test/infer/gapi_infer_ie_test.cpp
View File

@ -233,6 +233,115 @@ TEST(TestAgeGenderIE, InferBasicImage)
normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
}
struct InferWithReshape: public ::testing::Test {
cv::gapi::ie::detail::ParamDesc params;
cv::Mat m_in_mat;
std::vector<cv::Rect> m_roi_list;
std::vector<size_t> reshape_dims;
std::vector<cv::Mat> m_out_ie_ages;
std::vector<cv::Mat> m_out_ie_genders;
std::vector<cv::Mat> m_out_gapi_ages;
std::vector<cv::Mat> m_out_gapi_genders;
using AGInfo = std::tuple<cv::GMat, cv::GMat>;
G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
InferenceEngine::CNNNetwork net;
InferenceEngine::Core plugin;
InferWithReshape() {
// FIXME: it must be cv::imread(findDataFile("../dnn/grace_hopper_227.png", false));
m_in_mat = cv::Mat(cv::Size(320, 240), CV_8UC3);
cv::randu(m_in_mat, 0, 255);
m_out_gapi_ages.resize(1);
m_out_gapi_genders.resize(1);
// both ROIs point to the same face, with a slightly changed geometry
m_roi_list = {
cv::Rect(cv::Point{64, 60}, cv::Size{ 96, 96}),
cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}),
};
// New dimensions for "data" input
reshape_dims = {1, 3, 70, 70};
initDLDTDataPath();
params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
params.device_id = "CPU";
plugin = cv::gimpl::ie::wrap::getPlugin(params);
net = cv::gimpl::ie::wrap::readNetwork(params);
setNetParameters(net);
net.reshape({{"data", reshape_dims}});
}
void inferROIs(IE::Blob::Ptr blob) {
auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
auto infer_request = this_network.CreateInferRequest();
for (auto &&rc : m_roi_list) {
const auto ie_rc = IE::ROI {
0u
, static_cast<std::size_t>(rc.x)
, static_cast<std::size_t>(rc.y)
, static_cast<std::size_t>(rc.width)
, static_cast<std::size_t>(rc.height)
};
infer_request.SetBlob("data", IE::make_shared_blob(blob, ie_rc));
infer_request.Infer();
using namespace cv::gapi::ie::util;
m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone());
m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone());
}
}
void infer(cv::Mat& in, const bool with_roi = false) {
if (!with_roi) {
auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
auto infer_request = this_network.CreateInferRequest();
infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in));
infer_request.Infer();
using namespace cv::gapi::ie::util;
m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone());
m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone());
} else {
auto frame_blob = cv::gapi::ie::util::to_ie(in);
inferROIs(frame_blob);
}
}
void validate() {
// Validate with IE itself (avoid DNN module dependency here)
GAPI_Assert(!m_out_gapi_ages.empty());
ASSERT_EQ(m_out_gapi_genders.size(), m_out_gapi_ages.size());
ASSERT_EQ(m_out_gapi_ages.size(), m_out_ie_ages.size());
ASSERT_EQ(m_out_gapi_genders.size(), m_out_ie_genders.size());
const size_t size = m_out_gapi_ages.size();
for (size_t i = 0; i < size; ++i) {
normAssert(m_out_ie_ages [i], m_out_gapi_ages [i], "Test age output");
normAssert(m_out_ie_genders[i], m_out_gapi_genders[i], "Test gender output");
}
}
}; // InferWithReshape
struct InferWithReshapeNV12: public InferWithReshape {
cv::Mat m_in_uv;
cv::Mat m_in_y;
void SetUp() {
cv::Size sz{320, 240};
m_in_y = cv::Mat{sz, CV_8UC1};
cv::randu(m_in_y, 0, 255);
m_in_uv = cv::Mat{sz / 2, CV_8UC2};
cv::randu(m_in_uv, 0, 255);
setNetParameters(net, true);
net.reshape({{"data", reshape_dims}});
auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv);
inferROIs(frame_blob);
}
};
struct ROIList: public ::testing::Test {
cv::gapi::ie::detail::ParamDesc params;
@ -1403,6 +1512,153 @@ TEST(Infer2EmptyList, TestStreamingInfer)
}
}
TEST_F(InferWithReshape, TestInfer)
{
// IE code
infer(m_in_mat);
// G-API code
cv::GMat in;
cv::GMat age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshape, TestInferInImage)
{
// Input image already has 70x70 size
cv::Mat rsz;
cv::resize(m_in_mat, rsz, cv::Size(70, 70));
// IE code
infer(rsz);
// G-API code
cv::GMat in;
cv::GMat age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({"data"});
// Reshape CNN input by input image size
comp.apply(cv::gin(rsz), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshape, TestInferForSingleLayer)
{
// IE code
infer(m_in_mat);
// G-API code
cv::GMat in;
cv::GMat age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" })
.cfgInputReshape("data", reshape_dims);
comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshape, TestInferList)
{
// IE code
infer(m_in_mat, true);
// G-API code
cv::GArray<cv::Rect> rr;
cv::GMat in;
cv::GArray<cv::GMat> age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
comp.apply(cv::gin(m_in_mat, m_roi_list),
cv::gout(m_out_gapi_ages, m_out_gapi_genders),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshape, TestInferList2)
{
// IE code
infer(m_in_mat, true);
// G-API code
cv::GArray<cv::Rect> rr;
cv::GMat in;
cv::GArray<cv::GMat> age, gender;
std::tie(age, gender) = cv::gapi::infer2<AgeGender>(in, rr);
cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
comp.apply(cv::gin(m_in_mat, m_roi_list),
cv::gout(m_out_gapi_ages, m_out_gapi_genders),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshape, TestInferListBGR)
{
// IE code
infer(m_in_mat, true);
// G-API code
cv::GArray<cv::Rect> rr;
cv::GFrame in;
cv::GArray<cv::GMat> age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
auto frame = MediaFrame::Create<TestMediaBGR>(m_in_mat);
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
comp.apply(cv::gin(frame, m_roi_list),
cv::gout(m_out_gapi_ages, m_out_gapi_genders),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
TEST_F(InferWithReshapeNV12, TestInferListYUV)
{
// G-API code
cv::GFrame in;
cv::GArray<cv::Rect> rr;
cv::GArray<cv::GMat> age, gender;
std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
auto frame = MediaFrame::Create<TestMediaNV12>(m_in_y, m_in_uv);
auto pp = cv::gapi::ie::Params<AgeGender> {
params.model_path, params.weights_path, params.device_id
}.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
comp.apply(cv::gin(frame, m_roi_list),
cv::gout(m_out_gapi_ages, m_out_gapi_genders),
cv::compile_args(cv::gapi::networks(pp)));
// Validate
validate();
}
} // namespace opencv_test
#endif // HAVE_INF_ENGINE