third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #21658 from sivanov-work:vpp_core_add_roi

Browse Source 
G-API: Add ROI processing in VPP preproc

* Add ROI in VPP prepro

* Apply comments
This commit is contained in:
Sergey Ivanov 2022-03-05 13:50:40 +03:00 committed by GitHub
parent 8fa429caef
commit 44c2c77548
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 206 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp
View File

@ -205,6 +205,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)
}
VPPPreprocEngine preproc_engine(std::move(policy));
cv::gapi::wip::Data out;
cv::util::optional<cv::Rect> empty_roi;
TEST_CYCLE()
{
source_ptr->pull(out);
@ -212,7 +213,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)
cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
pp_session sess = preproc_engine.initialize_preproc(param.value(),
required_frame_param);
(void)preproc_engine.run_sync(sess, frame);
(void)preproc_engine.run_sync(sess, frame, empty_roi);
}
SANITY_CHECK_NOTHING();
@ -269,6 +270,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)
}
VPPPreprocEngine preproc_engine(std::move(policy));
cv::gapi::wip::Data out;
cv::util::optional<cv::Rect> empty_roi;
TEST_CYCLE()
{
source_ptr->pull(out);
@ -276,7 +278,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)
cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
pp_session sess = preproc_engine.initialize_preproc(param.value(),
required_frame_param);
(void)preproc_engine.run_sync(sess, frame);
(void)preproc_engine.run_sync(sess, frame, empty_roi);
}
SANITY_CHECK_NOTHING();

48
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
View File

@ -34,6 +34,21 @@ bool FrameInfoComparator::equal_to(const mfxFrameInfo& lhs, const mfxFrameInfo&
return lhs == rhs;
}
void apply_roi(mfxFrameSurface1* surface_handle,
const cv::util::optional<cv::Rect> &opt_roi) {
if (opt_roi.has_value()) {
const cv::Rect &roi = opt_roi.value();
surface_handle->Info.CropX = static_cast<mfxU16>(roi.x);
surface_handle->Info.CropY = static_cast<mfxU16>(roi.y);
surface_handle->Info.CropW = static_cast<mfxU16>(roi.width);
surface_handle->Info.CropH = static_cast<mfxU16>(roi.height);
GAPI_LOG_DEBUG(nullptr, "applied ROI {" << surface_handle->Info.CropX <<
", " << surface_handle->Info.CropY << "}, "
"{ " << surface_handle->Info.CropX + surface_handle->Info.CropW <<
", " << surface_handle->Info.CropY + surface_handle->Info.CropH << "}");
}
}
VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& accel) :
ProcessingEngineBase(std::move(accel)) {
GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine");
@ -57,31 +72,25 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
my_sess.sync_in_queue.pop();
auto *vpp_suface = my_sess.processing_surface_ptr.lock()->get_handle();
/* TODO: consider CROP/ROI here
static int x_offset = 0;
static int y_offset = 0;
dec_surface->Info.CropX = x_offset;
dec_surface->Info.CropY = y_offset;
dec_surface->Info.CropW = 100 + x_offset++;
dec_surface->Info.CropH = 100 + y_offset++;
*/
session_type::outgoing_task vpp_pending_op {pending_op.sync_handle, nullptr};
apply_roi(pending_op.decoded_surface_ptr, pending_op.roi);
mfxSyncPoint vpp_sync_handle{};
my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session,
pending_op.decoded_surface_ptr,
vpp_suface,
nullptr, &vpp_pending_op.sync_handle);
vpp_pending_op.vpp_surface_ptr = vpp_suface;
nullptr, &vpp_sync_handle);
session_type::outgoing_task vpp_pending_op {vpp_sync_handle,
vpp_suface,
std::move(pending_op) };
GAPI_LOG_DEBUG(nullptr, "Got VPP async operation" <<
", sync id: " <<
vpp_pending_op.sync_handle <<
", dec surface: " <<
pending_op.decoded_surface_ptr <<
vpp_pending_op.original_surface_ptr <<
", trans surface: " <<
vpp_pending_op.vpp_surface_ptr <<
", status: " <<
mfxstatus_to_string(my_sess.last_status));
// NB: process status
if (my_sess.last_status == MFX_ERR_MORE_SURFACE ||
my_sess.last_status == MFX_ERR_NONE) {
@ -131,6 +140,7 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
// put frames in ready queue on success
if (MFX_ERR_NONE == sess.last_status) {
pending_op.release_frame();
on_frame_ready(my_sess, pending_op.vpp_surface_ptr);
}
}
@ -327,8 +337,8 @@ VPPPreprocEngine::initialize_session(mfxSession,
return {};
}
cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame) {
cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &roi) {
std::shared_ptr<EngineSession> pp_sess_impl = sess.get<EngineSession>();
if (!pp_sess_impl) {
// bypass case
@ -347,8 +357,10 @@ cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::Medi
// schedule decoded surface into preproc queue
session_type::incoming_task in_preproc_request {nullptr,
vpl_adapter->get_surface()->get_handle(),
in_frame};
vpl_adapter->get_surface()->get_handle(),
vpl_adapter->get_surface()->get_info(),
in_frame,
roi};
s->sync_in_queue.emplace(in_preproc_request);
// invoke pipeline to transform decoded surface into preprocessed surface

3
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp
View File

@ -47,7 +47,8 @@ public:
const GFrameDesc& required_frame_descr) override;
cv::MediaFrame run_sync(const pp_session &session_handle,
const cv::MediaFrame& in_frame) override;
const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &opt_roi) override;
private:
std::map<mfxFrameInfo, session_ptr_type, FrameInfoComparator> preproc_session_map;

18
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp
View File

@ -60,6 +60,24 @@ void VPPPreprocSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key)
const mfxFrameInfo& VPPPreprocSession::get_video_param() const {
return mfx_vpp_out_param.vpp.Out;
}
VPPPreprocSession::outgoing_task::outgoing_task(mfxSyncPoint acquired_sync_handle,
mfxFrameSurface1* acquired_surface_ptr,
VPPPreprocSession::incoming_task &&in) :
sync_handle(acquired_sync_handle),
vpp_surface_ptr(acquired_surface_ptr),
original_surface_ptr(in.decoded_surface_ptr),
original_frame_info(std::move(in.decoded_frame_info)),
original_frame(in.decoded_frame_copy) {
}
void VPPPreprocSession::outgoing_task::release_frame() {
// restore initial surface params
memcpy(&(original_surface_ptr->Info),
&original_frame_info, sizeof(Surface::info_t));
// release references on frame adapter
original_frame = cv::MediaFrame();
}
} // namespace onevpl
} // namespace wip
} // namespace gapi

12
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp
View File

@ -41,12 +41,24 @@ private:
struct incoming_task {
mfxSyncPoint sync_handle;
mfxFrameSurface1* decoded_surface_ptr;
Surface::info_t decoded_frame_info;
cv::MediaFrame decoded_frame_copy;
cv::util::optional<cv::Rect> roi;
};
struct outgoing_task {
outgoing_task() = default;
outgoing_task(mfxSyncPoint acquired_sync_handle,
mfxFrameSurface1* acquired_surface_ptr,
incoming_task &&in);
mfxSyncPoint sync_handle;
mfxFrameSurface1* vpp_surface_ptr;
mfxFrameSurface1* original_surface_ptr;
void release_frame();
private:
Surface::info_t original_frame_info;
cv::MediaFrame original_frame;
};
std::queue<incoming_task> sync_in_queue;

3
modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp
View File

@ -27,7 +27,8 @@ struct IPreprocEngine {
initialize_preproc(const pp_params& initial_frame_param,
const GFrameDesc& required_frame_descr) = 0;
virtual cv::MediaFrame
run_sync(const pp_session &sess, const cv::MediaFrame& in_frame) = 0;
run_sync(const pp_session &sess, const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &opt_roi = {}) = 0;
};
} // namespace wip
} // namespace gapi

217
modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp
View File

@ -181,6 +181,8 @@ using acceleration_t = int;
using out_frame_info_t = cv::GFrameDesc;
using preproc_args_t = std::tuple<source_t, decoder_t, acceleration_t, out_frame_info_t>;
static cv::util::optional<cv::Rect> empty_roi;
class VPPPreprocParams : public ::testing::TestWithParam<preproc_args_t> {};
preproc_args_t files[] = {
@ -246,7 +248,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
required_frame_param);
// 2) make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame);
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess,
first_decoded_frame,
empty_roi);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
ASSERT_FALSE(first_frame_decoded_desc == first_outcome_pp_desc);
@ -278,7 +282,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame);
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess,
decoded_frame,
empty_roi);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
@ -291,8 +297,95 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
ASSERT_NE(frames_processed_count, 1);
}
void decode_function(cv::gapi::wip::onevpl::VPLLegacyDecodeEngine &decode_engine,
cv::gapi::wip::onevpl::ProcessingEngineBase::session_ptr sess_ptr,
SafeQueue &queue, size_t &decoded_number) {
// decode first frame
{
cv::MediaFrame decoded_frame;
ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine));
queue.push(std::move(decoded_frame));
}
TEST_P(VPPPreprocParams, functional_different_threads)
// launch pipeline
try {
while(true) {
queue.push(extract_decoded_frame(sess_ptr->session, decode_engine));
decoded_number++;
}
} catch (...) {}
// send stop
queue.push_stop();
}
void preproc_function(cv::gapi::wip::onevpl::VPPPreprocEngine &preproc_engine, SafeQueue&queue,
size_t &preproc_number, const out_frame_info_t &required_frame_param,
const cv::util::optional<cv::Rect> &roi_rect = {}) {
using namespace cv::gapi::wip;
using namespace cv::gapi::wip::onevpl;
// create preproc session based on frame description & network info
cv::MediaFrame first_decoded_frame = queue.pop();
cv::util::optional<pp_params> first_pp_params = preproc_engine.is_applicable(first_decoded_frame);
ASSERT_TRUE(first_pp_params.has_value());
pp_session first_pp_sess =
preproc_engine.initialize_preproc(first_pp_params.value(),
required_frame_param);
// make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess,
first_decoded_frame,
roi_rect);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
// do not hold media frames because they share limited DX11 surface pool resources
first_decoded_frame = cv::MediaFrame();
first_pp_frame = cv::MediaFrame();
// launch pipeline
bool in_progress = false;
// let's allow counting of preprocessed frames to check this value later:
// Currently, it looks redundant to implement any kind of gracefull shutdown logic
// in this test - so let's apply agreement that media source is processed
// succesfully when preproc_number != 1 in result.
// Specific validation logic which adhere to explicit counter value may be implemented
// in particular test scope
preproc_number = 1;
try {
while(true) {
cv::MediaFrame decoded_frame = queue.pop();
if (SafeQueue::is_stop(decoded_frame)) {
break;
}
in_progress = true;
cv::util::optional<pp_params> params = preproc_engine.is_applicable(decoded_frame);
ASSERT_TRUE(params.has_value());
ASSERT_TRUE(0 == memcmp(&params.value(), &first_pp_params.value(), sizeof(pp_params)));
pp_session pp_sess = preproc_engine.initialize_preproc(params.value(),
required_frame_param);
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame, empty_roi);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
preproc_number++;
}
} catch (...) {}
// test if interruption has happened
ASSERT_FALSE(in_progress);
ASSERT_NE(preproc_number, 1);
}
using roi_t = cv::util::optional<cv::Rect>;
using preproc_roi_args_t = decltype(std::tuple_cat(std::declval<preproc_args_t>(),
std::declval<std::tuple<roi_t>>()));
class VPPPreprocROIParams : public ::testing::TestWithParam<preproc_roi_args_t> {};
TEST_P(VPPPreprocROIParams, functional_roi_different_threads)
{
using namespace cv::gapi::wip;
using namespace cv::gapi::wip::onevpl;
@ -300,7 +393,8 @@ TEST_P(VPPPreprocParams, functional_different_threads)
decoder_t decoder_id;
acceleration_t accel;
out_frame_info_t required_frame_param;
std::tie(file_path, decoder_id, accel, required_frame_param) = GetParam();
roi_t opt_roi;
std::tie(file_path, decoder_id, accel, required_frame_param, opt_roi) = GetParam();
file_path = findDataFile(file_path);
@ -338,87 +432,52 @@ TEST_P(VPPPreprocParams, functional_different_threads)
size_t decoded_number = 1;
size_t preproc_number = 0;
std::thread decode_thread([&decode_engine, sess_ptr,
&queue, &decoded_number] () {
// decode first frame
{
cv::MediaFrame decoded_frame;
ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine));
queue.push(std::move(decoded_frame));
}
// launch pipeline
try {
while(true) {
queue.push(extract_decoded_frame(sess_ptr->session, decode_engine));
decoded_number++;
}
} catch (...) {}
// send stop
queue.push_stop();
});
std::thread preproc_thread([&preproc_engine, &queue, &preproc_number, required_frame_param] () {
// create preproc session based on frame description & network info
cv::MediaFrame first_decoded_frame = queue.pop();
cv::util::optional<pp_params> first_pp_params = preproc_engine.is_applicable(first_decoded_frame);
ASSERT_TRUE(first_pp_params.has_value());
pp_session first_pp_sess =
preproc_engine.initialize_preproc(first_pp_params.value(), required_frame_param);
// make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
// do not hold media frames because they share limited DX11 surface pool resources
first_decoded_frame = cv::MediaFrame();
first_pp_frame = cv::MediaFrame();
// launch pipeline
bool in_progress = false;
// let's allow counting of preprocessed frames to check this value later:
// Currently, it looks redundant to implement any kind of gracefull shutdown logic
// in this test - so let's apply agreement that media source is processed
// succesfully when preproc_number != 1 in result
preproc_number = 1;
try {
while(true) {
cv::MediaFrame decoded_frame = queue.pop();
if (SafeQueue::is_stop(decoded_frame)) {
break;
}
in_progress = true;
cv::util::optional<pp_params> params = preproc_engine.is_applicable(decoded_frame);
ASSERT_TRUE(params.has_value());
ASSERT_TRUE(0 == memcmp(&params.value(), &first_pp_params.value(), sizeof(pp_params)));
pp_session pp_sess = preproc_engine.initialize_preproc(params.value(),
required_frame_param);
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
preproc_number++;
}
} catch (...) {}
// test if interruption has happened
ASSERT_FALSE(in_progress);
ASSERT_NE(preproc_number, 1);
});
std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr,
std::ref(queue), std::ref(decoded_number));
std::thread preproc_thread(preproc_function, std::ref(preproc_engine),
std::ref(queue), std::ref(preproc_number),
std::cref(required_frame_param),
std::cref(opt_roi));
decode_thread.join();
preproc_thread.join();
ASSERT_EQ(preproc_number, decoded_number);
}
INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngine, VPPPreprocParams,
testing::ValuesIn(files));
preproc_roi_args_t files_w_roi[] = {
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{0,0,50,50}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{0,0,100,100}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{100,100,200,200}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{cv::Rect{0,0,100,100}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{cv::Rect{100,100,200,200}}}
};
INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams,
testing::ValuesIn(files_w_roi));
using VPPInnerPreprocParams = VPPPreprocParams;
TEST_P(VPPInnerPreprocParams, functional_inner_preproc_size)