diff --git a/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp b/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp index 513d4d1f56..eacdef6eba 100644 --- a/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp +++ b/modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp @@ -205,6 +205,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance) } VPPPreprocEngine preproc_engine(std::move(policy)); cv::gapi::wip::Data out; + cv::util::optional empty_roi; TEST_CYCLE() { source_ptr->pull(out); @@ -212,7 +213,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance) cv::util::optional param = preproc_engine.is_applicable(frame); pp_session sess = preproc_engine.initialize_preproc(param.value(), required_frame_param); - (void)preproc_engine.run_sync(sess, frame); + (void)preproc_engine.run_sync(sess, frame, empty_roi); } SANITY_CHECK_NOTHING(); @@ -269,6 +270,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance) } VPPPreprocEngine preproc_engine(std::move(policy)); cv::gapi::wip::Data out; + cv::util::optional empty_roi; TEST_CYCLE() { source_ptr->pull(out); @@ -276,7 +278,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance) cv::util::optional param = preproc_engine.is_applicable(frame); pp_session sess = preproc_engine.initialize_preproc(param.value(), required_frame_param); - (void)preproc_engine.run_sync(sess, frame); + (void)preproc_engine.run_sync(sess, frame, empty_roi); } SANITY_CHECK_NOTHING(); diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp index 1fb9bd4195..7de363fad5 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp @@ -34,6 +34,21 @@ bool FrameInfoComparator::equal_to(const mfxFrameInfo& lhs, const mfxFrameInfo& return lhs == rhs; } +void apply_roi(mfxFrameSurface1* surface_handle, + const cv::util::optional &opt_roi) { + if (opt_roi.has_value()) { + const cv::Rect &roi = opt_roi.value(); + surface_handle->Info.CropX = static_cast(roi.x); + surface_handle->Info.CropY = static_cast(roi.y); + surface_handle->Info.CropW = static_cast(roi.width); + surface_handle->Info.CropH = static_cast(roi.height); + GAPI_LOG_DEBUG(nullptr, "applied ROI {" << surface_handle->Info.CropX << + ", " << surface_handle->Info.CropY << "}, " + "{ " << surface_handle->Info.CropX + surface_handle->Info.CropW << + ", " << surface_handle->Info.CropY + surface_handle->Info.CropH << "}"); + } +} + VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr&& accel) : ProcessingEngineBase(std::move(accel)) { GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine"); @@ -57,31 +72,25 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr&& acce my_sess.sync_in_queue.pop(); auto *vpp_suface = my_sess.processing_surface_ptr.lock()->get_handle(); - /* TODO: consider CROP/ROI here - static int x_offset = 0; - static int y_offset = 0; - dec_surface->Info.CropX = x_offset; - dec_surface->Info.CropY = y_offset; - dec_surface->Info.CropW = 100 + x_offset++; - dec_surface->Info.CropH = 100 + y_offset++; - */ - session_type::outgoing_task vpp_pending_op {pending_op.sync_handle, nullptr}; + apply_roi(pending_op.decoded_surface_ptr, pending_op.roi); + + mfxSyncPoint vpp_sync_handle{}; my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session, pending_op.decoded_surface_ptr, vpp_suface, - nullptr, &vpp_pending_op.sync_handle); - vpp_pending_op.vpp_surface_ptr = vpp_suface; - + nullptr, &vpp_sync_handle); + session_type::outgoing_task vpp_pending_op {vpp_sync_handle, + vpp_suface, + std::move(pending_op) }; GAPI_LOG_DEBUG(nullptr, "Got VPP async operation" << ", sync id: " << vpp_pending_op.sync_handle << ", dec surface: " << - pending_op.decoded_surface_ptr << + vpp_pending_op.original_surface_ptr << ", trans surface: " << vpp_pending_op.vpp_surface_ptr << ", status: " << mfxstatus_to_string(my_sess.last_status)); - // NB: process status if (my_sess.last_status == MFX_ERR_MORE_SURFACE || my_sess.last_status == MFX_ERR_NONE) { @@ -131,6 +140,7 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr&& acce // put frames in ready queue on success if (MFX_ERR_NONE == sess.last_status) { + pending_op.release_frame(); on_frame_ready(my_sess, pending_op.vpp_surface_ptr); } } @@ -327,8 +337,8 @@ VPPPreprocEngine::initialize_session(mfxSession, return {}; } -cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame) { - +cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame, + const cv::util::optional &roi) { std::shared_ptr pp_sess_impl = sess.get(); if (!pp_sess_impl) { // bypass case @@ -347,8 +357,10 @@ cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::Medi // schedule decoded surface into preproc queue session_type::incoming_task in_preproc_request {nullptr, - vpl_adapter->get_surface()->get_handle(), - in_frame}; + vpl_adapter->get_surface()->get_handle(), + vpl_adapter->get_surface()->get_info(), + in_frame, + roi}; s->sync_in_queue.emplace(in_preproc_request); // invoke pipeline to transform decoded surface into preprocessed surface diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp index c4be48708a..b1d0cee264 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp @@ -47,7 +47,8 @@ public: const GFrameDesc& required_frame_descr) override; cv::MediaFrame run_sync(const pp_session &session_handle, - const cv::MediaFrame& in_frame) override; + const cv::MediaFrame& in_frame, + const cv::util::optional &opt_roi) override; private: std::map preproc_session_map; diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp index 059b7caea7..2695a26049 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp @@ -60,6 +60,24 @@ void VPPPreprocSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key) const mfxFrameInfo& VPPPreprocSession::get_video_param() const { return mfx_vpp_out_param.vpp.Out; } + +VPPPreprocSession::outgoing_task::outgoing_task(mfxSyncPoint acquired_sync_handle, + mfxFrameSurface1* acquired_surface_ptr, + VPPPreprocSession::incoming_task &&in) : + sync_handle(acquired_sync_handle), + vpp_surface_ptr(acquired_surface_ptr), + original_surface_ptr(in.decoded_surface_ptr), + original_frame_info(std::move(in.decoded_frame_info)), + original_frame(in.decoded_frame_copy) { +} + +void VPPPreprocSession::outgoing_task::release_frame() { + // restore initial surface params + memcpy(&(original_surface_ptr->Info), + &original_frame_info, sizeof(Surface::info_t)); + // release references on frame adapter + original_frame = cv::MediaFrame(); +} } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp index 1f873fda56..b6800c3f76 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp @@ -41,12 +41,24 @@ private: struct incoming_task { mfxSyncPoint sync_handle; mfxFrameSurface1* decoded_surface_ptr; + Surface::info_t decoded_frame_info; cv::MediaFrame decoded_frame_copy; + cv::util::optional roi; }; struct outgoing_task { + outgoing_task() = default; + outgoing_task(mfxSyncPoint acquired_sync_handle, + mfxFrameSurface1* acquired_surface_ptr, + incoming_task &&in); mfxSyncPoint sync_handle; mfxFrameSurface1* vpp_surface_ptr; + + mfxFrameSurface1* original_surface_ptr; + void release_frame(); + private: + Surface::info_t original_frame_info; + cv::MediaFrame original_frame; }; std::queue sync_in_queue; diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp index 4997a04562..be347a258f 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp @@ -27,7 +27,8 @@ struct IPreprocEngine { initialize_preproc(const pp_params& initial_frame_param, const GFrameDesc& required_frame_descr) = 0; virtual cv::MediaFrame - run_sync(const pp_session &sess, const cv::MediaFrame& in_frame) = 0; + run_sync(const pp_session &sess, const cv::MediaFrame& in_frame, + const cv::util::optional &opt_roi = {}) = 0; }; } // namespace wip } // namespace gapi diff --git a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp index c43dfa9496..a0a66c7b93 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp @@ -181,6 +181,8 @@ using acceleration_t = int; using out_frame_info_t = cv::GFrameDesc; using preproc_args_t = std::tuple; +static cv::util::optional empty_roi; + class VPPPreprocParams : public ::testing::TestWithParam {}; preproc_args_t files[] = { @@ -246,7 +248,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread) required_frame_param); // 2) make preproc using incoming decoded frame & preproc session - cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame); + cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, + first_decoded_frame, + empty_roi); cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc(); ASSERT_FALSE(first_frame_decoded_desc == first_outcome_pp_desc); @@ -278,7 +282,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread) ASSERT_EQ(pp_sess.get().get(), first_pp_sess.get().get()); - cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame); + cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, + decoded_frame, + empty_roi); cv::GFrameDesc pp_desc = pp_frame.desc(); ASSERT_TRUE(pp_desc == first_outcome_pp_desc); in_progress = false; @@ -291,8 +297,95 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread) ASSERT_NE(frames_processed_count, 1); } +void decode_function(cv::gapi::wip::onevpl::VPLLegacyDecodeEngine &decode_engine, + cv::gapi::wip::onevpl::ProcessingEngineBase::session_ptr sess_ptr, + SafeQueue &queue, size_t &decoded_number) { + // decode first frame + { + cv::MediaFrame decoded_frame; + ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine)); + queue.push(std::move(decoded_frame)); + } -TEST_P(VPPPreprocParams, functional_different_threads) + // launch pipeline + try { + while(true) { + queue.push(extract_decoded_frame(sess_ptr->session, decode_engine)); + decoded_number++; + } + } catch (...) {} + + // send stop + queue.push_stop(); +} + +void preproc_function(cv::gapi::wip::onevpl::VPPPreprocEngine &preproc_engine, SafeQueue&queue, + size_t &preproc_number, const out_frame_info_t &required_frame_param, + const cv::util::optional &roi_rect = {}) { + using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; + // create preproc session based on frame description & network info + cv::MediaFrame first_decoded_frame = queue.pop(); + cv::util::optional first_pp_params = preproc_engine.is_applicable(first_decoded_frame); + ASSERT_TRUE(first_pp_params.has_value()); + pp_session first_pp_sess = + preproc_engine.initialize_preproc(first_pp_params.value(), + required_frame_param); + + // make preproc using incoming decoded frame & preproc session + cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, + first_decoded_frame, + roi_rect); + cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc(); + + // do not hold media frames because they share limited DX11 surface pool resources + first_decoded_frame = cv::MediaFrame(); + first_pp_frame = cv::MediaFrame(); + + // launch pipeline + bool in_progress = false; + // let's allow counting of preprocessed frames to check this value later: + // Currently, it looks redundant to implement any kind of gracefull shutdown logic + // in this test - so let's apply agreement that media source is processed + // succesfully when preproc_number != 1 in result. + // Specific validation logic which adhere to explicit counter value may be implemented + // in particular test scope + preproc_number = 1; + try { + while(true) { + cv::MediaFrame decoded_frame = queue.pop(); + if (SafeQueue::is_stop(decoded_frame)) { + break; + } + in_progress = true; + + cv::util::optional params = preproc_engine.is_applicable(decoded_frame); + ASSERT_TRUE(params.has_value()); + ASSERT_TRUE(0 == memcmp(¶ms.value(), &first_pp_params.value(), sizeof(pp_params))); + + pp_session pp_sess = preproc_engine.initialize_preproc(params.value(), + required_frame_param); + ASSERT_EQ(pp_sess.get().get(), + first_pp_sess.get().get()); + + cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame, empty_roi); + cv::GFrameDesc pp_desc = pp_frame.desc(); + ASSERT_TRUE(pp_desc == first_outcome_pp_desc); + in_progress = false; + preproc_number++; + } + } catch (...) {} + + // test if interruption has happened + ASSERT_FALSE(in_progress); + ASSERT_NE(preproc_number, 1); +} + +using roi_t = cv::util::optional; +using preproc_roi_args_t = decltype(std::tuple_cat(std::declval(), + std::declval>())); +class VPPPreprocROIParams : public ::testing::TestWithParam {}; +TEST_P(VPPPreprocROIParams, functional_roi_different_threads) { using namespace cv::gapi::wip; using namespace cv::gapi::wip::onevpl; @@ -300,7 +393,8 @@ TEST_P(VPPPreprocParams, functional_different_threads) decoder_t decoder_id; acceleration_t accel; out_frame_info_t required_frame_param; - std::tie(file_path, decoder_id, accel, required_frame_param) = GetParam(); + roi_t opt_roi; + std::tie(file_path, decoder_id, accel, required_frame_param, opt_roi) = GetParam(); file_path = findDataFile(file_path); @@ -338,87 +432,52 @@ TEST_P(VPPPreprocParams, functional_different_threads) size_t decoded_number = 1; size_t preproc_number = 0; - std::thread decode_thread([&decode_engine, sess_ptr, - &queue, &decoded_number] () { - // decode first frame - { - cv::MediaFrame decoded_frame; - ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine)); - queue.push(std::move(decoded_frame)); - } - - // launch pipeline - try { - while(true) { - queue.push(extract_decoded_frame(sess_ptr->session, decode_engine)); - decoded_number++; - } - } catch (...) {} - - // send stop - queue.push_stop(); - }); - - std::thread preproc_thread([&preproc_engine, &queue, &preproc_number, required_frame_param] () { - // create preproc session based on frame description & network info - cv::MediaFrame first_decoded_frame = queue.pop(); - cv::util::optional first_pp_params = preproc_engine.is_applicable(first_decoded_frame); - ASSERT_TRUE(first_pp_params.has_value()); - pp_session first_pp_sess = - preproc_engine.initialize_preproc(first_pp_params.value(), required_frame_param); - - // make preproc using incoming decoded frame & preproc session - cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame); - cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc(); - - // do not hold media frames because they share limited DX11 surface pool resources - first_decoded_frame = cv::MediaFrame(); - first_pp_frame = cv::MediaFrame(); - - // launch pipeline - bool in_progress = false; - // let's allow counting of preprocessed frames to check this value later: - // Currently, it looks redundant to implement any kind of gracefull shutdown logic - // in this test - so let's apply agreement that media source is processed - // succesfully when preproc_number != 1 in result - preproc_number = 1; - try { - while(true) { - cv::MediaFrame decoded_frame = queue.pop(); - if (SafeQueue::is_stop(decoded_frame)) { - break; - } - in_progress = true; - - cv::util::optional params = preproc_engine.is_applicable(decoded_frame); - ASSERT_TRUE(params.has_value()); - ASSERT_TRUE(0 == memcmp(¶ms.value(), &first_pp_params.value(), sizeof(pp_params))); - - pp_session pp_sess = preproc_engine.initialize_preproc(params.value(), - required_frame_param); - ASSERT_EQ(pp_sess.get().get(), - first_pp_sess.get().get()); - - cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame); - cv::GFrameDesc pp_desc = pp_frame.desc(); - ASSERT_TRUE(pp_desc == first_outcome_pp_desc); - in_progress = false; - preproc_number++; - } - } catch (...) {} - - // test if interruption has happened - ASSERT_FALSE(in_progress); - ASSERT_NE(preproc_number, 1); - }); + std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr, + std::ref(queue), std::ref(decoded_number)); + std::thread preproc_thread(preproc_function, std::ref(preproc_engine), + std::ref(queue), std::ref(preproc_number), + std::cref(required_frame_param), + std::cref(opt_roi)); decode_thread.join(); preproc_thread.join(); ASSERT_EQ(preproc_number, decoded_number); } -INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngine, VPPPreprocParams, - testing::ValuesIn(files)); +preproc_roi_args_t files_w_roi[] = { + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{cv::Rect{0,0,50,50}}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{cv::Rect{0,0,100,100}}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h264", + MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}}, + roi_t{cv::Rect{100,100,200,200}}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{cv::Rect{0,0,100,100}}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{}}, + preproc_roi_args_t {"highgui/video/big_buck_bunny.h265", + MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11, + out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}}, + roi_t{cv::Rect{100,100,200,200}}} +}; + +INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams, + testing::ValuesIn(files_w_roi)); + using VPPInnerPreprocParams = VPPPreprocParams; TEST_P(VPPInnerPreprocParams, functional_inner_preproc_size)