diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index f57ff6299d..a689bc3e81 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -191,6 +191,8 @@ set(gapi_srcs src/streaming/onevpl/engine/processing_engine_base.cpp src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp src/streaming/onevpl/engine/decode/decode_session.cpp + src/streaming/onevpl/engine/transcode/transcode_engine_legacy.cpp + src/streaming/onevpl/engine/transcode/transcode_session.cpp src/streaming/onevpl/demux/async_mfp_demux_data_provider.cpp src/streaming/onevpl/data_provider_dispatcher.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp index bfd922496a..d93b4c561d 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -110,6 +110,62 @@ struct GAPI_EXPORTS CfgParam { static CfgParam create_implementation(uint32_t value); static CfgParam create_implementation(const char* value); + + static constexpr const char *vpp_frames_pool_size_name() { return "vpp_frames_pool_size"; } + static CfgParam create_vpp_frames_pool_size(size_t value); + + static constexpr const char *vpp_in_width_name() { return "vpp.In.Width"; } + static CfgParam create_vpp_in_width(uint16_t value); + + static constexpr const char *vpp_in_height_name() { return "vpp.In.Height"; } + static CfgParam create_vpp_in_height(uint16_t value); + + static constexpr const char *vpp_in_crop_x_name() { return "vpp.In.CropX"; } + static CfgParam create_vpp_in_crop_x(uint16_t value); + + static constexpr const char *vpp_in_crop_y_name() { return "vpp.In.CropY"; } + static CfgParam create_vpp_in_crop_y(uint16_t value); + + static constexpr const char *vpp_in_crop_w_name() { return "vpp.In.CropW"; } + static CfgParam create_vpp_in_crop_w(uint16_t value); + + static constexpr const char *vpp_in_crop_h_name() { return "vpp.In.CropH"; } + static CfgParam create_vpp_in_crop_h(uint16_t value); + + + static constexpr const char *vpp_out_fourcc_name() { return "vpp.Out.FourCC"; } + static CfgParam create_vpp_out_fourcc(uint32_t value); + + static constexpr const char *vpp_out_chroma_format_name() { return "vpp.Out.ChromaFormat"; } + static CfgParam create_vpp_out_chroma_format(uint16_t value); + + static constexpr const char *vpp_out_width_name() { return "vpp.Out.Width"; } + static CfgParam create_vpp_out_width(uint16_t value); + + static constexpr const char *vpp_out_height_name() { return "vpp.Out.Height"; } + static CfgParam create_vpp_out_height(uint16_t value); + + static constexpr const char *vpp_out_crop_x_name() { return "vpp.Out.CropX"; } + static CfgParam create_vpp_out_crop_x(uint16_t value); + + static constexpr const char *vpp_out_crop_y_name() { return "vpp.Out.CropY"; } + static CfgParam create_vpp_out_crop_y(uint16_t value); + + static constexpr const char *vpp_out_crop_w_name() { return "vpp.Out.CropW"; } + static CfgParam create_vpp_out_crop_w(uint16_t value); + + static constexpr const char *vpp_out_crop_h_name() { return "vpp.Out.CropH"; } + static CfgParam create_vpp_out_crop_h(uint16_t value); + + static constexpr const char *vpp_out_pic_struct_name() { return "vpp.Out.PicStruct"; } + static CfgParam create_vpp_out_pic_struct(uint16_t value); + + static constexpr const char *vpp_out_framerate_n_name() { return "vpp.Out.FrameRateExtN"; } + static CfgParam create_vpp_out_framerate_n(uint32_t value); + + static constexpr const char *vpp_out_framerate_d_name() { return "vpp.Out.FrameRateExtD"; } + static CfgParam create_vpp_out_framerate_d(uint32_t value); + /** * Create generic onevp::GSource configuration parameter. * diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 06950bcabe..80327e2d59 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -45,10 +45,15 @@ const std::string keys = "{ faced | AUTO | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }" "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automaticaly if 0 }" - "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}"; - + "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}" + "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}" + "{ source_preproc_enable | 0 | Turn on OneVPL source frame preprocessing using network input description instead of IE plugin preprocessing}"; namespace { +bool is_gpu(const std::string &device_name) { + return device_name.find("GPU") != std::string::npos; +} + std::string get_weights_path(const std::string &model_path) { const auto EXT_LEN = 4u; const auto sz = model_path.size(); @@ -123,8 +128,9 @@ using GRect = cv::GOpaque; using GSize = cv::GOpaque; using GPrims = cv::GArray; -G_API_OP(LocateROI, , "sample.custom.locate-roi") { - static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { +G_API_OP(LocateROI, )>, "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &, + std::reference_wrapper) { return cv::empty_gopaque_desc(); } }; @@ -145,18 +151,30 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { // but only crops the input image to square (this is // the most convenient aspect ratio for detectors to use) - static void run(const cv::Size& in_size, cv::Rect &out_rect) { + static void run(const cv::Size& in_size, + std::reference_wrapper device_id_ref, + cv::Rect &out_rect) { // Identify the central point & square size (- some padding) - const auto center = cv::Point{in_size.width/2, in_size.height/2}; - auto sqside = std::min(in_size.width, in_size.height); + // NB: GPU plugin in InferenceEngine doesn't support ROI at now + if (!is_gpu(device_id_ref.get())) { + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); - // Now build the central square ROI - out_rect = cv::Rect{ center.x - sqside/2 - , center.y - sqside/2 - , sqside - , sqside - }; + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } else { + // use whole frame for GPU device + out_rect = cv::Rect{ 0 + , 0 + , in_size.width + , in_size.height + }; + } } }; @@ -193,11 +211,14 @@ int main(int argc, char *argv[]) { } // get file name - std::string file_path = cmd.get("input"); - const std::string output = cmd.get("output"); + const auto file_path = cmd.get("input"); + const auto output = cmd.get("output"); const auto face_model_path = cmd.get("facem"); const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); - const auto source_queue_capacity = cmd.get("frames_pool_size"); + const auto source_decode_queue_capacity = cmd.get("frames_pool_size"); + const auto source_vpp_queue_capacity = cmd.get("vpp_frames_pool_size"); + const auto vpl_source_preproc_enable = cmd.get("source_preproc_enable"); + const auto device_id = cmd.get("faced"); // check ouput file extension if (!output.empty()) { @@ -214,6 +235,12 @@ int main(int argc, char *argv[]) { try { std::string line; while (std::getline(params_list, line, ';')) { + if (vpl_source_preproc_enable == 0) { + if (line.find("vpp.") != std::string::npos) { + // skip VPP preprocessing primitives if not requested + continue; + } + } source_cfgs.push_back(cfg::create_from_string(line)); } } catch (const std::exception& ex) { @@ -221,11 +248,13 @@ int main(int argc, char *argv[]) { return -1; } - if (source_queue_capacity != 0) { - source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_queue_capacity)); + if (source_decode_queue_capacity != 0) { + source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_decode_queue_capacity)); + } + if (source_vpp_queue_capacity != 0) { + source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity)); } - const std::string& device_id = cmd.get("faced"); auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR get_weights_path(face_model_path), // path to weights @@ -247,7 +276,7 @@ int main(int argc, char *argv[]) { auto dx11_dev = createCOMPtrGuard(); auto dx11_ctx = createCOMPtrGuard(); - if (device_id.find("GPU") != std::string::npos) { + if (is_gpu(device_id)) { auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; @@ -294,11 +323,25 @@ int main(int argc, char *argv[]) { #endif // HAVE_D3D11 #endif // HAVE_DIRECTX // set ctx_config for GPU device only - no need in case of CPU device type - if (device_id.find("GPU") != std::string::npos) { + if (is_gpu(device_id)) { InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, {"VA_DEVICE", accel_device_ptr} }); face_net.cfgContextParams(ctx_config); + face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }}); + + std::cout <<"/*******************************************************/\n" + "ATTENTION: GPU Inference Engine preprocessing is not vital as expected!" + " Please consider param \"source_preproc_enable=1\" and specify " + " appropriated media frame transformation using oneVPL::VPP primitives" + " which force onevpl::GSource to produce tranformed media frames." + " For exploring list of supported transformations please find out " + " vpp_* related stuff in" + " gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp" + " Pay attention that to obtain expected result In this case VPP " + " transformation must match network input params." + " Please vote/create issue about exporting network params using GAPI\n" + "/******************************************************/" << std::endl; } #endif // HAVE_INF_ENGINE @@ -314,7 +357,7 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - if (device_id.find("GPU") != std::string::npos) { + if (is_gpu(device_id)) { cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, device_id, accel_device_ptr, @@ -334,7 +377,7 @@ int main(int argc, char *argv[]) { // Now build the graph cv::GFrame in; auto size = cv::gapi::streaming::size(in); - auto roi = custom::LocateROI::on(size); + auto roi = custom::LocateROI::on(size, std::cref(device_id)); auto blob = cv::gapi::infer(roi, in); cv::GArray rcs = cv::gapi::parseSSD(blob, size, 0.5f, true, true); auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); @@ -397,6 +440,8 @@ typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &l std::string name = line.substr(0, name_endline_pos); std::string value = line.substr(name_endline_pos + 1); - return cv::gapi::wip::onevpl::CfgParam::create(name, value); + return cv::gapi::wip::onevpl::CfgParam::create(name, value, + /* vpp params strongly optional */ + name.find("vpp.") == std::string::npos); } } diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index d1e9c1eb5b..d088ee0617 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -507,7 +507,6 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { "Remote blob is supported for MediaFrame only"); cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); - auto ie_core = cv::gimpl::ie::wrap::getCore(); using ParamType = std::pair; using NV12ParamType = std::pair; @@ -529,7 +528,6 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { #else return IE::make_shared_blob(y_blob, uv_blob); #endif - } inline IE::Blob::Ptr extractBlob(IECallContext& ctx, @@ -571,6 +569,19 @@ static void setBlob(InferenceEngine::InferRequest& req, } } +static void setROIBlob(InferenceEngine::InferRequest& req, + const std::string& layer_name, + const IE::Blob::Ptr& blob, + const cv::Rect &roi, + const IECallContext& ctx) { + if (ctx.uu.params.device_id.find("GPU") != std::string::npos) { + GAPI_LOG_DEBUG(nullptr, "Skip ROI blob creation for device_id: " << + ctx.uu.params.device_id << ", layer: " << layer_name); + setBlob(req, layer_name, blob, ctx); + } else { + setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx); + } +} } // anonymous namespace std::vector cv::gimpl::ie::IECompiled::createInferRequests() { @@ -1125,10 +1136,9 @@ struct InferROI: public cv::detail::KernelTag { // it should be treated as image IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); - setBlob(req, - *(ctx->uu.params.input_names.begin()), - IE::make_shared_blob(this_blob, toIE(this_roi)), - *ctx); + setROIBlob(req, + *(ctx->uu.params.input_names.begin()), + this_blob, this_roi, *ctx); // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp index 2cdf1c2b44..ad0e5bf667 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -210,30 +210,29 @@ VPLCPUAccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_s } VPLCPUAccelerationPolicy::pool_key_t -VPLCPUAccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) { +VPLCPUAccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxFrameInfo& info) { // External (application) allocation of decode surfaces GAPI_LOG_DEBUG(nullptr, "Query mfxFrameAllocRequest.NumFrameSuggested: " << alloc_request.NumFrameSuggested << ", mfxFrameAllocRequest.Type: " << alloc_request.Type); - mfxU32 singleSurfaceSize = utils::GetSurfaceSize_(param.mfx.FrameInfo.FourCC, - param.mfx.FrameInfo.Width, - param.mfx.FrameInfo.Height); + mfxU32 singleSurfaceSize = utils::GetSurfaceSize_(info.FourCC, + info.Width, + info.Height); if (!singleSurfaceSize) { throw std::runtime_error("Cannot determine surface size for: fourCC: " + - std::to_string(param.mfx.FrameInfo.FourCC) + - ", width: " + std::to_string(param.mfx.FrameInfo.Width) + - ", height: " + std::to_string(param.mfx.FrameInfo.Height)); + std::to_string(info.FourCC) + + ", width: " + std::to_string(info.Width) + + ", height: " + std::to_string(info.Height)); } - const auto &frameInfo = param.mfx.FrameInfo; auto surface_creator = - [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, + [&info] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, size_t out_buf_size) -> surface_ptr_t { - return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? - utils::create_surface_RGB4_(frameInfo, out_buf_ptr, out_buf_ptr_offset, + return (info.FourCC == MFX_FOURCC_RGB4) ? + utils::create_surface_RGB4_(info, out_buf_ptr, out_buf_ptr_offset, out_buf_size) : - utils::create_surface_other_(frameInfo, out_buf_ptr, out_buf_ptr_offset, + utils::create_surface_other_(info, out_buf_ptr, out_buf_ptr_offset, out_buf_size);}; return create_surface_pool(alloc_request.NumFrameSuggested, diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp index fdc0afd4bf..8a2061dce0 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -32,7 +32,7 @@ struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolic void init(session_t session) override; void deinit(session_t session) override; pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator); - pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) override; + pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxFrameInfo& info) override; surface_weak_ptr_t get_free_surface(pool_key_t key) override; size_t get_free_surface_count(pool_key_t key) const override; size_t get_surface_count(pool_key_t key) const override; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index f528190ad5..02720f3774 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -98,9 +98,7 @@ void VPLDX11AccelerationPolicy::deinit(session_t session) { VPLDX11AccelerationPolicy::pool_key_t VPLDX11AccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc_req, - mfxVideoParam& param) { - param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY; - + mfxFrameInfo& info) { // allocate textures by explicit request mfxFrameAllocResponse mfxResponse; mfxStatus sts = on_alloc(&alloc_req, &mfxResponse); @@ -120,7 +118,7 @@ VPLDX11AccelerationPolicy::create_surface_pool(const mfxFrameAllocRequest& alloc pool_t pool(numSurfaces); for (int i = 0; i < numSurfaces; i++) { std::unique_ptr handle(new mfxFrameSurface1 {}); - handle->Info = param.mfx.FrameInfo; + handle->Info = info; handle->Data.MemId = mfxResponse.mids[i]; pool.push_back(Surface::create_surface(std::move(handle), table_it->second)); @@ -261,24 +259,54 @@ mfxStatus VPLDX11AccelerationPolicy::on_alloc(const mfxFrameAllocRequest *reques desc.Format = colorFormat; desc.SampleDesc.Count = 1; desc.Usage = D3D11_USAGE_DEFAULT; - desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + desc.MiscFlags = 0; desc.BindFlags = D3D11_BIND_DECODER; + if ((MFX_MEMTYPE_FROM_VPPIN & request->Type) && (DXGI_FORMAT_YUY2 == desc.Format) || + (DXGI_FORMAT_B8G8R8A8_UNORM == desc.Format) || + (DXGI_FORMAT_R10G10B10A2_UNORM == desc.Format) || + (DXGI_FORMAT_R16G16B16A16_UNORM == desc.Format)) { + desc.BindFlags = D3D11_BIND_RENDER_TARGET; + } + + if ((MFX_MEMTYPE_FROM_VPPOUT & request->Type) || + (MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET & request->Type)) { + desc.BindFlags = D3D11_BIND_RENDER_TARGET; + } + if (request->Type & MFX_MEMTYPE_SHARED_RESOURCE) { desc.BindFlags |= D3D11_BIND_SHADER_RESOURCE; desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; } - ComPtrGuard main_texture = createCOMPtrGuard(); + if (DXGI_FORMAT_P8 == desc.Format) { + desc.BindFlags = 0; + } + + size_t main_textures_count = 1; + if (D3D11_BIND_RENDER_TARGET & desc.BindFlags) { + GAPI_LOG_DEBUG(nullptr, "Use array of testures instead of texture array"); + desc.ArraySize = 1; + main_textures_count = request->NumFrameSuggested; + } + + // create GPU textures HRESULT err = S_OK; - { - ID3D11Texture2D *pTexture2D = nullptr; - err = hw_handle->CreateTexture2D(&desc, nullptr, &pTexture2D); - if (FAILED(err)) { - GAPI_LOG_WARNING(nullptr, "Cannot create texture, error: " + std::to_string(HRESULT_CODE(err))); - return MFX_ERR_MEMORY_ALLOC; + std::vector> main_textures; + main_textures.reserve(main_textures_count); + for (size_t i = 0; i < main_textures_count; i++) { + ComPtrGuard main_texture = createCOMPtrGuard(); + { + ID3D11Texture2D *pTexture2D = nullptr; + err = hw_handle->CreateTexture2D(&desc, nullptr, &pTexture2D); + if (FAILED(err)) { + GAPI_LOG_WARNING(nullptr, "Cannot create texture by index: " << i << + ", error: " << std::to_string(HRESULT_CODE(err))); + return MFX_ERR_MEMORY_ALLOC; + } + main_texture.reset(pTexture2D); } - main_texture.reset(pTexture2D); + main_textures.push_back(std::move(main_texture)); } // create staging texture to read it from @@ -308,7 +336,7 @@ mfxStatus VPLDX11AccelerationPolicy::on_alloc(const mfxFrameAllocRequest *reques DX11AllocationRecord::create(request->NumFrameSuggested, device_context, allocator, - std::move(main_texture), + std::move(main_textures), std::move(staging_textures))); if (!inserted_it.second) { GAPI_LOG_WARNING(nullptr, "Cannot assign allocation by id: " + std::to_string(request->AllocId) + @@ -363,7 +391,7 @@ mfxStatus VPLDX11AccelerationPolicy::on_get_hdl(mfxMemId mid, mfxHDL *handle) { pPair->second = static_cast(reinterpret_cast( static_cast(data->get_subresource()))); - GAPI_LOG_DEBUG(nullptr, "texture : " << pPair->first << ", sub id: " << pPair->second); + GAPI_LOG_DEBUG(nullptr, "ID3D11Texture2D : " << pPair->first << ", sub id: " << pPair->second); return MFX_ERR_NONE; } diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index e053089587..893698eb36 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -43,7 +43,7 @@ struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolic void init(session_t session) override; void deinit(session_t session) override; pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, - mfxVideoParam& param) override; + mfxFrameInfo& info) override; surface_weak_ptr_t get_free_surface(pool_key_t key) override; size_t get_free_surface_count(pool_key_t key) const override; size_t get_surface_count(pool_key_t key) const override; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp index a9059c29ef..b1d7c25bb1 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -54,7 +54,7 @@ struct VPLAccelerationPolicy // for existing workspace in existing pool (see realloc) // thus it is not implemented, // PLEASE provide initial memory area large enough - virtual pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxVideoParam& param) = 0; + virtual pool_key_t create_surface_pool(const mfxFrameAllocRequest& alloc_request, mfxFrameInfo& info) = 0; virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; virtual size_t get_free_surface_count(pool_key_t key) const = 0; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp index 3bbfb25b0a..574860e03d 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.cpp @@ -96,6 +96,7 @@ void LockAdapter::unlock_write(mfxMemId mid, mfxFrameData &data) { SharedLock* LockAdapter::set_adaptee(SharedLock* new_impl) { SharedLock* old_impl = impl; + GAPI_DbgAssert(old_impl == nullptr || new_impl == nullptr && "Must not be previous impl"); impl = new_impl; return old_impl; } @@ -355,13 +356,14 @@ DX11AllocationRecord::~DX11AllocationRecord() { GAPI_LOG_DEBUG(nullptr, "release final referenced texture: " << texture_ptr.get()); } -void DX11AllocationRecord::init(unsigned int items, - ID3D11DeviceContext* origin_ctx, +void DX11AllocationRecord::init(unsigned int items, ID3D11DeviceContext* origin_ctx, mfxFrameAllocator origin_allocator, - ComPtrGuard&& texture, + std::vector> &&textures, std::vector> &&staging_textures) { + GAPI_DbgAssert(items != 0 && "Cannot create DX11AllocationRecord with empty items"); GAPI_DbgAssert(items == staging_textures.size() && "Allocation items count and staging size are not equal"); + GAPI_DbgAssert(textures.size() != 1 ? items == textures.size() : true && "Allocation items count and staging size are not equal"); GAPI_DbgAssert(origin_ctx && "Cannot create DX11AllocationItem for empty origin_ctx"); auto shared_allocator_copy = origin_allocator; @@ -374,13 +376,22 @@ void DX11AllocationRecord::init(unsigned int items, shared_allocator_copy.pthis = nullptr; - GAPI_LOG_DEBUG(nullptr, "subresources count: " << items << ", text: " << texture.get()); + GAPI_LOG_DEBUG(nullptr, "subresources count: " << items); resources.reserve(items); - // no AddRef here, because DX11AllocationRecord receive ownership it here - texture_ptr = createCOMSharedPtrGuard(std::move(texture)); + + if (textures.size() == 1) { + texture_ptr = createCOMSharedPtrGuard(std::move(textures[0])); + } for(unsigned int i = 0; i < items; i++) { - resources.emplace_back(new DX11AllocationItem(get_ptr(), origin_ctx, shared_allocator_copy, - texture_ptr, i, std::move(staging_textures[i]))); + if (textures.size() == 1) { + GAPI_LOG_DEBUG(nullptr, "subresources: [" << i <<", " << items << "], ID3D11Texture2D: " << texture_ptr.get()); + resources.emplace_back(new DX11AllocationItem(get_ptr(), origin_ctx, shared_allocator_copy, + texture_ptr, i, std::move(staging_textures[i]))); + } else { + GAPI_LOG_DEBUG(nullptr, "subresources: [" << i <<", " << items << "], ID3D11Texture2D: " << textures[i].get()); + resources.emplace_back(new DX11AllocationItem(get_ptr(), origin_ctx, shared_allocator_copy, + std::move(textures[i]), 0, std::move(staging_textures[i]))); + } } } diff --git a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp index 46ddff86a4..c68a08a3f8 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/dx11_alloc_resource.hpp @@ -133,10 +133,10 @@ struct DX11AllocationRecord : public std::enable_shared_from_this&& texture, std::vector> &&staging_textures); - + std::vector>&& textures, std::vector> &&staging_textures); std::vector resources; ComSharedPtrGuard texture_ptr; }; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp index 04cf10c8d7..6afa2cf0b6 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp @@ -48,7 +48,8 @@ VPLMediaFrameDX11Adapter::VPLMediaFrameDX11Adapter(std::shared_ptr surf Surface::data_t& data = parent_surface_ptr->get_data(); GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << ", w: " << info.Width << ", h: " << info.Height << - ", p: " << data.Pitch); + ", p: " << data.Pitch << + ", frame id: " << reinterpret_cast(this)); switch(info.FourCC) { case MFX_FOURCC_I420: @@ -72,6 +73,9 @@ VPLMediaFrameDX11Adapter::~VPLMediaFrameDX11Adapter() { // Each VPLMediaFrameDX11Adapter releases mfx surface counter // The last VPLMediaFrameDX11Adapter releases shared Surface pointer // The last surface pointer releases workspace memory + + GAPI_LOG_DEBUG(nullptr, "destroy frame id: " << reinterpret_cast(this)); + Surface::data_t& data = parent_surface_ptr->get_data(); LockAdapter* alloc_data = reinterpret_cast(data.MemId); alloc_data->set_adaptee(nullptr); @@ -155,30 +159,44 @@ MediaFrame::View VPLMediaFrameDX11Adapter::access(MediaFrame::Access mode) { } cv::util::any VPLMediaFrameDX11Adapter::blobParams() const { + /*GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not fully integrated" + "in OpenVINO InferenceEngine and would be temporary disable.");*/ #ifdef HAVE_INF_ENGINE - GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not fully operable " - "in G-API streaming. Please waiting for future PRs"); - Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); NativeHandleAdapter* native_handle_getter = reinterpret_cast(data.MemId); mfxHDLPair handle{}; native_handle_getter->get_handle(data.MemId, reinterpret_cast(handle)); - InferenceEngine::ParamMap params{{"SHARED_MEM_TYPE", "VA_SURFACE"}, - {"DEV_OBJECT_HANDLE", handle.first}, - {"COLOR_FORMAT", InferenceEngine::ColorFormat::NV12}, - {"VA_PLANE", + GAPI_Assert(frame_desc.fmt == MediaFormat::NV12 && + "blobParams() for VPLMediaFrameDX11Adapter supports NV12 only"); + + InferenceEngine::ParamMap y_params{{"SHARED_MEM_TYPE", "VA_SURFACE"}, + {"DEV_OBJECT_HANDLE", handle.first}, + {"COLOR_FORMAT", InferenceEngine::ColorFormat::NV12}, + {"VA_PLANE", static_cast( reinterpret_cast( reinterpret_cast( handle.second)))}};//, - const Surface::info_t& info = parent_surface_ptr->get_info(); - InferenceEngine::TensorDesc tdesc({InferenceEngine::Precision::U8, - {1, 3, static_cast(info.Height), - static_cast(info.Width)}, - InferenceEngine::Layout::NCHW}); - return std::make_pair(tdesc, params); + InferenceEngine::TensorDesc y_tdesc({InferenceEngine::Precision::U8, + {1, 1, static_cast(info.Height), + static_cast(info.Width)}, + InferenceEngine::Layout::NHWC}); + + InferenceEngine::ParamMap uv_params = y_params; + uv_params["MEM_HANDLE"] = handle.first; + uv_params["VA_PLANE"] = static_cast( + reinterpret_cast( + reinterpret_cast( + handle.second))) + 1; + InferenceEngine::TensorDesc uv_tdesc({InferenceEngine::Precision::U8, + {1, 2, static_cast(info.Height) / 2, + static_cast(info.Width) / 2}, + InferenceEngine::Layout::NHWC}); + return std::make_pair(std::make_pair(y_tdesc, y_params), + std::make_pair(uv_tdesc, uv_params)); #else GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not implemented"); #endif // HAVE_INF_ENGINE diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp index 599f751358..b13f9cadb1 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -118,6 +118,82 @@ CfgParam CfgParam::create_implementation(const char* value) { return CfgParam::create(CfgParam::implementation_name(), std::string(value)); } +CfgParam CfgParam::create_vpp_frames_pool_size(size_t value) { + // NB: cast to uint64_t because CfgParam inner variant works over + // uint64_t instead of size_t and mirrored VPL types variety + // but size_t looks more friendly for C++ high-level development + return CfgParam::create(CfgParam::vpp_frames_pool_size_name(), + static_cast(value), false); +} + +CfgParam CfgParam::create_vpp_in_width(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_width_name(), value, false); +} + +CfgParam CfgParam::create_vpp_in_height(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_height_name(), value, false); +} + +CfgParam CfgParam::create_vpp_in_crop_x(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_crop_x_name(), value, false); +} + +CfgParam CfgParam::create_vpp_in_crop_y(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_crop_y_name(), value, false); +} + +CfgParam CfgParam::create_vpp_in_crop_w(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_crop_w_name(), value, false); +} + +CfgParam CfgParam::create_vpp_in_crop_h(uint16_t value) { + return CfgParam::create(CfgParam::vpp_in_crop_h_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_fourcc(uint32_t value) { + return CfgParam::create(CfgParam::vpp_out_fourcc_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_chroma_format(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_chroma_format_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_width(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_width_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_height(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_height_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_crop_x(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_crop_x_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_crop_y(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_crop_y_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_crop_w(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_crop_w_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_crop_h(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_crop_h_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_pic_struct(uint16_t value) { + return CfgParam::create(CfgParam::vpp_out_pic_struct_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_framerate_n(uint32_t value) { + return CfgParam::create(CfgParam::vpp_out_framerate_n_name(), value, false); +} + +CfgParam CfgParam::create_vpp_out_framerate_d(uint32_t value) { + return CfgParam::create(CfgParam::vpp_out_framerate_d_name(), value, false); +} + CfgParam& CfgParam::operator=(const CfgParam& src) { if (this != &src) { m_priv = src.m_priv; diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp index 07c639faa2..d748825b1c 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp @@ -35,6 +35,12 @@ struct ParamCreator { return create_impl(name, value); } private: + mfxVariant create_impl(const std::string&, mfxU16 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_U16; + ret.Data.U16 = value; + return ret; + } mfxVariant create_impl(const std::string&, mfxU32 value) { mfxVariant ret; ret.Type = MFX_VARIANT_TYPE_U32; @@ -53,6 +59,10 @@ private: ret.Data.U64 = value; return ret; } + mfxVariant create_impl(const std::string&, const std::string&) { + GAPI_Assert(false && "Something wrong: you should not create mfxVariant " + "from string directly - native type is lost in this case"); + } }; template @@ -86,6 +96,76 @@ std::vector get_params_from_string(const std::string& str) { ret.push_back(creator.create(name, cstr_to_mfx_version(value.c_str()))); } else if (name == CfgParam::frames_pool_size_name()) { ret.push_back(creator.create(name, strtoull_or_throw(value.c_str()), false)); + } else if (name == CfgParam::vpp_frames_pool_size_name()) { + ret.push_back(creator.create(name, strtoull_or_throw(value.c_str()), false)); + } else if (name == CfgParam::vpp_in_width_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_in_height_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_in_crop_w_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_in_crop_h_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_in_crop_x_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_in_crop_y_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_fourcc_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_chroma_format_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_width_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_height_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_crop_w_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_crop_h_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_crop_x_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_crop_y_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_pic_struct_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_framerate_n_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); + } else if (name == CfgParam::vpp_out_framerate_d_name()) { + ret.push_back(creator.create(name, + static_cast(strtoul_or_throw(value.c_str())), + false)); } else { GAPI_LOG_DEBUG(nullptr, "Cannot parse configuration param, name: " << name << ", value: " << value); @@ -128,6 +208,45 @@ mfxVariant cfg_param_to_mfx_variant(const CfgParam& cfg_val) { return ret; } +void extract_optional_param_by_name(const std::string &name, + const std::vector &in_params, + cv::util::optional &out_param) { + auto it = std::find_if(in_params.begin(), in_params.end(), [&name] (const CfgParam& value) { + return value.get_name() == name; + }); + if (it != in_params.end()) { + cv::util::visit(cv::util::overload_lambdas( + [&out_param](uint8_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](int8_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](uint16_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](int16_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](uint32_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](int32_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](uint64_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](int64_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](float_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](double_t value) { out_param = cv::util::make_optional(static_cast(value)); }, + [&out_param](void*) { GAPI_Assert(false && "`void*` is unsupported type"); }, + [&out_param](const std::string& value) { + out_param = cv::util::make_optional(strtoull_or_throw(value.c_str())); + }), + it->get_value()); + } +} + +unsigned long strtoul_or_throw(const char* str) { + char *end_ptr = nullptr; + errno = 0; + unsigned long ret = strtoul(str, &end_ptr, 10); + if ((end_ptr == str) || + ((ret == ULONG_MAX || ret == LONG_MIN) && errno == ERANGE)) { + // nothing parsed from the string, handle errors or exit + GAPI_LOG_WARNING(nullptr, "strtoul failed for: " << str); + GAPI_Assert(false && "strtoul_or_throw"); + } + return ret; +} + size_t strtoull_or_throw(const char* str) { char *end_ptr = nullptr; errno = 0; diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp index c5e7685756..e01d5c412a 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp @@ -31,6 +31,11 @@ struct ParamCreator { mfxVariant cfg_param_to_mfx_variant(const CfgParam& value); +void extract_optional_param_by_name(const std::string &name, + const std::vector& in_params, + cv::util::optional &out_param); + +unsigned long strtoul_or_throw(const char* str); size_t strtoull_or_throw(const char* str); int64_t strtoll_or_throw(const char* str); diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp index 6707a401b1..d8af94f939 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -26,6 +26,31 @@ namespace gapi { namespace wip { namespace onevpl { +void VPLLegacyDecodeEngine::try_modify_pool_size_request_param(const char* param_name, + size_t new_frames_count, + mfxFrameAllocRequest& request) { + if (new_frames_count < request.NumFrameMin) { + GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << param_name << "\": " << + new_frames_count << ". It must be equal or greater than " + "mfxFrameAllocRequest.NumFrameMin: " << request.NumFrameMin); + throw std::runtime_error(std::string("Invalid value of param: ") + + param_name + ", underflow"); + } else { + if (static_cast(std::numeric_limits::max()) < new_frames_count) { + GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << param_name << "\": " << + new_frames_count << ". It must not be greater than " << + std::numeric_limits::max()); + throw std::runtime_error(std::string("Invalid value of param: ") + + param_name + ", overflow"); + } + request.NumFrameSuggested = static_cast(new_frames_count); + GAPI_LOG_DEBUG(nullptr, "mfxFrameAllocRequest overriden by user input: " << + ", mfxFrameAllocRequest.NumFrameMin: " << request.NumFrameMin << + ", mfxFrameAllocRequest.NumFrameSuggested: " << request.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << request.Type); + } +} + VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr&& accel) : ProcessingEngineBase(std::move(accel)) { @@ -138,11 +163,12 @@ VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr& cfg_params, - std::shared_ptr provider) { - GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); +VPLLegacyDecodeEngine::SessionParam VPLLegacyDecodeEngine::prepare_session_param( + mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) { + + GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); // init session acceleration_policy->init(mfx_session); @@ -206,78 +232,55 @@ VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, ", mfxFrameAllocRequest.Type: " << decRequest.Type); // NB: override NumFrameSuggested preallocation size (how many frames we can hold) - size_t preallocated_frames_count = decRequest.NumFrameSuggested; - // NB: if you see bunch of WARNING about "cannot get free surface from pool" - // and have abundant RAM size then increase `preallocated_frames_count` + // if you see bunch of WARNING about "cannot get free surface from pool" + // and have abundant RAM size then increase `CfgParam::frames_pool_size_name()` // to keep more free surfaces in a round. Otherwise VPL decode pipeline will be waiting // till application is freeing unusable surface on its side. // - auto queue_capacity_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { - return value.get_name() == CfgParam::frames_pool_size_name(); - }); - if (queue_capacity_it != cfg_params.end()) { - cv::util::visit(cv::util::overload_lambdas( - [&preallocated_frames_count](uint8_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](int8_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](uint16_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](int16_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](uint32_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](int32_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](uint64_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](int64_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](float_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](double_t value) { preallocated_frames_count = static_cast(value); }, - [&preallocated_frames_count](void*) { GAPI_Assert(false && "`void*` is unsupported type"); }, - [&preallocated_frames_count](const std::string& value) { - preallocated_frames_count = strtoull_or_throw(value.c_str()); - }), - queue_capacity_it->get_value()); - + cv::optional preallocated_frames_count_cfg; + extract_optional_param_by_name(CfgParam::frames_pool_size_name(), + cfg_params, + preallocated_frames_count_cfg); + if (preallocated_frames_count_cfg.has_value()) { GAPI_LOG_INFO(nullptr, "Try to use CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << - preallocated_frames_count << ", for session: " << mfx_session); + preallocated_frames_count_cfg.value() << ", for session: " << mfx_session); + try_modify_pool_size_request_param(CfgParam::frames_pool_size_name(), + preallocated_frames_count_cfg.value(), + decRequest); } - if (preallocated_frames_count < decRequest.NumFrameMin) { - GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << - preallocated_frames_count << ". It must be equal or greater than " - "mfxFrameAllocRequest.NumFrameMin: " << decRequest.NumFrameMin); - throw std::runtime_error(std::string("Invalid value of param: ") + - CfgParam::frames_pool_size_name() + ", underflow"); - } else { - if (static_cast(std::numeric_limits::max()) < preallocated_frames_count) { - GAPI_LOG_WARNING(nullptr, "Cannot proceed with CfgParam \"" << CfgParam::frames_pool_size_name() << "\": " << - preallocated_frames_count << ". It must not be equal than " << - std::numeric_limits::max()); - throw std::runtime_error(std::string("Invalid value of param: ") + - CfgParam::frames_pool_size_name() + ", overflow"); - } - decRequest.NumFrameSuggested = static_cast(preallocated_frames_count); - GAPI_LOG_DEBUG(nullptr, "mfxFrameAllocRequest overriden by user input for session: " << mfx_session << - ", mfxFrameAllocRequest.NumFrameMin: " << decRequest.NumFrameMin << - ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << - ", mfxFrameAllocRequest.Type: " << decRequest.Type); - } + decRequest.Type |= MFX_MEMTYPE_EXTERNAL_FRAME | MFX_MEMTYPE_FROM_DECODE | MFX_MEMTYPE_FROM_VPPIN; VPLAccelerationPolicy::pool_key_t decode_pool_key = - acceleration_policy->create_surface_pool(decRequest, mfxDecParams); + acceleration_policy->create_surface_pool(decRequest, mfxDecParams.mfx.FrameInfo); // Input parameters finished, now initialize decode // create decoder for session accoring to header recovered from source file + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); if (MFX_ERR_NONE != sts) { throw std::runtime_error("Error initializing Decode, error: " + mfxstatus_to_string(sts)); } - DecoderParams decoder_param {bitstream, mfxDecParams}; + return {decode_pool_key, {bitstream, mfxDecParams, preallocated_frames_count_cfg}}; +} + + +ProcessingEngineBase::session_ptr +VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) { + + SessionParam param = prepare_session_param(mfx_session, cfg_params, provider); // create session std::shared_ptr sess_ptr = register_session(mfx_session, - std::move(decoder_param), + std::move(param.decoder_params), provider); - sess_ptr->init_surface_pool(decode_pool_key); + sess_ptr->init_surface_pool(param.decode_pool_key); // prepare working decode surface sess_ptr->swap_surface(*this); return sess_ptr; diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp index f6a02db3db..1b7bee6a82 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp @@ -24,20 +24,31 @@ struct DecoderParams; struct IDataProvider; struct VPLAccelerationPolicy; -class VPLLegacyDecodeEngine : public ProcessingEngineBase { +class GAPI_EXPORTS VPLLegacyDecodeEngine : public ProcessingEngineBase { public: VPLLegacyDecodeEngine(std::unique_ptr&& accel); - session_ptr initialize_session(mfxSession mfx_session, - const std::vector& cfg_params, - std::shared_ptr provider) override; + virtual session_ptr initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) override; +protected: + struct SessionParam { + void* decode_pool_key; + DecoderParams decoder_params; + }; + + SessionParam prepare_session_param(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider); -private: ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; ExecutionStatus process_error(mfxStatus status, LegacyDecodeSession& sess); void on_frame_ready(LegacyDecodeSession& sess, mfxFrameSurface1* ready_surface); + static void try_modify_pool_size_request_param(const char* param_name, + size_t new_frames_count, + mfxFrameAllocRequest& request); }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp index bbb1378767..56e51ffd9f 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp @@ -11,7 +11,6 @@ #include "streaming/onevpl/engine/decode/decode_session.hpp" #include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" -#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/utils.hpp" @@ -75,8 +74,8 @@ Data::Meta LegacyDecodeSession::generate_frame_meta() { return meta; } -const mfxVideoParam& LegacyDecodeSession::get_video_param() const { - return mfx_decoder_param; +const mfxFrameInfo& LegacyDecodeSession::get_video_param() const { + return mfx_decoder_param.mfx.FrameInfo; } } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp index 476a575172..356f9851cd 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp @@ -26,9 +26,10 @@ struct IDataProvider; class Surface; struct VPLAccelerationPolicy; -class LegacyDecodeSession : public EngineSession { +class GAPI_EXPORTS LegacyDecodeSession : public EngineSession { public: friend class VPLLegacyDecodeEngine; + friend class VPLLegacyTranscodeEngine; //TODO: remove friend add method LegacyDecodeSession(mfxSession sess, DecoderParams&& decoder_param, std::shared_ptr provider); ~LegacyDecodeSession(); @@ -38,15 +39,15 @@ public: void init_surface_pool(VPLAccelerationPolicy::pool_key_t key); Data::Meta generate_frame_meta(); - const mfxVideoParam& get_video_param() const override; + virtual const mfxFrameInfo& get_video_param() const override; private: mfxVideoParam mfx_decoder_param; std::shared_ptr data_provider; VPLAccelerationPolicy::pool_key_t decoder_pool_id; mfxFrameAllocRequest request; +protected: std::weak_ptr procesing_surface_ptr; - using op_handle_t = std::pair; std::queue sync_queue; diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp index 67018d0fd7..8a1f4383eb 100644 --- a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp @@ -29,6 +29,11 @@ namespace onevpl { struct GAPI_EXPORTS DecoderParams { std::shared_ptr stream; mfxVideoParam param; + cv::optional preallocated_frames_count; +}; + +struct GAPI_EXPORTS TranscoderParams { + mfxVideoParam param; }; struct GAPI_EXPORTS EngineSession { @@ -41,7 +46,7 @@ struct GAPI_EXPORTS EngineSession { std::string error_code_to_str() const; virtual ~EngineSession(); - virtual const mfxVideoParam& get_video_param() const = 0; + virtual const mfxFrameInfo& get_video_param() const = 0; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp index 72f2f62fc4..35cd664219 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -36,7 +36,7 @@ ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession s session_ptr processing_session = sess_it->second; ExecutionData& exec_data = execution_table[session]; - GAPI_LOG_DEBUG(nullptr, "[" << session <<"] start op id: " << exec_data.op_id); + GAPI_LOG_DEBUG(nullptr, "[" << session << "] start op id: " << exec_data.op_id); ExecutionStatus status = execute_op(pipeline.at(exec_data.op_id), *processing_session); size_t old_op_id = exec_data.op_id++; if (exec_data.op_id == pipeline.size()) @@ -44,10 +44,10 @@ ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession s exec_data.op_id = 0; } cv::util::suppress_unused_warning(old_op_id); - GAPI_LOG_DEBUG(nullptr, "[" << session <<"] finish op id: " << old_op_id << - ", " << processing_session->error_code_to_str() << - ", " << ProcessingEngineBase::status_to_string(status) << - ", next op id: " << exec_data.op_id); + GAPI_LOG_DEBUG(nullptr, "[" << session << "] finish op id: " << old_op_id << + ", " << processing_session->error_code_to_str() << + ", " << ProcessingEngineBase::status_to_string(status) << + ", next op id: " << exec_data.op_id); if (status == ExecutionStatus::Failed) { @@ -81,7 +81,7 @@ const char* ProcessingEngineBase::status_to_string(ExecutionStatus status) ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::execute_op(operation_t& op, EngineSession& sess) { - return op(sess); + return op(sess); } size_t ProcessingEngineBase::get_ready_frames_count() const diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp index 059ef963de..cacc8bd748 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -67,16 +67,25 @@ protected: std::vector pipeline; std::unique_ptr acceleration_policy; - +public: virtual ExecutionStatus execute_op(operation_t& op, EngineSession& sess); template void create_pipeline(Ops&&...ops) { - GAPI_DbgAssert(pipeline.empty() && "Pipeline must be empty"); std::vector({std::forward(ops)...}).swap(pipeline); } + template + void inject_pipeline_operations(size_t in_position, Ops&&...ops) + { + GAPI_Assert(pipeline.size() >= in_position && + "Invalid position to inject pipeline operation"); + auto it = pipeline.begin(); + std::advance(it, in_position); + pipeline.insert(it, {std::forward(ops)...}); + } + template std::shared_ptr register_session(mfxSession key, SessionArgs&& ...args) diff --git a/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.cpp new file mode 100644 index 0000000000..36d36d5ec0 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.cpp @@ -0,0 +1,477 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include +#include "streaming/onevpl/data_provider_defines.hpp" + +#include "streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp" +#include "streaming/onevpl/engine/transcode/transcode_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +#define ALIGN16(value) (((value + 15) >> 4) << 4) + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template +bool set_vpp_param(const char* name, Type& out_vpp_param, + const std::map ¶ms_storage, + mfxSession session); + +template<> +bool set_vpp_param(const char* name, uint32_t& out_vpp_param, + const std::map ¶ms_storage, + mfxSession session) { + auto it = params_storage.find(name); + if (it != params_storage.end()) { + auto value = it->second.Data.U32; + GAPI_LOG_INFO(nullptr, "[" << session << "] set \"" << name << + "\": " << value); + out_vpp_param = value; + return true; + } + return false; +} + +template<> +bool set_vpp_param(const char* name, uint16_t& out_vpp_param, + const std::map ¶ms_storage, + mfxSession session) { + auto it = params_storage.find(name); + if (it != params_storage.end()) { + auto value = it->second.Data.U16; + GAPI_LOG_INFO(nullptr, "[" << session << "] set \"" << name << + "\": " << value); + out_vpp_param = value; + return true; + } + return false; +} + +std::map + VPLLegacyTranscodeEngine::get_vpp_params(const std::vector &cfg_params) { + std::map ret; + static const char* vpp_param_prefix {"vpp."}; + for (const auto ¶m : cfg_params) { + const char *param_name_cptr = param.get_name().c_str(); + if (strstr(param_name_cptr, vpp_param_prefix) == param_name_cptr) { + ret.emplace(param.get_name(), cfg_param_to_mfx_variant(param)); + } + } + GAPI_LOG_INFO(nullptr, "Detected VPP params count: [" << ret.size() << + "/" << cfg_params.size() << "]"); + return ret; +} + +VPLLegacyTranscodeEngine::VPLLegacyTranscodeEngine(std::unique_ptr&& accel) + : VPLLegacyDecodeEngine(std::move(accel)) { + + GAPI_LOG_INFO(nullptr, "Create Legacy Transcode Engine"); + //inject_pipeline_operations(2, + create_pipeline( + // 1) Read File + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyTranscodeSession &my_sess = static_cast(sess); + if (!my_sess.data_provider) { + my_sess.last_status = MFX_ERR_MORE_DATA; + return ExecutionStatus::Continue; + } + + my_sess.last_status = MFX_ERR_NONE; + if (!my_sess.data_provider->fetch_bitstream_data(my_sess.stream)) { + my_sess.last_status = MFX_ERR_MORE_DATA; + my_sess.data_provider.reset(); //close source + } + return ExecutionStatus::Continue; + }, + // 2) enqueue ASYNC decode operation + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyTranscodeSession &my_sess = static_cast(sess); + + // prepare sync object for new surface + LegacyTranscodeSession::op_handle_t sync_pair{}; + + // enqueue decode operation with current session surface + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + (my_sess.data_provider || (my_sess.stream && my_sess.stream->DataLength)) + ? my_sess.stream.get() + + : nullptr, /* No more data to read, start decode draining mode*/ + my_sess.procesing_surface_ptr.lock()->get_handle(), + &sync_pair.second, + &sync_pair.first); + + GAPI_LOG_DEBUG(nullptr, "START decode: " << + ", sync id: " << + sync_pair.first << + ", dec in surface: " << + my_sess.procesing_surface_ptr.lock()->get_handle() << + ", dec out surface: " << sync_pair.second << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); + + // process wait-like statuses in-place: + // It had better to use up all VPL decoding resources in pipeline + // as soon as possible. So waiting more free-surface or device free + while (my_sess.last_status == MFX_ERR_MORE_SURFACE || + my_sess.last_status == MFX_WRN_DEVICE_BUSY) { + try { + if (my_sess.last_status == MFX_ERR_MORE_SURFACE) { + my_sess.swap_surface(*this); + } + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + my_sess.stream.get(), + my_sess.procesing_surface_ptr.lock()->get_handle(), + &sync_pair.second, + &sync_pair.first); + + } catch (const std::runtime_error& ex) { + // NB: not an error, yield CPU ticks to check + // surface availability at a next phase. + // But print WARNING to notify user about pipeline stuck + GAPI_LOG_WARNING(nullptr, "[" << my_sess.session << + "] has no surface, reason: " << + ex.what()); + break; + } + } + + if (my_sess.last_status == MFX_ERR_NONE) { + my_sess.sync_queue.emplace(sync_pair); + } else if (my_sess.last_status != MFX_ERR_MORE_DATA) /* suppress MFX_ERR_MORE_DATA warning */ { + GAPI_LOG_WARNING(nullptr, "decode pending ops count: " << + my_sess.sync_queue.size() << + ", sync id: " << sync_pair.first << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); + } + return ExecutionStatus::Continue; + }, + // 3) transcode + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyTranscodeSession &my_sess = static_cast(sess); + + LegacyDecodeSession::op_handle_t last_op {}; + while (!my_sess.sync_queue.empty()) { + do { + if (!my_sess.vpp_surface_ptr.expired()) { + LegacyDecodeSession::op_handle_t pending_op = my_sess.sync_queue.front(); + GAPI_LOG_DEBUG(nullptr, "pending DEC ops count: " << + my_sess.sync_queue.size() << + ", sync id: " << + pending_op.first << + ", surface: " << + pending_op.second << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); + + my_sess.sync_queue.pop(); + auto *dec_surface = pending_op.second; + auto *vpp_suface = my_sess.vpp_surface_ptr.lock()->get_handle(); + my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session, + dec_surface, + vpp_suface, + nullptr, &pending_op.first); + pending_op.second = vpp_suface; + + GAPI_LOG_DEBUG(nullptr, "START transcode ops count: " << + my_sess.vpp_queue.size() << + ", sync id: " << + pending_op.first << + ", dec surface: " << + dec_surface << + ", trans surface: " << pending_op.second << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); + + if (my_sess.last_status == MFX_ERR_MORE_SURFACE || + my_sess.last_status == MFX_ERR_NONE) { + pending_op.second->Data.Locked++; // TODO -S- workaround + my_sess.vpp_queue.emplace(pending_op); + } + } + + try { + my_sess.swap_transcode_surface(*this); + } catch (const std::runtime_error& ex) { + // NB: not an error, yield CPU ticks to check + // surface availability at a next phase. + // But print WARNING to notify user about pipeline stuck + GAPI_LOG_WARNING(nullptr, "[" << my_sess.session << + "] has no VPP surface, reason: " << + ex.what()); + my_sess.vpp_surface_ptr.reset(); + break; + } + } while(my_sess.last_status == MFX_ERR_MORE_SURFACE); + + if (my_sess.vpp_surface_ptr.expired()) { + // TODO break main loop + break; + } + } + return ExecutionStatus::Continue; + }, + // 4) Wait for ASYNC decode result + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyTranscodeSession& my_sess = static_cast(sess); + do { + if (!my_sess.vpp_queue.empty()) { // FIFO: check the oldest async operation complete + LegacyDecodeSession::op_handle_t& pending_op = my_sess.vpp_queue.front(); + sess.last_status = MFXVideoCORE_SyncOperation(sess.session, pending_op.first, 0); + + GAPI_LOG_DEBUG(nullptr, "pending VPP ops count: " << + my_sess.vpp_queue.size() << + ", sync id: " << + pending_op.first << + ", surface: " << + pending_op.second << + ", status: " << + mfxstatus_to_string(my_sess.last_status)); + + // put frames in ready queue on success + if (MFX_ERR_NONE == sess.last_status) { + on_frame_ready(my_sess, pending_op.second); + } + } + } while (MFX_ERR_NONE == sess.last_status && !my_sess.vpp_queue.empty()); + return ExecutionStatus::Continue; + }, + // 5) Falls back on generic status procesing + [this] (EngineSession& sess) -> ExecutionStatus + { + return this->process_error(sess.last_status, static_cast(sess)); + } + ); +} + +ProcessingEngineBase::session_ptr +VPLLegacyTranscodeEngine::initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) { + // NB: obtain decoder params + VPLLegacyDecodeEngine::SessionParam decode_params = + prepare_session_param(mfx_session, cfg_params, provider); + + + // NB: create transcode params + const auto& mfxDecParams = decode_params.decoder_params.param; + + // NB: create transcode params: Out = In by default, In = initially decoded + mfxVideoParam mfxVPPParams{0}; + mfxVPPParams.vpp.In = mfxDecParams.mfx.FrameInfo; + mfxVPPParams.vpp.Out = mfxVPPParams.vpp.In; + + std::map cfg_vpp_params = + VPLLegacyTranscodeEngine::get_vpp_params(cfg_params); + + // override some in-params + if (set_vpp_param(CfgParam::vpp_in_width_name(), mfxVPPParams.vpp.In.Width, + cfg_vpp_params, mfx_session)) { + mfxVPPParams.vpp.In.Width = ALIGN16(mfxVPPParams.vpp.In.Width); + } + if (set_vpp_param(CfgParam::vpp_in_height_name(), mfxVPPParams.vpp.In.Height, + cfg_vpp_params, mfx_session)) { + mfxVPPParams.vpp.In.Height = ALIGN16(mfxVPPParams.vpp.In.Height); + } + set_vpp_param(CfgParam::vpp_in_crop_x_name(), mfxVPPParams.vpp.In.CropX, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_in_crop_y_name(), mfxVPPParams.vpp.In.CropY, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_in_crop_w_name(), mfxVPPParams.vpp.In.CropW, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_in_crop_h_name(), mfxVPPParams.vpp.In.CropH, + cfg_vpp_params, mfx_session); + + // override out params + set_vpp_param(CfgParam::vpp_out_fourcc_name(), mfxVPPParams.vpp.Out.FourCC, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_chroma_format_name(), mfxVPPParams.vpp.Out.ChromaFormat, + cfg_vpp_params, mfx_session); + if (set_vpp_param(CfgParam::vpp_out_width_name(), mfxVPPParams.vpp.Out.Width, + cfg_vpp_params, mfx_session)) { + mfxVPPParams.vpp.Out.Width = ALIGN16(mfxVPPParams.vpp.Out.Width); + } + if (set_vpp_param(CfgParam::vpp_out_height_name(), mfxVPPParams.vpp.Out.Height, + cfg_vpp_params, mfx_session)) { + mfxVPPParams.vpp.Out.Height = ALIGN16(mfxVPPParams.vpp.Out.Height); + } + set_vpp_param(CfgParam::vpp_out_crop_x_name(), mfxVPPParams.vpp.Out.CropX, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_crop_y_name(), mfxVPPParams.vpp.Out.CropY, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_crop_w_name(), mfxVPPParams.vpp.Out.CropW, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_crop_h_name(), mfxVPPParams.vpp.Out.CropH, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_pic_struct_name(), mfxVPPParams.vpp.Out.PicStruct, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_framerate_n_name(), mfxVPPParams.vpp.Out.FrameRateExtN, + cfg_vpp_params, mfx_session); + set_vpp_param(CfgParam::vpp_out_framerate_d_name(), mfxVPPParams.vpp.Out.FrameRateExtD, + cfg_vpp_params, mfx_session); + + VPLLegacyTranscodeEngine::validate_vpp_param(mfxVPPParams); + + if (mfxDecParams.IOPattern == MFX_IOPATTERN_OUT_VIDEO_MEMORY) { + mfxVPPParams.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY; + } else { + mfxVPPParams.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; + } + GAPI_LOG_INFO(nullptr, "Starting VPP initialization"); + + mfxFrameAllocRequest vppRequests[2]; + memset(&vppRequests, 0, sizeof(mfxFrameAllocRequest) * 2); + mfxStatus sts = MFXVideoVPP_QueryIOSurf(mfx_session, &mfxVPPParams, vppRequests); + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "cannot execute MFXVideoVPP_QueryIOSurf"); + throw std::runtime_error("Cannot execute MFXVideoVPP_QueryIOSurf, error: " + + mfxstatus_to_string(sts)); + } + + // NB: override NumFrameSuggested preallocation size (how many frames we can hold) + // if you see bunch of WARNING about "cannot get free surface from pool" + // and have abundant RAM size then increase `CfgParam::vpp_frames_pool_size_name()` + // to keep more free surfaces in a round. Otherwise VPL decode pipeline will be waiting + // till application is freeing unusable surface on its side. + cv::optional preallocated_frames_count_cfg; + extract_optional_param_by_name(CfgParam::vpp_frames_pool_size_name(), + cfg_params, + preallocated_frames_count_cfg); + if (preallocated_frames_count_cfg.has_value()) { + GAPI_LOG_INFO(nullptr, "Try to use CfgParam \"" << CfgParam::vpp_frames_pool_size_name() << "\": " << + preallocated_frames_count_cfg.value() << ", for session: " << mfx_session); + try_modify_pool_size_request_param(CfgParam::vpp_frames_pool_size_name(), + preallocated_frames_count_cfg.value(), + vppRequests[1]); + + } + + // NB: Assing ID as upper limit descendant to distinguish specific VPP allocation + // from decode allocations witch started from 0: by local module convention + vppRequests[1].AllocId = std::numeric_limits::max(); + + vppRequests[1].Type |= MFX_MEMTYPE_FROM_VPPIN; + VPLAccelerationPolicy::pool_key_t vpp_out_pool_key = + acceleration_policy->create_surface_pool(vppRequests[1], mfxVPPParams.vpp.Out); + + sts = MFXVideoVPP_Init(mfx_session, &mfxVPPParams); + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "cannot Init VPP"); + throw std::runtime_error("Cannot init VPP, error: " + + mfxstatus_to_string(sts)); + } + + // create engine session + TranscoderParams transcoder_param {mfxVPPParams}; + std::shared_ptr sess_ptr = + register_session(mfx_session, + std::move(decode_params.decoder_params), + std::move(transcoder_param), + provider); + + sess_ptr->init_surface_pool(decode_params.decode_pool_key); + sess_ptr->init_transcode_surface_pool(vpp_out_pool_key); + + // prepare working surfaces + sess_ptr->swap_surface(*this); + sess_ptr->swap_transcode_surface(*this); + return sess_ptr; +} + +void VPLLegacyTranscodeEngine::validate_vpp_param(const mfxVideoParam& mfxVPPParams) { + GAPI_LOG_INFO(nullptr, "Starting VPP param validation"); + if (mfxVPPParams.vpp.In.Width < mfxVPPParams.vpp.In.CropW + mfxVPPParams.vpp.In.CropX) { + GAPI_LOG_WARNING(nullptr, "Invalid vonfiguration params: sum \"" << + CfgParam::vpp_in_crop_w_name() << + "\": " << mfxVPPParams.vpp.In.CropW << " and \"" << + CfgParam::vpp_in_crop_x_name() << + "\": " << mfxVPPParams.vpp.In.CropX << + " must be less or equal to \"" << + CfgParam::vpp_in_width_name() << "\": " << + mfxVPPParams.vpp.In.Width); + GAPI_Assert(false && "Invalid VPP params combination: Width & Crop"); + } + + if (mfxVPPParams.vpp.In.Height < mfxVPPParams.vpp.In.CropH + mfxVPPParams.vpp.In.CropY) { + GAPI_LOG_WARNING(nullptr, "Invalid vonfiguration params: sum \"" << + CfgParam::vpp_in_crop_h_name() << + "\": " << mfxVPPParams.vpp.In.CropH << " and \"" << + CfgParam::vpp_in_crop_y_name() << + "\": " << mfxVPPParams.vpp.In.CropY << + " must be less or equal to \"" << + CfgParam::vpp_in_height_name() << "\": " << + mfxVPPParams.vpp.In.Height); + GAPI_Assert(false && "Invalid VPP params combination: Height & Crop"); + } + + if (mfxVPPParams.vpp.Out.Width < mfxVPPParams.vpp.Out.CropW + mfxVPPParams.vpp.Out.CropX) { + GAPI_LOG_WARNING(nullptr, "Invalid vonfiguration params: sum \"" << + CfgParam::vpp_out_crop_w_name() << + "\": " << mfxVPPParams.vpp.Out.CropW << " and \"" << + CfgParam::vpp_out_crop_x_name() << + "\": " << mfxVPPParams.vpp.Out.CropX << + " must be less or equal to \"" << + CfgParam::vpp_out_width_name() << "\": " << + mfxVPPParams.vpp.Out.Width); + GAPI_Assert(false && "Invalid VPP params combination: Width & Crop"); + } + + if (mfxVPPParams.vpp.Out.Height < mfxVPPParams.vpp.Out.CropH + mfxVPPParams.vpp.Out.CropY) { + GAPI_LOG_WARNING(nullptr, "Invalid vonfiguration params: sum \"" << + CfgParam::vpp_out_crop_h_name() << + "\": " << mfxVPPParams.vpp.Out.CropH << " and \"" << + CfgParam::vpp_out_crop_y_name() << + "\": " << mfxVPPParams.vpp.Out.CropY << + " must be less or equal to \"" << + CfgParam::vpp_out_height_name() << "\": " << + mfxVPPParams.vpp.Out.Height); + GAPI_Assert(false && "Invalid VPP params combination: Height & Crop"); + } + + GAPI_LOG_INFO(nullptr, "Finished VPP param validation"); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyTranscodeEngine::execute_op(operation_t& op, EngineSession& sess) { + return op(sess); +} + +void VPLLegacyTranscodeEngine::on_frame_ready(LegacyTranscodeSession& sess, + mfxFrameSurface1* ready_surface) +{ + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "], frame ready"); + + // manage memory ownership rely on acceleration policy + ready_surface->Data.Locked--; // TODO -S- workaround + auto frame_adapter = acceleration_policy->create_frame_adapter(sess.vpp_out_pool_id, + ready_surface); + ready_frames.emplace(cv::MediaFrame(std::move(frame_adapter)), sess.generate_frame_meta()); + + // pop away synced out object + sess.vpp_queue.pop(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp new file mode 100644 index 0000000000..cf0621dd93 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_TRANSCODE_ENGINE_LEGACY_HPP +#define GAPI_STREAMING_ONVPL_TRANSCODE_ENGINE_LEGACY_HPP +#include +#include + +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class LegacyTranscodeSession; +struct IDataProvider; +struct VPLAccelerationPolicy; + +class GAPI_EXPORTS VPLLegacyTranscodeEngine : public VPLLegacyDecodeEngine { +public: + + VPLLegacyTranscodeEngine(std::unique_ptr&& accel); + session_ptr initialize_session(mfxSession mfx_session, + const std::vector& cfg_params, + std::shared_ptr provider) override; + + static std::map get_vpp_params(const std::vector &cfg_params); +private: + ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; + + void on_frame_ready(LegacyTranscodeSession& sess, + mfxFrameSurface1* ready_surface); + void validate_vpp_param(const mfxVideoParam& mfxVPPParams); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.cpp new file mode 100644 index 0000000000..9fcabc7e10 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.cpp @@ -0,0 +1,70 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include "streaming/onevpl/engine/transcode/transcode_session.hpp" +#include "streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" + +#include "logger.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +LegacyTranscodeSession::LegacyTranscodeSession(mfxSession sess, + DecoderParams&& decoder_param, + TranscoderParams&& transcoder_param, + std::shared_ptr provider) : + LegacyDecodeSession(sess, std::move(decoder_param), std::move(provider)), + mfx_transcoder_param(std::move(transcoder_param.param)) +{ +} + +LegacyTranscodeSession::~LegacyTranscodeSession() +{ + GAPI_LOG_INFO(nullptr, "Close Transcode for session: " << session); + MFXVideoVPP_Close(session); +} + +void LegacyTranscodeSession::init_transcode_surface_pool(VPLAccelerationPolicy::pool_key_t key) { + GAPI_Assert(key && "Init transcode pull with empty key"); + vpp_out_pool_id = key; +} + +void LegacyTranscodeSession::swap_transcode_surface(VPLLegacyTranscodeEngine& engine) { + VPLAccelerationPolicy* acceleration_policy = engine.get_accel(); + GAPI_Assert(acceleration_policy && "Empty acceleration_policy"); + try { + auto cand = acceleration_policy->get_free_surface(vpp_out_pool_id).lock(); + + GAPI_LOG_DEBUG(nullptr, "[" << session << "] swap surface" + ", old: " << (!vpp_surface_ptr.expired() + ? vpp_surface_ptr.lock()->get_handle() + : nullptr) << + ", new: "<< cand->get_handle()); + + vpp_surface_ptr = cand; + } catch (const std::runtime_error& ex) { + GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what()); + + // Delegate exception processing on caller + throw; + } +} + +const mfxFrameInfo& LegacyTranscodeSession::get_video_param() const { + return mfx_transcoder_param.vpp.Out; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.hpp new file mode 100644 index 0000000000..aa6f70c587 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/transcode/transcode_session.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_ENGINE_TRANSCODE_SESSION_HPP +#define GAPI_STREAMING_ONVPL_ENGINE_TRANSCODE_SESSION_HPP + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/engine/decode/decode_session.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct IDataProvider; +class Surface; +struct VPLAccelerationPolicy; + +class GAPI_EXPORTS LegacyTranscodeSession : public LegacyDecodeSession { +public: + friend class VPLLegacyTranscodeEngine; + + LegacyTranscodeSession(mfxSession sess, DecoderParams&& decoder_param, + TranscoderParams&& transcoder_param, + std::shared_ptr provider); + ~LegacyTranscodeSession(); + + void init_transcode_surface_pool(VPLAccelerationPolicy::pool_key_t key); + void swap_transcode_surface(VPLLegacyTranscodeEngine& engine); + const mfxFrameInfo& get_video_param() const override; +private: + mfxVideoParam mfx_transcoder_param; + + VPLAccelerationPolicy::pool_key_t vpp_out_pool_id; + std::weak_ptr vpp_surface_ptr; + std::queue vpp_queue; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_ENGINE_TRANSCODE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp index cfa1245916..10171999a0 100644 --- a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp @@ -18,7 +18,7 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { -struct FileDataProvider : public IDataProvider { +struct GAPI_EXPORTS FileDataProvider : public IDataProvider { using file_ptr = std::unique_ptr; FileDataProvider(const std::string& file_path, diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index fd2a401957..d1ff15b06d 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -8,6 +8,7 @@ #include #include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp" #include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" #include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" #include "streaming/onevpl/utils.hpp" @@ -106,6 +107,20 @@ GSource::Priv::Priv(std::shared_ptr provider, GAPI_Assert(false && "MFXSetConfigFilterProperty failed"); } + mfx_param.Type = MFX_VARIANT_TYPE_U32; + mfx_param.Data.U32 = MFX_EXTBUFF_VPP_SCALING; + sts = MFXSetConfigFilterProperty(cfg_inst, + (mfxU8 *)"mfxImplDescription.mfxVPPDescription.filter.FilterFourCC", + mfx_param); + + if (sts != MFX_ERR_NONE ) + { + GAPI_LOG_WARNING(nullptr, "MFXSetConfigFilterProperty failed, error: " << + mfxstatus_to_string(sts) << + " - for \"mfxImplDescription.mfxVPPDescription.filter.FilterFourCC\""); + GAPI_Assert(false && "MFXSetConfigFilterProperty failed"); + } + ++cfg_param_it; } @@ -204,7 +219,12 @@ GSource::Priv::Priv(std::shared_ptr provider, "GSource mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION" " - is not implemented"); } else { - engine.reset(new VPLLegacyDecodeEngine(std::move(acceleration))); + const auto& transcode_params = VPLLegacyTranscodeEngine::get_vpp_params(preferred_params); + if (!transcode_params.empty()) { + engine.reset(new VPLLegacyTranscodeEngine(std::move(acceleration))); + } else { + engine.reset(new VPLLegacyDecodeEngine(std::move(acceleration))); + } } } @@ -212,13 +232,13 @@ GSource::Priv::Priv(std::shared_ptr provider, auto engine_session_ptr = engine->initialize_session(mfx_session, cfg_params, provider); - const mfxVideoParam& video_param = engine_session_ptr->get_video_param(); + const mfxFrameInfo& video_param = engine_session_ptr->get_video_param(); // set valid description description.size = cv::Size { - video_param.mfx.FrameInfo.Width, - video_param.mfx.FrameInfo.Height}; - switch(video_param.mfx.FrameInfo.FourCC) { + video_param.Width, + video_param.Height}; + switch(video_param.FourCC) { case MFX_FOURCC_I420: throw std::runtime_error("Cannot parse GMetaArg description: MediaFrame doesn't support I420 type"); case MFX_FOURCC_NV12: @@ -226,7 +246,7 @@ GSource::Priv::Priv(std::shared_ptr provider, break; default: throw std::runtime_error("Cannot parse GMetaArg description: MediaFrame unknown 'fmt' type: " + - std::to_string(video_param.mfx.FrameInfo.FourCC)); + std::to_string(video_param.FourCC)); } description_is_valid = true; diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 36711bf9a0..76a66a63f4 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -73,8 +73,8 @@ const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type); mfxU32 cstr_to_mfx_version(const char* cstr); -std::string mfxstatus_to_string(int64_t err); -std::string mfxstatus_to_string(mfxStatus err); +std::string GAPI_EXPORTS mfxstatus_to_string(int64_t err); +std::string GAPI_EXPORTS mfxstatus_to_string(mfxStatus err); std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc); diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index c62f58eecf..51fb9f276a 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -29,6 +29,7 @@ #ifdef HAVE_ONEVPL #include +#include "streaming/onevpl/file_data_provider.hpp" #include "streaming/onevpl/cfg_param_device_selector.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" @@ -37,8 +38,15 @@ #include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" #include "streaming/onevpl/accelerators/dx11_alloc_resource.hpp" #include "streaming/onevpl/accelerators/utils/shared_lock.hpp" -#include "streaming/onevpl/engine/processing_engine_base.hpp" -#include "streaming/onevpl/engine/engine_session.hpp" +#define private public +#define protected public +#include "streaming/onevpl/engine/transcode/transcode_engine_legacy.hpp" +#include "streaming/onevpl/engine/transcode/transcode_session.hpp" +#undef protected +#undef private +#include "logger.hpp" + +#define ALIGN16(value) (((value + 15) >> 4) << 4) namespace opencv_test { @@ -63,9 +71,9 @@ struct TestProcessingSession : public cv::gapi::wip::onevpl::EngineSession { EngineSession(mfx_session, {}) { } - const mfxVideoParam& get_video_param() const override { + const mfxFrameInfo& get_video_param() const override { static mfxVideoParam empty; - return empty; + return empty.mfx.FrameInfo; } }; @@ -581,7 +589,7 @@ TEST(OneVPL_Source_DX11_Accel, Init) // Allocate surfaces for decoder VPLAccelerationPolicy::pool_key_t key = accel.create_surface_pool(request, - mfxDecParams); + mfxDecParams.mfx.FrameInfo); auto cand_surface = accel.get_free_surface(key).lock(); sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); @@ -594,6 +602,212 @@ TEST(OneVPL_Source_DX11_Accel, Init) MFXClose(mfx_session); MFXUnload(mfx_handle); } + +TEST(OneVPL_Source_DX11_Accel_VPL, Init) +{ + using namespace cv::gapi::wip::onevpl; + + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); + std::unique_ptr acceleration_policy (new VPLDX11AccelerationPolicy(std::make_shared(cfg_params_w_dx11))); + + mfxLoader mfx_handle = MFXLoad(); + + mfxConfig cfg_inst_0 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_0); + mfxVariant mfx_param_0; + mfx_param_0.Type = MFX_VARIANT_TYPE_U32; + mfx_param_0.Data.U32 = MFX_IMPL_TYPE_HARDWARE; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_0,(mfxU8 *)CfgParam::implementation_name(), + mfx_param_0), MFX_ERR_NONE); + + mfxConfig cfg_inst_1 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_1); + mfxVariant mfx_param_1; + mfx_param_1.Type = MFX_VARIANT_TYPE_U32; + mfx_param_1.Data.U32 = MFX_ACCEL_MODE_VIA_D3D11; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_1,(mfxU8 *)CfgParam::acceleration_mode_name(), + mfx_param_1), MFX_ERR_NONE); + + mfxConfig cfg_inst_2 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_2); + mfxVariant mfx_param_2; + mfx_param_2.Type = MFX_VARIANT_TYPE_U32; + mfx_param_2.Data.U32 = MFX_CODEC_HEVC; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_2,(mfxU8 *)CfgParam::decoder_id_name(), + mfx_param_2), MFX_ERR_NONE); + + mfxConfig cfg_inst_3 = MFXCreateConfig(mfx_handle); + EXPECT_TRUE(cfg_inst_3); + mfxVariant mfx_param_3; + mfx_param_3.Type = MFX_VARIANT_TYPE_U32; + mfx_param_3.Data.U32 = MFX_EXTBUFF_VPP_SCALING; + EXPECT_EQ(MFXSetConfigFilterProperty(cfg_inst_3, + (mfxU8 *)"mfxImplDescription.mfxVPPDescription.filter.FilterFourCC", + mfx_param_3), MFX_ERR_NONE); + // create session + mfxSession mfx_session{}; + mfxStatus sts = MFXCreateSession(mfx_handle, 0, &mfx_session); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // assign acceleration + EXPECT_NO_THROW(acceleration_policy->init(mfx_session)); + + // create proper bitstream + std::string file_path = findDataFile("highgui/video/big_buck_bunny.h265"); + std::shared_ptr data_provider(new FileDataProvider(file_path, + {CfgParam::create_decoder_id(MFX_CODEC_HEVC)})); + IDataProvider::mfx_codec_id_type decoder_id_name = data_provider->get_mfx_codec_id(); + + // Prepare video param + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = decoder_id_name; + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY; + + // try fetch & decode input data + sts = MFX_ERR_NONE; + std::shared_ptr bitstream{}; + do { + EXPECT_TRUE(data_provider->fetch_bitstream_data(bitstream)); + sts = MFXVideoDECODE_DecodeHeader(mfx_session, bitstream.get(), &mfxDecParams); + EXPECT_TRUE(MFX_ERR_NONE == sts || MFX_ERR_MORE_DATA == sts); + } while (sts == MFX_ERR_MORE_DATA && !data_provider->empty()); + + EXPECT_EQ(MFX_ERR_NONE, sts); + + mfxFrameAllocRequest request{}; + memset(&request, 0, sizeof(request)); + sts = MFXVideoDECODE_QueryIOSurf(mfx_session, &mfxDecParams, &request); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // Allocate surfaces for decoder + request.Type |= MFX_MEMTYPE_EXTERNAL_FRAME | MFX_MEMTYPE_FROM_DECODE | MFX_MEMTYPE_FROM_VPPIN; + VPLAccelerationPolicy::pool_key_t decode_pool_key = acceleration_policy->create_surface_pool(request, + mfxDecParams.mfx.FrameInfo); + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + EXPECT_EQ(MFX_ERR_NONE, sts); + + // initialize VPLL + mfxU16 vppOutImgWidth = 672; + mfxU16 vppOutImgHeight = 382; + + mfxVideoParam mfxVPPParams{0}; + mfxVPPParams.vpp.In = mfxDecParams.mfx.FrameInfo; + + mfxVPPParams.vpp.Out.FourCC = MFX_FOURCC_NV12; + mfxVPPParams.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV420; + mfxVPPParams.vpp.Out.Width = ALIGN16(vppOutImgWidth); + mfxVPPParams.vpp.Out.Height = ALIGN16(vppOutImgHeight); + mfxVPPParams.vpp.Out.CropX = 0; + mfxVPPParams.vpp.Out.CropY = 0; + mfxVPPParams.vpp.Out.CropW = vppOutImgWidth; + mfxVPPParams.vpp.Out.CropH = vppOutImgHeight; + mfxVPPParams.vpp.Out.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; + mfxVPPParams.vpp.Out.FrameRateExtN = 30; + mfxVPPParams.vpp.Out.FrameRateExtD = 1; + + mfxVPPParams.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY; + + mfxFrameAllocRequest vppRequests[2]; + memset(&vppRequests, 0, sizeof(mfxFrameAllocRequest) * 2); + EXPECT_EQ(MFXVideoVPP_QueryIOSurf(mfx_session, &mfxVPPParams, vppRequests), MFX_ERR_NONE); + + vppRequests[1].AllocId = 666; + VPLAccelerationPolicy::pool_key_t vpp_out_pool_key = + acceleration_policy->create_surface_pool(vppRequests[1], mfxVPPParams.vpp.Out); + EXPECT_EQ(MFXVideoVPP_Init(mfx_session, &mfxVPPParams), MFX_ERR_NONE); + + // finalize session creation + DecoderParams d_param{bitstream, mfxDecParams}; + TranscoderParams t_param{mfxVPPParams}; + VPLLegacyTranscodeEngine engine(std::move(acceleration_policy)); + std::shared_ptr sess_ptr = + engine.register_session( + mfx_session, + std::move(d_param), + std::move(t_param), + data_provider); + + sess_ptr->init_surface_pool(decode_pool_key); + sess_ptr->init_transcode_surface_pool(vpp_out_pool_key); + + // prepare working surfaces + sess_ptr->swap_surface(engine); + sess_ptr->swap_transcode_surface(engine); + + // launch pipeline + LegacyTranscodeSession & my_sess = *sess_ptr; + { + if (!my_sess.data_provider) { + my_sess.last_status = MFX_ERR_MORE_DATA; + } else { + my_sess.last_status = MFX_ERR_NONE; + if (!my_sess.data_provider->fetch_bitstream_data(my_sess.stream)) { + my_sess.last_status = MFX_ERR_MORE_DATA; + my_sess.data_provider.reset(); //close source + } + } + + // 2) enqueue ASYNC decode operation + // prepare sync object for new surface + LegacyTranscodeSession::op_handle_t sync_pair{}; + + // enqueue decode operation with current session surface + { + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + (my_sess.data_provider || (my_sess.stream && my_sess.stream->DataLength)) + ? my_sess.stream.get() + + : nullptr, /* No more data to read, start decode draining mode*/ + my_sess.procesing_surface_ptr.lock()->get_handle(), + &sync_pair.second, + &sync_pair.first); + + // process wait-like statuses in-place: + // It had better to use up all VPL decoding resources in pipeline + // as soon as possible. So waiting more free-surface or device free + while (my_sess.last_status == MFX_ERR_MORE_SURFACE || + my_sess.last_status == MFX_WRN_DEVICE_BUSY) { + try { + if (my_sess.last_status == MFX_ERR_MORE_SURFACE) { + my_sess.swap_surface(engine); + } + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + my_sess.stream.get(), + my_sess.procesing_surface_ptr.lock()->get_handle(), + &sync_pair.second, + &sync_pair.first); + + } catch (const std::runtime_error&) { + // NB: not an error, yield CPU ticks to check + // surface availability at a next phase. + break; + } + } + } + // 4) transcode + { + auto *dec_surface = sync_pair.second; + if(my_sess.vpp_surface_ptr.lock()) + { + mfxFrameSurface1* out_surf = my_sess.vpp_surface_ptr.lock()->get_handle(); + my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session, dec_surface, + out_surf, + nullptr, &sync_pair.first); + sync_pair.second = out_surf; + + my_sess.last_status = MFXVideoCORE_SyncOperation(my_sess.session, sync_pair.first, 11000); + } + try { + my_sess.swap_transcode_surface(engine); + } catch (... ) { + my_sess.vpp_surface_ptr.reset(); + } + } + } +} #endif // HAVE_DIRECTX #endif // HAVE_D3D11