From ba8f9d8620de9ff6cd0b81ab422a4d5d08f5b4e0 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 20 Sep 2021 16:28:32 +0300 Subject: [PATCH] Merge pull request #20601 from sivanov-work:surf_bk_test G-API: oneVPL (simplification) added surface & frame adapter * added surface & frame adapter * Add FrameAdapter unut tests * Fix compilation after rebase * Fix compilation tests * Apply some review comments * Fix compile warning * Revert back CV_Func usage * Apply some comments --- modules/gapi/CMakeLists.txt | 2 + .../accelerators/accel_policy_interface.hpp | 59 ++++++ .../surface/cpu_frame_adapter.cpp | 117 +++++++++++ .../surface/cpu_frame_adapter.hpp | 41 ++++ .../onevpl/accelerators/surface/surface.cpp | 76 +++++++ .../onevpl/accelerators/surface/surface.hpp | 102 ++++++++++ .../gapi_streaming_vpl_core_test.cpp | 185 ++++++++++++++++++ 7 files changed, 582 insertions(+) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 08dee2f9af..56d051cf0f 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -169,6 +169,8 @@ set(gapi_srcs src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp src/streaming/onevpl/data_provider_interface_exception.cpp + src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp + src/streaming/onevpl/accelerators/surface/surface.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp new file mode 100644 index 0000000000..87b1246d25 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -0,0 +1,59 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +struct VPLAccelerationPolicy +{ + virtual ~VPLAccelerationPolicy() {} + + using pool_key_t = void*; + + using session_t = mfxSession; + using surface_t = Surface; + using surface_ptr_t = std::shared_ptr; + using surface_weak_ptr_t = std::weak_ptr; + using surface_ptr_ctr_t = std::function out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_ptr_size)>; + + virtual void init(session_t session) = 0; + virtual void deinit(session_t session) = 0; + + // Limitation: cannot give guarantee in succesful memory realloccation + // for existing workspace in existing pool (see realloc) + // thus it is not implemented, + // PLEASE provide initial memory area large enough + virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0; + + virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; + virtual size_t get_free_surface_count(pool_key_t key) const = 0; + virtual size_t get_surface_count(pool_key_t key) const = 0; + + virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) = 0; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp new file mode 100644 index 0000000000..3fd959ca8b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -0,0 +1,117 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { + +VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): + parent_surface_ptr(surface) { + + GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); + parent_surface_ptr->obtain_lock(); + + GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << + ", w: " << parent_surface_ptr->get_info().Width << + ", h: " << parent_surface_ptr->get_info().Height << + ", p: " << parent_surface_ptr->get_data().Pitch); +} + +VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { + + // Each VPLMediaFrameCPUAdapter releases mfx surface counter + // The last VPLMediaFrameCPUAdapter releases shared Surface pointer + // The last surface pointer releases workspace memory + parent_surface_ptr->release_lock(); +} + +cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const { + GFrameDesc desc; + const Surface::info_t& info = parent_surface_ptr->get_info(); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + + desc.size = cv::Size{info.Width, info.Height}; + return desc; +} + +MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { + const Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); + using stride_t = typename cv::MediaFrame::View::Strides::value_type; + + stride_t pitch = static_cast(data.Pitch); + switch(info.FourCC) { + case MFX_FOURCC_I420: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.U, + data.V, + nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch / 2, + pitch / 2, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + case MFX_FOURCC_NV12: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.UV, nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } +} + +cv::util::any VPLMediaFrameCPUAdapter::blobParams() const { + GAPI_Assert("VPLMediaFrameCPUAdapter::blobParams() is not implemented"); + return {}; +} + +void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::serialize() is not implemented"); +} + +void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp new file mode 100644 index 0000000000..16111dadaf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#include + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { +public: + // GAPI_EXPORTS for tests + GAPI_EXPORTS explicit VPLMediaFrameCPUAdapter(std::shared_ptr assoc_surface); + GAPI_EXPORTS ~VPLMediaFrameCPUAdapter(); + cv::GFrameDesc meta() const override; + MediaFrame::View access(MediaFrame::Access) override; + + // The default implementation does nothing + cv::util::any blobParams() const override; + void serialize(cv::gapi::s11n::IOStream&) override; + void deserialize(cv::gapi::s11n::IIStream&) override; +private: + std::shared_ptr parent_surface_ptr; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp new file mode 100644 index 0000000000..5d315412e1 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : + workspace_memory_ptr(associated_memory), + mfx_surface(std::move(surf)), + mirrored_locked_count() { + + GAPI_Assert(mfx_surface && "Surface is nullptr"); + mirrored_locked_count.store(mfx_surface->Data.Locked); + GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface << + ", locked count: " << mfx_surface->Data.Locked); +} + +Surface::~Surface() { + GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface << + ", worspace memory counter: " << workspace_memory_ptr.use_count()); +} + +std::shared_ptr Surface::create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory) { + surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)}; + return ret; +} + +Surface::handle_t* Surface::get_handle() const { + return mfx_surface.get(); +} + +const Surface::info_t& Surface::get_info() const { + return mfx_surface->Info; +} + +const Surface::data_t& Surface::get_data() const { + return mfx_surface->Data; +} + +size_t Surface::get_locks_count() const { + return mirrored_locked_count.load(); +} + +size_t Surface::obtain_lock() { + size_t locked_count = mirrored_locked_count.fetch_add(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + mfx_surface->Data.Locked = static_cast(locked_count + 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count + 1); + return locked_count; // return preceding value +} + +size_t Surface::release_lock() { + size_t locked_count = mirrored_locked_count.fetch_sub(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + GAPI_Assert(locked_count && "Surface lock counter is invalid"); + mfx_surface->Data.Locked = static_cast(locked_count - 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count - 1); + return locked_count; // return preceding value +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp new file mode 100644 index 0000000000..3fc30099cf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + + +namespace cv { +namespace gapi { +namespace wip { + +/** + * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. + * + * Surface has no own memory and shares accelerator allocated memory using reference counter semantics. + * So it lives till a last memory consumer (surface/accelerator/media frame) lives. + * This approach allows to support different scenarious in releasing allocated memory + * + * VPL surface `mfxFrameSurface1` support Lock-Free semantics and application MUST NOT operate with a + * surface in locked state. But VPL inner counter is not threadsafe so it would be failed in any concurrent scenario. + * std::atomic counter introduced in a way to overcome this problem. + * But only few scenarious for concurrency are supported here because it is not assumed to implement entire Surface in + * for a fully multithread approach. + * Supported concurrency scenarios deal only with transaction pair: @ref Surface::get_locks_count() against + * @ref Surface::release_lock() - which may be called from different threads. On the other hand @ref Surface::get_locks_count() against + * @ref Surface::obtain_lock() happens in single thread only. Surface doesn't support shared ownership that + * because it doesn't require thread safe guarantee between transactions: + * - @ref Surface::obtain_lock() against @ref Surface::obtain_lock() + * - @ref Surface::obtain_lock() against @ref Surface::release_lock() + * - @ref Surface::release_lock() against @ref Surface::release_lock() + */ +class Surface { + using handle_t = mfxFrameSurface1; + + std::shared_ptr workspace_memory_ptr; + std::unique_ptr mfx_surface; + std::atomic mirrored_locked_count; +public: + using info_t = mfxFrameInfo; + using data_t = mfxFrameData; + + // GAPI_EXPORTS for tests + GAPI_EXPORTS static std::shared_ptr create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory); + GAPI_EXPORTS ~Surface(); + + GAPI_EXPORTS handle_t* get_handle() const; + GAPI_EXPORTS const info_t& get_info() const; + GAPI_EXPORTS const data_t& get_data() const; + + /** + * Extract value thread-safe lock counter (see @ref Surface description). + * It's usual situation that counter may be instantly decreased in other thread after this method called. + * We need instantaneous value. This method syncronized in inter-threading way with @ref Surface::release_lock() + * + * @return fetched locks count. + */ + GAPI_EXPORTS size_t get_locks_count() const; + + /** + * Atomically increase value of thread-safe lock counter (see @ref Surface description). + * This method is single-threaded happens-after @ref Surface::get_locks_count() and + * multi-threaded happens-before @ref Surface::release_lock() + * + * @return locks count just before its increasing. + */ + GAPI_EXPORTS size_t obtain_lock(); + + /** + * Atomically decrease value of thread-safe lock counter (see @ref Surface description). + * This method is synchronized with @ref Surface::get_locks_count() and + * multi-threaded happens-after @ref Surface::obtain_lock() + * + * @return locks count just before its decreasing. + */ + GAPI_EXPORTS size_t release_lock(); +private: + Surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory); +}; + +using surface_ptr_t = std::shared_ptr; +using surface_weak_ptr_t = std::weak_ptr; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp new file mode 100644 index 0000000000..be905029ba --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -0,0 +1,185 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" + +namespace opencv_test +{ +namespace +{ +TEST(OneVPL_Source_Surface, InitSurface) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + mfxFrameSurface1 *mfx_core_handle = handle.get(); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(reinterpret_cast(surf->get_handle()), + reinterpret_cast(mfx_core_handle)); + EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(surf->get_locks_count(), 0); +} + +TEST(OneVPL_Source_Surface, ConcurrentLock) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + // MFX internal limitation: do not exceede U16 range + // so I16 is using here + int16_t lock_counter = std::numeric_limits::max() - 1; + std::promise barrier; + std::future sync = barrier.get_future(); + + + std::thread worker_thread([&barrier, surf, lock_counter] () { + barrier.set_value(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + }); + sync.wait(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + + worker_thread.join(); + EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); +} + +TEST(OneVPL_Source_Surface, MemoryLifeTime) +{ + using namespace cv::gapi::wip; + + // create preallocate surface memory + std::unique_ptr preallocated_memory_ptr(new char); + std::shared_ptr associated_memory (preallocated_memory_ptr.get(), + [&preallocated_memory_ptr] (void* ptr) { + EXPECT_TRUE(preallocated_memory_ptr); + EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + preallocated_memory_ptr.reset(); + }); + + // generate surfaces + constexpr size_t surface_num = 10000; + std::vector> surfaces(surface_num); + std::generate(surfaces.begin(), surfaces.end(), [surface_num, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // destroy surfaces + { + std::thread deleter_thread([&surfaces]() { + surfaces.clear(); + }); + deleter_thread.join(); + } + + // workspace memory must be alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory != nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // generate surfaces again + 1 + constexpr size_t surface_num_plus_one = 10001; + surfaces.resize(surface_num_plus_one); + std::generate(surfaces.begin(), surfaces.end(), [surface_num_plus_one, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // remember one surface + std::shared_ptr last_surface = surfaces.back(); + + // destroy another surfaces + surfaces.clear(); + + // destroy associated_memory + associated_memory.reset(); + + // workspace memory must be still alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory == nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // destroy last surface + last_surface.reset(); + + // workspace memory must be freed + EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); +} + +TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + { + VPLMediaFrameCPUAdapter adapter(surf); + EXPECT_EQ(surf->get_locks_count(), 1); + } + EXPECT_EQ(surf->get_locks_count(), 0); +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL