Merge pull request #20601 from sivanov-work:surf_bk_test

G-API: oneVPL (simplification) added surface & frame adapter

* added surface & frame adapter

* Add FrameAdapter unut tests

* Fix compilation after rebase

* Fix compilation tests

* Apply some review comments

* Fix compile warning

* Revert back CV_Func usage

* Apply some comments

modules/gapi/CMakeLists.txt

@@ -169,6 +169,8 @@ set(gapi_srcs
     src/streaming/onevpl/file_data_provider.cpp
     src/streaming/onevpl/cfg_params.cpp
     src/streaming/onevpl/data_provider_interface_exception.cpp
+    src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp
+    src/streaming/onevpl/accelerators/surface/surface.cpp
 
     # Utils (ITT tracing)
     src/utils/itt.cpp

modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp

@@ -0,0 +1,59 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP
+#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP
+
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+#include <opencv2/gapi/media.hpp>
+
+#ifdef HAVE_ONEVPL
+#include <vpl/mfxvideo.h>
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+class Surface;
+struct VPLAccelerationPolicy
+{
+    virtual ~VPLAccelerationPolicy() {}
+
+    using pool_key_t = void*;
+
+    using session_t = mfxSession;
+    using surface_t = Surface;
+    using surface_ptr_t = std::shared_ptr<surface_t>;
+    using surface_weak_ptr_t = std::weak_ptr<surface_t>;
+    using surface_ptr_ctr_t = std::function<surface_ptr_t(std::shared_ptr<void> out_buf_ptr,
+                                                          size_t out_buf_ptr_offset,
+                                                          size_t out_buf_ptr_size)>;
+
+    virtual void init(session_t session) = 0;
+    virtual void deinit(session_t session) = 0;
+
+    // Limitation: cannot give guarantee in succesful memory realloccation
+    // for existing workspace in existing pool (see realloc)
+    // thus it is not implemented,
+    // PLEASE provide initial memory area large enough
+    virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0;
+
+    virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0;
+    virtual size_t get_free_surface_count(pool_key_t key) const = 0;
+    virtual size_t get_surface_count(pool_key_t key) const = 0;
+
+    virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key,
+                                                            mfxFrameSurface1* surface) = 0;
+};
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#endif // HAVE_ONEVPL
+#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP

modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp

@@ -0,0 +1,117 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp"
+#include "streaming/onevpl/accelerators/surface/surface.hpp"
+#include "logger.hpp"
+
+#ifdef HAVE_ONEVPL
+
+#if (MFX_VERSION >= 2000)
+#include <vpl/mfxdispatcher.h>
+#endif
+
+#include <vpl/mfx.h>
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr<Surface> surface):
+    parent_surface_ptr(surface) {
+
+    GAPI_Assert(parent_surface_ptr && "Surface is nullptr");
+    parent_surface_ptr->obtain_lock();
+
+    GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() <<
+                            ", w: " << parent_surface_ptr->get_info().Width <<
+                            ", h: " << parent_surface_ptr->get_info().Height <<
+                            ", p: " << parent_surface_ptr->get_data().Pitch);
+}
+
+VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() {
+
+    // Each VPLMediaFrameCPUAdapter releases mfx surface counter
+    // The last VPLMediaFrameCPUAdapter releases shared Surface pointer
+    // The last surface pointer releases workspace memory
+    parent_surface_ptr->release_lock();
+}
+
+cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const {
+    GFrameDesc desc;
+    const Surface::info_t& info = parent_surface_ptr->get_info();
+    switch(info.FourCC)
+    {
+        case MFX_FOURCC_I420:
+            throw std::runtime_error("MediaFrame doesn't support I420 type");
+            break;
+        case MFX_FOURCC_NV12:
+            desc.fmt = MediaFormat::NV12;
+            break;
+        default:
+            throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC));
+    }
+
+    desc.size = cv::Size{info.Width, info.Height};
+    return desc;
+}
+
+MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) {
+    const Surface::data_t& data = parent_surface_ptr->get_data();
+    const Surface::info_t& info = parent_surface_ptr->get_info();
+    using stride_t = typename cv::MediaFrame::View::Strides::value_type;
+
+    stride_t pitch = static_cast<stride_t>(data.Pitch);
+    switch(info.FourCC) {
+        case MFX_FOURCC_I420:
+        {
+            cv::MediaFrame::View::Ptrs pp = {
+                data.Y,
+                data.U,
+                data.V,
+                nullptr
+                };
+            cv::MediaFrame::View::Strides ss = {
+                    pitch,
+                    pitch / 2,
+                    pitch / 2, 0u
+                };
+            return cv::MediaFrame::View(std::move(pp), std::move(ss));
+        }
+        case MFX_FOURCC_NV12:
+        {
+            cv::MediaFrame::View::Ptrs pp = {
+                data.Y,
+                data.UV, nullptr, nullptr
+                };
+            cv::MediaFrame::View::Strides ss = {
+                    pitch,
+                    pitch, 0u, 0u
+                };
+            return cv::MediaFrame::View(std::move(pp), std::move(ss));
+        }
+            break;
+        default:
+            throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC));
+    }
+}
+
+cv::util::any VPLMediaFrameCPUAdapter::blobParams() const {
+    GAPI_Assert("VPLMediaFrameCPUAdapter::blobParams() is not implemented");
+    return {};
+}
+
+void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) {
+    GAPI_Assert("VPLMediaFrameCPUAdapter::serialize() is not implemented");
+}
+
+void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) {
+    GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented");
+}
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+#endif // HAVE_ONEVPL

modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp

@@ -0,0 +1,41 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP
+#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP
+#include <memory>
+
+#include <opencv2/gapi/media.hpp>
+#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS
+
+#ifdef HAVE_ONEVPL
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+class Surface;
+class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter {
+public:
+    // GAPI_EXPORTS for tests
+    GAPI_EXPORTS explicit VPLMediaFrameCPUAdapter(std::shared_ptr<Surface> assoc_surface);
+    GAPI_EXPORTS ~VPLMediaFrameCPUAdapter();
+    cv::GFrameDesc meta() const override;
+    MediaFrame::View access(MediaFrame::Access) override;
+
+    // The default implementation does nothing
+    cv::util::any blobParams() const override;
+    void serialize(cv::gapi::s11n::IOStream&) override;
+    void deserialize(cv::gapi::s11n::IIStream&) override;
+private:
+    std::shared_ptr<Surface> parent_surface_ptr;
+};
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#endif // HAVE_ONEVPL
+#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP

modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp

@@ -0,0 +1,76 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include "streaming/onevpl/accelerators/surface/surface.hpp"
+#include "logger.hpp"
+
+#ifdef HAVE_ONEVPL
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+Surface::Surface(std::unique_ptr<handle_t>&& surf, std::shared_ptr<void> associated_memory) :
+    workspace_memory_ptr(associated_memory),
+    mfx_surface(std::move(surf)),
+    mirrored_locked_count() {
+
+    GAPI_Assert(mfx_surface && "Surface is nullptr");
+    mirrored_locked_count.store(mfx_surface->Data.Locked);
+    GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface <<
+                            ", locked count: " << mfx_surface->Data.Locked);
+}
+
+Surface::~Surface() {
+    GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface <<
+                            ", worspace memory counter: " << workspace_memory_ptr.use_count());
+}
+
+std::shared_ptr<Surface> Surface::create_surface(std::unique_ptr<handle_t>&& surf,
+                                                 std::shared_ptr<void> accociated_memory) {
+    surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)};
+    return ret;
+}
+
+Surface::handle_t* Surface::get_handle() const {
+    return mfx_surface.get();
+}
+
+const Surface::info_t& Surface::get_info() const {
+    return mfx_surface->Info;
+}
+
+const Surface::data_t& Surface::get_data() const {
+    return mfx_surface->Data;
+}
+
+size_t Surface::get_locks_count() const {
+    return mirrored_locked_count.load();
+}
+
+size_t Surface::obtain_lock() {
+    size_t locked_count = mirrored_locked_count.fetch_add(1);
+    GAPI_Assert(locked_count < std::numeric_limits<mfxU16>::max() && "Too many references ");
+    mfx_surface->Data.Locked = static_cast<mfxU16>(locked_count + 1);
+    GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() <<
+                            ", locked times: " << locked_count + 1);
+    return locked_count; // return preceding value
+}
+
+size_t Surface::release_lock() {
+    size_t locked_count = mirrored_locked_count.fetch_sub(1);
+    GAPI_Assert(locked_count < std::numeric_limits<mfxU16>::max() && "Too many references ");
+    GAPI_Assert(locked_count && "Surface lock counter is invalid");
+    mfx_surface->Data.Locked = static_cast<mfxU16>(locked_count - 1);
+    GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() <<
+                            ", locked times: " << locked_count - 1);
+    return locked_count; // return preceding value
+}
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#endif // HAVE_ONEVPL

modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp

@@ -0,0 +1,102 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP
+#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP
+
+#include <atomic>
+#include <memory>
+
+#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS
+
+#ifdef HAVE_ONEVPL
+#if (MFX_VERSION >= 2000)
+#include <vpl/mfxdispatcher.h>
+#endif
+
+#include <vpl/mfx.h>
+
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+/**
+ * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`.
+ *
+ * Surface has no own memory and shares accelerator allocated memory using reference counter semantics.
+ * So it lives till a last memory consumer (surface/accelerator/media frame) lives.
+ * This approach allows to support different scenarious in releasing allocated memory
+ *
+ * VPL surface `mfxFrameSurface1` support Lock-Free semantics and application MUST NOT operate with a
+ * surface in locked state. But VPL inner counter is not threadsafe so it would be failed in any concurrent scenario.
+ * std::atomic counter introduced in a way to overcome this problem.
+ * But only few scenarious for concurrency are supported here because it is not assumed to implement entire Surface in
+ * for a fully multithread approach.
+ * Supported concurrency scenarios deal only with transaction pair: @ref Surface::get_locks_count() against
+ * @ref Surface::release_lock() - which may be called from different threads. On the other hand @ref Surface::get_locks_count() against
+ * @ref Surface::obtain_lock() happens in single thread only. Surface doesn't support shared ownership that
+ * because it doesn't require thread safe guarantee between transactions:
+ * - @ref Surface::obtain_lock() against @ref Surface::obtain_lock()
+ * - @ref Surface::obtain_lock() against @ref Surface::release_lock()
+ * - @ref Surface::release_lock() against @ref Surface::release_lock()
+ */
+class Surface {
+    using handle_t = mfxFrameSurface1;
+
+    std::shared_ptr<void> workspace_memory_ptr;
+    std::unique_ptr<handle_t> mfx_surface;
+    std::atomic<size_t> mirrored_locked_count;
+public:
+    using info_t = mfxFrameInfo;
+    using data_t = mfxFrameData;
+
+    // GAPI_EXPORTS for tests
+    GAPI_EXPORTS static std::shared_ptr<Surface> create_surface(std::unique_ptr<handle_t>&& surf,
+                                                                std::shared_ptr<void> accociated_memory);
+    GAPI_EXPORTS ~Surface();
+
+    GAPI_EXPORTS handle_t* get_handle() const;
+    GAPI_EXPORTS const info_t& get_info() const;
+    GAPI_EXPORTS const data_t& get_data() const;
+
+    /**
+     * Extract value thread-safe lock counter (see @ref Surface description).
+     * It's usual situation that counter may be instantly decreased in other thread after this method called.
+     * We need instantaneous value. This method syncronized in inter-threading way with @ref Surface::release_lock()
+     *
+     * @return fetched locks count.
+     */
+    GAPI_EXPORTS size_t get_locks_count() const;
+
+    /**
+     * Atomically increase value of thread-safe lock counter (see @ref Surface description).
+     * This method is single-threaded happens-after @ref Surface::get_locks_count() and
+     * multi-threaded happens-before @ref Surface::release_lock()
+     *
+     * @return locks count just before its increasing.
+     */
+    GAPI_EXPORTS size_t obtain_lock();
+
+    /**
+     * Atomically decrease value of thread-safe lock counter (see @ref Surface description).
+     * This method is synchronized with @ref Surface::get_locks_count() and
+     * multi-threaded happens-after @ref Surface::obtain_lock()
+     *
+     * @return locks count just before its decreasing.
+     */
+    GAPI_EXPORTS size_t release_lock();
+private:
+    Surface(std::unique_ptr<handle_t>&& surf, std::shared_ptr<void> accociated_memory);
+};
+
+using surface_ptr_t = std::shared_ptr<Surface>;
+using surface_weak_ptr_t = std::weak_ptr<Surface>;
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+#endif // HAVE_ONEVPL
+#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP

modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp

@@ -0,0 +1,185 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+
+#include "../test_precomp.hpp"
+
+#include "../common/gapi_tests_common.hpp"
+
+#include <future>
+
+#include <opencv2/gapi/cpu/core.hpp>
+#include <opencv2/gapi/cpu/imgproc.hpp>
+
+#include <opencv2/gapi/fluid/core.hpp>
+#include <opencv2/gapi/fluid/imgproc.hpp>
+#include <opencv2/gapi/fluid/gfluidkernel.hpp>
+
+#include <opencv2/gapi/ocl/core.hpp>
+#include <opencv2/gapi/ocl/imgproc.hpp>
+
+#include <opencv2/gapi/streaming/cap.hpp>
+#include <opencv2/gapi/streaming/desync.hpp>
+#include <opencv2/gapi/streaming/format.hpp>
+
+#include <opencv2/gapi/streaming/onevpl/source.hpp>
+
+#ifdef HAVE_ONEVPL
+#include "streaming/onevpl/accelerators/surface/surface.hpp"
+#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp"
+
+namespace opencv_test
+{
+namespace
+{
+TEST(OneVPL_Source_Surface, InitSurface)
+{
+    using namespace cv::gapi::wip;
+
+    // create raw MFX handle
+    std::unique_ptr<mfxFrameSurface1> handle(new mfxFrameSurface1{});
+    mfxFrameSurface1 *mfx_core_handle = handle.get();
+
+    // create preallocate surface memory: empty for test
+    std::shared_ptr<void> associated_memory {};
+    auto surf = Surface::create_surface(std::move(handle), associated_memory);
+
+    // check self consistency
+    EXPECT_EQ(reinterpret_cast<void*>(surf->get_handle()),
+              reinterpret_cast<void*>(mfx_core_handle));
+    EXPECT_EQ(surf->get_locks_count(), 0);
+    EXPECT_EQ(surf->obtain_lock(), 0);
+    EXPECT_EQ(surf->get_locks_count(), 1);
+    EXPECT_EQ(surf->release_lock(), 1);
+    EXPECT_EQ(surf->get_locks_count(), 0);
+}
+
+TEST(OneVPL_Source_Surface, ConcurrentLock)
+{
+    using namespace cv::gapi::wip;
+
+    // create raw MFX handle
+    std::unique_ptr<mfxFrameSurface1> handle(new mfxFrameSurface1{});
+
+    // create preallocate surface memory: empty for test
+    std::shared_ptr<void> associated_memory {};
+    auto surf = Surface::create_surface(std::move(handle), associated_memory);
+
+    // check self consistency
+    EXPECT_EQ(surf->get_locks_count(), 0);
+
+    // MFX internal limitation: do not exceede U16 range
+    // so I16 is using here
+    int16_t lock_counter = std::numeric_limits<int16_t>::max() - 1;
+    std::promise<void> barrier;
+    std::future<void> sync = barrier.get_future();
+
+
+    std::thread worker_thread([&barrier, surf, lock_counter] () {
+        barrier.set_value();
+
+        // concurrent lock
+        for (int16_t i = 0; i < lock_counter; i ++) {
+            surf->obtain_lock();
+        }
+    });
+    sync.wait();
+
+    // concurrent lock
+    for (int16_t i = 0; i < lock_counter; i ++) {
+            surf->obtain_lock();
+    }
+
+    worker_thread.join();
+    EXPECT_EQ(surf->get_locks_count(), lock_counter * 2);
+}
+
+TEST(OneVPL_Source_Surface, MemoryLifeTime)
+{
+    using namespace cv::gapi::wip;
+
+    // create preallocate surface memory
+    std::unique_ptr<char> preallocated_memory_ptr(new char);
+    std::shared_ptr<void> associated_memory (preallocated_memory_ptr.get(),
+                                             [&preallocated_memory_ptr] (void* ptr) {
+                                                    EXPECT_TRUE(preallocated_memory_ptr);
+                                                    EXPECT_EQ(preallocated_memory_ptr.get(), ptr);
+                                                    preallocated_memory_ptr.reset();
+                                            });
+
+    // generate surfaces
+    constexpr size_t surface_num = 10000;
+    std::vector<std::shared_ptr<Surface>> surfaces(surface_num);
+    std::generate(surfaces.begin(), surfaces.end(), [surface_num, associated_memory](){
+        std::unique_ptr<mfxFrameSurface1> handle(new mfxFrameSurface1{});
+        return Surface::create_surface(std::move(handle), associated_memory);
+    });
+
+    // destroy surfaces
+    {
+        std::thread deleter_thread([&surfaces]() {
+            surfaces.clear();
+        });
+        deleter_thread.join();
+    }
+
+    // workspace memory must be alive
+    EXPECT_EQ(surfaces.size(), 0);
+    EXPECT_TRUE(associated_memory != nullptr);
+    EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr);
+
+    // generate surfaces again + 1
+    constexpr size_t surface_num_plus_one = 10001;
+    surfaces.resize(surface_num_plus_one);
+    std::generate(surfaces.begin(), surfaces.end(), [surface_num_plus_one, associated_memory](){
+        std::unique_ptr<mfxFrameSurface1> handle(new mfxFrameSurface1{});
+        return Surface::create_surface(std::move(handle), associated_memory);
+    });
+
+    // remember one surface
+    std::shared_ptr<Surface> last_surface = surfaces.back();
+
+    // destroy another surfaces
+    surfaces.clear();
+
+    // destroy associated_memory
+    associated_memory.reset();
+
+    // workspace memory must be still alive
+    EXPECT_EQ(surfaces.size(), 0);
+    EXPECT_TRUE(associated_memory == nullptr);
+    EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr);
+
+    // destroy last surface
+    last_surface.reset();
+
+    // workspace memory must be freed
+    EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr);
+}
+
+TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter)
+{
+    using namespace cv::gapi::wip;
+
+    // create raw MFX handle
+    std::unique_ptr<mfxFrameSurface1> handle(new mfxFrameSurface1{});
+
+    // create preallocate surface memory: empty for test
+    std::shared_ptr<void> associated_memory {};
+    auto surf = Surface::create_surface(std::move(handle), associated_memory);
+
+    // check consistency
+    EXPECT_EQ(surf->get_locks_count(), 0);
+
+    {
+        VPLMediaFrameCPUAdapter adapter(surf);
+        EXPECT_EQ(surf->get_locks_count(), 1);
+    }
+    EXPECT_EQ(surf->get_locks_count(), 0);
+}
+}
+} // namespace opencv_test
+#endif // HAVE_ONEVPL