diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index c1f58ee22b..2ad7be9ad2 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -157,6 +157,9 @@ set(gapi_srcs src/api/s11n.cpp src/backends/common/serialization.cpp + # Streaming backend + src/backends/streaming/gstreamingbackend.cpp + # Python bridge src/backends/ie/bindings_ie.cpp ) diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index 8445746710..e4b2021479 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -33,8 +33,9 @@ #include #include -// Include this file here to avoid cyclic dependency between +// Include these files here to avoid cyclic dependency between // Desync & GKernel & GComputation & GStreamingCompiled. #include +#include #endif // OPENCV_GAPI_HPP diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 5dd70bd2e8..6ddcb7270c 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -101,6 +101,7 @@ public: const cv::Scalar& inVal(int input); cv::Scalar& outValR(int output); // FIXME: Avoid cv::Scalar s = ctx.outValR() + cv::MediaFrame& outFrame(int output); template std::vector& outVecR(int output) // FIXME: the same issue { return outVecRef(output).wref(); @@ -258,6 +259,13 @@ template<> struct get_out return ctx.outValR(idx); } }; +template<> struct get_out +{ + static cv::MediaFrame& get(GCPUContext &ctx, int idx) + { + return ctx.outFrame(idx); + } +}; template struct get_out> { static std::vector& get(GCPUContext &ctx, int idx) diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 0838573b56..d1482da8e4 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -210,6 +210,7 @@ using GRunArgP = util::variant< cv::Mat*, cv::RMat*, cv::Scalar*, + cv::MediaFrame*, cv::detail::VectorRef, cv::detail::OpaqueRef >; diff --git a/modules/gapi/include/opencv2/gapi/gcall.hpp b/modules/gapi/include/opencv2/gapi/gcall.hpp index 511eca1408..8d1b8d6010 100644 --- a/modules/gapi/include/opencv2/gapi/gcall.hpp +++ b/modules/gapi/include/opencv2/gapi/gcall.hpp @@ -11,6 +11,7 @@ #include // GArg #include // GMat #include // GScalar +#include // GFrame #include // GArray #include // GOpaque @@ -41,6 +42,7 @@ public: GMat yield (int output = 0); GMatP yieldP (int output = 0); GScalar yieldScalar(int output = 0); + GFrame yieldFrame (int output = 0); template GArray yieldArray(int output = 0) { diff --git a/modules/gapi/include/opencv2/gapi/gframe.hpp b/modules/gapi/include/opencv2/gapi/gframe.hpp index f555a93aa3..13fd5d6d29 100644 --- a/modules/gapi/include/opencv2/gapi/gframe.hpp +++ b/modules/gapi/include/opencv2/gapi/gframe.hpp @@ -62,6 +62,9 @@ struct GAPI_EXPORTS GFrameDesc static inline GFrameDesc empty_gframe_desc() { return GFrameDesc{}; } /** @} */ +class MediaFrame; +GAPI_EXPORTS GFrameDesc descr_of(const MediaFrame &frame); + GAPI_EXPORTS std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &desc); } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index 0ec7dd07c0..2c44c67a96 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -90,6 +90,10 @@ namespace detail { static inline cv::GOpaque yield(cv::GCall &call, int i) { return call.yieldOpaque(i); } }; + template<> struct Yield + { + static inline cv::GFrame yield(cv::GCall &call, int i) { return call.yieldFrame(i); } + }; //////////////////////////////////////////////////////////////////////////// // Helper classes which brings outputMeta() marshalling to kernel @@ -239,8 +243,6 @@ public: using InArgs = std::tuple; using OutArgs = std::tuple; - static_assert(!cv::detail::contains::value, "Values of GFrame type can't be used as operation outputs"); - static R on(Args... args) { cv::GCall call(GKernel{ K::id() diff --git a/modules/gapi/include/opencv2/gapi/streaming/format.hpp b/modules/gapi/include/opencv2/gapi/streaming/format.hpp new file mode 100644 index 0000000000..8bec5dc813 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/format.hpp @@ -0,0 +1,52 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_GSTREAMING_FORMAT_HPP +#define OPENCV_GAPI_GSTREAMING_FORMAT_HPP + +#include // GKernelPackage + +namespace cv { +namespace gapi { +namespace streaming { + +cv::gapi::GKernelPackage kernels(); +cv::gapi::GBackend backend(); + +// FIXME: Make a generic kernel +G_API_OP(GCopy, , "org.opencv.streaming.copy") +{ + static GFrameDesc outMeta(const GFrameDesc& in) { return in; } +}; + +G_API_OP(GBGR, , "org.opencv.streaming.BGR") +{ + static GMatDesc outMeta(const GFrameDesc& in) { return GMatDesc{CV_8U, 3, in.size}; } +}; + +/** @brief Gets copy from the input frame + +@note Function textual ID is "org.opencv.streaming.copy" + +@param in Input frame +@return Copy of the input frame +*/ +GAPI_EXPORTS cv::GFrame copy(const cv::GFrame& in); + +/** @brief Gets bgr plane from input frame + +@note Function textual ID is "org.opencv.streaming.BGR" + +@param in Input frame +@return Image in BGR format +*/ +GAPI_EXPORTS cv::GMat BGR (const cv::GFrame& in); + +} // namespace streaming +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMING_FORMAT_HPP diff --git a/modules/gapi/src/api/gbackend.cpp b/modules/gapi/src/api/gbackend.cpp index fd4a5eb38b..1e7b8a2a8d 100644 --- a/modules/gapi/src/api/gbackend.cpp +++ b/modules/gapi/src/api/gbackend.cpp @@ -211,6 +211,9 @@ void bindOutArg(Mag& mag, const RcDesc &rc, const GRunArgP &arg, HandleRMat hand } break; } + case GShape::GFRAME: + mag.template slot()[rc.id] = *util::get(arg); + break; case GShape::GARRAY: mag.template slot()[rc.id] = util::get(arg); break; @@ -319,6 +322,9 @@ cv::GRunArgP getObjPtr(Mag& mag, const RcDesc &rc, bool is_umat) // debugging this!!!1 return GRunArgP(const_cast(mag) .template slot().at(rc.id)); + case GShape::GFRAME: + return GRunArgP(&mag.template slot()[rc.id]); + default: util::throw_error(std::logic_error("Unsupported GShape type")); break; @@ -345,6 +351,12 @@ void writeBack(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) break; } + case GShape::GFRAME: + { + *util::get(g_arg) = mag.template slot().at(rc.id); + break; + } + default: util::throw_error(std::logic_error("Unsupported GShape type")); break; diff --git a/modules/gapi/src/api/gcall.cpp b/modules/gapi/src/api/gcall.cpp index 6a2121bd36..618107f346 100644 --- a/modules/gapi/src/api/gcall.cpp +++ b/modules/gapi/src/api/gcall.cpp @@ -69,6 +69,11 @@ cv::detail::GOpaqueU cv::GCall::yieldOpaque(int output) return cv::detail::GOpaqueU(m_priv->m_node, output); } +cv::GFrame cv::GCall::yieldFrame(int output) +{ + return cv::GFrame(m_priv->m_node, output); +} + cv::GCall::Priv& cv::GCall::priv() { return *m_priv; diff --git a/modules/gapi/src/api/gframe.cpp b/modules/gapi/src/api/gframe.cpp index 3f228f1a65..1acaa9b766 100644 --- a/modules/gapi/src/api/gframe.cpp +++ b/modules/gapi/src/api/gframe.cpp @@ -8,6 +8,7 @@ #include "precomp.hpp" #include +#include #include "api/gorigin.hpp" @@ -34,6 +35,10 @@ bool GFrameDesc::operator== (const GFrameDesc &rhs) const { return fmt == rhs.fmt && size == rhs.size; } +GFrameDesc descr_of(const cv::MediaFrame &frame) { + return frame.desc(); +} + std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &d) { os << '['; switch (d.fmt) { diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp index ec7674a14d..0c7c6462ee 100644 --- a/modules/gapi/src/api/gproto.cpp +++ b/modules/gapi/src/api/gproto.cpp @@ -146,6 +146,7 @@ cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp) #endif // !defined(GAPI_STANDALONE) case GRunArgP::index_of(): return GMetaArg(cv::descr_of(*util::get(argp))); case GRunArgP::index_of(): return GMetaArg(descr_of(*util::get(argp))); + case GRunArgP::index_of(): return GMetaArg(descr_of(*util::get(argp))); case GRunArgP::index_of(): return GMetaArg(util::get(argp).descr_of()); case GRunArgP::index_of(): return GMetaArg(util::get(argp).descr_of()); default: util::throw_error(std::logic_error("Unsupported GRunArgP type")); @@ -163,6 +164,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp) case GRunArgP::index_of(): return util::holds_alternative(meta) && util::get(meta).canDescribe(*util::get(argp)); case GRunArgP::index_of(): return meta == GMetaArg(cv::descr_of(*util::get(argp))); + case GRunArgP::index_of(): return meta == GMetaArg(cv::descr_of(*util::get(argp))); case GRunArgP::index_of(): return meta == GMetaArg(util::get(argp).descr_of()); case GRunArgP::index_of(): return meta == GMetaArg(util::get(argp).descr_of()); default: util::throw_error(std::logic_error("Unsupported GRunArgP type")); @@ -288,6 +290,8 @@ const void* cv::gimpl::proto::ptr(const GRunArgP &arg) return cv::util::get(arg).ptr(); case GRunArgP::index_of(): return cv::util::get(arg).ptr(); + case GRunArgP::index_of(): + return static_cast(cv::util::get(arg)); default: util::throw_error(std::logic_error("Unknown GRunArgP type!")); } diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp index af7bd19dd1..66bf27260d 100644 --- a/modules/gapi/src/api/kernels_streaming.cpp +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -7,6 +7,8 @@ #include "precomp.hpp" #include +#include + #include cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { @@ -72,3 +74,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { // connected to a desynchronized data object, and this sole last_written_value // object will feed both branches of the streaming executable. } + +cv::GFrame cv::gapi::streaming::copy(const cv::GFrame& in) { + return cv::gapi::streaming::GCopy::on(in); +} + +cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) { + return cv::gapi::streaming::GBGR::on(in); +} diff --git a/modules/gapi/src/backends/cpu/gcpukernel.cpp b/modules/gapi/src/backends/cpu/gcpukernel.cpp index 0d8d7379b6..4497952c87 100644 --- a/modules/gapi/src/backends/cpu/gcpukernel.cpp +++ b/modules/gapi/src/backends/cpu/gcpukernel.cpp @@ -41,6 +41,11 @@ cv::detail::OpaqueRef& cv::GCPUContext::outOpaqueRef(int output) return util::get(m_results.at(output)); } +cv::MediaFrame& cv::GCPUContext::outFrame(int output) +{ + return *util::get(m_results.at(output)); +} + cv::GCPUKernel::GCPUKernel() { } diff --git a/modules/gapi/src/backends/streaming/gstreamingbackend.cpp b/modules/gapi/src/backends/streaming/gstreamingbackend.cpp new file mode 100644 index 0000000000..d5f042de0b --- /dev/null +++ b/modules/gapi/src/backends/streaming/gstreamingbackend.cpp @@ -0,0 +1,203 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include +#include // throw_error +#include // kernels + +#include "api/gbackend_priv.hpp" +#include "backends/common/gbackend.hpp" + +#include "gstreamingbackend.hpp" +#include "gstreamingkernel.hpp" + +namespace { + +struct StreamingCreateFunction +{ + static const char *name() { return "StreamingCreateFunction"; } + cv::gapi::streaming::CreateActorFunction createActorFunction; +}; + +using StreamingGraph = ade::TypedGraph + < cv::gimpl::Op + , StreamingCreateFunction + >; + +using ConstStreamingGraph = ade::ConstTypedGraph + < cv::gimpl::Op + , StreamingCreateFunction + >; + + +class GStreamingIntrinExecutable final: public cv::gimpl::GIslandExecutable +{ + virtual void run(std::vector &&, + std::vector &&) override { + GAPI_Assert(false && "Not implemented"); + } + + virtual void run(GIslandExecutable::IInput &in, + GIslandExecutable::IOutput &out) override; + + virtual bool allocatesOutputs() const override { return true; } + // Return an empty RMat since we will reuse the input. + // There is no need to allocate and copy 4k image here. + virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; } + + virtual bool canReshape() const override { return true; } + virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override { + // Do nothing here + } + +public: + GStreamingIntrinExecutable(const ade::Graph &, + const std::vector &); + + const ade::Graph& m_g; + cv::gimpl::GModel::ConstGraph m_gm; + cv::gapi::streaming::IActor::Ptr m_actor; +}; + +void GStreamingIntrinExecutable::run(GIslandExecutable::IInput &in, + GIslandExecutable::IOutput &out) +{ + m_actor->run(in, out); +} + +class GStreamingBackendImpl final: public cv::gapi::GBackend::Priv +{ + virtual void unpackKernel(ade::Graph &graph, + const ade::NodeHandle &op_node, + const cv::GKernelImpl &impl) override + { + StreamingGraph gm(graph); + const auto &kimpl = cv::util::any_cast(impl.opaque); + gm.metadata(op_node).set(StreamingCreateFunction{kimpl.createActorFunction}); + } + + virtual EPtr compile(const ade::Graph &graph, + const cv::GCompileArgs &, + const std::vector &nodes) const override + { + return EPtr{new GStreamingIntrinExecutable(graph, nodes)}; + } + + virtual bool controlsMerge() const override + { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override + { + return false; + } +}; + +GStreamingIntrinExecutable::GStreamingIntrinExecutable(const ade::Graph& g, + const std::vector& nodes) + : m_g(g), m_gm(m_g) +{ + using namespace cv::gimpl; + const auto is_op = [this](const ade::NodeHandle &nh) + { + return m_gm.metadata(nh).get().t == NodeType::OP; + }; + + auto it = std::find_if(nodes.begin(), nodes.end(), is_op); + GAPI_Assert(it != nodes.end() && "No operators found for this island?!"); + + ConstStreamingGraph cag(m_g); + m_actor = cag.metadata(*it).get().createActorFunction(); + + // Ensure this the only op in the graph + if (std::any_of(it+1, nodes.end(), is_op)) + { + cv::util::throw_error + (std::logic_error + ("Internal error: Streaming subgraph has multiple operations")); + } +} + +} // anonymous namespace + +cv::gapi::GBackend cv::gapi::streaming::backend() +{ + static cv::gapi::GBackend this_backend(std::make_shared()); + return this_backend; +} + +cv::gapi::GKernelPackage cv::gapi::streaming::kernels() +{ + return cv::gapi::kernels(); +} + +void cv::gimpl::Copy::Actor::run(cv::gimpl::GIslandExecutable::IInput &in, + cv::gimpl::GIslandExecutable::IOutput &out) +{ + while (true) + { + const auto in_msg = in.get(); + if (cv::util::holds_alternative(in_msg)) + { + out.post(cv::gimpl::EndOfStream{}); + return; + } + + const cv::GRunArgs &in_args = cv::util::get(in_msg); + GAPI_Assert(in_args.size() == 1u); + + cv::GRunArgP out_arg = out.get(0); + *cv::util::get(out_arg) = cv::util::get(in_args[0]); + out.post(std::move(out_arg)); + } +} + +void cv::gimpl::BGR::Actor::run(cv::gimpl::GIslandExecutable::IInput &in, + cv::gimpl::GIslandExecutable::IOutput &out) +{ + while (true) + { + const auto in_msg = in.get(); + if (cv::util::holds_alternative(in_msg)) + { + out.post(cv::gimpl::EndOfStream{}); + return; + } + + const cv::GRunArgs &in_args = cv::util::get(in_msg); + GAPI_Assert(in_args.size() == 1u); + + cv::GRunArgP out_arg = out.get(0); + auto frame = cv::util::get(in_args[0]); + const auto& desc = frame.desc(); + + auto& rmat = *cv::util::get(out_arg); + switch (desc.fmt) + { + case cv::MediaFormat::BGR: + rmat = cv::make_rmat(frame); + break; + case cv::MediaFormat::NV12: + { + cv::Mat bgr; + auto view = frame.access(cv::MediaFrame::Access::R); + cv::Mat y_plane (desc.size, CV_8UC1, view.ptr[0]); + cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1]); + cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12); + rmat = cv::make_rmat(bgr); + break; + } + default: + cv::util::throw_error( + std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR")); + } + out.post(std::move(out_arg)); + } +} diff --git a/modules/gapi/src/backends/streaming/gstreamingbackend.hpp b/modules/gapi/src/backends/streaming/gstreamingbackend.hpp new file mode 100644 index 0000000000..bb2100c159 --- /dev/null +++ b/modules/gapi/src/backends/streaming/gstreamingbackend.hpp @@ -0,0 +1,89 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_GSTREAMINGBACKEND_HPP +#define OPENCV_GAPI_GSTREAMINGBACKEND_HPP + +#include +#include +#include "gstreamingkernel.hpp" + +namespace cv { +namespace gimpl { + +struct RMatMediaBGRAdapter final: public cv::RMat::Adapter +{ + RMatMediaBGRAdapter(cv::MediaFrame frame) : m_frame(frame) { }; + + virtual cv::RMat::View access(cv::RMat::Access a) override + { + auto view = m_frame.access(a == cv::RMat::Access::W ? cv::MediaFrame::Access::W + : cv::MediaFrame::Access::R); + auto ptr = reinterpret_cast(view.ptr[0]); + auto stride = view.stride[0]; + + std::shared_ptr view_ptr = + std::make_shared(std::move(view)); + auto callback = [view_ptr]() mutable { view_ptr.reset(); }; + + return cv::RMat::View(desc(), ptr, stride, callback); + } + + virtual cv::GMatDesc desc() const override + { + const auto& desc = m_frame.desc(); + GAPI_Assert(desc.fmt == cv::MediaFormat::BGR); + return cv::GMatDesc{CV_8U, 3, desc.size}; + } + + cv::MediaFrame m_frame; +}; + +struct Copy: public cv::detail::KernelTag +{ + using API = cv::gapi::streaming::GCopy; + + static gapi::GBackend backend() { return cv::gapi::streaming::backend(); } + + class Actor final: public cv::gapi::streaming::IActor + { + public: + explicit Actor() {} + virtual void run(cv::gimpl::GIslandExecutable::IInput &in, + cv::gimpl::GIslandExecutable::IOutput &out) override; + }; + + static cv::gapi::streaming::IActor::Ptr create() + { + return cv::gapi::streaming::IActor::Ptr(new Actor()); + } + + static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; }; +}; + +struct BGR: public cv::detail::KernelTag +{ + using API = cv::gapi::streaming::GBGR; + static gapi::GBackend backend() { return cv::gapi::streaming::backend(); } + + class Actor final: public cv::gapi::streaming::IActor { + public: + explicit Actor() {} + virtual void run(cv::gimpl::GIslandExecutable::IInput &in, + cv::gimpl::GIslandExecutable::IOutput&out) override; + }; + + static cv::gapi::streaming::IActor::Ptr create() + { + return cv::gapi::streaming::IActor::Ptr(new Actor()); + } + static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; }; +}; + +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMINGBACKEND_HPP diff --git a/modules/gapi/src/backends/streaming/gstreamingkernel.hpp b/modules/gapi/src/backends/streaming/gstreamingkernel.hpp new file mode 100644 index 0000000000..4732262aa0 --- /dev/null +++ b/modules/gapi/src/backends/streaming/gstreamingkernel.hpp @@ -0,0 +1,37 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#ifndef OPENCV_GAPI_GSTREAMINGKERNEL_HPP +#define OPENCV_GAPI_GSTREAMINGKERNEL_HPP + +#include "compiler/gislandmodel.hpp" + +namespace cv { +namespace gapi { +namespace streaming { + +class IActor { +public: + using Ptr = std::shared_ptr; + + virtual void run(cv::gimpl::GIslandExecutable::IInput &in, + cv::gimpl::GIslandExecutable::IOutput &out) = 0; + + virtual ~IActor() = default; +}; + +using CreateActorFunction = std::function; +struct GStreamingKernel +{ + CreateActorFunction createActorFunction; +}; + +} // namespace streaming +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMINGKERNEL_HPP diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index 4d050dbabd..1f1cbf9dbf 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -43,6 +43,7 @@ #include // ...Imgproc #include // ...and Video kernel implementations #include // render::ocv::backend() +#include // streaming::kernels() #endif // !defined(GAPI_STANDALONE) // @@ -72,7 +73,8 @@ namespace combine(cv::gapi::core::cpu::kernels(), cv::gapi::imgproc::cpu::kernels(), cv::gapi::video::cpu::kernels(), - cv::gapi::render::ocv::kernels()); + cv::gapi::render::ocv::kernels(), + cv::gapi::streaming::kernels()); #else cv::gapi::GKernelPackage(); #endif // !defined(GAPI_STANDALONE) diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index be0869e663..59b19d4252 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -20,7 +20,7 @@ namespace gimpl // FIXME: GAPI_EXPORTS is here only due to tests and Windows linker issues // FIXME: It seems it clearly duplicates the GStreamingCompiled and -// GStreamingExecutable APIs so is highly redundant now. +// GStreamingIntrinExecutable APIs so is highly redundant now. // Same applies to GCompiled/GCompiled::Priv/GExecutor. class GAPI_EXPORTS GStreamingCompiled::Priv { diff --git a/modules/gapi/src/compiler/passes/intrin.cpp b/modules/gapi/src/compiler/passes/intrin.cpp index 5d2707570a..56f2db69e0 100644 --- a/modules/gapi/src/compiler/passes/intrin.cpp +++ b/modules/gapi/src/compiler/passes/intrin.cpp @@ -201,7 +201,7 @@ void traceDown(cv::gimpl::GModel::Graph &g, // Streaming case: ensure the graph has proper isolation of the // desynchronized parts, set proper Edge metadata hints for -// GStreamingExecutable +// GStreamingIntrinExecutable void apply(cv::gimpl::GModel::Graph &g) { using namespace cv::gimpl; diff --git a/modules/gapi/src/compiler/passes/streaming.cpp b/modules/gapi/src/compiler/passes/streaming.cpp index 6e982e2553..9d5dd713c4 100644 --- a/modules/gapi/src/compiler/passes/streaming.cpp +++ b/modules/gapi/src/compiler/passes/streaming.cpp @@ -32,7 +32,7 @@ namespace cv { namespace gimpl { namespace passes { * connected to a new "Sink" node which becomes its _consumer_. * * These extra nodes are required to streamline the queues - * initialization by the GStreamingExecutable and its derivatives. + * initialization by the GStreamingIntrinExecutable and its derivatives. */ void addStreaming(ade::passes::PassContext &ctx) { diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 70686699d0..cfb4a527df 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -144,6 +144,9 @@ void sync_data(cv::GRunArgs &results, cv::GRunArgsP &outputs) case T::index_of(): cv::util::get(out_obj).mov(cv::util::get(res_obj)); break; + case T::index_of(): + *cv::util::get(out_obj) = std::move(cv::util::get(res_obj)); + break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. break; @@ -636,6 +639,13 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput ret_val = cv::GRunArgP(rr); } break; + case cv::GShape::GFRAME: + { + cv::MediaFrame frame; + out_arg = cv::GRunArg(std::move(frame)); + ret_val = cv::GRunArgP(&cv::util::get(out_arg)); + } + break; default: cv::util::throw_error(std::logic_error("Unsupported GShape")); } diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 8370aee262..25d31f172e 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -21,6 +21,7 @@ #include #include +#include namespace opencv_test { @@ -113,6 +114,111 @@ GAPI_OCV_KERNEL(OCVDelay, Delay) { } }; +class TestMediaBGR final: public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + using Cb = cv::MediaFrame::View::Callback; + Cb m_cb; + + public: + explicit TestMediaBGR(cv::Mat m, Cb cb = [](){}) + : m_mat(m), m_cb(cb) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, cv::Size(m_mat.cols, m_mat.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb}); + } +}; + +class TestMediaNV12 final: public cv::MediaFrame::IAdapter { + cv::Mat m_y; + cv::Mat m_uv; +public: + TestMediaNV12(cv::Mat y, cv::Mat uv) : m_y(y), m_uv(uv) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size(m_y.cols, m_y.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { + m_y.ptr(), m_uv.ptr(), nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + m_y.step, m_uv.step, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } +}; + +class BGRSource : public cv::gapi::wip::GCaptureSource { +public: + explicit BGRSource(const std::string& pipeline) + : cv::gapi::wip::GCaptureSource(pipeline) { + } + + bool pull(cv::gapi::wip::Data& data) { + if (cv::gapi::wip::GCaptureSource::pull(data)) { + data = cv::MediaFrame::Create(cv::util::get(data)); + return true; + } + return false; + } + + GMetaArg descr_of() const override { + return cv::GMetaArg{cv::GFrameDesc{cv::MediaFormat::BGR, + cv::util::get( + cv::gapi::wip::GCaptureSource::descr_of()).size}}; + } +}; + +void cvtBGR2NV12(const cv::Mat& bgr, cv::Mat& y, cv::Mat& uv) { + cv::Size frame_sz = bgr.size(); + cv::Size half_sz = frame_sz / 2; + + cv::Mat yuv; + cv::cvtColor(bgr, yuv, cv::COLOR_BGR2YUV_I420); + + // Copy Y plane + yuv.rowRange(0, frame_sz.height).copyTo(y); + + // Merge sampled U and V planes + std::vector dims = {half_sz.height, half_sz.width}; + auto start = frame_sz.height; + auto range_h = half_sz.height/2; + std::vector uv_planes = { + yuv.rowRange(start, start + range_h) .reshape(0, dims), + yuv.rowRange(start + range_h, start + range_h*2).reshape(0, dims) + }; + cv::merge(uv_planes, uv); +} + +class NV12Source : public cv::gapi::wip::GCaptureSource { +public: + explicit NV12Source(const std::string& pipeline) + : cv::gapi::wip::GCaptureSource(pipeline) { +} + +bool pull(cv::gapi::wip::Data& data) { + if (cv::gapi::wip::GCaptureSource::pull(data)) { + cv::Mat bgr = cv::util::get(data); + cv::Mat y, uv; + cvtBGR2NV12(bgr, y, uv); + data = cv::MediaFrame::Create(y, uv); + return true; + } + return false; +} + +GMetaArg descr_of() const override { + return cv::GMetaArg{cv::GFrameDesc{cv::MediaFormat::NV12, + cv::util::get( + cv::gapi::wip::GCaptureSource::descr_of()).size}}; +} +}; + } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -1247,7 +1353,6 @@ TEST(GAPI_Streaming_Desync, SmokeTest_Streaming) } EXPECT_EQ(100u, out1_hits); // out1 must be available for all frames EXPECT_LE(out2_hits, out1_hits); // out2 must appear less times than out1 - std::cout << "Got " << out1_hits << " out1's and " << out2_hits << " out2's" << std::endl; } TEST(GAPI_Streaming_Desync, SmokeTest_Streaming_TwoParts) @@ -1588,4 +1693,167 @@ TEST(GAPI_Streaming_Desync, DesyncObjectConsumedByTwoIslandsViaSameDesync) { EXPECT_NO_THROW(c.compileStreaming(cv::compile_args(p))); } +TEST(GAPI_Streaming, CopyFrame) +{ + initTestDataPath(); + std::string filepath = findDataFile("cv/video/768x576.avi"); + + cv::GFrame in; + auto out = cv::gapi::streaming::copy(in); + + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + auto cc = comp.compileStreaming(); + cc.setSource(filepath); + + cv::VideoCapture cap; + cap.open(filepath); + if (!cap.isOpened()) + throw SkipTestException("Video file can not be opened"); + + cv::MediaFrame frame; + cv::Mat ocv_mat; + std::size_t num_frames = 0u; + std::size_t max_frames = 10u; + + cc.start(); + while (cc.pull(cv::gout(frame)) && num_frames < max_frames) + { + auto view = frame.access(cv::MediaFrame::Access::R); + cv::Mat gapi_mat(frame.desc().size, CV_8UC3, view.ptr[0]); + num_frames++; + cap >> ocv_mat; + + EXPECT_EQ(0, cvtest::norm(ocv_mat, gapi_mat, NORM_INF)); + } +} + +TEST(GAPI_Streaming, Reshape) +{ + initTestDataPath(); + std::string filepath = findDataFile("cv/video/768x576.avi"); + + cv::GFrame in; + auto out = cv::gapi::streaming::copy(in); + + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + auto cc = comp.compileStreaming(); + cc.setSource(filepath); + + cv::VideoCapture cap; + cap.open(filepath); + if (!cap.isOpened()) + throw SkipTestException("Video file can not be opened"); + + cv::MediaFrame frame; + cv::Mat ocv_mat; + std::size_t num_frames = 0u; + std::size_t max_frames = 10u; + + cc.start(); + while (cc.pull(cv::gout(frame)) && num_frames < max_frames) + { + auto view = frame.access(cv::MediaFrame::Access::R); + cv::Mat gapi_mat(frame.desc().size, CV_8UC3, view.ptr[0]); + num_frames++; + cap >> ocv_mat; + + EXPECT_EQ(0, cvtest::norm(ocv_mat, gapi_mat, NORM_INF)); + } + + // Reshape the graph meta + filepath = findDataFile("cv/video/1920x1080.avi"); + cc.stop(); + cc.setSource(filepath); + + cap.open(filepath); + if (!cap.isOpened()) + throw SkipTestException("Video file can not be opened"); + + cv::MediaFrame frame2; + cv::Mat ocv_mat2; + + num_frames = 0u; + + cc.start(); + while (cc.pull(cv::gout(frame2)) && num_frames < max_frames) + { + auto view = frame2.access(cv::MediaFrame::Access::R); + cv::Mat gapi_mat(frame2.desc().size, CV_8UC3, view.ptr[0]); + num_frames++; + cap >> ocv_mat2; + + EXPECT_EQ(0, cvtest::norm(ocv_mat2, gapi_mat, NORM_INF)); + } +} + +TEST(GAPI_Streaming, AccessBGRFromBGRFrame) +{ + initTestDataPath(); + std::string filepath = findDataFile("cv/video/768x576.avi"); + + cv::GFrame in; + auto out = cv::gapi::streaming::BGR(in); + + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + auto cc = comp.compileStreaming(); + cc.setSource(filepath); + + cv::VideoCapture cap; + cap.open(filepath); + if (!cap.isOpened()) + throw SkipTestException("Video file can not be opened"); + + cv::Mat ocv_mat, gapi_mat; + std::size_t num_frames = 0u; + std::size_t max_frames = 10u; + + cc.start(); + while (cc.pull(cv::gout(gapi_mat)) && num_frames < max_frames) + { + num_frames++; + cap >> ocv_mat; + + EXPECT_EQ(0, cvtest::norm(ocv_mat, gapi_mat, NORM_INF)); + } +} + +TEST(GAPI_Streaming, AccessBGRFromNV12Frame) +{ + initTestDataPath(); + std::string filepath = findDataFile("cv/video/768x576.avi"); + + cv::GFrame in; + auto out = cv::gapi::streaming::BGR(in); + + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + auto cc = comp.compileStreaming(); + cc.setSource(filepath); + + cv::VideoCapture cap; + cap.open(filepath); + if (!cap.isOpened()) + throw SkipTestException("Video file can not be opened"); + + cv::Mat ocv_mat, gapi_mat; + std::size_t num_frames = 0u; + std::size_t max_frames = 10u; + + cc.start(); + while (cc.pull(cv::gout(gapi_mat)) && num_frames < max_frames) + { + num_frames++; + cap >> ocv_mat; + + cv::Mat y, uv; + cvtBGR2NV12(ocv_mat, y, uv); + cv::cvtColorTwoPlane(y, uv, ocv_mat, cv::COLOR_YUV2BGR_NV12); + + EXPECT_EQ(0, cvtest::norm(ocv_mat, gapi_mat, NORM_INF)); + } +} + } // namespace opencv_test