diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index d95f255951..ee275fe1af 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -57,6 +57,7 @@ file(GLOB gapi_ext_hdrs set(gapi_srcs # Front-end part + src/api/grunarg.cpp src/api/gorigin.cpp src/api/gmat.cpp src/api/garray.cpp @@ -131,18 +132,19 @@ set(gapi_srcs src/backends/ie/giebackend.cpp src/backends/ie/giebackend/giewrapper.cpp - # ONNX Backend. + # ONNX backend src/backends/onnx/gonnxbackend.cpp - # Render Backend. + # Render backend src/backends/render/grenderocv.cpp src/backends/render/ft_render.cpp - #PlaidML Backend + # PlaidML Backend src/backends/plaidml/gplaidmlcore.cpp src/backends/plaidml/gplaidmlbackend.cpp - # Compound + # Common backend code + src/backends/common/gmetabackend.cpp src/backends/common/gcompoundbackend.cpp src/backends/common/gcompoundkernel.cpp diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 67ce0d990c..0838573b56 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -9,12 +9,14 @@ #define OPENCV_GAPI_GARG_HPP #include +#include #include #include #include #include +#include #include #include @@ -93,7 +95,7 @@ using GArgs = std::vector; // FIXME: Express as M::type // FIXME: Move to a separate file! -using GRunArg = util::variant< +using GRunArgBase = util::variant< #if !defined(GAPI_STANDALONE) cv::UMat, #endif // !defined(GAPI_STANDALONE) @@ -105,6 +107,61 @@ using GRunArg = util::variant< cv::detail::OpaqueRef, cv::MediaFrame >; + +namespace detail { +template +struct in_variant; + +template +struct in_variant > + : std::integral_constant::value > { +}; +} // namespace detail + +struct GAPI_EXPORTS GRunArg: public GRunArgBase +{ + // Metadata information here + using Meta = std::unordered_map; + Meta meta; + + // Mimic the old GRunArg semantics here, old of the times when + // GRunArg was an alias to variant<> + GRunArg(); + GRunArg(const cv::GRunArg &arg); + GRunArg(cv::GRunArg &&arg); + + GRunArg& operator= (const GRunArg &arg); + GRunArg& operator= (GRunArg &&arg); + + template + GRunArg(const T &t, + const Meta &m = Meta{}, + typename std::enable_if< detail::in_variant::value, int>::type = 0) + : GRunArgBase(t) + , meta(m) + { + } + template + GRunArg(T &&t, + const Meta &m = Meta{}, + typename std::enable_if< detail::in_variant::value, int>::type = 0) + : GRunArgBase(std::move(t)) + , meta(m) + { + } + template auto operator= (const T &t) + -> typename std::enable_if< detail::in_variant::value, cv::GRunArg>::type& + { + GRunArgBase::operator=(t); + return *this; + } + template auto operator= (T&& t) + -> typename std::enable_if< detail::in_variant::value, cv::GRunArg>::type& + { + GRunArgBase::operator=(std::move(t)); + return *this; + } +}; using GRunArgs = std::vector; // TODO: Think about the addition operator @@ -129,11 +186,13 @@ namespace gapi namespace wip { /** - * @brief This aggregate type represents all types which G-API can handle (via variant). + * @brief This aggregate type represents all types which G-API can + * handle (via variant). * - * It only exists to overcome C++ language limitations (where a `using`-defined class can't be forward-declared). + * It only exists to overcome C++ language limitations (where a + * `using`-defined class can't be forward-declared). */ -struct Data: public GRunArg +struct GAPI_EXPORTS Data: public GRunArg { using GRunArg::GRunArg; template diff --git a/modules/gapi/include/opencv2/gapi/gopaque.hpp b/modules/gapi/include/opencv2/gapi/gopaque.hpp index 6ab28910d6..6117971768 100644 --- a/modules/gapi/include/opencv2/gapi/gopaque.hpp +++ b/modules/gapi/include/opencv2/gapi/gopaque.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -119,6 +120,7 @@ namespace detail virtual void mov(BasicOpaqueRef &ref) = 0; virtual const void* ptr() const = 0; + virtual void set(const cv::util::any &a) = 0; }; template class OpaqueRefT final: public BasicOpaqueRef @@ -212,6 +214,10 @@ namespace detail } virtual const void* ptr() const override { return &rref(); } + + virtual void set(const cv::util::any &a) override { + wref() = util::any_cast(a); + } }; // This class strips type information from OpaqueRefT<> and makes it usable @@ -285,6 +291,13 @@ namespace detail // May be used to uniquely identify this object internally const void *ptr() const { return m_ref->ptr(); } + + // Introduced for in-graph meta handling + OpaqueRef& operator= (const cv::util::any &a) + { + m_ref->set(a); + return *this; + } }; } // namespace detail diff --git a/modules/gapi/include/opencv2/gapi/streaming/cap.hpp b/modules/gapi/include/opencv2/gapi/streaming/cap.hpp index 9781ef1ffb..aad6af618c 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/cap.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/cap.hpp @@ -21,9 +21,11 @@ * Note for developers: please don't put videoio dependency in G-API * because of this file. */ +#include #include #include +#include namespace cv { namespace gapi { @@ -55,6 +57,7 @@ protected: cv::VideoCapture cap; cv::Mat first; bool first_pulled = false; + int64_t counter = 0; void prep() { @@ -80,19 +83,26 @@ protected: GAPI_Assert(!first.empty()); first_pulled = true; data = first; // no need to clone here since it was cloned already - return true; } - - if (!cap.isOpened()) return false; - - cv::Mat frame; - if (!cap.read(frame)) + else { - // end-of-stream happened - return false; + if (!cap.isOpened()) return false; + + cv::Mat frame; + if (!cap.read(frame)) + { + // end-of-stream happened + return false; + } + // Same reason to clone as in prep() + data = frame.clone(); } - // Same reason to clone as in prep() - data = frame.clone(); + // Tag data with seq_id/ts + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + data.meta[cv::gapi::streaming::meta_tag::timestamp] = int64_t{dur.count()}; + data.meta[cv::gapi::streaming::meta_tag::seq_id] = int64_t{counter++}; return true; } diff --git a/modules/gapi/include/opencv2/gapi/streaming/meta.hpp b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp new file mode 100644 index 0000000000..cbcfc3aa37 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp @@ -0,0 +1,79 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#ifndef OPENCV_GAPI_GSTREAMING_META_HPP +#define OPENCV_GAPI_GSTREAMING_META_HPP + +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace streaming { + +// FIXME: the name is debatable +namespace meta_tag { +static constexpr const char * timestamp = "org.opencv.gapi.meta.timestamp"; +static constexpr const char * seq_id = "org.opencv.gapi.meta.seq_id"; +} // namespace meta_tag + +namespace detail { +struct GMeta { + static const char *id() { + return "org.opencv.streaming.meta"; + } + // A universal yield for meta(), same as in GDesync + template + static std::tuple yield(cv::GCall &call, cv::detail::Seq) { + return std::make_tuple(cv::detail::Yield::yield(call, IIs)...); + } + // Also a universal outMeta stub here + static GMetaArgs getOutMeta(const GMetaArgs &args, const GArgs &) { + return args; + } +}; +} // namespace detail + +template +cv::GOpaque meta(G g, const std::string &tag) { + using O = cv::GOpaque; + cv::GKernel k{ + detail::GMeta::id() // kernel id + , tag // kernel tag. Use meta tag here + , &detail::GMeta::getOutMeta // outMeta callback + , {cv::detail::GTypeTraits::shape} // output Shape + , {cv::detail::GTypeTraits::op_kind} // input data kinds + , {cv::detail::GObtainCtor::get()} // output template ctors + }; + cv::GCall call(std::move(k)); + call.pass(g); + return std::get<0>(detail::GMeta::yield(call, cv::detail::MkSeq<1>::type())); +} + +template +cv::GOpaque timestamp(G g) { + return meta(g, meta_tag::timestamp); +} + +template +cv::GOpaque seq_id(G g) { + return meta(g, meta_tag::seq_id); +} + +template +cv::GOpaque seqNo(G g) { + // Old name, compatibility only + return seq_id(g); +} + +} // namespace streaming +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMING_META_HPP diff --git a/modules/gapi/src/api/gbackend.cpp b/modules/gapi/src/api/gbackend.cpp index 6b8d0fcbee..fd4a5eb38b 100644 --- a/modules/gapi/src/api/gbackend.cpp +++ b/modules/gapi/src/api/gbackend.cpp @@ -143,6 +143,14 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle if (handleRMat == HandleRMat::SKIP) return; GAPI_Assert(arg.index() == GRunArg::index_of()); bindRMat(mag, rc, util::get(arg), RMat::Access::R); + + // FIXME: Here meta may^WWILL be copied multiple times! + // Replace it is reference-counted object? + mag.meta()[rc.id] = arg.meta; + mag.meta()[rc.id] = arg.meta; +#if !defined(GAPI_STANDALONE) + mag.meta()[rc.id] = arg.meta; +#endif break; } @@ -154,19 +162,23 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle case GRunArg::index_of() : mag_scalar = util::get(arg); break; default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } + mag.meta()[rc.id] = arg.meta; break; } case GShape::GARRAY: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; case GShape::GOPAQUE: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; case GShape::GFRAME: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; default: @@ -250,13 +262,23 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref) // Wrap associated CPU object (either host or an internal one) switch (ref.shape) { - case GShape::GMAT: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GSCALAR: return GRunArg(mag.template slot().at(ref.id)); + case GShape::GMAT: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GSCALAR: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); // Note: .at() is intentional for GArray and GOpaque as objects MUST be already there // (and constructed by either bindIn/Out or resetInternal) - case GShape::GARRAY: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GOPAQUE: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GFRAME: return GRunArg(mag.template slot().at(ref.id)); + case GShape::GARRAY: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GOPAQUE: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GFRAME: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); default: util::throw_error(std::logic_error("Unsupported GShape type")); break; diff --git a/modules/gapi/src/api/grunarg.cpp b/modules/gapi/src/api/grunarg.cpp new file mode 100644 index 0000000000..30ae2adbc0 --- /dev/null +++ b/modules/gapi/src/api/grunarg.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" +#include + +cv::GRunArg::GRunArg() { +} + +cv::GRunArg::GRunArg(const cv::GRunArg &arg) + : cv::GRunArgBase(static_cast(arg)) + , meta(arg.meta) { +} + +cv::GRunArg::GRunArg(cv::GRunArg &&arg) + : cv::GRunArgBase(std::move(static_cast(arg))) + , meta(std::move(arg.meta)) { +} + +cv::GRunArg& cv::GRunArg::operator= (const cv::GRunArg &arg) { + cv::GRunArgBase::operator=(static_cast(arg)); + meta = arg.meta; + return *this; +} + +cv::GRunArg& cv::GRunArg::operator= (cv::GRunArg &&arg) { + cv::GRunArgBase::operator=(std::move(static_cast(arg))); + meta = std::move(arg.meta); + return *this; +} diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp index 4914715fa7..576168db53 100644 --- a/modules/gapi/src/backends/common/gbackend.hpp +++ b/modules/gapi/src/backends/common/gbackend.hpp @@ -62,6 +62,8 @@ namespace magazine { template struct Class { template using MapT = std::unordered_map; + using MapM = std::unordered_map; + template MapT& slot() { return std::get::value>(slots); @@ -70,8 +72,17 @@ namespace magazine { { return std::get::value>(slots); } + template MapM& meta() + { + return metas[ade::util::type_list_index::value]; + } + template const MapM& meta() const + { + return metas[ade::util::type_list_index::value]; + } private: std::tuple...> slots; + std::array metas; }; } // namespace magazine diff --git a/modules/gapi/src/backends/common/gmetabackend.cpp b/modules/gapi/src/backends/common/gmetabackend.cpp new file mode 100644 index 0000000000..5364152b65 --- /dev/null +++ b/modules/gapi/src/backends/common/gmetabackend.cpp @@ -0,0 +1,105 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" + +#include // compile args +#include // any +#include // GMeta + +#include "compiler/gobjref.hpp" // RcDesc +#include "compiler/gmodel.hpp" // GModel, Op +#include "backends/common/gbackend.hpp" +#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! + +#include "backends/common/gmetabackend.hpp" + +namespace { + +class GraphMetaExecutable final: public cv::gimpl::GIslandExecutable { + std::string m_meta_tag; + +public: + GraphMetaExecutable(const ade::Graph& g, + const std::vector& nodes); + bool canReshape() const override; + void reshape(ade::Graph&, const cv::GCompileArgs&) override; + + void run(std::vector &&input_objs, + std::vector &&output_objs) override; +}; + +bool GraphMetaExecutable::canReshape() const { + return true; +} +void GraphMetaExecutable::reshape(ade::Graph&, const cv::GCompileArgs&) { + // do nothing here +} + +GraphMetaExecutable::GraphMetaExecutable(const ade::Graph& g, + const std::vector& nodes) { + // There may be only one node in the graph + GAPI_Assert(nodes.size() == 1u); + + cv::gimpl::GModel::ConstGraph cg(g); + const auto &op = cg.metadata(nodes[0]).get(); + GAPI_Assert(op.k.name == cv::gapi::streaming::detail::GMeta::id()); + m_meta_tag = op.k.tag; +} + +void GraphMetaExecutable::run(std::vector &&input_objs, + std::vector &&output_objs) { + GAPI_Assert(input_objs.size() == 1u); + GAPI_Assert(output_objs.size() == 1u); + + const cv::GRunArg in_arg = input_objs[0].second; + cv::GRunArgP out_arg = output_objs[0].second; + + auto it = in_arg.meta.find(m_meta_tag); + if (it == in_arg.meta.end()) { + cv::util::throw_error + (std::logic_error("Run-time meta " + + m_meta_tag + + " is not found in object " + + std::to_string(static_cast(input_objs[0].first.shape)) + + "/" + + std::to_string(input_objs[0].first.id))); + } + cv::util::get(out_arg) = it->second; +} + +class GraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { + virtual void unpackKernel(ade::Graph &, + const ade::NodeHandle &, + const cv::GKernelImpl &) override { + // Do nothing here + } + + virtual EPtr compile(const ade::Graph& graph, + const cv::GCompileArgs&, + const std::vector& nodes, + const std::vector&, + const std::vector&) const override { + return EPtr{new GraphMetaExecutable(graph, nodes)}; + } +}; + +cv::gapi::GBackend graph_meta_backend() { + static cv::gapi::GBackend this_backend(std::make_shared()); + return this_backend; +} + +struct InGraphMetaKernel final: public cv::detail::KernelTag { + using API = cv::gapi::streaming::detail::GMeta; + static cv::gapi::GBackend backend() { return graph_meta_backend(); } + static int kernel() { return 42; } +}; + +} // anonymous namespace + +cv::gapi::GKernelPackage cv::gimpl::meta::kernels() { + return cv::gapi::kernels(); +} diff --git a/modules/gapi/src/backends/common/gmetabackend.hpp b/modules/gapi/src/backends/common/gmetabackend.hpp new file mode 100644 index 0000000000..56f61d0e3d --- /dev/null +++ b/modules/gapi/src/backends/common/gmetabackend.hpp @@ -0,0 +1,16 @@ +#ifndef OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP +#define OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP + +#include + +namespace cv { +namespace gimpl { +namespace meta { + +cv::gapi::GKernelPackage kernels(); + +} // namespace meta +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index eb75f44e0e..f6fa398c17 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -35,6 +35,7 @@ #include "executor/gexecutor.hpp" #include "executor/gstreamingexecutor.hpp" #include "backends/common/gbackend.hpp" +#include "backends/common/gmetabackend.hpp" // #if !defined(GAPI_STANDALONE) @@ -58,7 +59,8 @@ namespace for (const auto &b : pkg.backends()) { aux_pkg = combine(aux_pkg, b.priv().auxiliaryKernels()); } - return combine(pkg, aux_pkg); + // Always include built-in meta<> implementation + return combine(pkg, aux_pkg, cv::gimpl::meta::kernels()); }; auto has_use_only = cv::gapi::getCompileArg(args); diff --git a/modules/gapi/src/compiler/gislandmodel.cpp b/modules/gapi/src/compiler/gislandmodel.cpp index 9ffc605372..4d0feaea71 100644 --- a/modules/gapi/src/compiler/gislandmodel.cpp +++ b/modules/gapi/src/compiler/gislandmodel.cpp @@ -357,26 +357,21 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO for (auto &&it: ade::util::zip(ade::util::toRange(in_desc), ade::util::toRange(in_vector))) { - // FIXME: Not every Island expects a cv::Mat instead of own::Mat on input - // This kludge should go as a result of de-ownification const cv::GRunArg& in_data_orig = std::get<1>(it); cv::GRunArg in_data; -#if !defined(GAPI_STANDALONE) switch (in_data_orig.index()) { case cv::GRunArg::index_of(): - in_data = cv::GRunArg{cv::make_rmat(cv::util::get(in_data_orig))}; - break; - case cv::GRunArg::index_of(): - in_data = cv::GRunArg{(cv::util::get(in_data_orig))}; + // FIXME: This whole construct is ugly, from + // its writing to a need in this in general + in_data = cv::GRunArg{ cv::make_rmat(cv::util::get(in_data_orig)) + , in_data_orig.meta + }; break; default: in_data = in_data_orig; break; } -#else - in_data = in_data_orig; -#endif // GAPI_STANDALONE in_objs.emplace_back(std::get<0>(it), std::move(in_data)); } for (auto &&it: ade::util::indexed(ade::util::toRange(out_desc))) @@ -385,9 +380,27 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO out.get(ade::util::checked_cast(ade::util::index(it)))); } run(std::move(in_objs), std::move(out_objs)); + + // Propagate in-graph meta down to the graph + // Note: this is not a complete implementation! Mainly this is a stub + // and the proper implementation should come later. + // + // Propagating the meta information here has its pros and cons. + // Pros: it works here uniformly for both regular and streaming cases, + // also for the majority of old-fashioned (synchronous) backends + // Cons: backends implementing the asynchronous run(IInput,IOutput) + // won't get it out of the box + cv::GRunArg::Meta stub_meta; + for (auto &&in_arg : in_vector) + { + stub_meta.insert(in_arg.meta.begin(), in_arg.meta.end()); + } + // Report output objects as "ready" to the executor, also post + // calculated in-graph meta for the objects for (auto &&it: out_objs) { - out.post(std::move(it.second)); // report output objects as "ready" to the executor + out.meta(it.second, stub_meta); + out.post(std::move(it.second)); } } diff --git a/modules/gapi/src/compiler/gislandmodel.hpp b/modules/gapi/src/compiler/gislandmodel.hpp index c2e7b96d45..e8eb73692b 100644 --- a/modules/gapi/src/compiler/gislandmodel.hpp +++ b/modules/gapi/src/compiler/gislandmodel.hpp @@ -172,6 +172,10 @@ struct GIslandExecutable::IOutput: public GIslandExecutable::IODesc { virtual GRunArgP get(int idx) = 0; // Allocate (wrap) a new data object for output idx virtual void post(GRunArgP&&) = 0; // Release the object back to the framework (mark available) virtual void post(EndOfStream&&) = 0; // Post end-of-stream marker back to the framework + + // Assign accumulated metadata to the given output object. + // This method can only be called after get() and before post(). + virtual void meta(const GRunArgP&, const GRunArg::Meta &) = 0; }; // GIslandEmitter - a backend-specific thing which feeds data into diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index d9f5cfafe6..66f3b24771 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -12,6 +12,8 @@ #include #include + +#include "api/gproto_priv.hpp" // ptr(GRunArgP) #include "executor/gexecutor.hpp" #include "compiler/passes/passes.hpp" @@ -105,6 +107,9 @@ void bindInArgExec(Mag& mag, const RcDesc &rc, const GRunArg &arg) mag_rmat = util::get(arg); break; default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } + // FIXME: has to take extra care about meta here for this particuluar + // case, just because this function exists at all + mag.meta()[rc.id] = arg.meta; } void bindOutArgExec(Mag& mag, const RcDesc &rc, const GRunArgP &arg) @@ -131,7 +136,7 @@ cv::GRunArgP getObjPtrExec(Mag& mag, const RcDesc &rc) { return getObjPtr(mag, rc); } - return GRunArgP(&mag.template slot()[rc.id]); + return GRunArgP(&mag.slot()[rc.id]); } void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) @@ -155,6 +160,25 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } } + +void assignMetaStubExec(Mag& mag, const RcDesc &rc, const cv::GRunArg::Meta &meta) { + switch (rc.shape) + { + case GShape::GARRAY: mag.meta()[rc.id] = meta; break; + case GShape::GOPAQUE: mag.meta()[rc.id] = meta; break; + case GShape::GSCALAR: mag.meta()[rc.id] = meta; break; + case GShape::GFRAME: mag.meta()[rc.id] = meta; break; + case GShape::GMAT: + mag.meta() [rc.id] = meta; + mag.meta()[rc.id] = meta; +#if !defined(GAPI_STANDALONE) + mag.meta()[rc.id] = meta; +#endif + break; + default: util::throw_error(std::logic_error("Unsupported GShape type")); break; + } +} + } // anonymous namespace }}} // namespace cv::gimpl::magazine @@ -231,11 +255,28 @@ public: class cv::gimpl::GExecutor::Output final: public cv::gimpl::GIslandExecutable::IOutput { cv::gimpl::Mag &mag; - virtual GRunArgP get(int idx) override { return magazine::getObjPtrExec(mag, desc()[idx]); } - virtual void post(GRunArgP&&) override { } // Do nothing here - virtual void post(EndOfStream&&) override {} // Do nothing here too + std::unordered_map out_idx; + + GRunArgP get(int idx) override + { + auto r = magazine::getObjPtrExec(mag, desc()[idx]); + // Remember the output port for this output object + out_idx[cv::gimpl::proto::ptr(r)] = idx; + return r; + } + void post(GRunArgP&&) override { } // Do nothing here + void post(EndOfStream&&) override {} // Do nothing here too + void meta(const GRunArgP &out, const GRunArg::Meta &m) override + { + const auto idx = out_idx.at(cv::gimpl::proto::ptr(out)); + magazine::assignMetaStubExec(mag, desc()[idx], m); + } public: - Output(cv::gimpl::Mag &m, const std::vector &rcs) : mag(m) { set(rcs); } + Output(cv::gimpl::Mag &m, const std::vector &rcs) + : mag(m) + { + set(rcs); + } }; void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args) @@ -330,7 +371,7 @@ void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args) // Run the script for (auto &op : m_ops) { - // (5) + // (5), (6) Input i{m_res, op.in_objects}; Output o{m_res, op.out_objects}; op.isl_exec->run(i, o); diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 653d20e712..58789889a3 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -350,16 +350,14 @@ bool QueueReader::getInputVector(std::vector &in_queues, // value-initialized scalar) // It can also hold a constant value received with // Stop::Kind::CNST message (see above). - // FIXME: Variant move problem - isl_inputs[id] = const_cast(in_constants[id]); + isl_inputs[id] = in_constants[id]; continue; } q->pop(m_cmd[id]); if (!cv::util::holds_alternative(m_cmd[id])) { - // FIXME: Variant move problem - isl_inputs[id] = const_cast(cv::util::get(m_cmd[id])); + isl_inputs[id] = cv::util::get(m_cmd[id]); } else // A Stop sign { @@ -382,7 +380,7 @@ bool QueueReader::getInputVector(std::vector &in_queues, // NEXT time (on a next call to getInputVector()), the // "q==nullptr" check above will be triggered, but now // we need to make it manually: - isl_inputs[id] = const_cast(in_constants[id]); + isl_inputs[id] = in_constants[id]; } else { @@ -666,8 +664,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput Cmd cmd; if (cv::util::holds_alternative(post_iter->data)) { - // FIXME: That ugly VARIANT problem - cmd = Cmd{const_cast(cv::util::get(post_iter->data))}; + cmd = Cmd{cv::util::get(post_iter->data)}; } else { @@ -677,8 +674,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput } for (auto &&q : m_out_queues[out_idx]) { - // FIXME: This ugly VARIANT problem - q->push(const_cast(cmd)); + q->push(cmd); } post_iter = m_postings[out_idx].erase(post_iter); } @@ -708,6 +704,15 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput } } } + void meta(const cv::GRunArgP &out, const cv::GRunArg::Meta &m) override + { + const auto it = m_postIdx.find(cv::gimpl::proto::ptr(out)); + GAPI_Assert(it != m_postIdx.end()); + + const auto out_iter = it->second.second; + cv::util::get(out_iter->data).meta = m; + } + public: explicit StreamingOutput(const cv::GMetaArgs &metas, std::vector< std::vector > &out_queues, @@ -769,6 +774,7 @@ void islandActorThread(std::vector in_rcs, // void collectorThread(std::vector in_queues, std::vector in_mapping, const std::size_t out_size, + const bool handle_stop, Q& out_queue) { // These flags are static now: regardless if the sync or @@ -783,9 +789,14 @@ void collectorThread(std::vector in_queues, while (true) { cv::GRunArgs this_result(out_size); - if (!qr.getResultsVector(in_queues, in_mapping, out_size, this_result)) + const bool ok = qr.getResultsVector(in_queues, in_mapping, out_size, this_result); + if (!ok) { - out_queue.push(Cmd{Stop{}}); + if (handle_stop) + { + out_queue.push(Cmd{Stop{}}); + } + // Terminate the thread anyway return; } out_queue.push(Cmd{Result{std::move(this_result), flags}}); @@ -1263,12 +1274,22 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) // If there are desynchronized parts in the graph, there may be // multiple theads polling every separate (desynchronized) // branch in the graph individually. + const bool has_main_path = m_sink_sync.end() != + std::find(m_sink_sync.begin(), m_sink_sync.end(), -1); for (auto &&info : m_collector_map) { m_threads.emplace_back(collectorThread, info.second.queues, info.second.mapping, m_sink_queues.size(), + has_main_path ? info.first == -1 : true, // see below (*) std::ref(m_out_queue)); + + // (*) - there may be a problem with desynchronized paths when those work + // faster than the main path. In this case, the desync paths get "Stop" message + // earlier and thus broadcast it down to pipeline gets stopped when there is + // some "main path" data to process. This new collectorThread's flag regulates it: + // - desync paths should never post Stop message if there is a main path. + // - if there is no main path, than any desync path can terminate the execution. } state = State::READY; } diff --git a/modules/gapi/test/gapi_graph_meta_tests.cpp b/modules/gapi/test/gapi_graph_meta_tests.cpp new file mode 100644 index 0000000000..73c0da3c9e --- /dev/null +++ b/modules/gapi/test/gapi_graph_meta_tests.cpp @@ -0,0 +1,195 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include +#include + +#include "test_precomp.hpp" +#include "opencv2/gapi/streaming/meta.hpp" +#include "opencv2/gapi/streaming/cap.hpp" + +namespace opencv_test { + +namespace { +void initTestDataPath() { +#ifndef WINRT + static bool initialized = false; + if (!initialized) + { + // Since G-API has no own test data (yet), it is taken from the common space + const char* testDataPath = getenv("OPENCV_TEST_DATA_PATH"); + if (testDataPath != nullptr) { + cvtest::addDataSearchPath(testDataPath); + initialized = true; + } + } +#endif // WINRT +} +} // anonymous namespace + +TEST(GraphMeta, Trad_AccessInput) { + cv::GMat in; + cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); + cv::GOpaque out2 = cv::gapi::streaming::meta(in, "foo"); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); + + cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); + cv::Mat out_mat; + int out_meta = 0; + + // manually set metadata in the input fields + auto inputs = cv::gin(in_mat); + inputs[0].meta["foo"] = 42; + + graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); + EXPECT_EQ(42, out_meta); +} + +TEST(GraphMeta, Trad_AccessTmp) { + cv::GMat in; + cv::GMat tmp = cv::gapi::blur(in, cv::Size(3,3)); + cv::GMat out1 = tmp+1; + cv::GOpaque out2 = cv::gapi::streaming::meta(tmp, "bar"); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); + + cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); + cv::Mat out_mat; + float out_meta = 0.f; + + // manually set metadata in the input fields + auto inputs = cv::gin(in_mat); + inputs[0].meta["bar"] = 1.f; + + graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); + EXPECT_EQ(1.f, out_meta); +} + +TEST(GraphMeta, Trad_AccessOutput) { + cv::GMat in; + cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); + cv::GOpaque out2 = cv::gapi::streaming::meta(out1, "baz"); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); + + cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); + cv::Mat out_mat; + std::string out_meta; + + // manually set metadata in the input fields + auto inputs = cv::gin(in_mat); + + // NOTE: Assigning explicitly an std::string is important, + // otherwise a "const char*" will be stored and won't be + // translated properly by util::any since std::string is + // used within the graph. + inputs[0].meta["baz"] = std::string("opencv"); + + graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); + EXPECT_EQ("opencv", out_meta); +} + +TEST(GraphMeta, Streaming_AccessInput) { + initTestDataPath(); + + cv::GMat in; + cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); + cv::GOpaque out2 = cv::gapi::streaming::seq_id(in); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); + + auto ccomp = graph.compileStreaming(); + ccomp.setSource(findDataFile("cv/video/768x576.avi", false)); + ccomp.start(); + + cv::Mat out_mat; + int64_t out_meta = 0; + int64_t expected_counter = 0; + + while (ccomp.pull(cv::gout(out_mat, out_meta))) { + EXPECT_EQ(expected_counter, out_meta); + ++expected_counter; + } +} + +TEST(GraphMeta, Streaming_AccessOutput) { + initTestDataPath(); + + cv::GMat in; + cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); + cv::GOpaque out2 = cv::gapi::streaming::seq_id(out1); + cv::GOpaque out3 = cv::gapi::streaming::timestamp(out1); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3)); + + auto ccomp = graph.compileStreaming(); + ccomp.setSource(findDataFile("cv/video/768x576.avi", false)); + ccomp.start(); + + cv::Mat out_mat; + int64_t out_meta = 0; + int64_t out_timestamp = 0; + int64_t expected_counter = 0; + int64_t prev_timestamp = -1; + + while (ccomp.pull(cv::gout(out_mat, out_meta, out_timestamp))) { + EXPECT_EQ(expected_counter, out_meta); + ++expected_counter; + + EXPECT_NE(prev_timestamp, out_timestamp); + prev_timestamp = out_timestamp; + } +} + +TEST(GraphMeta, Streaming_AccessDesync) { + initTestDataPath(); + + cv::GMat in; + cv::GOpaque out1 = cv::gapi::streaming::seq_id(in); + cv::GOpaque out2 = cv::gapi::streaming::timestamp(in); + cv::GMat out3 = cv::gapi::blur(in, cv::Size(3,3)); + + cv::GMat tmp = cv::gapi::streaming::desync(in); + cv::GScalar mean = cv::gapi::mean(tmp); + cv::GOpaque out4 = cv::gapi::streaming::seq_id(mean); + cv::GOpaque out5 = cv::gapi::streaming::timestamp(mean); + cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3, out4, out5)); + + auto ccomp = graph.compileStreaming(); + ccomp.setSource(findDataFile("cv/video/768x576.avi", false)); + ccomp.start(); + + cv::optional out_sync_id; + cv::optional out_sync_ts; + cv::optional out_sync_mat; + + cv::optional out_desync_id; + cv::optional out_desync_ts; + + std::unordered_set sync_ids; + std::unordered_set desync_ids; + + while (ccomp.pull(cv::gout(out_sync_id, out_sync_ts, out_sync_mat, + out_desync_id, out_desync_ts))) { + if (out_sync_id.has_value()) { + CV_Assert(out_sync_ts.has_value()); + CV_Assert(out_sync_mat.has_value()); + sync_ids.insert(out_sync_id.value()); + } + if (out_desync_id.has_value()) { + CV_Assert(out_desync_ts.has_value()); + desync_ids.insert(out_desync_id.value()); + } + } + // Visually report that everything is really ok + std::cout << sync_ids.size() << " vs " << desync_ids.size() << std::endl; + + // Desync path should generate less objects than the synchronized one + EXPECT_GE(sync_ids.size(), desync_ids.size()); + + // ..but all desynchronized IDs must be present in the synchronized set + for (auto &&d_id : desync_ids) { + EXPECT_TRUE(sync_ids.count(d_id) > 0); + } +} + +} // namespace opencv_test