third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

G-API: Introduce streaming::desync and infer(ROI)

Browse Source 
- desync() is a new (and for now, the only one) intrinsic
  which splits the graph execution into asynchronous parts
  when running in Streaming mode;
- desync() makes no effect when compiling in Traditional mode;
- Added tests on desync() working in various scenarios;
- Extended GStreamingExecutor to support desync(); also extended
  GStreamingCompiled() with a new version of pull() returning a
  vector of optional values;
- Fixed various issues with storing the type information & proper
  construction callbacks for GArray<> and GOpaque;

- Introduced a new infer(Roi,GMat) overload with a sample;

- Introduced an internal API for Islands to control fusion
  procedure (to fuse or not to fuse);
- Introduced handleStopStream() callback for island executables;
- Added GCompileArgs to metadata of the graph (required for other
  features).
This commit is contained in:
Dmitry Matveev
2020-03-18 02:38:24 +03:00
parent f345ed564a
commit ca8bb8d053
40 changed files with 2827 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gapi/include/opencv2/gapi.hpp
+4
View File
@@ -33,4 +33,8 @@
#include <opencv2/gapi/gkernel.hpp>
#include <opencv2/gapi/operators.hpp>
// Include this file here to avoid cyclic dependency between
// Desync & GKernel & GComputation & GStreamingCompiled.
#include <opencv2/gapi/streaming/desync.hpp>
#endif // OPENCV_GAPI_HPP
modules/gapi/include/opencv2/gapi/garray.hpp
+17 -6
View File
@@ -284,6 +284,14 @@ namespace detail
return static_cast<VectorRefT<T>&>(*m_ref).rref();
}
// Check if was created for/from std::vector<T>
template <typename T> bool holds() const
{
if (!m_ref) return false;
using U = typename std::decay<T>::type;
return dynamic_cast<VectorRefT<U>*>(m_ref.get()) != nullptr;
}
void mov(VectorRef &v)
{
m_ref->mov(*v.m_ref);
@@ -341,15 +349,18 @@ public:
explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor
: m_ref(ref) { putDetails(); } // (used by GCall, not for users)
detail::GArrayU strip() const { return m_ref; }
/// @private
detail::GArrayU strip() const {
return m_ref;
}
/// @private
static void VCtor(detail::VectorRef& vref) {
vref.reset<HT>();
}
private:
static void VCTor(detail::VectorRef& vref) {
vref.reset<HT>();
vref.storeKind<HT>();
}
void putDetails() {
m_ref.setConstructFcn(&VCTor);
m_ref.setConstructFcn(&VCtor);
m_ref.specifyType<HT>(); // FIXME: to unify those 2 to avoid excessive dynamic_cast
m_ref.storeKind<HT>(); //
}
modules/gapi/include/opencv2/gapi/gkernel.hpp
+29 -30
View File
@@ -28,6 +28,7 @@ namespace cv {
using GShapes = std::vector<GShape>;
using GKinds = std::vector<cv::detail::OpaqueKind>;
using GCtors = std::vector<detail::HostCtor>;
// GKernel describes kernel API to the system
// FIXME: add attributes of a kernel, (e.g. number and types
@@ -41,6 +42,7 @@ struct GAPI_EXPORTS GKernel
M outMeta; // generic adaptor to API::outMeta(...)
GShapes outShapes; // types (shapes) kernel's outputs
GKinds inKinds; // kinds of kernel's inputs (fixme: below)
GCtors outCtors; // captured constructors for template output types
};
// TODO: It's questionable if inKinds should really be here. Instead,
// this information could come from meta.
@@ -60,30 +62,27 @@ namespace detail
// yield() is used in graph construction time as a generic method to obtain
// lazy "return value" of G-API operations
//
namespace
template<typename T> struct Yield;
template<> struct Yield<cv::GMat>
{
template<typename T> struct Yield;
template<> struct Yield<cv::GMat>
{
static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); }
};
template<> struct Yield<cv::GMatP>
{
static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); }
};
template<> struct Yield<cv::GScalar>
{
static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); }
};
template<typename U> struct Yield<cv::GArray<U> >
{
static inline cv::GArray<U> yield(cv::GCall &call, int i) { return call.yieldArray<U>(i); }
};
template<typename U> struct Yield<cv::GOpaque<U> >
{
static inline cv::GOpaque<U> yield(cv::GCall &call, int i) { return call.yieldOpaque<U>(i); }
};
} // anonymous namespace
static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); }
};
template<> struct Yield<cv::GMatP>
{
static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); }
};
template<> struct Yield<cv::GScalar>
{
static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); }
};
template<typename U> struct Yield<cv::GArray<U> >
{
static inline cv::GArray<U> yield(cv::GCall &call, int i) { return call.yieldArray<U>(i); }
};
template<typename U> struct Yield<cv::GOpaque<U> >
{
static inline cv::GOpaque<U> yield(cv::GCall &call, int i) { return call.yieldOpaque<U>(i); }
};
////////////////////////////////////////////////////////////////////////////
// Helper classes which brings outputMeta() marshalling to kernel
@@ -215,7 +214,8 @@ public:
, K::tag()
, &K::getOutMeta
, {detail::GTypeTraits<R>::shape...}
, {detail::GTypeTraits<Args>::op_kind...}});
, {detail::GTypeTraits<Args>::op_kind...}
, {detail::GObtainCtor<R>::get()...}});
call.pass(args...); // TODO: std::forward() here?
return yield(call, typename detail::MkSeq<sizeof...(R)>::type());
}
@@ -240,7 +240,8 @@ public:
, K::tag()
, &K::getOutMeta
, {detail::GTypeTraits<R>::shape}
, {detail::GTypeTraits<Args>::op_kind...}});
, {detail::GTypeTraits<Args>::op_kind...}
, {detail::GObtainCtor<R>::get()}});
call.pass(args...);
return detail::Yield<R>::yield(call, 0);
}
@@ -459,11 +460,6 @@ namespace gapi {
std::vector<GTransform> m_transformations;
protected:
/// @private
// Check if package contains ANY implementation of a kernel API
// by API textual id.
bool includesAPI(const std::string &id) const;
/// @private
// Remove ALL implementations of the given API (identified by ID)
void removeAPI(const std::string &id);
@@ -566,6 +562,9 @@ namespace gapi {
return includesAPI(KAPI::id());
}
/// @private
bool includesAPI(const std::string &id) const;
// FIXME: The below comment is wrong, and who needs this function?
/**
* @brief Find a kernel (by its API)
modules/gapi/include/opencv2/gapi/gopaque.hpp
+15 -13
View File
@@ -295,25 +295,27 @@ namespace detail
template<typename T> class GOpaque
{
public:
GOpaque() { putDetails(); } // Empty constructor
explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor
: m_ref(ref) { putDetails(); } // (used by GCall, not for users)
detail::GOpaqueU strip() const { return m_ref; }
private:
// Host type (or Flat type) - the type this GOpaque is actually
// specified to.
using HT = typename detail::flatten_g<util::decay_t<T>>::type;
static void CTor(detail::OpaqueRef& ref) {
ref.reset<HT>();
ref.storeKind<HT>();
GOpaque() { putDetails(); } // Empty constructor
explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor
: m_ref(ref) { putDetails(); } // (used by GCall, not for users)
/// @private
detail::GOpaqueU strip() const {
return m_ref;
}
/// @private
static void Ctor(detail::OpaqueRef& ref) {
ref.reset<HT>();
}
private:
void putDetails() {
m_ref.setConstructFcn(&CTor);
m_ref.specifyType<HT>(); // FIXME: to unify those 2 to avoid excessive dynamic_cast
m_ref.storeKind<HT>(); //
m_ref.setConstructFcn(&Ctor);
m_ref.specifyType<HT>();
m_ref.storeKind<HT>();
}
detail::GOpaqueU m_ref;
modules/gapi/include/opencv2/gapi/gstreaming.hpp
+122
View File
@@ -8,15 +8,99 @@
#ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP
#define OPENCV_GAPI_GSTREAMING_COMPILED_HPP
#include <memory>
#include <vector>
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/own/assert.hpp>
#include <opencv2/gapi/util/optional.hpp>
#include <opencv2/gapi/garg.hpp>
#include <opencv2/gapi/streaming/source.hpp>
namespace cv {
template<class T> using optional = cv::util::optional<T>;
namespace detail {
template<typename T> struct wref_spec {
using type = T;
};
template<typename T> struct wref_spec<std::vector<T> > {
using type = T;
};
template<typename RefHolder>
struct OptRef {
struct OptHolder {
virtual void mov(RefHolder &h) = 0;
virtual void reset() = 0;
virtual ~OptHolder() = default;
using Ptr = std::shared_ptr<OptHolder>;
};
template<class T> struct Holder final: OptHolder {
std::reference_wrapper<cv::optional<T> > m_opt_ref;
explicit Holder(cv::optional<T>& opt) : m_opt_ref(std::ref(opt)) {
}
virtual void mov(RefHolder &h) override {
using U = typename wref_spec<T>::type;
m_opt_ref.get() = cv::util::make_optional(std::move(h.template wref<U>()));
}
virtual void reset() override {
m_opt_ref.get().reset();
}
};
template<class T>
explicit OptRef(cv::optional<T>& t) : m_opt{new Holder<T>(t)} {}
void mov(RefHolder &h) { m_opt->mov(h); }
void reset() { m_opt->reset();}
private:
typename OptHolder::Ptr m_opt;
};
using OptionalVectorRef = OptRef<cv::detail::VectorRef>;
using OptionalOpaqueRef = OptRef<cv::detail::OpaqueRef>;
} // namespace detail
// TODO: Keep it in sync with GRunArgP (derive the type automatically?)
using GOptRunArgP = util::variant<
optional<cv::Mat>*,
optional<cv::RMat>*,
optional<cv::Scalar>*,
cv::detail::OptionalVectorRef,
cv::detail::OptionalOpaqueRef
>;
using GOptRunArgsP = std::vector<GOptRunArgP>;
namespace detail {
template<typename T> inline GOptRunArgP wrap_opt_arg(optional<T>& arg) {
// By default, T goes to an OpaqueRef. All other types are specialized
return GOptRunArgP{OptionalOpaqueRef(arg)};
}
template<typename T> inline GOptRunArgP wrap_opt_arg(optional<std::vector<T> >& arg) {
return GOptRunArgP{OptionalVectorRef(arg)};
}
template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Mat> &m) {
return GOptRunArgP{&m};
}
template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Scalar> &s) {
return GOptRunArgP{&s};
}
} // namespace detail
// Now cv::gout() may produce an empty vector (see "dynamic graphs"), so
// there may be a conflict between these two. State here that Opt version
// _must_ have at least one input for this overload
template<typename T, typename... Ts>
inline GOptRunArgsP gout(optional<T>&arg, optional<Ts>&... args)
{
return GOptRunArgsP{ detail::wrap_opt_arg(arg), detail::wrap_opt_arg(args)... };
}
/**
* \addtogroup gapi_main_classes
* @{
@@ -169,6 +253,44 @@ public:
// NB: Used from python
GAPI_WRAP std::tuple<bool, cv::GRunArgs> pull();
/**
* @brief Get some next available data from the pipeline.
*
* This method takes a vector of cv::optional object. An object is
* assigned to some value if this value is available (ready) at
* the time of the call, and resets the object to empty() if it is
* not.
*
* This is a blocking method which guarantees that some data has
* been written to the output vector on return.
*
* Using this method only makes sense if the graph has
* desynchronized parts (see cv::gapi::desync). If there is no
* desynchronized parts in the graph, the behavior of this
* method is identical to the regular pull() (all data objects are
* produced synchronously in the output vector).
*
* Use gout() to create an output parameter vector.
*
* Output vectors must have the same number of elements as defined
* in the cv::GComputation protocol (at the moment of its
* construction). Shapes of elements also must conform to protocol
* (e.g. cv::optional<cv::Mat> needs to be passed where cv::GMat
* has been declared as output, and so on). Run-time exception is
* generated on type mismatch.
*
* This method writes new data into objects passed via output
* vector. If there is no data ready yet, this method blocks. Use
* try_pull() if you need a non-blocking version.
*
* @param outs vector of output parameters to obtain.
* @return true if next result has been obtained,
* false marks end of the stream.
*
* @sa cv::gapi::desync
*/
bool pull(cv::GOptRunArgsP &&outs);
/**
* @brief Try to get the next processed frame from the pipeline.
*
modules/gapi/include/opencv2/gapi/gtype_traits.hpp
+23
View File
@@ -191,6 +191,29 @@ namespace detail
template<typename T> using wrap_gapi_helper = WrapValue<typename std::decay<T>::type>;
template<typename T> using wrap_host_helper = WrapValue<typename std::decay<g_type_of_t<T> >::type>;
// Union type for various user-defined type constructors (GArray<T>,
// GOpaque<T>, etc)
//
// TODO: Replace construct-only API with a more generic one (probably
// with bits of introspection)
//
// Not required for non-user-defined types (GMat, GScalar, etc)
using HostCtor = util::variant
< util::monostate
, detail::ConstructVec
, detail::ConstructOpaque
>;
template<typename T> struct GObtainCtor {
static HostCtor get() { return HostCtor{}; }
};
template<typename T> struct GObtainCtor<GArray<T> > {
static HostCtor get() { return HostCtor{ConstructVec{&GArray<T>::VCtor}}; };
};
template<typename T> struct GObtainCtor<GOpaque<T> > {
static HostCtor get() { return HostCtor{ConstructOpaque{&GOpaque<T>::Ctor}}; };
};
} // namespace detail
} // namespace cv
modules/gapi/include/opencv2/gapi/infer.hpp
+55 -8
View File
@@ -2,7 +2,7 @@
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
// Copyright (C) 2019-2020 Intel Corporation
#ifndef OPENCV_GAPI_INFER_HPP
@@ -77,6 +77,9 @@ public:
using ResultL = std::tuple< cv::GArray<R>... >;
using APIList = std::function<ResultL(cv::GArray<cv::Rect>, Args...)>;
// FIXME: Args... must be limited to a single GMat
using APIRoi = std::function<Result(cv::GOpaque<cv::Rect>, Args...)>;
};
// Single-return-value network definition (specialized base class)
@@ -92,6 +95,9 @@ public:
using ResultL = cv::GArray<R>;
using APIList = std::function<ResultL(cv::GArray<cv::Rect>, Args...)>;
// FIXME: Args... must be limited to a single GMat
using APIRoi = std::function<Result(cv::GOpaque<cv::Rect>, Args...)>;
};
// APIList2 is also template to allow different calling options
@@ -114,10 +120,10 @@ struct InferAPIList2 {
// a particular backend, not by a network itself.
struct GInferBase {
static constexpr const char * id() {
return "org.opencv.dnn.infer"; // Universal stub
return "org.opencv.dnn.infer"; // Universal stub
}
static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
return GMetaArgs{}; // One more universal stub
return GMetaArgs{}; // One more universal stub
}
};
@@ -164,15 +170,25 @@ private:
std::shared_ptr<Priv> m_priv;
};
/** @} */
// Base "InferROI" kernel.
// All notes from "Infer" kernel apply here as well.
struct GInferROIBase {
static constexpr const char * id() {
return "org.opencv.dnn.infer-roi"; // Universal stub
}
static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
return GMetaArgs{}; // One more universal stub
}
};
// Base "Infer list" kernel.
// All notes from "Infer" kernel apply here as well.
struct GInferListBase {
static constexpr const char * id() {
return "org.opencv.dnn.infer-roi"; // Universal stub
return "org.opencv.dnn.infer-roi-list-1"; // Universal stub
}
static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
return GMetaArgs{}; // One more universal stub
return GMetaArgs{}; // One more universal stub
}
};
@@ -180,10 +196,10 @@ struct GInferListBase {
// All notes from "Infer" kernel apply here as well.
struct GInferList2Base {
static constexpr const char * id() {
return "org.opencv.dnn.infer-roi-list"; // Universal stub
return "org.opencv.dnn.infer-roi-list-2"; // Universal stub
}
static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) {
return GMetaArgs{}; // One more universal stub
return GMetaArgs{}; // One more universal stub
}
};
@@ -200,6 +216,19 @@ struct GInfer final
static constexpr const char* tag() { return Net::tag(); }
};
// A specific roi-inference kernel. API (::on()) is fixed here and
// verified against Net.
template<typename Net>
struct GInferROI final
: public GInferROIBase
, public detail::KernelTypeMedium< GInferROI<Net>
, typename Net::APIRoi > {
using GInferROIBase::getOutMeta; // FIXME: name lookup conflict workaround?
static constexpr const char* tag() { return Net::tag(); }
};
// A generic roi-list inference kernel. API (::on()) is derived from
// the Net template parameter (see more in infer<> overload).
template<typename Net>
@@ -238,6 +267,23 @@ struct GInferList2 final
namespace cv {
namespace gapi {
/** @brief Calculates response for the specified network (template
* parameter) for the specified region in the source image.
* Currently expects a single-input network only.
*
* @tparam A network type defined with G_API_NET() macro.
* @param in input image where to take ROI from.
* @param roi an object describing the region of interest
* in the source image. May be calculated in the same graph dynamically.
* @return an object of return type as defined in G_API_NET().
* If a network has multiple return values (defined with a tuple), a tuple of
* objects of appropriate type is returned.
* @sa G_API_NET()
*/
template<typename Net>
typename Net::Result infer(cv::GOpaque<cv::Rect> roi, cv::GMat in) {
return GInferROI<Net>::on(roi, in);
}
/** @brief Calculates responses for the specified network (template
* parameter) for every region in the source image.
@@ -328,7 +374,8 @@ infer(const std::string& tag, const GInferInputs& inputs)
tag,
GInferBase::getOutMeta,
{}, // outShape will be filled later
std::move(kinds)
std::move(kinds),
{}, // outCtors will be filled later
});
call->setArgs(std::move(input_args));
modules/gapi/include/opencv2/gapi/streaming/desync.hpp
+84
View File
@@ -0,0 +1,84 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP
#define OPENCV_GAPI_GSTREAMING_DESYNC_HPP
#include <tuple>
#include <opencv2/gapi/util/util.hpp>
#include <opencv2/gapi/gtype_traits.hpp>
#include <opencv2/gapi/garg.hpp>
#include <opencv2/gapi/gcall.hpp>
#include <opencv2/gapi/gkernel.hpp>
namespace cv {
namespace gapi {
namespace streaming {
namespace detail {
struct GDesync {
static const char *id() {
return "org.opencv.streaming.desync";
}
// An universal yield for desync.
// Yields output objects according to the input Types...
// Reuses gkernel machinery.
// FIXME: This function can be generic and declared in gkernel.hpp
// (it is there already, but a part of GKernelType[M]
template<typename... R, int... IIs>
static std::tuple<R...> yield(cv::GCall &call, cv::detail::Seq<IIs...>) {
return std::make_tuple(cv::detail::Yield<R>::yield(call, IIs)...);
}
};
template<typename G>
G desync(const G &g) {
cv::GKernel k{
GDesync::id() // kernel id
, "" // kernel tag
, [](const GMetaArgs &a, const GArgs &) {return a;} // outMeta callback
, {cv::detail::GTypeTraits<G>::shape} // output Shape
, {cv::detail::GTypeTraits<G>::op_kind} // input data kinds
, {cv::detail::GObtainCtor<G>::get()} // output template ctors
};
cv::GCall call(std::move(k));
call.pass(g);
return std::get<0>(GDesync::yield<G>(call, cv::detail::MkSeq<1>::type()));
}
} // namespace detail
/**
* @brief Starts a desynchronized branch in the graph.
*
* This operation takes a single G-API data object and returns a
* graph-level "duplicate" of this object.
*
* Operations which use this data object can be desynchronized
* from the rest of the graph.
*
* This operation has no effect when a GComputation is compiled with
* regular cv::GComputation::compile(), since cv::GCompiled objects
* always produce their full output vectors.
*
* This operation only makes sense when a GComputation is compiled in
* straming mode with cv::GComputation::compileStreaming(). If this
* operation is used and there are desynchronized outputs, the user
* should use a special version of cv::GStreamingCompiled::pull()
* which produces an array of cv::util::optional<> objects.
*
* @note This feature is highly experimental now and is currently
* limited to a single GMat argument only.
*/
GAPI_EXPORTS GMat desync(const GMat &g);
} // namespace streaming
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_GSTREAMING_DESYNC_HPP