From 7566921364f70063e57478ab846d3dc1b9b6a5b4 Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Fri, 26 Jun 2020 22:41:29 +0300 Subject: [PATCH] Merge pull request #17020 from dbudniko:dbudniko/serialization_backend G-API Serialization routines * Serialization backend in tests, initial version * S11N/00: A Great Rename - "Serialization" is too long and too error-prone to type, so now it is renamed to "s11n" everywhere; - Same applies to "SRLZ"; - Tests also renamed to start with 'S11N.*' (easier to run); - Also updated copyright years in new files to 2020. * S11N/01: Some basic interface segregation - Moved some details (low-level functions) out of serialization.hpp; - Introduced I::IStream and I::OStream interfaces; - Implemented those via the existing [De]SerializationStream classes; - Moved all operators to use interfaces instead of classes; - Moved the htonl/ntohl handling out of operators (to the classes). The implementation didn't change much, it is a subject to the further refactoring * S11N/02: Basic operator reorg, basic tests, vector support - Reorganized operators on atomic types to follow >>/<< model (put them closer in the code for the respective types); - Introduce more operators for basic (scalar) types; - Drop all vector s11n overloads -- replace with a generic (template-based) one; - Introduced a new test suite where low-level s11n functionality is tested (for the basic types). * S11N/03: Operators reorganization - Sorted the Opaque types enum by complexity; - Reorganized the existing operators for basic types, also ordered by complexity; - Organized operators in three groups (Basics, OpenCV, G-API); - Added a generic serialization for variant<>; - Reimplemented some of the existing operators (for OpenCV and G-API data structures); - Introduced new operators for cv::gimpl data types. These operators (and so, the data structures) are not yet used in the graph dump/reconstruction routine, it will be done as a next step. * S11N/04: The Great Clean-up - Drop the duplicates of GModel data structures from the serialization, serialize the GModel data structures themselve instead (hand-written code replaced with operators). - Also removed usuned code for printing, etc. * S11N/05: Internal API Clean-up - Minimize the serialization API to just Streams and Operators; - Refactor and fix the graph serialization (deconstruction and reconstruction) routines, fix data addressing problems there; - Move the serialization.[ch]pp files to the core G-API library * S11N/06: Top-level API introduction - !!!This is likely the most invasive commit in the series!!! - Introduced a top-level API to serialize and deserialize a GComputation - Extended the compiler to support both forms of a GComputation: an expession based and a deserialized one. This has led to changes in the cv::GComputation::Priv and in its dependent components (even the transformation tests); - Had to extend the kernel API (GKernel) with extra information on operations (mainly `outMeta`) which was only available for expression based graphs. Now the `outMeta` can be taken from kernels too (and for the deserialized graphs it is the only way); - Revisited the internal serialization API, had to expose previously hidden entities (like `GSerialized`); - Extended the serialized graph info with new details (object counter, protocol). Added unordered_map generic serialization for that; - Reworked the very first pipeline test to be "proper"; GREEN now, the rest is to be reworked in the next iteration. * S11N/07: Tests reworked - Moved the sample pipeline tests w/serialization to test the public API (`cv::gapi::serialize`, then followed by `cv::gapi::deserialize<>`). All GREEN. - As a consequence, dropped the "Serialization" test backend as no longer necessary. * S11N/08: Final touches - Exposed the C++ native data types at Streams level; - Switched the ByteMemoryIn/OutStreams to store data in `char` internally (2x less memory for sample pipelines); - Fixed and refactored Mat dumping to the stream; - Renamed S11N pipeline tests to their new meaning. * linux build fix * fix RcDesc and int uint warnings * more Linux build fix * white space and virtual android error fix (attempt) * more warnings to be fixed * android warnings fix attempt * one more attempt for android build fix * android warnings one more fix * return back override * avoid size_t * static deserialize * and how do you like this, elon? anonymous namespace to fix android warning. * static inline * trying to fix standalone build * mat dims fix * fix mat r/w for standalone Co-authored-by: Dmitry Matveev --- modules/gapi/CMakeLists.txt | 10 + .../include/opencv2/gapi/cpu/gcpukernel.hpp | 15 +- modules/gapi/include/opencv2/gapi/garg.hpp | 3 + .../include/opencv2/gapi/gcomputation.hpp | 14 + modules/gapi/include/opencv2/gapi/gkernel.hpp | 13 +- .../include/opencv2/gapi/gtype_traits.hpp | 24 + modules/gapi/include/opencv2/gapi/s11n.hpp | 38 ++ modules/gapi/src/api/gcomputation.cpp | 27 +- modules/gapi/src/api/gcomputation_priv.hpp | 21 +- modules/gapi/src/api/s11n.cpp | 20 + .../src/backends/common/serialization.cpp | 611 ++++++++++++++++++ .../src/backends/common/serialization.hpp | 313 +++++++++ modules/gapi/src/compiler/gcompiler.cpp | 80 ++- modules/gapi/src/compiler/gcompiler.hpp | 2 + modules/gapi/src/compiler/gmodel.hpp | 21 +- modules/gapi/src/compiler/passes/kernels.cpp | 11 + .../src/compiler/passes/transformations.cpp | 3 +- modules/gapi/test/gapi_transform_tests.cpp | 6 +- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 128 ++++ .../test/s11n/gapi_sample_pipelines_s11n.cpp | 285 ++++++++ 20 files changed, 1592 insertions(+), 53 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/s11n.hpp create mode 100644 modules/gapi/src/api/s11n.cpp create mode 100644 modules/gapi/src/backends/common/serialization.cpp create mode 100644 modules/gapi/src/backends/common/serialization.hpp create mode 100644 modules/gapi/test/s11n/gapi_s11n_tests.cpp create mode 100644 modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index b4fb879d5f..b30bee1872 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -128,6 +128,10 @@ set(gapi_srcs # Compound src/backends/common/gcompoundbackend.cpp src/backends/common/gcompoundkernel.cpp + + # Serialization API and routines + src/api/s11n.cpp + src/backends/common/serialization.cpp ) ocv_add_dispatched_file(backends/fluid/gfluidimgproc_func SSE4_1 AVX2) @@ -156,6 +160,7 @@ if(OPENCV_GAPI_INF_ENGINE) list(APPEND __test_extra_deps ${INF_ENGINE_TARGET}) endif() ocv_add_accuracy_tests(${__test_extra_deps}) + # FIXME: test binary is linked with ADE directly since ADE symbols # are not exported from libopencv_gapi.so in any form - thus # there're two copies of ADE code in memory when tests run (!) @@ -183,5 +188,10 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() +if(WIN32) + # Required for htonl/ntohl on Windows + target_link_libraries(${the_module} PRIVATE wsock32 ws2_32) +endif() + ocv_add_perf_tests() ocv_add_samples() diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 37986fd065..86ceace19f 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -479,9 +479,10 @@ class gapi::cpu::GOCVFunctor : public gapi::GFunctor { public: using Impl = std::function; + using Meta = cv::GKernel::M; - GOCVFunctor(const char* id, const Impl& impl) - : gapi::GFunctor(id), impl_{GCPUKernel(impl)} + GOCVFunctor(const char* id, const Meta &meta, const Impl& impl) + : gapi::GFunctor(id), impl_{GCPUKernel(impl), meta} { } @@ -497,14 +498,20 @@ template gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(Callable& c) { using P = detail::OCVCallHelper; - return GOCVFunctor(K::id(), std::bind(&P::callFunctor, std::placeholders::_1, std::ref(c))); + return GOCVFunctor{ K::id() + , &K::getOutMeta + , std::bind(&P::callFunctor, std::placeholders::_1, std::ref(c)) + }; } template gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(const Callable& c) { using P = detail::OCVCallHelper; - return GOCVFunctor(K::id(), std::bind(&P::callFunctor, std::placeholders::_1, c)); + return GOCVFunctor{ K::id() + , &K::getOutMeta + , std::bind(&P::callFunctor, std::placeholders::_1, c) + }; } //! @endcond diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index ef460c1439..980806f3f8 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -46,6 +46,7 @@ public: template::value, int>::type = 0> explicit GArg(const T &t) : kind(detail::GTypeTraits::kind) + , opaque_kind(detail::GOpaqueTraits::kind) , value(detail::wrap_gapi_helper::wrap(t)) { } @@ -53,6 +54,7 @@ public: template::value, int>::type = 0> explicit GArg(T &&t) : kind(detail::GTypeTraits::type>::kind) + , opaque_kind(detail::GOpaqueTraits::type>::kind) , value(detail::wrap_gapi_helper::wrap(t)) { } @@ -78,6 +80,7 @@ public: } detail::ArgKind kind = detail::ArgKind::OPAQUE_VAL; + detail::OpaqueKind opaque_kind = detail::OpaqueKind::CV_UNKNOWN; protected: util::any value; diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp index 20789ccc66..2f0e685ae6 100644 --- a/modules/gapi/include/opencv2/gapi/gcomputation.hpp +++ b/modules/gapi/include/opencv2/gapi/gcomputation.hpp @@ -36,6 +36,16 @@ namespace detail using last_type_t = typename last_type::type; } +// Forward-declare the serialization objects +namespace gimpl { +namespace s11n { +namespace I { + struct IStream; + struct OStream; +} // namespace I +} // namespace s11n +} // namespace gimpl + /** * \addtogroup gapi_main_classes * @{ @@ -495,6 +505,10 @@ public: Priv& priv(); /// @private const Priv& priv() const; + /// @private + explicit GComputation(cv::gimpl::s11n::I::IStream &); + /// @private + void serialize(cv::gimpl::s11n::I::OStream &) const; protected: diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index 478d7d34d1..8fc029e683 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2018-2020 Intel Corporation #ifndef OPENCV_GAPI_GKERNEL_HPP @@ -45,6 +45,7 @@ struct GAPI_EXPORTS GKernel struct GAPI_EXPORTS GKernelImpl { util::any opaque; // backend-specific opaque info + GKernel::M outMeta; // for deserialized graphs, the outMeta is taken here }; template class GKernelTypeM; @@ -456,12 +457,12 @@ namespace gapi { /// @private // Partial include() specialization for kernels template - typename std::enable_if<(std::is_base_of::value), void>::type + typename std::enable_if<(std::is_base_of::value), void>::type includeHelper() { auto backend = KImpl::backend(); auto kernel_id = KImpl::API::id(); - auto kernel_impl = GKernelImpl{KImpl::kernel()}; + auto kernel_impl = GKernelImpl{KImpl::kernel(), &KImpl::API::getOutMeta}; removeAPI(kernel_id); m_id_kernels[kernel_id] = std::make_pair(backend, kernel_impl); @@ -470,7 +471,7 @@ namespace gapi { /// @private // Partial include() specialization for transformations template - typename std::enable_if<(std::is_base_of::value), void>::type + typename std::enable_if<(std::is_base_of::value), void>::type includeHelper() { m_transformations.emplace_back(TImpl::transformation()); @@ -509,7 +510,7 @@ namespace gapi { template bool includes() const { - static_assert(std::is_base_of::value, + static_assert(std::is_base_of::value, "includes() can be applied to kernels only"); auto kernel_it = m_id_kernels.find(KImpl::API::id()); @@ -624,7 +625,7 @@ namespace gapi { { // FIXME: currently there is no check that transformations' signatures are unique // and won't be any intersection in graph compilation stage - static_assert(detail::all_unique::value, "Kernels API must be unique"); + static_assert(cv::detail::all_unique::value, "Kernels API must be unique"); GKernelPackage pkg; diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index c0acef0ceb..1dd6146a24 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -41,6 +41,30 @@ namespace detail GOPAQUE, // a cv::GOpaqueU (note - exactly GOpaqueU, not GOpaque!) }; + enum class OpaqueKind: int + { + CV_UNKNOWN, // Unknown, generic, opaque-to-GAPI data type unsupported in graph seriallization + CV_BOOL, // bool user G-API data + CV_INT, // int user G-API data + CV_DOUBLE, // double user G-API data + CV_POINT, // cv::Point user G-API data + CV_SIZE, // cv::Size user G-API data + CV_RECT, // cv::Rect user G-API data + CV_SCALAR, // cv::Scalar user G-API data + CV_MAT, // cv::Mat user G-API data + }; + + template struct GOpaqueTraits; + template struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UNKNOWN; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_DOUBLE; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_SIZE; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_BOOL; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_SCALAR; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_POINT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_MAT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_RECT; }; + // Describe G-API types (G-types) with traits. Mostly used by // cv::GArg to store meta information about types passed into // operation arguments. Please note that cv::GComputation is diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp new file mode 100644 index 0000000000..689eda5899 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -0,0 +1,38 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_S11N_HPP +#define OPENCV_GAPI_S11N_HPP + +#include +#include + +namespace cv { +namespace gapi { + +namespace detail { + GAPI_EXPORTS cv::GComputation getGraph(const std::vector &p); +} // namespace detail + +GAPI_EXPORTS std::vector serialize(const cv::GComputation &c); +//namespace{ + +template static inline +T deserialize(const std::vector &p); + +//} //ananymous namespace + +template<> inline +cv::GComputation deserialize(const std::vector &p) { + return detail::getGraph(p); +} + + + +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_S11N_HPP diff --git a/modules/gapi/src/api/gcomputation.cpp b/modules/gapi/src/api/gcomputation.cpp index 5dac47b7cb..60119f717a 100644 --- a/modules/gapi/src/api/gcomputation.cpp +++ b/modules/gapi/src/api/gcomputation.cpp @@ -56,19 +56,38 @@ cv::GComputation::GComputation(const std::vector &ins, const std::vector &outs) : m_priv(new Priv()) { + Priv::Expr e; const auto wrap = [](cv::GMat m) { return GProtoArg(m); }; - ade::util::transform(ins, std::back_inserter(m_priv->m_ins), wrap); - ade::util::transform(outs, std::back_inserter(m_priv->m_outs), wrap); + ade::util::transform(ins, std::back_inserter(e.m_ins), wrap); + ade::util::transform(outs, std::back_inserter(e.m_outs), wrap); + m_priv->m_shape = std::move(e); } cv::GComputation::GComputation(cv::GProtoInputArgs &&ins, cv::GProtoOutputArgs &&outs) : m_priv(new Priv()) { - m_priv->m_ins = std::move(ins.m_args); - m_priv->m_outs = std::move(outs.m_args); + m_priv->m_shape = Priv::Expr{ + std::move(ins.m_args) + , std::move(outs.m_args) + }; } +cv::GComputation::GComputation(cv::gimpl::s11n::I::IStream &is) + : m_priv(new Priv()) +{ + m_priv->m_shape = gimpl::s11n::deserialize(is); +} + +void cv::GComputation::serialize(cv::gimpl::s11n::I::OStream &os) const +{ + // Build a basic GModel and write the whole thing to the stream + auto pG = cv::gimpl::GCompiler::makeGraph(*m_priv); + std::vector nhs(pG->nodes().begin(), pG->nodes().end()); + gimpl::s11n::serialize(os, *pG, nhs); +} + + cv::GCompiled cv::GComputation::compile(GMetaArgs &&metas, GCompileArgs &&args) { // FIXME: Cache gcompiled per parameters here? diff --git a/modules/gapi/src/api/gcomputation_priv.hpp b/modules/gapi/src/api/gcomputation_priv.hpp index 13d1b9afa2..c3160b4b2e 100644 --- a/modules/gapi/src/api/gcomputation_priv.hpp +++ b/modules/gapi/src/api/gcomputation_priv.hpp @@ -8,20 +8,37 @@ #ifndef OPENCV_GAPI_GCOMPUTATION_PRIV_HPP #define OPENCV_GAPI_GCOMPUTATION_PRIV_HPP +#include + +#include "opencv2/gapi/util/variant.hpp" + #include "opencv2/gapi.hpp" #include "opencv2/gapi/gcall.hpp" #include "opencv2/gapi/util/variant.hpp" +#include "backends/common/serialization.hpp" + namespace cv { class GComputation::Priv { public: + struct Expr { + cv::GProtoArgs m_ins; + cv::GProtoArgs m_outs; + }; + + using Dump = cv::gimpl::s11n::GSerialized; + + using Shape = cv::util::variant + < Expr // An expression-based graph + , Dump // A deserialized graph + >; + GCompiled m_lastCompiled; GMetaArgs m_lastMetas; // TODO: make GCompiled remember its metas? - GProtoArgs m_ins; - GProtoArgs m_outs; + Shape m_shape; }; } diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp new file mode 100644 index 0000000000..c17b78c3cb --- /dev/null +++ b/modules/gapi/src/api/s11n.cpp @@ -0,0 +1,20 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include + +#include "backends/common/serialization.hpp" + +std::vector cv::gapi::serialize(const cv::GComputation &c) { + cv::gimpl::s11n::ByteMemoryOutStream os; + c.serialize(os); + return os.data(); +} + +cv::GComputation cv::gapi::detail::getGraph(const std::vector &p) { + cv::gimpl::s11n::ByteMemoryInStream is(p); + return cv::GComputation(is); +} diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp new file mode 100644 index 0000000000..eb503f2c94 --- /dev/null +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -0,0 +1,611 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include // set +#include // map +#include // indexed + +#ifdef _WIN32 +#include // htonl, ntohl +#else +#include // htonl, ntohl +#endif + +#include + +#include "backends/common/serialization.hpp" + +namespace cv { +namespace gimpl { +namespace s11n { +namespace { + +void putData(GSerialized& s, const GModel::ConstGraph& cg, const ade::NodeHandle &nh) { + const auto gdata = cg.metadata(nh).get(); + const auto it = ade::util::find_if(s.m_datas, [&gdata](const cv::gimpl::Data &cd) { + return cd.rc == gdata.rc && cd.shape == gdata.shape; + }); + if (s.m_datas.end() == it) { + s.m_datas.push_back(gdata); + } +} + +void putOp(GSerialized& s, const GModel::ConstGraph& cg, const ade::NodeHandle &nh) { + const auto& op = cg.metadata(nh).get(); + for (const auto &in_nh : nh->inNodes()) { putData(s, cg, in_nh); } + for (const auto &out_nh : nh->outNodes()) { putData(s, cg, out_nh); } + s.m_ops.push_back(op); +} + +void mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { + GModel::Graph gm(g); + auto nh = gm.createNode(); + gm.metadata(nh).set(NodeType{NodeType::DATA}); + gm.metadata(nh).set(data); +} + +void mkOpNode(ade::Graph& g, const cv::gimpl::Op& op) { + GModel::Graph gm(g); + auto nh = gm.createNode(); + gm.metadata(nh).set(NodeType{NodeType::OP}); + gm.metadata(nh).set(op); +} + +void linkNodes(ade::Graph& g) { + std::map dataNodes; + GModel::Graph gm(g); + + for (const auto& nh : g.nodes()) { + if (gm.metadata(nh).get().t == NodeType::DATA) { + const auto &d = gm.metadata(nh).get(); + const auto rc = cv::gimpl::RcDesc{d.rc, d.shape, d.ctor}; + dataNodes[rc] = nh; + } + } + + for (const auto& nh : g.nodes()) { + if (gm.metadata(nh).get().t == NodeType::OP) { + const auto& op = gm.metadata(nh).get(); + for (const auto& in : ade::util::indexed(op.args)) { + const auto& arg = ade::util::value(in); + if (arg.kind == cv::detail::ArgKind::GOBJREF) { + const auto idx = ade::util::index(in); + const auto rc = arg.get(); + const auto& in_nh = dataNodes.at(rc); + const auto& in_eh = g.link(in_nh, nh); + gm.metadata(in_eh).set(Input{idx}); + } + } + + for (const auto& out : ade::util::indexed(op.outs)) { + const auto idx = ade::util::index(out); + const auto rc = ade::util::value(out); + const auto& out_nh = dataNodes.at(rc); + const auto& out_eh = g.link(nh, out_nh); + gm.metadata(out_eh).set(Output{idx}); + } + } + } +} + +void relinkProto(ade::Graph& g) { + // identify which node handles map to the protocol + // input/output object in the reconstructed graph + using S = std::set; // FIXME: use ... + using M = std::map; // FIXME: unordered! + + cv::gimpl::GModel::Graph gm(g); + auto &proto = gm.metadata().get(); + + const S set_in(proto.inputs.begin(), proto.inputs.end()); + const S set_out(proto.outputs.begin(), proto.outputs.end()); + M map_in, map_out; + + // Associate the protocol node handles with their resource identifiers + for (auto &&nh : gm.nodes()) { + if (gm.metadata(nh).get().t == cv::gimpl::NodeType::DATA) { + const auto &d = gm.metadata(nh).get(); + const auto rc = cv::gimpl::RcDesc{d.rc, d.shape, d.ctor}; + if (set_in.count(rc) > 0) { + GAPI_DbgAssert(set_out.count(rc) == 0); + map_in[rc] = nh; + } else if (set_out.count(rc) > 0) { + GAPI_DbgAssert(set_in.count(rc) == 0); + map_out[rc] = nh; + } + } + } + + // Reconstruct the protocol vectors, ordered + proto.in_nhs.reserve(proto.inputs.size()); + proto.in_nhs.clear(); + proto.out_nhs.reserve(proto.outputs.size()); + proto.out_nhs.clear(); + for (auto &rc : proto.inputs) { proto.in_nhs .push_back(map_in .at(rc)); } + for (auto &rc : proto.outputs) { proto.out_nhs.push_back(map_out.at(rc)); } +} + +} // anonymous namespace + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// Graph dump operators + +// OpenCV types //////////////////////////////////////////////////////////////// + +I::OStream& operator<< (I::OStream& os, const cv::Point &pt) { + return os << pt.x << pt.y; +} +I::IStream& operator>> (I::IStream& is, cv::Point& pt) { + return is >> pt.x >> pt.y; +} + +I::OStream& operator<< (I::OStream& os, const cv::Size &sz) { + return os << sz.width << sz.height; +} +I::IStream& operator>> (I::IStream& is, cv::Size& sz) { + return is >> sz.width >> sz.height; +} + +I::OStream& operator<< (I::OStream& os, const cv::Rect &rc) { + return os << rc.x << rc.y << rc.width << rc.height; +} +I::IStream& operator>> (I::IStream& is, cv::Rect& rc) { + return is >> rc.x >> rc.y >> rc.width >> rc.height; +} + +I::OStream& operator<< (I::OStream& os, const cv::Scalar &s) { + return os << s.val[0] << s.val[1] << s.val[2] << s.val[3]; +} +I::IStream& operator>> (I::IStream& is, cv::Scalar& s) { + return is >> s.val[0] >> s.val[1] >> s.val[2] >> s.val[3]; +} + +namespace +{ + +#if !defined(GAPI_STANDALONE) +template + void write_plain(I::OStream &os, const T *arr, std::size_t sz) { + for (auto &&it : ade::util::iota(sz)) os << arr[it]; +} +template + void read_plain(I::IStream &is, T *arr, std::size_t sz) { + for (auto &&it : ade::util::iota(sz)) is >> arr[it]; +} +template +void write_mat_data(I::OStream &os, const cv::Mat &m) { + // Write every row individually (handles the case when Mat is a view) + for (auto &&r : ade::util::iota(m.rows)) { + write_plain(os, m.ptr(r), m.cols*m.channels()); + } +} +template +void read_mat_data(I::IStream &is, cv::Mat &m) { + // Write every row individually (handles the case when Mat is aligned) + for (auto &&r : ade::util::iota(m.rows)) { + read_plain(is, m.ptr(r), m.cols*m.channels()); + } +} +#else +void write_plain(I::OStream &os, const uchar *arr, std::size_t sz) { + for (auto &&it : ade::util::iota(sz)) os << arr[it]; +} +void read_plain(I::IStream &is, uchar *arr, std::size_t sz) { + for (auto &&it : ade::util::iota(sz)) is >> arr[it]; +} +template +void write_mat_data(I::OStream &os, const cv::Mat &m) { + // Write every row individually (handles the case when Mat is a view) + for (auto &&r : ade::util::iota(m.rows)) { + write_plain(os, m.ptr(r), m.cols*m.channels()*sizeof(T)); + } +} +template +void read_mat_data(I::IStream &is, cv::Mat &m) { + // Write every row individually (handles the case when Mat is aligned) + for (auto &&r : ade::util::iota(m.rows)) { + read_plain(is, m.ptr(r), m.cols*m.channels()*sizeof(T)); + } +} +#endif +} // namespace + +I::OStream& operator<< (I::OStream& os, const cv::Mat &m) { +#if !defined(GAPI_STANDALONE) + GAPI_Assert(m.size.dims() == 2 && "Only 2D images are supported now"); +#else + GAPI_Assert(m.dims.size() == 2 && "Only 2D images are supported now"); +#endif + os << m.rows << m.cols << m.type(); + switch (m.depth()) { + case CV_8U: write_mat_data< uint8_t>(os, m); break; + case CV_8S: write_mat_data< char>(os, m); break; + case CV_16U: write_mat_data(os, m); break; + case CV_16S: write_mat_data< int16_t>(os, m); break; + case CV_32S: write_mat_data< int32_t>(os, m); break; + case CV_32F: write_mat_data< float>(os, m); break; + case CV_64F: write_mat_data< double>(os, m); break; + default: GAPI_Assert(false && "Unsupported Mat depth"); + } + return os; +} +I::IStream& operator>> (I::IStream& is, cv::Mat& m) { + int rows = -1, cols = -1, type = 0; + is >> rows >> cols >> type; + m.create(cv::Size(cols, rows), type); + switch (m.depth()) { + case CV_8U: read_mat_data< uint8_t>(is, m); break; + case CV_8S: read_mat_data< char>(is, m); break; + case CV_16U: read_mat_data(is, m); break; + case CV_16S: read_mat_data< int16_t>(is, m); break; + case CV_32S: read_mat_data< int32_t>(is, m); break; + case CV_32F: read_mat_data< float>(is, m); break; + case CV_64F: read_mat_data< double>(is, m); break; + default: GAPI_Assert(false && "Unsupported Mat depth"); + } + return is; +} + +// G-API types ///////////////////////////////////////////////////////////////// + +// Stubs (empty types) + +I::OStream& operator<< (I::OStream& os, cv::util::monostate ) {return os;} +I::IStream& operator>> (I::IStream& is, cv::util::monostate &) {return is;} + +I::OStream& operator<< (I::OStream& os, const cv::GScalarDesc &) {return os;} +I::IStream& operator>> (I::IStream& is, cv::GScalarDesc &) {return is;} + +I::OStream& operator<< (I::OStream& os, const cv::GOpaqueDesc &) {return os;} +I::IStream& operator>> (I::IStream& is, cv::GOpaqueDesc &) {return is;} + +I::OStream& operator<< (I::OStream& os, const cv::GArrayDesc &) {return os;} +I::IStream& operator>> (I::IStream& is, cv::GArrayDesc &) {return is;} + +// Enums and structures + +namespace { +template I::OStream& put_enum(I::OStream& os, E e) { + return os << static_cast(e); +} +template I::IStream& get_enum(I::IStream& is, E &e) { + int x{}; is >> x; e = static_cast(x); + return is; +} +} // anonymous namespace + +I::OStream& operator<< (I::OStream& os, cv::GShape sh) { + return put_enum(os, sh); +} +I::IStream& operator>> (I::IStream& is, cv::GShape &sh) { + return get_enum(is, sh); +} +I::OStream& operator<< (I::OStream& os, cv::detail::ArgKind k) { + return put_enum(os, k); +} +I::IStream& operator>> (I::IStream& is, cv::detail::ArgKind &k) { + return get_enum(is, k); +} +I::OStream& operator<< (I::OStream& os, cv::detail::OpaqueKind k) { + return put_enum(os, k); +} +I::IStream& operator>> (I::IStream& is, cv::detail::OpaqueKind &k) { + return get_enum(is, k); +} +I::OStream& operator<< (I::OStream& os, cv::gimpl::Data::Storage s) { + return put_enum(os, s); +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::Data::Storage &s) { + return get_enum(is, s); +} + + +I::OStream& operator<< (I::OStream& os, const cv::GArg &arg) { + // Only GOBJREF and OPAQUE_VAL kinds can be serialized/deserialized + GAPI_Assert( arg.kind == cv::detail::ArgKind::OPAQUE_VAL + || arg.kind == cv::detail::ArgKind::GOBJREF); + + os << arg.kind << arg.opaque_kind; + if (arg.kind == cv::detail::ArgKind::GOBJREF) { + os << arg.get(); + } else { + GAPI_Assert(arg.kind == cv::detail::ArgKind::OPAQUE_VAL); + GAPI_Assert(arg.opaque_kind != cv::detail::OpaqueKind::CV_UNKNOWN); + switch (arg.opaque_kind) { + case cv::detail::OpaqueKind::CV_BOOL: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_INT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_DOUBLE: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_POINT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_SIZE: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_RECT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_SCALAR: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_MAT: os << arg.get(); break; + default: GAPI_Assert(false && "GArg: Unsupported (unknown?) opaque value type"); + } + } + return os; +} +I::IStream& operator>> (I::IStream& is, cv::GArg &arg) { + is >> arg.kind >> arg.opaque_kind; + + // Only GOBJREF and OPAQUE_VAL kinds can be serialized/deserialized + GAPI_Assert( arg.kind == cv::detail::ArgKind::OPAQUE_VAL + || arg.kind == cv::detail::ArgKind::GOBJREF); + + if (arg.kind == cv::detail::ArgKind::GOBJREF) { + cv::gimpl::RcDesc rc; + is >> rc; + arg = (GArg(rc)); + } else { + GAPI_Assert(arg.kind == cv::detail::ArgKind::OPAQUE_VAL); + GAPI_Assert(arg.opaque_kind != cv::detail::OpaqueKind::CV_UNKNOWN); + switch (arg.opaque_kind) { +#define HANDLE_CASE(E,T) case cv::detail::OpaqueKind::CV_##E: \ + { T t{}; is >> t; arg = (cv::GArg(t)); } break + HANDLE_CASE(BOOL , bool); + HANDLE_CASE(INT , int); + HANDLE_CASE(DOUBLE , double); + HANDLE_CASE(POINT , cv::Point); + HANDLE_CASE(SIZE , cv::Size); + HANDLE_CASE(RECT , cv::Rect); + HANDLE_CASE(SCALAR , cv::Scalar); + HANDLE_CASE(MAT , cv::Mat); +#undef HANDLE_CASE + default: GAPI_Assert(false && "GArg: Unsupported (unknown?) opaque value type"); + } + } + return is; +} + + +I::OStream& operator<< (I::OStream& os, const cv::GKernel &k) { + return os << k.name << k.tag << k.outShapes; +} +I::IStream& operator>> (I::IStream& is, cv::GKernel &k) { + return is >> const_cast(k.name) + >> const_cast(k.tag) + >> const_cast(k.outShapes); +} + + +I::OStream& operator<< (I::OStream& os, const cv::GMatDesc &d) { + return os << d.depth << d.chan << d.size << d.planar << d.dims; +} +I::IStream& operator>> (I::IStream& is, cv::GMatDesc &d) { + return is >> d.depth >> d.chan >> d.size >> d.planar >> d.dims; +} + + +I::OStream& operator<< (I::OStream& os, const cv::gimpl::RcDesc &rc) { + // FIXME: HostCtor is not serialized! + return os << rc.id << rc.shape; +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::RcDesc &rc) { + // FIXME: HostCtor is not deserialized! + return is >> rc.id >> rc.shape; +} + + +I::OStream& operator<< (I::OStream& os, const cv::gimpl::Op &op) { + return os << op.k << op.args << op.outs; +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::Op &op) { + return is >> op.k >> op.args >> op.outs; +} + + +I::OStream& operator<< (I::OStream& os, const cv::gimpl::Data &d) { + // FIXME: HostCtor is not stored here!! + // FIXME: Storage may be incorrect for subgraph-to-graph process + return os << d.shape << d.rc << d.meta << d.storage; +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::Data &d) { + // FIXME: HostCtor is not stored here!! + // FIXME: Storage may be incorrect for subgraph-to-graph process + return is >> d.shape >> d.rc >> d.meta >> d.storage; +} + + +I::OStream& operator<< (I::OStream& os, const cv::gimpl::DataObjectCounter &c) { + return os << c.m_next_data_id; +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::DataObjectCounter &c) { + return is >> c.m_next_data_id; +} + + +I::OStream& operator<< (I::OStream& os, const cv::gimpl::Protocol &p) { + // NB: in_nhs/out_nhs are not written! + return os << p.inputs << p.outputs; +} +I::IStream& operator>> (I::IStream& is, cv::gimpl::Protocol &p) { + // NB: in_nhs/out_nhs are reconstructed at a later phase + return is >> p.inputs >> p.outputs; +} + + +void serialize( I::OStream& os + , const ade::Graph &g + , const std::vector &nodes) { + cv::gimpl::GModel::ConstGraph cg(g); + GSerialized s; + for (auto &nh : nodes) { + switch (cg.metadata(nh).get().t) + { + case NodeType::OP: putOp (s, cg, nh); break; + case NodeType::DATA: putData(s, cg, nh); break; + default: util::throw_error(std::logic_error("Unknown NodeType")); + } + } + s.m_counter = cg.metadata().get(); + s.m_proto = cg.metadata().get(); + os << s.m_ops << s.m_datas << s.m_counter << s.m_proto; +} + +GSerialized deserialize(I::IStream &is) { + GSerialized s; + is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto; + return s; +} + +void reconstruct(const GSerialized &s, ade::Graph &g) { + GAPI_Assert(g.nodes().empty()); + for (const auto& d : s.m_datas) cv::gimpl::s11n::mkDataNode(g, d); + for (const auto& op : s.m_ops) cv::gimpl::s11n::mkOpNode(g, op); + cv::gimpl::s11n::linkNodes(g); + + cv::gimpl::GModel::Graph gm(g); + gm.metadata().set(s.m_counter); + gm.metadata().set(s.m_proto); + cv::gimpl::s11n::relinkProto(g); + gm.metadata().set(cv::gimpl::Deserialized{}); +} + +//////////////////////////////////////////////////////////////////////////////// +// Streams ///////////////////////////////////////////////////////////////////// + +const std::vector& ByteMemoryOutStream::data() const { + return m_storage; +} +I::OStream& ByteMemoryOutStream::operator<< (uint32_t atom) { + m_storage.push_back(0xFF & (atom)); + m_storage.push_back(0xFF & (atom >> 8)); + m_storage.push_back(0xFF & (atom >> 16)); + m_storage.push_back(0xFF & (atom >> 24)); + return *this; +} +I::OStream& ByteMemoryOutStream::operator<< (bool atom) { + m_storage.push_back(atom ? 1 : 0); + return *this; +} +I::OStream& ByteMemoryOutStream::operator<< (char atom) { + m_storage.push_back(atom); + return *this; +} +I::OStream& ByteMemoryOutStream::operator<< (unsigned char atom) { + return *this << static_cast(atom); +} +I::OStream& ByteMemoryOutStream::operator<< (short atom) { + static_assert(sizeof(short) == 2, "Expecting sizeof(short) == 2"); + m_storage.push_back(0xFF & (atom)); + m_storage.push_back(0xFF & (atom >> 8)); + return *this; +} +I::OStream& ByteMemoryOutStream::operator<< (unsigned short atom) { + return *this << static_cast(atom); +} +I::OStream& ByteMemoryOutStream::operator<< (int atom) { + static_assert(sizeof(int) == 4, "Expecting sizeof(int) == 4"); + return *this << static_cast(atom); +} +//I::OStream& ByteMemoryOutStream::operator<< (std::size_t atom) { +// // NB: type truncated! +// return *this << static_cast(atom); +//} +I::OStream& ByteMemoryOutStream::operator<< (float atom) { + static_assert(sizeof(float) == 4, "Expecting sizeof(float) == 4"); + uint32_t tmp = 0u; + memcpy(&tmp, &atom, sizeof(float)); + return *this << static_cast(htonl(tmp)); +} +I::OStream& ByteMemoryOutStream::operator<< (double atom) { + static_assert(sizeof(double) == 8, "Expecting sizeof(double) == 8"); + uint32_t tmp[2] = {0u}; + memcpy(tmp, &atom, sizeof(double)); + *this << static_cast(htonl(tmp[0])); + *this << static_cast(htonl(tmp[1])); + return *this; +} +I::OStream& ByteMemoryOutStream::operator<< (const std::string &str) { + //*this << static_cast(str.size()); // N.B. Put type explicitly + *this << static_cast(str.size()); // N.B. Put type explicitly + for (auto c : str) *this << c; + return *this; +} + +ByteMemoryInStream::ByteMemoryInStream(const std::vector &data) + : m_storage(data) { +} +I::IStream& ByteMemoryInStream::operator>> (uint32_t &atom) { + check(sizeof(uint32_t)); + uint8_t x[4]; + x[0] = static_cast(m_storage[m_idx++]); + x[1] = static_cast(m_storage[m_idx++]); + x[2] = static_cast(m_storage[m_idx++]); + x[3] = static_cast(m_storage[m_idx++]); + atom = ((x[0]) | (x[1] << 8) | (x[2] << 16) | (x[3] << 24)); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (bool& atom) { + check(sizeof(char)); + atom = (m_storage[m_idx++] == 0) ? false : true; + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (char &atom) { + check(sizeof(char)); + atom = m_storage[m_idx++]; + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (unsigned char &atom) { + char c{}; + *this >> c; + atom = static_cast(c); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (short &atom) { + static_assert(sizeof(short) == 2, "Expecting sizeof(short) == 2"); + check(sizeof(short)); + uint8_t x[2]; + x[0] = static_cast(m_storage[m_idx++]); + x[1] = static_cast(m_storage[m_idx++]); + atom = ((x[0]) | (x[1] << 8)); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (unsigned short &atom) { + short s{}; + *this >> s; + atom = static_cast(s); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (int& atom) { + static_assert(sizeof(int) == 4, "Expecting sizeof(int) == 4"); + atom = static_cast(getU32()); + return *this; +} +//I::IStream& ByteMemoryInStream::operator>> (std::size_t& atom) { +// // NB. Type was truncated! +// atom = static_cast(getU32()); +// return *this; +//} +I::IStream& ByteMemoryInStream::operator>> (float& atom) { + static_assert(sizeof(float) == 4, "Expecting sizeof(float) == 4"); + uint32_t tmp = ntohl(getU32()); + memcpy(&atom, &tmp, sizeof(float)); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (double& atom) { + static_assert(sizeof(double) == 8, "Expecting sizeof(double) == 8"); + uint32_t tmp[2] = {ntohl(getU32()), ntohl(getU32())}; + memcpy(&atom, tmp, sizeof(double)); + return *this; +} +I::IStream& ByteMemoryInStream::operator>> (std::string& str) { + //std::size_t sz = 0u; + uint32_t sz = 0u; + *this >> sz; + if (sz == 0u) { + str.clear(); + } else { + str.resize(sz); + for (auto &&i : ade::util::iota(sz)) { *this >> str[i]; } + } + return *this; +} + +} // namespace s11n +} // namespace gimpl +} // namespace cv diff --git a/modules/gapi/src/backends/common/serialization.hpp b/modules/gapi/src/backends/common/serialization.hpp new file mode 100644 index 0000000000..a88c8c2cde --- /dev/null +++ b/modules/gapi/src/backends/common/serialization.hpp @@ -0,0 +1,313 @@ +#ifndef OPENCV_GAPI_COMMON_SERIALIZATION_HPP +#define OPENCV_GAPI_COMMON_SERIALIZATION_HPP + +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include +#include +#include + +#include // used in the vector<> + +#include "compiler/gmodel.hpp" + +namespace cv { +namespace gimpl { +namespace s11n { + +struct GSerialized { + std::vector m_ops; + std::vector m_datas; + cv::gimpl::DataObjectCounter m_counter; + cv::gimpl::Protocol m_proto; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Stream interfaces, so far temporary +namespace I { + struct GAPI_EXPORTS OStream { + virtual ~OStream() = default; + + // Define the native support for basic C++ types at the API level: + virtual OStream& operator<< (bool) = 0; + virtual OStream& operator<< (char) = 0; + virtual OStream& operator<< (unsigned char) = 0; + virtual OStream& operator<< (short) = 0; + virtual OStream& operator<< (unsigned short) = 0; + virtual OStream& operator<< (int) = 0; + //virtual OStream& operator<< (std::size_t) = 0; + virtual OStream& operator<< (uint32_t) = 0; + virtual OStream& operator<< (float) = 0; + virtual OStream& operator<< (double) = 0; + virtual OStream& operator<< (const std::string&) = 0; + }; + + struct GAPI_EXPORTS IStream { + virtual ~IStream() = default; + + virtual IStream& operator>> (bool &) = 0; + virtual IStream& operator>> (char &) = 0; + virtual IStream& operator>> (unsigned char &) = 0; + virtual IStream& operator>> (short &) = 0; + virtual IStream& operator>> (unsigned short &) = 0; + virtual IStream& operator>> (int &) = 0; + virtual IStream& operator>> (float &) = 0; + virtual IStream& operator>> (double &) = 0; + //virtual IStream& operator>> (std::size_t &) = 0; + virtual IStream& operator >> (uint32_t &) = 0; + virtual IStream& operator>> (std::string &) = 0; + }; +} // namespace I + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// S11N operators +// Note: operators for basic types are defined in IStream/OStream + +// OpenCV types //////////////////////////////////////////////////////////////// + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::Point &pt); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::Point &pt); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::Size &sz); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::Size &sz); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::Rect &rc); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::Rect &rc); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::Scalar &s); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::Scalar &s); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::Mat &m); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::Mat &m); + +// G-API types ///////////////////////////////////////////////////////////////// + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, cv::util::monostate ); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::util::monostate &); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, cv::GShape shape); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GShape &shape); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, cv::detail::ArgKind k); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::detail::ArgKind &k); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, cv::detail::OpaqueKind k); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::detail::OpaqueKind &k); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, cv::gimpl::Data::Storage s); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::Data::Storage &s); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::gimpl::DataObjectCounter &c); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::DataObjectCounter &c); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::gimpl::Protocol &p); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::Protocol &p); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GArg &arg); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GArg &arg); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GKernel &k); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GKernel &k); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GMatDesc &d); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GMatDesc &d); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GScalarDesc &); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GScalarDesc &); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GOpaqueDesc &); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GOpaqueDesc &); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::GArrayDesc &); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::GArrayDesc &); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::gimpl::RcDesc &rc); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::RcDesc &rc); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::gimpl::Op &op); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::Op &op); + +GAPI_EXPORTS I::OStream& operator<< (I::OStream& os, const cv::gimpl::Data &op); +GAPI_EXPORTS I::IStream& operator>> (I::IStream& is, cv::gimpl::Data &op); + +// The top-level serialization routine. +// Note it is just a single function which takes a GModel and a list of nodes +// and writes the data to the stream (recursively) +GAPI_EXPORTS void serialize( I::OStream& os + , const ade::Graph &g + , const std::vector &nodes); + +// The top-level deserialization routineS. +// Unfortunately the deserialization is a two-step process: +// 1. First we decode a stream into some intermediate representation +// (called "GSerialized"); +// 2. Then we produce an ade::Graph from this intermediate representation. +// +// An ade::Graph can't be produced from the stream immediately +// since every GCompiled object has its own unique ade::Graph, so +// we can't do it once and for all since every compilation process +// is individual and _is_ altering the ade::Graph state (structure and metadata). +// At the same time, we can't hold the reference to "is" within the GComputation +// forever since this input stream may be associated with an external resource +// and have side effects. +// +// Summarizing, the `deserialize()` happens *once per GComputation* immediately +// during the cv::gapi::deserialize(), and `reconstruct()` happens +// on every compilation process issued for this GComputation. +GAPI_EXPORTS GSerialized deserialize(I::IStream& is); +GAPI_EXPORTS void reconstruct(const GSerialized &s, ade::Graph &g); + +// Legacy ////////////////////////////////////////////////////////////////////// + + +// Generic: vector serialization /////////////////////////////////////////////// +template +I::OStream& operator<< (I::OStream& os, const std::vector &ts) { + //const std::size_t sz = ts.size(); // explicitly specify type + const uint32_t sz = (uint32_t)ts.size(); // explicitly specify type + os << sz; + for (auto &&v : ts) os << v; + return os; +} +template +I::IStream& operator>> (I::IStream& is, std::vector &ts) { + //std::size_t sz = 0u; + uint32_t sz = 0u; + is >> sz; + if (sz == 0u) { + ts.clear(); + } else { + ts.resize(sz); + for (auto &&i : ade::util::iota(sz)) is >> ts[i]; + } + return is; +} + +// Generic: unordered_map serialization //////////////////////////////////////// +template +I::OStream& operator<< (I::OStream& os, const std::unordered_map &m) { + //const std::size_t sz = m.size(); // explicitly specify type + const uint32_t sz = (uint32_t)m.size(); // explicitly specify type + os << sz; + for (auto &&it : m) os << it.first << it.second; + return os; +} +template +I::IStream& operator>> (I::IStream& is, std::unordered_map &m) { + m.clear(); + //std::size_t sz = 0u; + uint32_t sz = 0u; + is >> sz; + if (sz != 0u) { + for (auto &&i : ade::util::iota(sz)) { + (void) i; + K k{}; + V v{}; + is >> k >> v; + m.insert({k,v}); + } + GAPI_Assert(sz == m.size()); + } + return is; +} + +// Generic: variant serialization ////////////////////////////////////////////// +namespace detail { // FIXME: breaks old code +template +I::OStream& put_v(I::OStream&, const V&, std::size_t) { + GAPI_Assert(false && "variant>>: requested index is invalid"); +}; +template +I::OStream& put_v(I::OStream& os, const V& v, std::size_t x) { + return (x == 0u) + ? os << cv::util::get(v) + : put_v(os, v, x-1); +} +template +I::IStream& get_v(I::IStream&, V&, std::size_t, std::size_t) { + GAPI_Assert(false && "variant<<: requested index is invalid"); +} +template +I::IStream& get_v(I::IStream& is, V& v, std::size_t i, std::size_t gi) { + if (i == gi) { + X x{}; + is >> x; + v = std::move(x); + return is; + } else return get_v(is, v, i+1, gi); +} +} // namespace detail FIXME: breaks old code + +template +I::OStream& operator<< (I::OStream& os, const cv::util::variant &v) { + os << (uint32_t)v.index(); + return detail::put_v, Ts...>(os, v, v.index()); +} +template +I::IStream& operator>> (I::IStream& is, cv::util::variant &v) { + int idx = -1; + is >> idx; + GAPI_Assert(idx >= 0 && idx < (int)sizeof...(Ts)); + return detail::get_v, Ts...>(is, v, 0u, idx); +} + +// FIXME: Basic Stream implementaions ////////////////////////////////////////// + +// Basic in-memory stream implementations. +class GAPI_EXPORTS ByteMemoryOutStream final: public I::OStream { + std::vector m_storage; + + //virtual I::OStream& operator << (uint32_t) override; + //virtual I::OStream& operator<< (uint32_t) final; +public: + const std::vector& data() const; + + virtual I::OStream& operator<< (bool) override; + virtual I::OStream& operator<< (char) override; + virtual I::OStream& operator<< (unsigned char) override; + virtual I::OStream& operator<< (short) override; + virtual I::OStream& operator<< (unsigned short) override; + virtual I::OStream& operator<< (int) override; + //virtual I::OStream& operator<< (std::size_t) override; + virtual I::OStream& operator<< (float) override; + virtual I::OStream& operator<< (double) override; + virtual I::OStream& operator<< (const std::string&) override; + virtual I::OStream& operator<< (uint32_t) override; +}; + +class GAPI_EXPORTS ByteMemoryInStream final: public I::IStream { + const std::vector& m_storage; + size_t m_idx = 0u; + + void check(std::size_t n) { (void) n; GAPI_DbgAssert(m_idx+n-1 < m_storage.size()); } + uint32_t getU32() { uint32_t v{}; *this >> v; return v; }; + + //virtual I::IStream& operator>> (uint32_t &) final; + +public: + explicit ByteMemoryInStream(const std::vector &data); + + virtual I::IStream& operator>> (bool &) override; + virtual I::IStream& operator>> (char &) override; + virtual I::IStream& operator>> (unsigned char &) override; + virtual I::IStream& operator>> (short &) override; + virtual I::IStream& operator>> (unsigned short &) override; + virtual I::IStream& operator>> (int &) override; + virtual I::IStream& operator>> (float &) override; + virtual I::IStream& operator>> (double &) override; + //virtual I::IStream& operator>> (std::size_t &) override; + virtual I::IStream& operator >> (uint32_t &) override; + virtual I::IStream& operator>> (std::string &) override; +}; + + +} // namespace s11n +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_COMMON_SERIALIZATION_HPP diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index 21486b3b14..9fa984c4cc 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -111,24 +111,6 @@ namespace return result; } - // Creates ADE graph from input/output proto args - std::unique_ptr makeGraph(const cv::GProtoArgs &ins, const cv::GProtoArgs &outs) { - std::unique_ptr pG(new ade::Graph); - ade::Graph& g = *pG; - - cv::gimpl::GModel::Graph gm(g); - cv::gimpl::GModel::init(gm); - cv::gimpl::GModelBuilder builder(g); - auto proto_slots = builder.put(ins, outs); - - // Store Computation's protocol in metadata - cv::gimpl::Protocol p; - std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots; - gm.metadata().set(p); - - return pG; - } - using adeGraphs = std::vector>; // Creates ADE graphs (patterns and substitutes) from pkg's transformations @@ -146,14 +128,10 @@ namespace ade::util::toRange(patterns), ade::util::toRange(substitutes))) { const auto& t = std::get<0>(it); - auto& p = std::get<1>(it); - auto& s = std::get<2>(it); - - auto pattern_comp = t.pattern(); - p = makeGraph(pattern_comp.priv().m_ins, pattern_comp.priv().m_outs); - - auto substitute_comp = t.substitute(); - s = makeGraph(substitute_comp.priv().m_ins, substitute_comp.priv().m_outs); + auto& p = std::get<1>(it); + auto& s = std::get<2>(it); + p = cv::gimpl::GCompiler::makeGraph(t.pattern().priv()); + s = cv::gimpl::GCompiler::makeGraph(t.substitute().priv()); } } @@ -311,11 +289,19 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c, void cv::gimpl::GCompiler::validateInputMeta() { - if (m_metas.size() != m_c.priv().m_ins.size()) + // FIXME: implement testing/accessor methods at the Priv's API level? + if (!util::holds_alternative(m_c.priv().m_shape)) + { + GAPI_LOG_WARNING(NULL, "Metadata validation is not implemented yet for" + " deserialized graphs!"); + return; + } + const auto &c_expr = util::get(m_c.priv().m_shape); + if (m_metas.size() != c_expr.m_ins.size()) { util::throw_error(std::logic_error ("COMPILE: GComputation interface / metadata mismatch! " - "(expected " + std::to_string(m_c.priv().m_ins.size()) + ", " + "(expected " + std::to_string(c_expr.m_ins.size()) + ", " "got " + std::to_string(m_metas.size()) + " meta arguments)")); } @@ -343,7 +329,7 @@ void cv::gimpl::GCompiler::validateInputMeta() return false; // should never happen }; - for (const auto &meta_arg_idx : ade::util::indexed(ade::util::zip(m_metas, m_c.priv().m_ins))) + for (const auto &meta_arg_idx : ade::util::indexed(ade::util::zip(m_metas, c_expr.m_ins))) { const auto &meta = std::get<0>(ade::util::value(meta_arg_idx)); const auto &proto = std::get<1>(ade::util::value(meta_arg_idx)); @@ -362,7 +348,15 @@ void cv::gimpl::GCompiler::validateInputMeta() void cv::gimpl::GCompiler::validateOutProtoArgs() { - for (const auto &out_pos : ade::util::indexed(m_c.priv().m_outs)) + // FIXME: implement testing/accessor methods at the Priv's API level? + if (!util::holds_alternative(m_c.priv().m_shape)) + { + GAPI_LOG_WARNING(NULL, "Output parameter validation is not implemented yet for" + " deserialized graphs!"); + return; + } + const auto &c_expr = util::get(m_c.priv().m_shape); + for (const auto &out_pos : ade::util::indexed(c_expr.m_outs)) { const auto &node = proto::origin_of(ade::util::value(out_pos)).node; if (node.shape() != cv::GNode::NodeShape::CALL) @@ -383,7 +377,7 @@ cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::generateGraph() validateInputMeta(); } validateOutProtoArgs(); - auto g = makeGraph(m_c.priv().m_ins, m_c.priv().m_outs); + auto g = makeGraph(m_c.priv()); if (!m_metas.empty()) { GModel::Graph(*g).metadata().set(OriginalInputMeta{m_metas}); @@ -512,3 +506,27 @@ void cv::gimpl::GCompiler::runMetaPasses(ade::Graph &g, const cv::GMetaArgs &met } engine.runPasses(g); } + +// Creates ADE graph from input/output proto args OR from its +// deserialized form +cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::makeGraph(const cv::GComputation::Priv &priv) { + std::unique_ptr pG(new ade::Graph); + ade::Graph& g = *pG; + + if (cv::util::holds_alternative(priv.m_shape)) { + auto c_expr = cv::util::get(priv.m_shape); + cv::gimpl::GModel::Graph gm(g); + cv::gimpl::GModel::init(gm); + cv::gimpl::GModelBuilder builder(g); + auto proto_slots = builder.put(c_expr.m_ins, c_expr.m_outs); + + // Store Computation's protocol in metadata + cv::gimpl::Protocol p; + std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots; + gm.metadata().set(p); + } else if (cv::util::holds_alternative(priv.m_shape)) { + auto c_dump = cv::util::get(priv.m_shape); + cv::gimpl::s11n::reconstruct(c_dump, g); + } + return pG; +} diff --git a/modules/gapi/src/compiler/gcompiler.hpp b/modules/gapi/src/compiler/gcompiler.hpp index f111d1618a..85b05d54db 100644 --- a/modules/gapi/src/compiler/gcompiler.hpp +++ b/modules/gapi/src/compiler/gcompiler.hpp @@ -58,6 +58,8 @@ public: GCompiled produceCompiled(GPtr &&pg); // Produce GCompiled from processed GModel GStreamingCompiled produceStreamingCompiled(GPtr &&pg); // Produce GStreamingCompiled from processed GMbodel static void runMetaPasses(ade::Graph &g, const cv::GMetaArgs &metas); + + static GPtr makeGraph(const cv::GComputation::Priv &); }; }} diff --git a/modules/gapi/src/compiler/gmodel.hpp b/modules/gapi/src/compiler/gmodel.hpp index bb327d00a4..1b16079a7d 100644 --- a/modules/gapi/src/compiler/gmodel.hpp +++ b/modules/gapi/src/compiler/gmodel.hpp @@ -73,7 +73,7 @@ struct Data HostCtor ctor; // T-specific helper to deal with unknown types in our code // FIXME: Why rc+shape+meta is not represented as RcDesc here? - enum class Storage + enum class Storage: int { INTERNAL, // data object is not listed in GComputation protocol INPUT, // data object is listed in GComputation protocol as Input @@ -138,7 +138,9 @@ class DataObjectCounter public: static const char* name() { return "DataObjectCounter"; } int GetNewId(GShape shape) { return m_next_data_id[shape]++; } -private: + + // NB: private!!! but used in the serialization + // couldn't get the `friend` stuff working correctly -- DM std::unordered_map m_next_data_id; }; @@ -166,6 +168,19 @@ struct Streaming static const char *name() { return "StreamingFlag"; } }; + +// This is a graph-global flag indicating this graph is compiled +// after the deserialization. Some bits of information may be +// unavailable (mainly callbacks) so let sensitive passes obtain +// the required information in their special way. +// +// FIXME: Probably a better design can be suggested. +struct Deserialized +{ + static const char *name() { return "DeserializedFlag"; } +}; + + // Backend-specific inference parameters for a neural network. // Since these parameters are set on compilation stage (not // on a construction stage), these parameters are bound lately @@ -213,6 +228,7 @@ namespace GModel , ActiveBackends , CustomMetaFunction , Streaming + , Deserialized >; // FIXME: How to define it based on GModel??? @@ -234,6 +250,7 @@ namespace GModel , ActiveBackends , CustomMetaFunction , Streaming + , Deserialized >; // FIXME: diff --git a/modules/gapi/src/compiler/passes/kernels.cpp b/modules/gapi/src/compiler/passes/kernels.cpp index 84b3621f90..69b339fb1e 100644 --- a/modules/gapi/src/compiler/passes/kernels.cpp +++ b/modules/gapi/src/compiler/passes/kernels.cpp @@ -150,6 +150,17 @@ void cv::gimpl::passes::resolveKernels(ade::passes::PassContext &ctx, selected_backend.priv().unpackKernel(ctx.graph, nh, selected_impl); op.backend = selected_backend; active_backends.insert(selected_backend); + + if (gr.metadata().contains()) + { + // Trick: in this case, the op.k.outMeta is by default + // missing. Take it from the resolved kernel + GAPI_Assert(op.k.outMeta == nullptr); + const_cast(op.k.outMeta) = selected_impl.outMeta; + } else { + // Sanity check: the metadata funciton must be present + GAPI_Assert(op.k.outMeta != nullptr); + } } } gr.metadata().set(ActiveBackends{active_backends}); diff --git a/modules/gapi/src/compiler/passes/transformations.cpp b/modules/gapi/src/compiler/passes/transformations.cpp index f3b963858e..62407fe5a8 100644 --- a/modules/gapi/src/compiler/passes/transformations.cpp +++ b/modules/gapi/src/compiler/passes/transformations.cpp @@ -77,7 +77,8 @@ bool tryToSubstitute(ade::Graph& main, // 2. build substitute graph inside the main graph cv::gimpl::GModelBuilder builder(main); - const auto& proto_slots = builder.put(substitute.priv().m_ins, substitute.priv().m_outs); + auto expr = cv::util::get(substitute.priv().m_shape); + const auto& proto_slots = builder.put(expr.m_ins, expr.m_outs); Protocol substituteP; std::tie(substituteP.inputs, substituteP.outputs, substituteP.in_nhs, substituteP.out_nhs) = proto_slots; diff --git a/modules/gapi/test/gapi_transform_tests.cpp b/modules/gapi/test/gapi_transform_tests.cpp index ad1a6aae0b..4077008f68 100644 --- a/modules/gapi/test/gapi_transform_tests.cpp +++ b/modules/gapi/test/gapi_transform_tests.cpp @@ -198,7 +198,7 @@ TEST(KernelPackageTransform, gmat_gsc_garray_in_gmat2_out) auto tr = gmat_gsc_garray_in_gmat2_out::transformation(); auto check = [](const cv::GComputation &comp){ - const auto &p = comp.priv(); + const auto &p = cv::util::get(comp.priv().m_shape); EXPECT_EQ(3u, p.m_ins.size()); EXPECT_EQ(2u, p.m_outs.size()); @@ -221,7 +221,7 @@ TEST(KernelPackageTransform, gmat_gsc_gopaque_in_gmat2_out) auto tr = gmat_gsc_gopaque_in_gmat2_out::transformation(); auto check = [](const cv::GComputation &comp){ - const auto &p = comp.priv(); + const auto &p = cv::util::get(comp.priv().m_shape); EXPECT_EQ(3u, p.m_ins.size()); EXPECT_EQ(2u, p.m_outs.size()); @@ -270,7 +270,7 @@ namespace template void args_check(const cv::GComputation &comp) { - const auto &p = comp.priv(); + const auto &p = cv::util::get(comp.priv().m_shape); EXPECT_EQ(1u, p.m_ins.size()); EXPECT_EQ(1u, p.m_outs.size()); arg_check(p.m_ins[0]); diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp new file mode 100644 index 0000000000..5c4fd2a425 --- /dev/null +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -0,0 +1,128 @@ +#include "../test_precomp.hpp" + +#include "backends/common/serialization.hpp" + +namespace opencv_test { + +struct S11N_Basic: public ::testing::Test { + template void put(T &&t) { + cv::gimpl::s11n::ByteMemoryOutStream os; + os << t; + m_buffer = os.data(); + } + + template T get() { + // FIXME: This stream API needs a fix-up + cv::gimpl::s11n::ByteMemoryInStream is(m_buffer); + T t{}; + is >> t; + return t; + } + +private: + std::vector m_buffer; +}; + +TEST_F(S11N_Basic, Test_int_pos) { + int x = 42; + put(x); + EXPECT_EQ(x, get()); +} + +TEST_F(S11N_Basic, Test_int_neg) { + int x = -42; + put(x); + EXPECT_EQ(x, get()); +} + +TEST_F(S11N_Basic, Test_fp32) { + float x = 3.14f; + put(x); + EXPECT_EQ(x, get()); +} + +TEST_F(S11N_Basic, Test_fp64) { + double x = 3.14; + put(x); + EXPECT_EQ(x, get()); +} + +TEST_F(S11N_Basic, Test_vector_int) { + std::vector v = {1,2,3}; + put(v); + EXPECT_EQ(v, get >()); +} + +TEST_F(S11N_Basic, Test_vector_cvSize) { + std::vector v = { + cv::Size(640, 480), + cv::Size(1280, 1024), + }; + put(v); + EXPECT_EQ(v, get >()); +} + +TEST_F(S11N_Basic, Test_vector_string) { + std::vector v = { + "hello", + "world", + "ok!" + }; + put(v); + EXPECT_EQ(v, get >()); +} + +TEST_F(S11N_Basic, Test_vector_empty) { + std::vector v; + put(v); + EXPECT_EQ(v, get >()); +} + +TEST_F(S11N_Basic, Test_variant) { + using S = std::string; + using V = cv::util::variant; + V v1{42}, v2{S{"hey"}}; + + put(v1); + EXPECT_EQ(v1, get()); + + put(v2); + EXPECT_EQ(v2, get()); +} + +TEST_F(S11N_Basic, Test_GArg_int) { + const int x = 42; + cv::GArg gs(x); + put(gs); + + cv::GArg gd = get(); + EXPECT_EQ(cv::detail::ArgKind::OPAQUE_VAL, gd.kind); + EXPECT_EQ(cv::detail::OpaqueKind::CV_INT, gd.opaque_kind); + EXPECT_EQ(x, gs.get()); +} + +TEST_F(S11N_Basic, Test_GArg_Point) { + const cv::Point pt{1,2}; + cv::GArg gs(pt); + put(gs); + + cv::GArg gd = get(); + EXPECT_EQ(cv::detail::ArgKind::OPAQUE_VAL, gd.kind); + EXPECT_EQ(cv::detail::OpaqueKind::CV_POINT, gd.opaque_kind); + EXPECT_EQ(pt, gs.get()); +} + +TEST_F(S11N_Basic, Test_Mat_full) { + auto mat = cv::Mat::eye(cv::Size(64,64), CV_8UC3); + put(mat); + EXPECT_EQ(0, cv::norm(mat, get(), cv::NORM_INF)); +} + +TEST_F(S11N_Basic, Test_Mat_view) { + auto mat = cv::Mat::eye(cv::Size(320,240), CV_8UC3); + auto view = mat(cv::Rect(10,15,123,70)); + put(view); + EXPECT_EQ(0, cv::norm(view, get(), cv::NORM_INF)); +} + +} // namespace opencv_test diff --git a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp new file mode 100644 index 0000000000..89956bfd28 --- /dev/null +++ b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp @@ -0,0 +1,285 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#include "../test_precomp.hpp" + +#include + +#include + +namespace opencv_test +{ + +TEST(S11N, Pipeline_Crop_Rect) +{ + cv::Rect rect_to{ 4,10,37,50 }; + cv::Size sz_in = cv::Size(1920, 1080); + cv::Size sz_out = cv::Size(37, 50); + cv::Mat in_mat = cv::Mat::eye(sz_in, CV_8UC1); + cv::Mat out_mat_gapi(sz_out, CV_8UC1); + cv::Mat out_mat_ocv(sz_out, CV_8UC1); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::crop(in, rect_to); + auto p = cv::gapi::serialize(cv::GComputation(in, out)); + auto c = cv::gapi::deserialize(p); + c.apply(in_mat, out_mat_gapi); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_mat_ocv = in_mat(rect_to); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); + } +} + + +TEST(S11N, Pipeline_Canny_Bool) +{ + const cv::Size sz_in(1280, 720); + cv::GMat in; + double thrLow = 120.0; + double thrUp = 240.0; + int apSize = 5; + bool l2gr = true; + cv::Mat in_mat = cv::Mat::eye(1280, 720, CV_8UC1); + cv::Mat out_mat_gapi(sz_in, CV_8UC1); + cv::Mat out_mat_ocv(sz_in, CV_8UC1); + + // G-API code ////////////////////////////////////////////////////////////// + auto out = cv::gapi::Canny(in, thrLow, thrUp, apSize, l2gr); + auto p = cv::gapi::serialize(cv::GComputation(in, out)); + auto c = cv::gapi::deserialize(p); + c.apply(in_mat, out_mat_gapi); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::Canny(in_mat, out_mat_ocv, thrLow, thrUp, apSize, l2gr); + } + // Comparison ////////////////////////////////////////////////////////////// + EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); +} + +TEST(S11N, Pipeline_Not) +{ + cv::GMat in; + auto p = cv::gapi::serialize(cv::GComputation(in, cv::gapi::bitwise_not(in))); + auto c = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Mat ref_mat = ~in_mat; + + cv::Mat out_mat; + c.apply(in_mat, out_mat); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + auto cc = c.compile(cv::descr_of(in_mat)); + cc(in_mat, out_mat); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); +} + +TEST(S11N, Pipeline_Sum_Scalar) +{ + cv::GMat in; + auto p = cv::gapi::serialize(cv::GComputation(in, cv::gapi::sum(in))); + auto c = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Scalar ref_scl = cv::sum(in_mat); + + cv::Scalar out_scl; + c.apply(in_mat, out_scl); + EXPECT_EQ(out_scl, ref_scl); + + out_scl = cv::Scalar(); + auto cc = c.compile(cv::descr_of(in_mat)); + cc(in_mat, out_scl); + EXPECT_EQ(out_scl, ref_scl); +} + +TEST(S11N, Pipeline_BinaryOp) +{ + cv::GMat a, b; + auto p = cv::gapi::serialize(cv::GComputation(a, b, cv::gapi::add(a, b))); + auto c = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Mat ref_mat = (in_mat + in_mat); + + cv::Mat out_mat; + c.apply(in_mat, in_mat, out_mat); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + auto cc = c.compile(cv::descr_of(in_mat), cv::descr_of(in_mat)); + cc(in_mat, in_mat, out_mat); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); +} + +TEST(S11N, Pipeline_Binary_Sum_Scalar) +{ + cv::GMat a, b; + auto p = cv::gapi::serialize(cv::GComputation(a, b, cv::gapi::sum(a + b))); + auto c = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Scalar ref_scl = cv::sum(in_mat + in_mat); + cv::Scalar out_scl; + c.apply(in_mat, in_mat, out_scl); + EXPECT_EQ(out_scl, ref_scl); + + out_scl = cv::Scalar(); + auto cc = c.compile(cv::descr_of(in_mat), cv::descr_of(in_mat)); + cc(in_mat, in_mat, out_scl); + EXPECT_EQ(out_scl, ref_scl); +} + +TEST(S11N, Pipeline_Sharpen) +{ + const cv::Size sz_in (1280, 720); + const cv::Size sz_out( 640, 480); + cv::Mat in_mat (sz_in, CV_8UC3); + in_mat = cv::Scalar(128, 33, 53); + + cv::Mat out_mat(sz_out, CV_8UC3); + cv::Mat out_mat_y; + cv::Mat out_mat_ocv(sz_out, CV_8UC3); + + float sharpen_coeffs[] = { + 0.0f, -1.f, 0.0f, + -1.0f, 5.f, -1.0f, + 0.0f, -1.f, 0.0f + }; + cv::Mat sharpen_kernel(3, 3, CV_32F, sharpen_coeffs); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto vga = cv::gapi::resize(in, sz_out); + auto yuv = cv::gapi::RGB2YUV(vga); + auto yuv_p = cv::gapi::split3(yuv); + auto y_sharp = cv::gapi::filter2D(std::get<0>(yuv_p), -1, sharpen_kernel); + auto yuv_new = cv::gapi::merge3(y_sharp, std::get<1>(yuv_p), std::get<2>(yuv_p)); + auto out = cv::gapi::YUV2RGB(yuv_new); + + auto p = cv::gapi::serialize(cv::GComputation(cv::GIn(in), cv::GOut(y_sharp, out))); + auto c = cv::gapi::deserialize(p); + c.apply(cv::gin(in_mat), cv::gout(out_mat_y, out_mat)); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::Mat smaller; + cv::resize(in_mat, smaller, sz_out); + + cv::Mat yuv_mat; + cv::cvtColor(smaller, yuv_mat, cv::COLOR_RGB2YUV); + std::vector yuv_planar(3); + cv::split(yuv_mat, yuv_planar); + cv::filter2D(yuv_planar[0], yuv_planar[0], -1, sharpen_kernel); + cv::merge(yuv_planar, yuv_mat); + cv::cvtColor(yuv_mat, out_mat_ocv, cv::COLOR_YUV2RGB); + } + + // Comparison ////////////////////////////////////////////////////////////// + { + cv::Mat diff = out_mat_ocv != out_mat; + std::vector diffBGR(3); + cv::split(diff, diffBGR); + EXPECT_EQ(0, cvtest::norm(diffBGR[0], NORM_INF)); + EXPECT_EQ(0, cvtest::norm(diffBGR[1], NORM_INF)); + EXPECT_EQ(0, cvtest::norm(diffBGR[2], NORM_INF)); + } + + // Metadata check ///////////////////////////////////////////////////////// + { + auto cc = c.compile(cv::descr_of(in_mat)); + auto metas = cc.outMetas(); + ASSERT_EQ(2u, metas.size()); + + auto out_y_meta = cv::util::get(metas[0]); + auto out_meta = cv::util::get(metas[1]); + + // Y-output + EXPECT_EQ(CV_8U, out_y_meta.depth); + EXPECT_EQ(1, out_y_meta.chan); + EXPECT_EQ(640, out_y_meta.size.width); + EXPECT_EQ(480, out_y_meta.size.height); + + // Final output + EXPECT_EQ(CV_8U, out_meta.depth); + EXPECT_EQ(3, out_meta.chan); + EXPECT_EQ(640, out_meta.size.width); + EXPECT_EQ(480, out_meta.size.height); + } +} + +TEST(S11N, Pipeline_CustomRGB2YUV) +{ + const cv::Size sz(1280, 720); + const int INS = 3; + std::vector in_mats(INS); + for (auto i : ade::util::iota(INS)) + { + in_mats[i].create(sz, CV_8U); + cv::randu(in_mats[i], cv::Scalar::all(0), cv::Scalar::all(255)); + } + + const int OUTS = 3; + std::vector out_mats_cv(OUTS); + std::vector out_mats_gapi(OUTS); + for (auto i : ade::util::iota(OUTS)) + { + out_mats_cv[i].create(sz, CV_8U); + out_mats_gapi[i].create(sz, CV_8U); + } + + // G-API code ////////////////////////////////////////////////////////////// + { + cv::GMat r, g, b; + cv::GMat y = 0.299f*r + 0.587f*g + 0.114f*b; + cv::GMat u = 0.492f*(b - y); + cv::GMat v = 0.877f*(r - y); + + auto p = cv::gapi::serialize(cv::GComputation({r, g, b}, {y, u, v})); + auto c = cv::gapi::deserialize(p); + c.apply(in_mats, out_mats_gapi); + } + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::Mat r = in_mats[0], g = in_mats[1], b = in_mats[2]; + cv::Mat y = 0.299f*r + 0.587f*g + 0.114f*b; + cv::Mat u = 0.492f*(b - y); + cv::Mat v = 0.877f*(r - y); + + out_mats_cv[0] = y; + out_mats_cv[1] = u; + out_mats_cv[2] = v; + } + + // Comparison ////////////////////////////////////////////////////////////// + { + const auto diff = [](cv::Mat m1, cv::Mat m2, int t) { + return cv::abs(m1 - m2) > t; + }; + + // FIXME: Not bit-accurate even now! + cv::Mat + diff_y = diff(out_mats_cv[0], out_mats_gapi[0], 2), + diff_u = diff(out_mats_cv[1], out_mats_gapi[1], 2), + diff_v = diff(out_mats_cv[2], out_mats_gapi[2], 2); + + EXPECT_EQ(0, cvtest::norm(diff_y, NORM_INF)); + EXPECT_EQ(0, cvtest::norm(diff_u, NORM_INF)); + EXPECT_EQ(0, cvtest::norm(diff_v, NORM_INF)); + } +} + +} // namespace opencv_test