third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #21504 from smirnov-alexey:as/oak_infer

Browse Source 
[GAPI] Support basic inference in OAK backend

* Combined commit which enables basic inference and other extra capabilities of OAK backend

* Remove unnecessary target options from the cmakelist
This commit is contained in:
Alexey Smirnov 2022-04-05 21:00:32 +03:00 committed by GitHub
parent b2e20a82ba
commit 7ed557497d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 871 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
modules/gapi/CMakeLists.txt
View File

@ -138,7 +138,7 @@ set(gapi_srcs
# OAK Backend (optional)
src/backends/oak/goak.cpp
src/backends/oak/goakbackend.cpp
src/backends/oak/goak_media_adapter.cpp
src/backends/oak/goak_memory_adapters.cpp
# OCL Backend (currently built-in)
src/backends/ocl/goclbackend.cpp
@ -375,13 +375,3 @@ if(HAVE_GAPI_ONEVPL)
endif()
endif()
endif()
if(HAVE_OAK)
# FIXME: consider better solution
if(TARGET example_gapi_oak_rgb_camera_encoding)
ocv_target_compile_definitions(example_gapi_oak_rgb_camera_encoding PRIVATE -DHAVE_OAK)
endif()
if(TARGET example_gapi_oak_small_hetero_pipeline)
ocv_target_compile_definitions(example_gapi_oak_small_hetero_pipeline PRIVATE -DHAVE_OAK)
endif()
endif()

66
modules/gapi/include/opencv2/gapi/oak/infer.hpp Normal file
View File

@ -0,0 +1,66 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2022 Intel Corporation
#ifndef OPENCV_GAPI_OAK_INFER_HPP
#define OPENCV_GAPI_OAK_INFER_HPP
#include <unordered_map>
#include <string>
#include <array>
#include <tuple>
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/core/cvdef.h> // GAPI_EXPORTS
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
namespace cv {
namespace gapi {
namespace oak {
namespace detail {
/**
* @brief This structure contains description of inference parameters
* which is specific to OAK models.
*/
struct ParamDesc {
std::string blob_file;
};
} // namespace detail
/**
* Contains description of inference parameters and kit of functions that
* fill this parameters.
*/
template<typename Net> class Params {
public:
/** @brief Class constructor.
Constructs Params based on model information and sets default values for other
inference description parameters.
@param model Path to model (.blob file)
*/
explicit Params(const std::string &model) {
desc.blob_file = model;
};
// BEGIN(G-API's network parametrization API)
GBackend backend() const { return cv::gapi::oak::backend(); }
std::string tag() const { return Net::tag(); }
cv::util::any params() const { return { desc }; }
// END(G-API's network parametrization API)
protected:
detail::ParamDesc desc;
};
} // namespace oak
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_OAK_INFER_HPP

29
modules/gapi/include/opencv2/gapi/oak/oak.hpp
View File

@ -89,28 +89,55 @@ G_API_OP(GSobelXY, <GFrame(GFrame, const cv::Mat&, const cv::Mat&)>, "org.opencv
}
};
G_API_OP(GCopy, <GFrame(GFrame)>, "org.opencv.oak.copy") {
static GFrameDesc outMeta(const GFrameDesc& in) {
return in;
}
};
// FIXME: add documentation on operations below
GAPI_EXPORTS GArray<uint8_t> encode(const GFrame& in, const EncoderConfig&);
GAPI_EXPORTS GFrame sobelXY(const GFrame& in,
const cv::Mat& hk,
const cv::Mat& vk);
GAPI_EXPORTS GFrame copy(const GFrame& in);
// OAK backend & kernels ////////////////////////////////////////////////////////
GAPI_EXPORTS cv::gapi::GBackend backend();
GAPI_EXPORTS cv::gapi::GKernelPackage kernels();
// Camera object ///////////////////////////////////////////////////////////////
struct GAPI_EXPORTS ColorCameraParams {};
struct GAPI_EXPORTS ColorCameraParams {
/**
* Format of the frame one gets from the camera
*/
bool interleaved = false;
// FIXME: extend
enum class BoardSocket: int { RGB, BGR };
BoardSocket board_socket = BoardSocket::RGB;
// FIXME: extend
enum class Resolution: int { THE_1080_P };
Resolution resolution = Resolution::THE_1080_P;
};
class GAPI_EXPORTS ColorCamera: public cv::gapi::wip::IStreamSource {
cv::MediaFrame m_dummy;
ColorCameraParams m_params;
virtual bool pull(cv::gapi::wip::Data &data) override;
virtual GMetaArg descr_of() const override;
public:
ColorCamera();
explicit ColorCamera(const ColorCameraParams& params);
};
} // namespace oak

122
modules/gapi/samples/oak_basic_infer.cpp Normal file
View File

@ -0,0 +1,122 @@
#include <algorithm>
#include <iostream>
#include <sstream>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/gapi.hpp>
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/imgproc.hpp>
#include <opencv2/gapi/infer.hpp>
#include <opencv2/gapi/infer/parsers.hpp>
#include <opencv2/gapi/render.hpp>
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/oak/infer.hpp>
const std::string keys =
"{ h help | | Print this help message }"
"{ detector | | Path to compiled .blob face detector model }"
"{ duration | 100 | Number of frames to pull from camera and run inference on }";
namespace custom {
G_API_NET(FaceDetector, <cv::GMat(cv::GFrame)>, "sample.custom.face-detector");
using GDetections = cv::GArray<cv::Rect>;
using GSize = cv::GOpaque<cv::Size>;
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
return cv::empty_array_desc();
}
};
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
// This kernel converts the rectangles into G-API's
// rendering primitives
static void run(const std::vector<cv::Rect> &in_face_rcs,
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
out_prims.clear();
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
return cv::gapi::wip::draw::Rect(rc, clr, 2);
};
for (auto &&rc : in_face_rcs) {
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
}
}
};
} // namespace custom
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
cmd.printMessage();
return 0;
}
const auto det_name = cmd.get<std::string>("detector");
const auto duration = cmd.get<int>("duration");
if (det_name.empty()) {
std::cerr << "FATAL: path to detection model is not provided for the sample."
<< "Please specify it with --detector options."
<< std::endl;
return 1;
}
// Prepare G-API kernels and networks packages:
auto detector = cv::gapi::oak::Params<custom::FaceDetector>(det_name);
auto networks = cv::gapi::networks(detector);
auto kernels = cv::gapi::combine(
cv::gapi::kernels<custom::OCVBBoxes>(),
cv::gapi::oak::kernels());
auto args = cv::compile_args(kernels, networks);
// Initialize graph structure
cv::GFrame in;
cv::GFrame copy = cv::gapi::oak::copy(in); // NV12 transfered to host + passthrough copy for infer
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(copy);
// infer is not affected by the actual copy here
cv::GMat blob = cv::gapi::infer<custom::FaceDetector>(copy);
// FIXME: OAK infer detects faces slightly out of frame bounds
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, false);
auto rendered = cv::gapi::wip::draw::renderFrame(copy, custom::BBoxes::on(rcs));
// on-the-fly conversion NV12->BGR
cv::GMat out = cv::gapi::streaming::BGR(rendered);
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out, rcs))
.compileStreaming(std::move(args));
// Graph execution
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
pipeline.start();
cv::Mat out_mat;
std::vector<cv::Rect> out_dets;
int frames = 0;
while (pipeline.pull(cv::gout(out_mat, out_dets))) {
std::string name = "oak_infer_frame_" + std::to_string(frames) + ".png";
cv::imwrite(name, out_mat);
if (!out_dets.empty()) {
std::cout << "Got " << out_dets.size() << " detections on frame #" << frames << std::endl;
}
++frames;
if (frames == duration) {
pipeline.stop();
break;
}
}
std::cout << "Pipeline finished. Processed " << frames << " frames" << std::endl;
return 0;
}

48
modules/gapi/samples/oak_copy.cpp Normal file
View File

@ -0,0 +1,48 @@
#include <opencv2/gapi.hpp>
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/cpu/core.hpp>
#include <opencv2/gapi/gframe.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
#include <opencv2/highgui.hpp> // CommandLineParser
const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.png | Path to the output file }";
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
cmd.printMessage();
return 0;
}
const std::string output_name = cmd.get<std::string>("output");
cv::GFrame in;
// Actually transfers data to host
cv::GFrame copy = cv::gapi::oak::copy(in);
// Default camera works only with nv12 format
cv::GMat out = cv::gapi::streaming::Y(copy);
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
cv::gapi::oak::kernels());
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
// Graph execution /////////////////////////////////////////////////////////
cv::Mat out_mat(1920, 1080, CV_8UC1);
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
pipeline.start();
// pull 1 frame
pipeline.pull(cv::gout(out_mat));
cv::imwrite(output_name, out_mat);
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}

10
modules/gapi/samples/oak_rgb_camera_encoding.cpp
View File

@ -13,8 +13,6 @@ const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.h265 | Path to the output .h265 video file }";
#ifdef HAVE_OAK
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
@ -60,11 +58,3 @@ int main(int argc, char *argv[]) {
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}
#else // HAVE_OAK
int main() {
GAPI_Assert(false && "Built without OAK support");
return -1;
}
#endif // HAVE_OAK

11
modules/gapi/samples/oak_small_hetero_pipeline.cpp
View File

@ -13,8 +13,6 @@ const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.png | Path to the output file }";
#ifdef HAVE_OAK
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
@ -58,12 +56,3 @@ int main(int argc, char *argv[]) {
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}
#else // HAVE_OAK
int main() {
GAPI_Assert(false && "Built without OAK support");
return -1;
}
#endif // HAVE_OAK

15
modules/gapi/src/backends/oak/goak.cpp
View File

@ -7,7 +7,7 @@
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
#include <thread>
#include <chrono>
@ -24,6 +24,10 @@ GFrame sobelXY(const GFrame& in, const cv::Mat& hk, const cv::Mat& vk) {
return GSobelXY::on(in, hk, vk);
}
GFrame copy(const GFrame& in) {
return GCopy::on(in);
}
// This is a dummy oak::ColorCamera class that just makes our pipelining
// machinery work. The real data comes from the physical camera which
// is handled by DepthAI library.
@ -31,6 +35,11 @@ ColorCamera::ColorCamera()
: m_dummy(cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>()) {
}
ColorCamera::ColorCamera(const ColorCameraParams& params)
: m_dummy(cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>()),
m_params(params) {
}
bool ColorCamera::pull(cv::gapi::wip::Data &data) {
// FIXME: Avoid passing this formal frame to the pipeline
std::this_thread::sleep_for(std::chrono::milliseconds(10));
@ -39,7 +48,9 @@ bool ColorCamera::pull(cv::gapi::wip::Data &data) {
}
cv::GMetaArg ColorCamera::descr_of() const {
return cv::GMetaArg{cv::descr_of(m_dummy)};
// FIXME: support other resolutions
GAPI_Assert(m_params.resolution == ColorCameraParams::Resolution::THE_1080_P);
return cv::GMetaArg{cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size{1920, 1080}}};
}
} // namespace oak

32
modules/gapi/src/backends/oak/goak_media_adapter.cpp → modules/gapi/src/backends/oak/goak_memory_adapters.cpp
View File

@ -4,17 +4,15 @@
//
// Copyright (C) 2021 Intel Corporation
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
namespace cv {
namespace gapi {
namespace oak {
OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer) {
OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer)
: m_sz(sz), m_fmt(fmt), m_buffer(buffer) {
GAPI_Assert(fmt == cv::MediaFormat::NV12 && "OAKMediaAdapter only supports NV12 format for now");
m_sz = sz;
m_fmt = fmt;
m_buffer = buffer;
}
MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) {
@ -27,6 +25,30 @@ MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) {
cv::GFrameDesc OAKMediaAdapter::OAKMediaAdapter::meta() const { return {m_fmt, m_sz}; }
OAKRMatAdapter::OAKRMatAdapter(const cv::Size& size,
int precision,
std::vector<float>&& buffer)
: m_size(size), m_precision(precision), m_buffer(buffer) {
GAPI_Assert(m_precision == CV_16F);
std::vector<int> wrapped_dims{1, 1, m_size.width, m_size.height};
// FIXME: check layout and add strides
m_desc = cv::GMatDesc(m_precision, wrapped_dims);
m_mat = cv::Mat(static_cast<int>(wrapped_dims.size()),
wrapped_dims.data(),
CV_16FC1, // FIXME: cover other precisions
m_buffer.data());
}
cv::GMatDesc OAKRMatAdapter::desc() const {
return m_desc;
}
cv::RMat::View OAKRMatAdapter::access(cv::RMat::Access) {
return cv::RMat::View{m_desc, m_mat.data};
}
} // namespace oak
} // namespace gapi
} // namespace cv

662
modules/gapi/src/backends/oak/goakbackend.cpp
View File

@ -2,9 +2,10 @@
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2021 Intel Corporation
// Copyright (C) 2021-2022 Intel Corporation
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
#include <opencv2/gapi/oak/oak.hpp> // kernels()
#ifdef HAVE_OAK
@ -18,21 +19,24 @@
#include <api/gbackend_priv.hpp>
#include <backends/common/gbackend.hpp>
#include <opencv2/gapi/infer.hpp> // GInferBase
#include <opencv2/gapi/streaming/meta.hpp> // streaming::meta_tag
#include "depthai/depthai.hpp"
#include <opencv2/gapi/oak/oak.hpp>
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
#include <opencv2/gapi/oak/infer.hpp> // infer params
namespace cv { namespace gimpl {
// Forward declaration
class GOAKContext;
struct OAKNodeInfo;
class OAKKernelParams;
class GOAKExecutable final: public GIslandExecutable {
friend class GOAKContext;
friend class OAKKernelParams;
virtual void run(std::vector<InObj>&&,
std::vector<OutObj>&&) override {
GAPI_Assert(false && "Not implemented");
@ -41,7 +45,8 @@ class GOAKExecutable final: public GIslandExecutable {
virtual void run(GIslandExecutable::IInput &in,
GIslandExecutable::IOutput &out) override;
void LinkToParents(ade::NodeHandle handle);
void linkToParent(ade::NodeHandle handle);
void linkCopy(ade::NodeHandle handle);
class ExtractTypeHelper : protected dai::Node {
public:
@ -61,6 +66,7 @@ class GOAKExecutable final: public GIslandExecutable {
std::shared_ptr<dai::node::XLinkOut> xlink_output;
std::shared_ptr<dai::DataOutputQueue> out_queue;
std::string out_queue_name;
size_t gapi_out_data_index;
};
cv::GArg packInArg(const GArg &arg, std::vector<ExtractTypeHelper::InputPtr>& oak_ins);
@ -79,11 +85,16 @@ class GOAKExecutable final: public GIslandExecutable {
cv::Size m_camera_size;
// Backend outputs
std::vector<OAKOutQueueInfo> m_out_queues;
std::unordered_map<ade::NodeHandle,
OAKOutQueueInfo,
ade::HandleHasher<ade::Node>> m_out_queues;
// Backend inputs
std::vector<std::pair<std::string, dai::Buffer>> m_in_queues;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> m_passthrough_copy_nodes;
// Note: dai::Pipeline should be the only one for the whole pipeline,
// so there is no way to insert any non-OAK node in graph between other OAK nodes.
// The only heterogeneous case possible is if we insert other backends after or before
@ -91,6 +102,14 @@ class GOAKExecutable final: public GIslandExecutable {
std::unique_ptr<dai::Device> m_device;
std::unique_ptr<dai::Pipeline> m_pipeline;
// Camera config
cv::gapi::oak::ColorCameraParams m_ccp;
// Infer info
std::unordered_map<ade::NodeHandle,
cv::gapi::oak::detail::ParamDesc,
ade::HandleHasher<ade::Node>> m_oak_infer_info;
public:
GOAKExecutable(const ade::Graph& g,
const cv::GCompileArgs& args,
@ -122,6 +141,12 @@ public:
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results);
GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
const cv::gapi::oak::detail::ParamDesc& infer_info,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results);
// Generic accessor API
template<typename T>
T& inArg(int input) { return m_args.at(input).get<T>(); }
@ -130,12 +155,14 @@ public:
InputPtr& in(int input);
OutputPtr& out(int output);
const std::unique_ptr<dai::Pipeline>& pipeline();
const std::unique_ptr<dai::Pipeline>& pipeline() const;
const cv::Size& camera_size() const;
const cv::gapi::oak::detail::ParamDesc& ii() const;
private:
const std::unique_ptr<dai::Pipeline>& m_pipeline;
const cv::Size& m_camera_size;
const cv::Size m_camera_size;
const cv::gapi::oak::detail::ParamDesc m_infer_info;
std::vector<cv::GArg>& m_args;
std::vector<OutputPtr>& m_outputs;
};
@ -144,9 +171,18 @@ GOAKContext::GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results)
: m_pipeline(pipeline), m_camera_size(camera_size), m_args(args), m_outputs(results) {}
: m_pipeline(pipeline), m_camera_size(camera_size),
m_args(args), m_outputs(results) {}
const std::unique_ptr<dai::Pipeline>& GOAKContext::pipeline() {
GOAKContext::GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
const cv::gapi::oak::detail::ParamDesc& infer_info,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results)
: m_pipeline(pipeline), m_camera_size(camera_size),
m_infer_info(infer_info), m_args(args), m_outputs(results) {}
const std::unique_ptr<dai::Pipeline>& GOAKContext::pipeline() const {
return m_pipeline;
}
@ -154,6 +190,10 @@ const cv::Size& GOAKContext::camera_size() const {
return m_camera_size;
}
const cv::gapi::oak::detail::ParamDesc& GOAKContext::ii() const {
return m_infer_info;
}
GOAKContext::InputPtr& GOAKContext::in(int input) {
return inArg<std::reference_wrapper<GOAKContext::InputPtr>>(input).get();
}
@ -162,6 +202,14 @@ GOAKContext::OutputPtr& GOAKContext::out(int output) {
return m_outputs.at(output);
}
class OAKKernelParams {
public:
const std::unique_ptr<dai::Pipeline>& pipeline;
const cv::Size& camera_size;
const cv::gapi::oak::detail::ParamDesc& infer_info;
std::vector<std::pair<std::string, dai::Buffer>>& in_queues;
};
namespace detail {
template<class T> struct get_in;
template<> struct get_in<cv::GFrame> {
@ -179,13 +227,10 @@ template<> struct get_out<cv::GFrame> {
template<typename U> struct get_out<cv::GArray<U>> {
static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); }
};
// FIXME: add support of other types
struct OAKKernelParams {
const std::unique_ptr<dai::Pipeline>& pipeline;
const cv::Size& camera_size;
std::vector<std::pair<std::string, dai::Buffer>>& m_in_queues;
template<> struct get_out<cv::GMat> {
static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); }
};
// FIXME: add support of other types
template<typename, typename, typename>
struct OAKCallHelper;
@ -200,6 +245,7 @@ struct OAKCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...> > {
, cv::detail::Seq<OIs...>) {
return Impl::put(OAKKernelParams{ctx.pipeline(),
ctx.camera_size(),
ctx.ii(),
in_queues_params},
get_in<Ins>::get(ctx, IIs)...,
get_out<Outs>::get(ctx, OIs)...);
@ -229,23 +275,84 @@ struct OAKComponent
static const char *name() { return "OAK Component"; }
GOAKKernel k;
};
}} // namespace gimpl // namespace cv
} // namespace gimpl
} // namespace cv
using OAKGraph = ade::TypedGraph
< cv::gimpl::OAKComponent
// FIXME: extend
< cv::gimpl::Protocol
, cv::gimpl::Op
, cv::gimpl::NetworkParams
, cv::gimpl::CustomMetaFunction
// OAK specific
, cv::gimpl::OAKComponent
>;
using ConstOAKGraph = ade::ConstTypedGraph
< cv::gimpl::OAKComponent
// FIXME: extend
< cv::gimpl::Protocol
, cv::gimpl::Op
, cv::gimpl::NetworkParams
, cv::gimpl::CustomMetaFunction
// OAK specific
, cv::gimpl::OAKComponent
>;
// This function links dai operation nodes - parent's output to child's input.
namespace
{
std::pair<dai::TensorInfo, dai::TensorInfo>
parseDaiInferMeta(const cv::gapi::oak::detail::ParamDesc& pd) {
dai::OpenVINO::Blob blob(pd.blob_file);
GAPI_Assert(blob.networkInputs.size() == 1);
GAPI_Assert(blob.networkOutputs.size() == 1);
return {blob.networkInputs.begin()->second,
blob.networkOutputs.begin()->second};
}
std::string
getDaiInferOutLayerName(const cv::gapi::oak::detail::ParamDesc& pd) {
dai::OpenVINO::Blob blob(pd.blob_file);
GAPI_Assert(blob.networkInputs.size() == 1);
GAPI_Assert(blob.networkOutputs.size() == 1);
return blob.networkOutputs.begin()->first;
}
} // anonymous namespace
// Custom meta function for OAK backend for infer
static cv::GMetaArgs customOutMeta(const ade::Graph &gr,
const ade::NodeHandle &nh,
const cv::GMetaArgs &/*in_metas*/,
const cv::GArgs &/*in_args*/) {
cv::GMetaArgs result;
const auto &np = ConstOAKGraph(gr).metadata(nh).get<cv::gimpl::NetworkParams>();
const auto &pd = cv::util::any_cast<cv::gapi::oak::detail::ParamDesc>(np.opaque);
// FIXME: Infer kernel and backend does rather the same
auto in_out_tensor_info = parseDaiInferMeta(pd);
GAPI_Assert(in_out_tensor_info.second.dataType ==
dai::TensorInfo::DataType::FP16);
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.second.order ==
dai::TensorInfo::StorageOrder::NCHW);
// FIXME: DAI returns vector<unsigned>, remove workaround
std::vector<int> wrapped_dims;
for (const auto& d : in_out_tensor_info.second.dims) {
wrapped_dims.push_back(d);
}
result = {cv::GMetaArg{cv::GMatDesc(CV_16F, 1, cv::Size(wrapped_dims[1], wrapped_dims[0]), false)}};
return result;
}
// This function links DAI operation nodes - parent's output to child's input.
// It utilizes G-API graph to search for operation's node it's previous operation in graph
// when links them in dai graph.
void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
// when links them in DAI graph.
void cv::gimpl::GOAKExecutable::linkToParent(ade::NodeHandle handle)
{
ade::NodeHandle parent;
for (const auto& data_nh : handle.get()->inNodes()) {
@ -253,6 +360,13 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
GAPI_Assert(data_nh.get()->inNodes().size() == 1);
parent = data_nh.get()->inNodes().front();
// Don't link if parent is copy - the case is handled differently
// in linkCopy
const auto& op = m_gm.metadata(parent).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
continue;
}
// Assuming that OAK nodes are aligned for linking.
// FIXME: potential rework might be needed then
// counterexample is found.
@ -269,6 +383,89 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
}
}
// This function links DAI operations for Copy OP in G-API graph
void cv::gimpl::GOAKExecutable::linkCopy(ade::NodeHandle handle) {
// 1. Check that there are no back-to-back Copy OPs in graph
auto copy_out = handle.get()->outNodes();
GAPI_Assert(copy_out.size() == 1);
for (const auto& copy_next_op : copy_out.front().get()->outNodes()) {
const auto& op = m_gm.metadata(copy_next_op).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
GAPI_Assert(false && "Back-to-back Copy operations are not supported in graph");
}
}
// 2. Link passthrough case
if (m_passthrough_copy_nodes.find(handle) != m_passthrough_copy_nodes.end()) {
ExtractTypeHelper::OutputPtr parent;
bool parent_is_camera = false;
// Copy has only 1 input data
GAPI_Assert(handle.get()->inNodes().size() == 1);
auto in_ops = handle.get()->inNodes().front().get()->inNodes();
if (in_ops.size() == 0) {
// No parent nodes - parent = camera
parent = &m_camera_input->video;
parent_is_camera = true;
} else {
// Data has only 1 input
GAPI_Assert(in_ops.size() == 1);
auto node = m_oak_nodes.at(in_ops.front());
// Should only have 1 output
GAPI_Assert(node.outputs.size() == 1);
parent = node.outputs[0];
}
// Now link DAI parent output to Copy's child's inputs ignoring the Copy operation
// FIXME: simplify this loop
auto copy_out_data = handle.get()->outNodes();
// Copy has only 1 output
GAPI_Assert(copy_out_data.size() == 1);
for (const auto& copy_next_op : copy_out_data.front().get()->outNodes()) {
if (m_oak_nodes.find(copy_next_op) != m_oak_nodes.end()) {
// FIXME: consider a better approach
if (parent_is_camera) {
if (m_oak_infer_info.find(copy_next_op) != m_oak_infer_info.end()) {
parent = &m_camera_input->preview;
} else {
parent = &m_camera_input->video;
}
}
// Found next Copy OP which needs to be linked to Copy's parent
GAPI_Assert(m_oak_nodes.at(copy_next_op).inputs.size() == 1 &&
"Internal OAK nodes are not aligned for linking (Copy operation)");
parent->link(*(m_oak_nodes.at(copy_next_op).inputs.front()));
}
}
}
// 3. Link output Copy case
if (m_out_queues.find(handle) != m_out_queues.end()) {
// DAI XLinkOutput node
auto xout = m_out_queues[handle].xlink_output->input;
// Find parent node
// FIXME: copypasted from case 2 above
ExtractTypeHelper::OutputPtr parent;
// Copy has only 1 input data
GAPI_Assert(handle.get()->inNodes().size() == 1);
auto in_ops = handle.get()->inNodes().front().get()->inNodes();
if (in_ops.size() == 0) {
// No parent nodes - parent = camera
parent = &m_camera_input->video;
} else {
// Data has only 1 input
GAPI_Assert(in_ops.size() == 1);
auto node = m_oak_nodes.at(in_ops.front());
// Should only have 1 output
GAPI_Assert(node.outputs.size() == 1);
parent = node.outputs[0];
}
// Link parent to xout
parent->link(xout);
}
}
cv::GArg
cv::gimpl::GOAKExecutable::packInArg(const GArg &arg,
std::vector<ExtractTypeHelper::InputPtr>& oak_ins) {
@ -298,9 +495,8 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc,
std::vector<ExtractTypeHelper::OutputPtr>& oak_outs) {
switch (rc.shape) {
case GShape::GFRAME:
oak_outs.push_back(nullptr);
break;
case GShape::GARRAY:
case GShape::GMAT:
oak_outs.push_back(nullptr);
break;
default:
@ -309,6 +505,33 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc,
}
}
namespace {
static dai::CameraBoardSocket extractCameraBoardSocket(cv::gapi::oak::ColorCameraParams ccp) {
switch (ccp.board_socket) {
case cv::gapi::oak::ColorCameraParams::BoardSocket::RGB:
return dai::CameraBoardSocket::RGB;
// FIXME: extend
default:
// basically unreachable
GAPI_Assert("Unsupported camera board socket");
return {};
}
}
static dai::ColorCameraProperties::SensorResolution
extractCameraResolution(cv::gapi::oak::ColorCameraParams ccp) {
switch (ccp.resolution) {
case cv::gapi::oak::ColorCameraParams::Resolution::THE_1080_P:
return dai::ColorCameraProperties::SensorResolution::THE_1080_P;
// FIXME: extend
default:
// basically unreachable
GAPI_Assert("Unsupported camera board socket");
return {};
}
}
} // anonymous namespace
cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle>& nodes,
@ -344,16 +567,52 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
}
}
m_ccp = cv::gimpl::getCompileArg<cv::gapi::oak::ColorCameraParams>(args)
.value_or(cv::gapi::oak::ColorCameraParams{});
// FIXME: change the hard-coded behavior (XLinkIn path)
auto camRgb = m_pipeline->create<dai::node::ColorCamera>();
// FIXME: extract camera compile arguments here and properly convert them for dai
camRgb->setBoardSocket(dai::CameraBoardSocket::RGB);
camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
camRgb->setBoardSocket(extractCameraBoardSocket(m_ccp));
camRgb->setResolution(extractCameraResolution(m_ccp));
camRgb->setInterleaved(m_ccp.interleaved);
// Extract infer params
for (const auto& nh : nodes) {
if (m_gm.metadata(nh).get<NodeType>().t == NodeType::OP) {
if (ConstOAKGraph(m_g).metadata(nh).contains<cv::gimpl::NetworkParams>()) {
const auto &np = ConstOAKGraph(m_g).metadata(nh).get<cv::gimpl::NetworkParams>();
const auto &pp = cv::util::any_cast<cv::gapi::oak::detail::ParamDesc>(np.opaque);
m_oak_infer_info[nh] = pp;
break;
}
}
}
// FIXME: handle multiple infers
if (!m_oak_infer_info.empty()) {
GAPI_Assert(m_oak_infer_info.size() == 1);
// FIXME: move to infer node?
auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second);
if (in_out_tensor_info.first.dataType ==
dai::TensorInfo::DataType::FP16 ||
in_out_tensor_info.first.dataType ==
dai::TensorInfo::DataType::FP32) {
camRgb->setFp16(true);
} else {
camRgb->setFp16(false);
}
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.first.order ==
dai::TensorInfo::StorageOrder::NCHW);
camRgb->setPreviewSize(in_out_tensor_info.first.dims[0], in_out_tensor_info.first.dims[1]);
}
// Set camera output. Fixme: consider working with other camera outputs
m_camera_input = camRgb;
// FIXME: change when other camera censors are introduced
std::tuple<int, int> video_size = camRgb->getVideoSize();
std::tuple<int, int> video_size = m_camera_input->getVideoSize();
m_camera_size = cv::Size{std::get<0>(video_size), std::get<1>(video_size)};
// Prepare XLinkOut nodes for each output object in graph
@ -361,7 +620,23 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
auto xout = m_pipeline->create<dai::node::XLinkOut>();
std::string xout_name = "xout" + std::to_string(i);
xout->setStreamName(xout_name);
m_out_queues.push_back({xout, nullptr, xout_name});
// Find parent OP's nh
ade::NodeHandle parent_op_nh;
for (const auto& nh : nodes) {
for (const auto& outdata : nh.get()->outNodes()) {
if (m_gm.metadata(outdata).get<NodeType>().t == NodeType::DATA) {
auto rc = m_gm.metadata(outdata).get<cv::gimpl::Data>().rc;
auto shape = m_gm.metadata(outdata).get<cv::gimpl::Data>().shape;
// Match outs_data with the actual operation
if (rc == outs_data[i].rc && shape == outs_data[i].shape) {
parent_op_nh = nh;
}
}
}
}
m_out_queues[parent_op_nh] = {xout, nullptr, xout_name, i};
}
// Create OAK node for each node in this backend
@ -375,33 +650,66 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
m_oak_nodes.at(nh).inputs.reserve(op.args.size());
m_oak_nodes.at(nh).outputs.reserve(op.outs.size());
// Copy operation in graph can fall into 3 cases:
// 1) Copy is an output of the island -
// in that case we link it to XLinkOut node from m_out_queues
// 2) Copy is between other two operations in the same OAK island -
// in that case we link its parent operation (could be camera) to
// the child one (those copy operations are placed in m_passthrough_copy_nodes)
// 3) Copy can fall into cases 1) and 2) at the same time
// Prepare passthrough Copy operations
if (op.k.name == "org.opencv.oak.copy") {
// Copy has only 1 output
auto copy_out = nh.get()->outNodes();
GAPI_Assert(copy_out.size() == 1);
for (const auto& copy_next_op : copy_out.front().get()->outNodes()) {
// Check that copy is a passthrough OP
if (std::find(nodes.begin(), nodes.end(), copy_next_op) != nodes.end()) {
m_passthrough_copy_nodes.insert(nh);
break;
}
}
}
std::vector<cv::GArg> in_ctx_args;
in_ctx_args.reserve(op.args.size());
for (auto &op_arg : op.args) in_ctx_args.push_back(packInArg(op_arg,
m_oak_nodes.at(nh).inputs));
m_oak_nodes.at(nh).inputs));
for (auto &&op_out : op.outs) packOutArg(op_out, m_oak_nodes.at(nh).outputs);
GAPI_Assert(!m_oak_nodes.at(nh).inputs.empty());
GAPI_Assert(!m_oak_nodes.at(nh).outputs.empty());
GOAKContext ctx(m_pipeline, m_camera_size, in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
GAPI_Assert(m_oak_nodes.at(nh).node != nullptr);
if (ConstOAKGraph(m_g).metadata(nh).contains<cv::gimpl::NetworkParams>()) {
GOAKContext ctx(m_pipeline, m_camera_size, m_oak_infer_info[nh],
in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
} else {
GOAKContext ctx(m_pipeline, m_camera_size,
in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
}
// Check that all inputs and outputs are properly filled after constructing kernels
// to then link it together
// FIXME: add more logging
const auto& node = m_oak_nodes.at(nh);
if (std::any_of(node.inputs.cbegin(), node.inputs.cend(),
[](ExtractTypeHelper::InputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI input are not set");
}
if (std::any_of(node.outputs.cbegin(), node.outputs.cend(),
[](ExtractTypeHelper::OutputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI outputs are not set");
const auto& node_info = m_oak_nodes.at(nh);
// Copy operations don't set their inputs/outputs properly
if (op.k.name != "org.opencv.oak.copy") {
GAPI_Assert(node_info.node != nullptr);
if (std::any_of(node_info.inputs.cbegin(), node_info.inputs.cend(),
[](ExtractTypeHelper::InputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI input are not set");
}
if (std::any_of(node_info.outputs.cbegin(), node_info.outputs.cend(),
[](ExtractTypeHelper::OutputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI outputs are not set");
}
}
}
}
@ -413,15 +721,26 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
ade::HandleHasher<ade::Node>> out_nodes;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> inter_nodes;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> copy_nodes;
// TODO: optimize this loop
for (const auto& node : m_oak_nodes) {
auto nh = node.first;
// Check if it's a Copy OP - will be handled differently when linking
GAPI_Assert(m_gm.metadata(nh).get<NodeType>().t == NodeType::OP);
const auto& op = m_gm.metadata(nh).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
copy_nodes.insert(nh);
continue;
}
// Fill input op nodes
for (const auto& d : ins_data) {
for (const auto& indata : nh.get()->inNodes()) {
auto rc = m_gm.metadata(indata).get<cv::gimpl::Data>().rc;
if (rc == d.rc) {
auto shape = m_gm.metadata(indata).get<cv::gimpl::Data>().shape;
if (rc == d.rc && shape == d.shape) {
in_nodes.insert(nh);
}
}
@ -430,7 +749,8 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
for (const auto& d : outs_data) {
for (const auto& outdata : nh.get()->outNodes()) {
auto rc = m_gm.metadata(outdata).get<cv::gimpl::Data>().rc;
if (rc == d.rc) {
auto shape = m_gm.metadata(outdata).get<cv::gimpl::Data>().shape;
if (rc == d.rc && shape == d.shape) {
out_nodes.insert(nh);
}
}
@ -446,42 +766,47 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
// 1. Link input nodes to camera
for (const auto& nh : in_nodes) {
GAPI_Assert(m_oak_nodes.at(nh).inputs.size() == 1);
// FIXME: covert other camera outputs
m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0]));
// FIXME: cover other camera outputs
// Link preview to infer, video to all other nodes
if (m_oak_infer_info.find(nh) == m_oak_infer_info.end()) {
m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0]));
} else {
m_camera_input->preview.link(*(m_oak_nodes.at(nh).inputs[0]));
}
}
// 2. Link output nodes to XLinkOut nodes
size_t out_counter = 0;
for (const auto& nh : out_nodes) {
GAPI_Assert(out_counter + m_oak_nodes.at(nh).outputs.size() <= m_out_queues.size());
for (const auto& out : m_oak_nodes.at(nh).outputs) {
out->link(m_out_queues[out_counter++].xlink_output->input);
out->link(m_out_queues[nh].xlink_output->input);
}
// Input nodes in OAK doesn't have parent operation - just camera (for now)
if (in_nodes.find(nh) == in_nodes.end()) {
LinkToParents(nh);
linkToParent(nh);
}
}
// 3. Link internal nodes to their parents
for (const auto& nh : inter_nodes) {
// Input nodes in OAK doesn't have parent operation - just camera (for now)
if (in_nodes.find(nh) == in_nodes.end()) {
LinkToParents(nh);
}
linkToParent(nh);
}
// 4. Link copy nodes
for (const auto& nh : copy_nodes) {
linkCopy(nh);
}
m_device = std::unique_ptr<dai::Device>(new dai::Device(*m_pipeline));
// Prepare OAK output queues
GAPI_Assert(m_out_queues.size() == outs_data.size());
for (const auto out_it : ade::util::indexed(outs_data))
for (const auto out_it : ade::util::indexed(m_out_queues))
{
auto& q = m_out_queues[ade::util::index(out_it)];
auto& q = ade::util::value(out_it).second;
GAPI_Assert(q.out_queue == nullptr); // shouldn't be not filled till this point
// FIXME: add queue parameters
// Currently: 30 - max DAI queue capacity, true - blocking queue
q.out_queue = m_device->getOutputQueue(q.out_queue_name, 30, true);
// Currently: 4 - max DAI queue capacity, true - blocking queue
q.out_queue = m_device->getOutputQueue(q.out_queue_name, 4, true);
}
}
@ -507,17 +832,25 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in,
q->send(in_q.second);
}
for (size_t i = 0; i < m_out_queues.size(); ++i) {
auto q = m_out_queues[i].out_queue;
// TODO: support other DAI types if needed
// Note: we utilize getData() method that returns std::vector of data
// on which we gain ownership
auto oak_frame = q->get<dai::ImgFrame>();
for (size_t i = 0; i < m_in_queues.size(); ++i) {
auto q = m_device->getInputQueue(m_in_queues[i].first);
q->send(m_in_queues[i].second);
}
auto out_arg = out.get(i);
for (const auto el : m_out_queues) {
const auto out_q = el.second;
auto& q = out_q.out_queue;
auto out_arg = out.get(out_q.gapi_out_data_index);
// FIXME: misc info to be utilized in switch below
cv::GRunArg::Meta meta;
std::shared_ptr<dai::ImgFrame> oak_frame;
switch(out_arg.index()) {
case cv::GRunArgP::index_of<cv::MediaFrame*>():
{
oak_frame = q->get<dai::ImgFrame>();
// FIXME: hard-coded NV12
*cv::util::get<cv::MediaFrame*>(out_arg) =
cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>(
@ -525,58 +858,63 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in,
static_cast<int>(oak_frame->getHeight())),
cv::MediaFormat::NV12,
std::move(oak_frame->getData()));
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
break;
}
case cv::GRunArgP::index_of<cv::detail::VectorRef>():
{
oak_frame = q->get<dai::ImgFrame>();
cv::util::get<cv::detail::VectorRef>(out_arg).wref<uint8_t>() = std::move(oak_frame->getData());
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
break;
}
case cv::GRunArgP::index_of<cv::RMat*>(): // only supported for infer
{
auto nn_data = q->get<dai::NNData>();
auto out_layer_name = getDaiInferOutLayerName(m_oak_infer_info.begin()->second);
auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second);
auto layer = std::move(nn_data->getLayerFp16(out_layer_name));
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.second.order ==
dai::TensorInfo::StorageOrder::NCHW);
// FIMXE: only 1-channel data is supported for now
GAPI_Assert(in_out_tensor_info.second.dims[2] == 1);
*cv::util::get<cv::RMat*>(out_arg) =
cv::make_rmat<cv::gapi::oak::OAKRMatAdapter>(
cv::Size(in_out_tensor_info.second.dims[1],
in_out_tensor_info.second.dims[0]),
CV_16F, // FIXME: cover other precisions
std::move(layer)
);
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = nn_data->getTimestamp();
meta[seq_id] = nn_data->getSequenceNum();
break;
}
// FIXME: Add support for remaining types
default:
GAPI_Assert(false && "Unsupported type in OAK backend");
}
using namespace cv::gapi::streaming::meta_tag;
cv::GRunArg::Meta meta;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
out.meta(out_arg, meta);
out.post(std::move(out_arg));
}
}
// Built-in kernels for OAK /////////////////////////////////////////////////////
class GOAKBackendImpl final : public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override {
OAKGraph gm(graph);
const auto &kimpl = cv::util::any_cast<cv::gimpl::GOAKKernel>(impl.opaque);
gm.metadata(op_node).set(cv::gimpl::OAKComponent{kimpl});
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle> &nodes,
const std::vector<cv::gimpl::Data>& ins_data,
const std::vector<cv::gimpl::Data>& outs_data) const override {
cv::gimpl::GModel::ConstGraph gm(graph);
// FIXME: pass streaming/non-streaming option to support non-camera case
// NB: how could we have non-OAK source in streaming mode, then OAK backend in
// streaming mode but without camera input?
if (!gm.metadata().contains<cv::gimpl::Streaming>()) {
GAPI_Assert(false && "OAK backend only supports Streaming mode for now");
}
return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)};
}
};
cv::gapi::GBackend cv::gapi::oak::backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GOAKBackendImpl>());
return this_backend;
}
namespace cv {
namespace gimpl {
namespace oak {
@ -604,10 +942,13 @@ static dai::VideoEncoderProperties::Profile convertEncProfile(cv::gapi::oak::Enc
// Kernels ///////////////////////////////////////////////////////////////
template<class Impl, class K>
class GOAKKernelImpl: public detail::OAKCallHelper<Impl, typename K::InArgs, typename K::OutArgs>
// FIXME: consider a better solution - hard-coded API
// Is there a way to extract API from somewhereelse/utilize structs
// like in streaming/infer backends (mainly infer and copy operations)
template<class Impl, class K, class InArgs = typename K::InArgs, class OutArgs = typename K::OutArgs>
class GOAKKernelImpl: public detail::OAKCallHelper<Impl, InArgs, OutArgs>
, public cv::detail::KernelTag {
using P = detail::OAKCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
using P = detail::OAKCallHelper<Impl, InArgs, OutArgs>;
public:
using API = K;
static cv::gapi::GBackend backend() { return cv::gapi::oak::backend(); }
@ -617,17 +958,51 @@ public:
#define GAPI_OAK_KERNEL(Name, API) \
struct Name: public cv::gimpl::oak::GOAKKernelImpl<Name, API>
#define GAPI_OAK_FIXED_API_KERNEL(Name, API, InArgs, OutArgs) \
struct Name: public cv::gimpl::oak::GOAKKernelImpl<Name, API, InArgs, OutArgs>
namespace {
GAPI_OAK_FIXED_API_KERNEL(GOAKInfer, cv::GInferBase, std::tuple<cv::GFrame>, std::tuple<cv::GMat>) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
GOAKContext::OutputPtr& out) {
auto nn = params.pipeline->create<dai::node::NeuralNetwork>();
nn->input.setBlocking(true);
nn->input.setQueueSize(1);
// FIXME: add G-API built-in preproc here (currently it's only setPreviewSize() on the camera node)
// Note: for some reason currently it leads to:
// "Fatal error. Please report to developers. Log: 'ImageManipHelper' '61'"
nn->setBlobPath(params.infer_info.blob_file);
in = &(nn->input);
out = &(nn->out);
return nn;
}
};
GAPI_OAK_KERNEL(GOAKCopy, cv::gapi::oak::GCopy) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams&,
GOAKContext::InputPtr&,
GOAKContext::OutputPtr&) {
// Do nothing in Copy OP since it's either already represented
// by XLinkOut node (bonded to output queues) or it's a passthrough OP
return nullptr;
}
};
GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::detail::OAKKernelParams& params,
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
const cv::gapi::oak::EncoderConfig& cfg,
GOAKContext::OutputPtr& out) {
auto videoEnc = params.pipeline->create<dai::node::VideoEncoder>();
// FIXME: convert all the parameters to dai
videoEnc->setDefaultProfilePreset(cfg.width, cfg.height,
cfg.frameRate,
videoEnc->setDefaultProfilePreset(cfg.frameRate,
convertEncProfile(cfg.profile));
in = &(videoEnc->input);
@ -638,7 +1013,7 @@ GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) {
};
GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::detail::OAKKernelParams& params,
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
const cv::Mat& hk,
const cv::Mat& vk,
@ -664,7 +1039,7 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
xinEdgeCfg->out.link(edgeDetector->inputConfig);
params.m_in_queues.push_back({"sobel_cfg", cfg});
params.in_queues.push_back({"sobel_cfg", cfg});
in = &(edgeDetector->inputImage);
out = &(edgeDetector->outputImage);
@ -672,11 +1047,55 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
return edgeDetector;
}
};
} // anonymous namespace
} // namespace oak
} // namespace gimpl
} // namespace cv
class GOAKBackendImpl final : public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override {
using namespace cv::gimpl;
OAKGraph gm(graph);
const auto &kimpl = cv::util::any_cast<GOAKKernel>(impl.opaque);
gm.metadata(op_node).set(OAKComponent{kimpl});
// Set custom meta for infer
if (gm.metadata(op_node).contains<cv::gimpl::NetworkParams>()) {
gm.metadata(op_node).set(CustomMetaFunction{customOutMeta});
}
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle> &nodes,
const std::vector<cv::gimpl::Data>& ins_data,
const std::vector<cv::gimpl::Data>& outs_data) const override {
cv::gimpl::GModel::ConstGraph gm(graph);
// FIXME: pass streaming/non-streaming option to support non-camera case
// NB: how could we have non-OAK source in streaming mode, then OAK backend in
// streaming mode but without camera input?
if (!gm.metadata().contains<cv::gimpl::Streaming>()) {
GAPI_Assert(false && "OAK backend only supports Streaming mode for now");
}
return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)};
}
virtual cv::GKernelPackage auxiliaryKernels() const override {
return cv::gapi::kernels< cv::gimpl::oak::GOAKInfer
>();
}
};
cv::gapi::GBackend cv::gapi::oak::backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GOAKBackendImpl>());
return this_backend;
}
namespace cv {
namespace gapi {
namespace oak {
@ -684,6 +1103,7 @@ namespace oak {
cv::gapi::GKernelPackage kernels() {
return cv::gapi::kernels< cv::gimpl::oak::GOAKEncFrame
, cv::gimpl::oak::GOAKSobelXY
, cv::gimpl::oak::GOAKCopy
>();
}
@ -697,13 +1117,17 @@ namespace cv {
namespace gapi {
namespace oak {
cv::gapi::GKernelPackage kernels();
cv::gapi::GKernelPackage kernels() {
GAPI_Assert(false && "Built without OAK support");
return {};
}
cv::gapi::GBackend backend() {
GAPI_Assert(false && "Built without OAK support");
static cv::gapi::GBackend this_backend(nullptr);
return this_backend;
}
} // namespace oak
} // namespace gapi
} // namespace cv

22
modules/gapi/src/backends/oak/oak_media_adapter.hpp → modules/gapi/src/backends/oak/oak_memory_adapters.hpp
View File

@ -10,11 +10,15 @@
#include <memory>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/rmat.hpp>
namespace cv {
namespace gapi {
namespace oak {
// Used for OAK backends outputs only.
// Filled from DepthAI's ImgFrame type and owns the memory.
// Used mainly for CV operations.
class GAPI_EXPORTS OAKMediaAdapter final : public cv::MediaFrame::IAdapter {
public:
OAKMediaAdapter() = default;
@ -28,6 +32,24 @@ private:
std::vector<uint8_t> m_buffer;
};
// Used for OAK backends outputs only.
// Filled from DepthAI's NNData type and owns the memory.
// Used only for infer operations.
class GAPI_EXPORTS OAKRMatAdapter final : public cv::RMat::Adapter {
public:
OAKRMatAdapter() = default;
OAKRMatAdapter(const cv::Size& size, int precision, std::vector<float>&& buffer);
cv::GMatDesc desc() const override;
cv::RMat::View access(cv::RMat::Access) override;
~OAKRMatAdapter() = default;
private:
cv::Size m_size;
int m_precision;
std::vector<float> m_buffer;
cv::GMatDesc m_desc;
cv::Mat m_mat;
};
} // namespace oak
} // namespace gapi
} // namespace cv

2
modules/gapi/src/compiler/gmodel.hpp
View File

@ -70,7 +70,7 @@ struct Data
// FIXME: This is a _pure_ duplication of RcDesc now! (except storage)
GShape shape; // FIXME: Probably to be replaced by GMetaArg?
int rc;
int rc; // rc is unique but local to shape
GMetaArg meta;
HostCtor ctor; // T-specific helper to deal with unknown types in our code
cv::detail::OpaqueKind kind; // FIXME: is needed to store GArray/GOpaque type