third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #19325 from AsyaPronina:asyadev/extract_y

Browse Source 
[G-API] Added Y, UV accessors for cv::MediaFrame
This commit is contained in:
Alexander Alekhin
2021-03-24 16:37:05 +03:00
committed by GitHub
parent a00d6b8377 b0d66e06f4
commit 7949983c41
6 changed files with 533 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gapi/src/backends/common/gbackend.hpp
+32 -13
View File
@@ -58,32 +58,51 @@ namespace gimpl {
struct Data;
struct RcDesc;
struct GAPI_EXPORTS RMatMediaAdapterBGR final: public cv::RMat::Adapter
struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::Adapter
{
explicit RMatMediaAdapterBGR(const cv::MediaFrame& frame) : m_frame(frame) { };
using MapDescF = std::function<cv::GMatDesc(const GFrameDesc&)>;
using MapDataF = std::function<cv::Mat(const GFrameDesc&, const cv::MediaFrame::View&)>;
RMatMediaFrameAdapter(const cv::MediaFrame& frame,
const MapDescF& frameDescToMatDesc,
const MapDataF& frameViewToMat) :
m_frame(frame),
m_frameDesc(frame.desc()),
m_frameDescToMatDesc(frameDescToMatDesc),
m_frameViewToMat(frameViewToMat)
{ }
virtual cv::RMat::View access(cv::RMat::Access a) override
{
auto view = m_frame.access(a == cv::RMat::Access::W ? cv::MediaFrame::Access::W
: cv::MediaFrame::Access::R);
auto ptr = reinterpret_cast<uchar*>(view.ptr[0]);
auto stride = view.stride[0];
auto rmatToFrameAccess = [](cv::RMat::Access rmatAccess) {
switch(rmatAccess) {
case cv::RMat::Access::R:
return cv::MediaFrame::Access::R;
case cv::RMat::Access::W:
return cv::MediaFrame::Access::W;
default:
cv::util::throw_error(std::logic_error("cv::RMat::Access::R or "
"cv::RMat::Access::W can only be mapped to cv::MediaFrame::Access!"));
}
};
std::shared_ptr<cv::MediaFrame::View> view_ptr =
std::make_shared<cv::MediaFrame::View>(std::move(view));
auto callback = [view_ptr]() mutable { view_ptr.reset(); };
auto fv = m_frame.access(rmatToFrameAccess(a));
return cv::RMat::View(desc(), ptr, stride, callback);
auto fvHolder = std::make_shared<cv::MediaFrame::View>(std::move(fv));
auto callback = [fvHolder]() mutable { fvHolder.reset(); };
return asView(m_frameViewToMat(m_frame.desc(), *fvHolder), callback);
}
virtual cv::GMatDesc desc() const override
{
const auto& desc = m_frame.desc();
GAPI_Assert(desc.fmt == cv::MediaFormat::BGR);
return cv::GMatDesc{CV_8U, 3, desc.size};
return m_frameDescToMatDesc(m_frameDesc);
}
cv::MediaFrame m_frame;
cv::GFrameDesc m_frameDesc;
MapDescF m_frameDescToMatDesc;
MapDataF m_frameViewToMat;
};
modules/gapi/src/backends/streaming/gstreamingbackend.cpp
+191 -33
View File
@@ -4,6 +4,8 @@
//
// Copyright (C) 2020 Intel Corporation
#include <mutex>
#if !defined(GAPI_STANDALONE)
#include <opencv2/imgproc.hpp>
#endif // !defined(GAPI_STANDALONE)
@@ -11,6 +13,7 @@
#include <opencv2/gapi/util/throw.hpp> // throw_error
#include <opencv2/gapi/streaming/format.hpp> // kernels
#include "logger.hpp"
#include "api/gbackend_priv.hpp"
#include "backends/common/gbackend.hpp"
@@ -197,16 +200,47 @@ cv::gapi::GKernelPackage cv::gimpl::streaming::kernels()
#if !defined(GAPI_STANDALONE)
class GAccessorActorBase : public cv::gapi::streaming::IActor {
public:
explicit GAccessorActorBase(const cv::GCompileArgs&) {}
virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out) override {
const auto in_msg = in.get();
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
{
out.post(cv::gimpl::EndOfStream{});
return;
}
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
GAPI_Assert(in_args.size() == 1u);
auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
cv::GRunArgP out_arg = out.get(0);
auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
extractRMat(frame, rmat);
out.meta(out_arg, in_args[0].meta);
out.post(std::move(out_arg));
}
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) = 0;
protected:
std::once_flag m_warnFlag;
};
struct GOCVBGR: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GBGR;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public cv::gapi::streaming::IActor {
public:
explicit Actor(const cv::GCompileArgs&) {}
virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput&out) override;
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
@@ -216,49 +250,173 @@ struct GOCVBGR: public cv::detail::KernelTag
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVBGR::Actor::run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out)
void GOCVBGR::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto in_msg = in.get();
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
{
out.post(cv::gimpl::EndOfStream{});
return;
}
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
GAPI_Assert(in_args.size() == 1u);
cv::GRunArgP out_arg = out.get(0);
auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
const auto& desc = frame.desc();
auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
rmat = cv::make_rmat<cv::gimpl::RMatMediaAdapterBGR>(frame);
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 3, d.size); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size, CV_8UC3, v.ptr[0], v.stride[0]);
});
break;
}
case cv::MediaFormat::NV12:
{
cv::Mat bgr;
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat y_plane (desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
break;
}
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from NV12 to BGR will happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat-s from NV12 cv::MediaFrame for free, you may use "
"cv::gapi::streaming::Y and cv::gapi::streaming::UV accessors.\n");
});
cv::Mat bgr;
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat y_plane (desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR"));
}
out.post(std::move(out_arg));
}
struct GOCVY: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GY;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVY::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto& desc = frame.desc();
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 Y plane will "
"happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
"cv::gapi::streaming::BGR accessor.\n");
});
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
cv::Mat yuv;
cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(yuv.rowRange(0, desc.size.height));
break;
}
case cv::MediaFormat::NV12:
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 1, d.size); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size, CV_8UC1, v.ptr[0], v.stride[0]);
});
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::Y"));
}
}
struct GOCVUV: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GUV;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVUV::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto& desc = frame.desc();
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 UV plane will "
"happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
"cv::gapi::streaming::BGR accessor.\n");
});
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
cv::Mat yuv;
cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
cv::Mat uv;
std::vector<int> dims = { desc.size.height / 2,
desc.size.width / 2 };
auto start = desc.size.height;
auto range_h = desc.size.height / 4;
std::vector<cv::Mat> uv_planes = {
yuv.rowRange(start, start + range_h).reshape(0, dims),
yuv.rowRange(start + range_h, start + range_h * 2).reshape(0, dims)
};
cv::merge(uv_planes, uv);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(uv);
break;
}
case cv::MediaFormat::NV12:
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 2, d.size / 2); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size / 2, CV_8UC2, v.ptr[1], v.stride[1]);
});
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::UV"));
}
}
cv::gapi::GKernelPackage cv::gapi::streaming::kernels()
{
return cv::gapi::kernels<GOCVBGR>();
return cv::gapi::kernels<GOCVBGR, GOCVY, GOCVUV>();
}
#else