third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #20614 from mshabunin:use-onevpl-load

Browse Source 
videoio: use oneVPL load mechanism, encoder bitrate estimation

* videoio: updated oneVPL support - use mfxLoad

* videoio: advanced bitrate estimation for MFX encoder

* videoio: improved MediaSDK/oneVPL/libva detection

* videoio(ffmpeg): don't try oneVPL

* videoio(test): tune checks of videoio_mfx.read_write_raw tests

Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
This commit is contained in:
Maksim Shabunin 2021-10-21 16:56:11 +03:00 committed by GitHub
parent 14d5098ca2
commit 7febec49b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 148 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
modules/videoio/cmake/detect_msdk.cmake
View File

@ -3,16 +3,23 @@ set(MFX_DEFS "")
if(NOT HAVE_MFX)
find_package(VPL QUIET)
if(VPL_FOUND)
set(MFX_INCLUDE_DIRS "")
set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}")
set(HAVE_MFX TRUE)
list(APPEND MFX_DEFS "HAVE_ONEVPL")
message(STATUS "VPL_VERSION: ${VPL_VERSION}")
# NOTE: oneVPL since 2021.4 have version 2.4, version 2021.1 does not work
if (VPL_VERSION VERSION_LESS "2021.2" AND VPL_VERSION VERSION_GREATER "2021.0")
message(STATUS "VPL version is too old (${VPL_DIR} : ${VPL_VERSION}) - skipping")
else()
set(MFX_INCLUDE_DIRS "")
set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}")
set(HAVE_MFX TRUE)
list(APPEND MFX_DEFS "HAVE_ONEVPL")
endif()
endif()
endif()
set(paths)
if(NOT HAVE_MFX)
set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT")
set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT" ${paths})
if(MSVC)
if(MSVC_VERSION LESS 1900)
set(vs_suffix)
@ -31,7 +38,7 @@ if(NOT HAVE_MFX)
NO_DEFAULT_PATH)
find_library(MFX_LIBRARY NAMES mfx libmfx${vs_suffix}
PATHS ${paths}
PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}"
PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}" "lib"
NO_DEFAULT_PATH)
if(MFX_INCLUDE AND MFX_LIBRARY)
set(HAVE_MFX TRUE)
@ -46,10 +53,11 @@ if(NOT HAVE_MFX AND PKG_CONFIG_FOUND)
endif()
if(HAVE_MFX AND UNIX)
set(paths "${VA_ROOT_DIR}" ENV "VA_ROOT_DIR" ${paths})
foreach(mode NO_DEFAULT_PATH "")
find_path(MFX_va_INCLUDE va/va.h PATHS ${paths} PATH_SUFFIXES "include" ${mode})
find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode})
find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode})
find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode})
find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode})
if(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY)
list(APPEND MFX_INCLUDE_DIRS "${MFX_va_INCLUDE}")
list(APPEND MFX_LIBRARIES "${MFX_va_LIBRARY}" "${MFX_va_drm_LIBRARY}")
@ -61,6 +69,7 @@ if(HAVE_MFX AND UNIX)
unset(MFX_va_drm_LIBRARY CACHE)
endforeach()
if(NOT(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY))
message(STATUS "libva not found - turning MFX OFF")
set(HAVE_MFX FALSE)
endif()

4
modules/videoio/src/cap_ffmpeg_hw.hpp
View File

@ -13,6 +13,10 @@
#endif
#include <sstream>
#if defined(HAVE_MFX) && defined(HAVE_ONEVPL)
#undef HAVE_MFX // libav's hwcontext_qsv.h doesn't expect oneVPL headers
#endif
#ifdef HAVE_D3D11
#define D3D11_NO_HELPERS
#include <d3d11.h>

35
modules/videoio/src/cap_mfx_common.cpp
View File

@ -14,11 +14,13 @@
using namespace std;
using namespace cv;
#ifndef HAVE_ONEVPL
static mfxIMPL getImpl()
{
static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_IMPL", MFX_IMPL_AUTO_ANY);
return (mfxIMPL)res;
}
#endif
static size_t getExtraSurfaceNum()
{
@ -32,19 +34,46 @@ static size_t getPoolTimeoutSec()
return res;
}
#ifdef HAVE_ONEVPL
// oneVPL loader singleton (HW implementation only)
static mfxLoader setupVPLLoader()
{
mfxLoader instance = MFXLoad();
mfxConfig cfg = MFXCreateConfig(instance);
mfxVariant impl;
impl.Type = MFX_VARIANT_TYPE_U32;
impl.Data.U32 = MFX_IMPL_TYPE_HARDWARE;
MFXSetConfigFilterProperty(cfg, (const mfxU8*)"mfxImplDescription.Impl", impl);
DBG(cerr << "MFX Load: " << instance << endl);
return instance;
}
mfxLoader getVPLLoaderInstance()
{
static mfxLoader instance = setupVPLLoader();
return instance;
}
#endif
//==================================================================================================
bool DeviceHandler::init(MFXVideoSession &session)
bool DeviceHandler::init(MFXVideoSession_WRAP &session)
{
mfxStatus res = MFX_ERR_NONE;
mfxIMPL impl = getImpl();
mfxVersion ver = { {19, 1} };
#ifdef HAVE_ONEVPL
res = session.CreateSession();
DBG(cout << "MFX CreateSession: " << res << endl);
#else
mfxIMPL impl = getImpl();
res = session.Init(impl, &ver);
DBG(cout << "MFX SessionInit: " << res << endl);
res = session.QueryIMPL(&impl);
DBG(cout << "MFX QueryIMPL: " << res << " => " << asHex(impl) << endl);
#endif
res = session.QueryVersion(&ver);
DBG(cout << "MFX QueryVersion: " << res << " => " << ver.Major << "." << ver.Minor << endl);
@ -77,7 +106,7 @@ VAHandle::~VAHandle() {
}
}
bool VAHandle::initDeviceSession(MFXVideoSession &session) {
bool VAHandle::initDeviceSession(MFXVideoSession_WRAP &session) {
int majorVer = 0, minorVer = 0;
VAStatus va_res = vaInitialize(display, &majorVer, &minorVer);
DBG(cout << "vaInitialize: " << va_res << endl << majorVer << '.' << minorVer << endl);

45
modules/videoio/src/cap_mfx_common.hpp
View File

@ -12,12 +12,18 @@
#include <fstream>
#include <sstream>
CV_SUPPRESS_DEPRECATED_START
# if defined(_MSC_VER)
# pragma warning(push)
# pragma warning(disable:4201) // nonstandard extension used: nameless struct/union
# endif
#ifdef HAVE_ONEVPL
# include <vpl/mfxcommon.h>
# include <vpl/mfxstructures.h>
# include <vpl/mfxvideo++.h>
# include <vpl/mfxvp8.h>
# include <vpl/mfxjpeg.h>
# include <vpl/mfxdispatcher.h>
#else
# include <mfxcommon.h>
# include <mfxstructures.h>
@ -28,6 +34,10 @@
# include <mfxplugin++.h>
# endif
#endif
# if defined(_MSC_VER)
# pragma warning(pop)
# endif
CV_SUPPRESS_DEPRECATED_END
// //
// Debug helpers //
@ -176,10 +186,29 @@ inline void cleanup(T * &ptr)
//==================================================================================================
#ifdef HAVE_ONEVPL
mfxLoader getVPLLoaderInstance();
#endif
//==================================================================================================
class MFXVideoSession_WRAP : public MFXVideoSession
{
#ifdef HAVE_ONEVPL
public:
mfxStatus CreateSession()
{
return MFXCreateSession(getVPLLoaderInstance(), 0, &m_session);
}
#endif
};
//==================================================================================================
class Plugin
{
public:
static Plugin * loadEncoderPlugin(MFXVideoSession &session, mfxU32 codecId)
static Plugin * loadEncoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId)
{
#ifdef HAVE_MFX_PLUGIN
static const mfxPluginUID hevc_enc_uid = { 0x6f, 0xad, 0xc7, 0x91, 0xa0, 0xc2, 0xeb, 0x47, 0x9a, 0xb6, 0xdc, 0xd5, 0xea, 0x9d, 0xa3, 0x47 };
@ -190,7 +219,7 @@ public:
#endif
return 0;
}
static Plugin * loadDecoderPlugin(MFXVideoSession &session, mfxU32 codecId)
static Plugin * loadDecoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId)
{
#ifdef HAVE_MFX_PLUGIN
static const mfxPluginUID hevc_dec_uid = { 0x33, 0xa6, 0x1c, 0x0b, 0x4c, 0x27, 0x45, 0x4c, 0xa8, 0xd8, 0x5d, 0xde, 0x75, 0x7c, 0x6f, 0x8e };
@ -213,9 +242,9 @@ private:
mfxStatus res;
private:
#ifdef HAVE_MFX_PLUGIN
MFXVideoSession &session;
MFXVideoSession_WRAP &session;
mfxPluginUID uid;
Plugin(MFXVideoSession &_session, mfxPluginUID _uid) : session(_session), uid(_uid)
Plugin(MFXVideoSession_WRAP &_session, mfxPluginUID _uid) : session(_session), uid(_uid)
{
res = MFXVideoUSER_Load(session, &uid, 1);
}
@ -298,9 +327,9 @@ public:
class DeviceHandler {
public:
virtual ~DeviceHandler() {}
bool init(MFXVideoSession &session);
bool init(MFXVideoSession_WRAP &session);
protected:
virtual bool initDeviceSession(MFXVideoSession &session) = 0;
virtual bool initDeviceSession(MFXVideoSession_WRAP &session) = 0;
};
@ -340,7 +369,7 @@ public:
private:
VAHandle(const VAHandle &);
VAHandle &operator=(const VAHandle &);
bool initDeviceSession(MFXVideoSession &session) CV_OVERRIDE;
bool initDeviceSession(MFXVideoSession_WRAP &session) CV_OVERRIDE;
private:
VADisplay display;
int file;
@ -360,7 +389,7 @@ public:
private:
DXHandle(const DXHandle &);
DXHandle &operator=(const DXHandle &);
bool initDeviceSession(MFXVideoSession &) CV_OVERRIDE { return true; }
bool initDeviceSession(MFXVideoSession_WRAP &) CV_OVERRIDE { return true; }
};
#endif // _WIN32

16
modules/videoio/src/cap_mfx_reader.cpp
View File

@ -41,7 +41,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename)
// Init device and session
deviceHandler = createDeviceHandler();
session = new MFXVideoSession();
session = new MFXVideoSession_WRAP();
if (!deviceHandler->init(*session))
{
MSG(cerr << "MFX: Can't initialize session" << endl);
@ -87,11 +87,11 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename)
return;
}
// Adjust parameters
// Adjust parameters - COMMENTED: h265 decoder resets crop size to 0 (oneVPL/Win)
res = decoder->Query(&params, &params);
DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo);
CV_Assert(res >= MFX_ERR_NONE);
//res = decoder->Query(&params, &params);
//DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo);
//CV_Assert(res >= MFX_ERR_NONE);
// Init surface pool
@ -105,7 +105,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename)
// Init decoder
res = decoder->Init(&params);
DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo);
DBG(cout << "MFX decoder Init: " << res << endl << params.mfx.FrameInfo);
if (res < MFX_ERR_NONE)
{
MSG(cerr << "MFX: Failed to init decoder: " << res << endl);
@ -113,6 +113,10 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename)
}
frameSize = Size(params.mfx.FrameInfo.CropW, params.mfx.FrameInfo.CropH);
if (frameSize == Size(0, 0)) // sometimes Crop size is 0
{
frameSize = Size(params.mfx.FrameInfo.Width, params.mfx.FrameInfo.Height);
}
good = true;
}

4
modules/videoio/src/cap_mfx_reader.hpp
View File

@ -8,7 +8,7 @@
#include "precomp.hpp"
class MFXVideoSession;
class MFXVideoSession_WRAP;
class Plugin;
class DeviceHandler;
class ReadBitstream;
@ -27,7 +27,7 @@ public:
bool isOpened() const CV_OVERRIDE;
int getCaptureDomain() CV_OVERRIDE;
private:
MFXVideoSession *session;
MFXVideoSession_WRAP *session;
Plugin *plugin;
DeviceHandler *deviceHandler;
ReadBitstream *bs;

31
modules/videoio/src/cap_mfx_writer.cpp
View File

@ -11,6 +11,31 @@
using namespace std;
using namespace cv;
static float estimateBitrate(int codecId, size_t pixelNum, float fps)
{
float bitrate = 0.f;
const float mp = pixelNum / 1000000.f;
if (codecId == MFX_CODEC_MPEG2)
{
bitrate = (mp * 43) * fps + 360;
}
else if (codecId == MFX_CODEC_AVC)
{
bitrate = (mp * 140 + 19) * pow(fps, 0.60f);
}
else if (codecId == MFX_CODEC_HEVC)
{
bitrate = (mp * 63 + 45) * pow(fps, 0.60f);
}
else
{
MSG(cerr << "MFX encoder Bitrate estimation FAILED" << endl);
}
DBG(cout << "MFX encoder Bitrate estimation (" << mp << " MP x " << fps << " fps): " << bitrate << endl);
return bitrate;
}
static size_t getBitrateDivisor()
{
static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_BITRATE_DIVISOR", 300);
@ -61,7 +86,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc,
// Init device and session
deviceHandler = createDeviceHandler();
session = new MFXVideoSession();
session = new MFXVideoSession_WRAP();
if (!deviceHandler->init(*session))
{
MSG(cerr << "MFX: Can't initialize session" << endl);
@ -90,7 +115,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc,
memset(&params, 0, sizeof(params));
params.mfx.CodecId = codecId;
params.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED;
params.mfx.TargetKbps = saturate_cast<mfxU16>((frameSize.area() * fps) / (42.6666 * getBitrateDivisor())); // TODO: set in options
params.mfx.TargetKbps = saturate_cast<mfxU16>(estimateBitrate(codecId, frameSize.area(), (float)fps) * 300 / getBitrateDivisor()); // TODO: set in options
params.mfx.RateControlMethod = MFX_RATECONTROL_VBR;
params.mfx.FrameInfo.FrameRateExtN = cvRound(fps * 1000);
params.mfx.FrameInfo.FrameRateExtD = 1000;
@ -122,7 +147,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc,
// Init encoder
res = encoder->Init(&params);
DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo);
DBG(cout << "MFX encoder Init: " << res << endl << params.mfx.FrameInfo);
if (res < MFX_ERR_NONE)
{
MSG(cerr << "MFX: Failed to init encoder: " << res << endl);

4
modules/videoio/src/cap_mfx_writer.hpp
View File

@ -7,7 +7,7 @@
#include "precomp.hpp"
class MFXVideoSession;
class MFXVideoSession_WRAP;
class Plugin;
class DeviceHandler;
class WriteBitstream;
@ -33,7 +33,7 @@ private:
VideoWriter_IntelMFX & operator=(const VideoWriter_IntelMFX &);
private:
MFXVideoSession *session;
MFXVideoSession_WRAP *session;
Plugin *plugin;
DeviceHandler *deviceHandler;
WriteBitstream *bs;

22
modules/videoio/test/test_mfx.cpp
View File

@ -97,6 +97,11 @@ TEST_P(videoio_mfx, read_write_raw)
const String filename = cv::tempfile(ext);
const int fourcc = fourccByExt(ext);
// For some reason MPEG2 codec does not work well with this particular videostream at 1 FPS
// even with large bitrate values. Thus skipping this case.
if (FPS == 1. && fourcc == VideoWriter::fourcc('M', 'P', 'G', '2'))
throw SkipTestException("This configuration is not supported");
bool isColor = true;
std::queue<Mat> goodFrames;
@ -120,24 +125,29 @@ TEST_P(videoio_mfx, read_write_raw)
ASSERT_TRUE(cap.isOpened());
EXPECT_EQ(FRAME_SIZE.width, cap.get(CAP_PROP_FRAME_WIDTH));
EXPECT_EQ(FRAME_SIZE.height, cap.get(CAP_PROP_FRAME_HEIGHT));
double psnrThreshold = (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) ? 27.0 : 29.5; // experimentally chosen value
for (int i = 0; i < FRAME_COUNT; ++i)
{
SCOPED_TRACE(i);
ASSERT_TRUE(cap.read(frame));
ASSERT_FALSE(frame.empty());
ASSERT_EQ(FRAME_SIZE.width, frame.cols);
ASSERT_EQ(FRAME_SIZE.height, frame.rows);
// verify
ASSERT_NE(goodFrames.size(), 0u);
const Mat &goodFrame = goodFrames.front();
const Mat goodFrame = goodFrames.front(); goodFrames.pop();
EXPECT_EQ(goodFrame.depth(), frame.depth());
EXPECT_EQ(goodFrame.channels(), frame.channels());
EXPECT_EQ(goodFrame.type(), frame.type());
double psnr = cvtest::PSNR(goodFrame, frame);
if (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2'))
EXPECT_GT(psnr, 31); // experimentally chosen value
else
EXPECT_GT(psnr, 33); // experimentally chosen value
goodFrames.pop();
if ((i == 1 || i == 4) && fourcc == VideoWriter::fourcc('H', '2', '6', '5'))
{
// ignore bugs of some HW/SW configurations:
// - (added 2021-10) i7-11700K, Win10, oneVPL 2021.4.0 / 2021.6.0
std::cout << "SKIP: bypass frame content check: i=" << i << " psnr=" << psnr << ", expected to be >= " << psnrThreshold << std::endl;
continue;
}
EXPECT_GE(psnr, psnrThreshold);
}
EXPECT_FALSE(cap.read(frame));
EXPECT_TRUE(frame.empty());