third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #18535 from joshdoe:gray16_gstreamer_writing

Browse Source 
Add CV_16UC1/GRAY16_LE support to GStreamer backend for VideoWriter

* videoio(backend): add Writer_open_with_params to plugin API

This will allow arbitrary parameters to be passed to plugin backends

* videoio(gstreamer): add GRAY16_LE/CV_16UC1 writing support to GStreamer

This introduces a new property VIDEOWRITER_PROP_DEPTH, which defaults to
CV_8U, but for GStreamer can be set to CV_16U.

Also, fix another test to not fail if plugin isn't found, copying logic
from the read_write test.

* videoio(plugin): fix handling plugins with previous API level

* videoio: coding style

* fix warning
This commit is contained in:
joshdoe 2020-12-05 16:28:07 -05:00 committed by GitHub
parent 753ccd6b17
commit 541a09b7ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 235 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/videoio/include/opencv2/videoio.hpp
View File

@ -194,8 +194,9 @@ enum VideoWriterProperties {
VIDEOWRITER_PROP_QUALITY = 1, //!< Current quality (0..100%) of the encoded videostream. Can be adjusted dynamically in some codecs.
VIDEOWRITER_PROP_FRAMEBYTES = 2, //!< (Read-only): Size of just encoded video frame. Note that the encoding order may be different from representation order.
VIDEOWRITER_PROP_NSTRIPES = 3, //!< Number of stripes for parallel encoding. -1 for auto detection.
VIDEOWRITER_PROP_IS_COLOR = 4 //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
VIDEOWRITER_PROP_IS_COLOR = 4, //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
//!< will work with grayscale frames.
VIDEOWRITER_PROP_DEPTH = 5 //!< Defaults to CV_8U.
};
//! @} videoio_flags_base

56
modules/videoio/src/backend_plugin.cpp
View File

@ -205,10 +205,15 @@ public:
FN_opencv_videoio_plugin_init_t fn_init = reinterpret_cast<FN_opencv_videoio_plugin_init_t>(lib_->getSymbol(init_name));
if (fn_init)
{
plugin_api_ = fn_init(ABI_VERSION, API_VERSION, NULL);
for (int supported_api_version = API_VERSION; supported_api_version >= 0; supported_api_version--)
{
plugin_api_ = fn_init(ABI_VERSION, supported_api_version, NULL);
if (plugin_api_)
break;
}
if (!plugin_api_)
{
CV_LOG_INFO(NULL, "Video I/O: plugin is incompatible: " << lib->getName());
CV_LOG_INFO(NULL, "Video I/O: plugin is incompatible (can't be initialized): " << lib->getName());
return;
}
if (plugin_api_->api_header.opencv_version_major != CV_VERSION_MAJOR)
@ -232,8 +237,29 @@ public:
plugin_api_ = NULL;
return;
}
// TODO Preview: add compatibility API/ABI checks
CV_LOG_INFO(NULL, "Video I/O: loaded plugin '" << plugin_api_->api_header.api_description << "'");
CV_LOG_INFO(NULL, "Video I/O: initialized '" << plugin_api_->api_header.api_description << "': built with "
<< cv::format("OpenCV %d.%d (ABI/API = %d/%d)",
plugin_api_->api_header.opencv_version_major, plugin_api_->api_header.opencv_version_minor,
plugin_api_->api_header.min_api_version, plugin_api_->api_header.api_version)
<< ", current OpenCV version is '" CV_VERSION "' (ABI/API = " << ABI_VERSION << "/" << API_VERSION << ")"
);
if (plugin_api_->api_header.min_api_version != ABI_VERSION) // future: range can be here
{
// actually this should never happen due to checks in plugin's init() function
CV_LOG_ERROR(NULL, "Video I/O: plugin is not supported due to incompatible ABI = " << plugin_api_->api_header.min_api_version);
plugin_api_ = NULL;
return;
}
if (plugin_api_->api_header.api_version != API_VERSION)
{
CV_LOG_INFO(NULL, "Video I/O: NOTE: plugin is supported, but there is API version mismath: "
<< cv::format("plugin API level (%d) != OpenCV API level (%d)", plugin_api_->api_header.api_version, API_VERSION));
if (plugin_api_->api_header.api_version < API_VERSION)
{
CV_LOG_INFO(NULL, "Video I/O: NOTE: some functionality may be unavailable due to lack of support by plugin implementation");
}
}
CV_LOG_INFO(NULL, "Video I/O: plugin is ready to use '" << plugin_api_->api_header.api_description << "'");
}
else
{
@ -508,11 +534,31 @@ public:
{
CV_Assert(plugin_api);
CvPluginWriter writer = NULL;
if (plugin_api->Writer_open)
if (plugin_api->api_header.api_version >= 1 && plugin_api->Writer_open_with_params)
{
CV_Assert(plugin_api->Writer_release);
CV_Assert(!filename.empty());
std::vector<int> vint_params = params.getIntVector();
int* c_params = &vint_params[0];
unsigned n_params = (unsigned)(vint_params.size() / 2);
if (CV_ERROR_OK == plugin_api->Writer_open_with_params(filename.c_str(), fourcc, fps, sz.width, sz.height, c_params, n_params, &writer))
{
CV_Assert(writer);
return makePtr<PluginWriter>(plugin_api, writer);
}
}
else if (plugin_api->Writer_open)
{
CV_Assert(plugin_api->Writer_release);
CV_Assert(!filename.empty());
const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
if (depth != CV_8U)
{
CV_LOG_WARNING(NULL, "Video I/O plugin doesn't support (due to lower API level) creation of VideoWriter with depth != CV_8U");
return Ptr<PluginWriter>();
}
if (CV_ERROR_OK == plugin_api->Writer_open(filename.c_str(), fourcc, fps, sz.width, sz.height, isColor, &writer))
{
CV_Assert(writer);

5
modules/videoio/src/cap_ffmpeg.cpp
View File

@ -396,7 +396,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
{
{
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
"FFmpeg OpenCV Video I/O plugin"
},
@ -411,7 +411,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
/* 9*/cv_writer_release,
/* 10*/cv_writer_get_prop,
/* 11*/cv_writer_set_prop,
/* 12*/cv_writer_write
/* 12*/cv_writer_write,
/* 13 Writer_open_with_params*/NULL
};
} // namespace

98
modules/videoio/src/cap_gstreamer.cpp
View File

@ -1240,8 +1240,8 @@ class CvVideoWriter_GStreamer : public CvVideoWriter
{
public:
CvVideoWriter_GStreamer()
: input_pix_fmt(0),
num_frames(0), framerate(0)
: ipl_depth(CV_8U)
, input_pix_fmt(0), num_frames(0), framerate(0)
{
}
virtual ~CvVideoWriter_GStreamer() CV_OVERRIDE
@ -1263,14 +1263,16 @@ public:
int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_GSTREAMER; }
bool open(const std::string &filename, int fourcc,
double fps, const Size &frameSize, bool isColor );
double fps, const Size &frameSize, bool isColor, int depth );
void close();
bool writeFrame( const IplImage* image ) CV_OVERRIDE;
int getIplDepth() const { return ipl_depth; }
protected:
const char* filenameToMimetype(const char* filename);
GSafePtr<GstElement> pipeline;
GSafePtr<GstElement> source;
int ipl_depth;
int input_pix_fmt;
int num_frames;
double framerate;
@ -1396,6 +1398,7 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename)
* \param fps desired framerate
* \param frameSize the size of the expected frames
* \param is_color color or grayscale
* \param depth the depth of the expected frames
* \return success
*
* We support 2 modes of operation. Either the user enters a filename and a fourcc
@ -1408,7 +1411,8 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename)
*
*/
bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
double fps, const cv::Size &frameSize, bool is_color )
double fps, const cv::Size &frameSize,
bool is_color, int depth )
{
// check arguments
CV_Assert(!filename.empty());
@ -1548,6 +1552,8 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
if (fourcc == CV_FOURCC('M','J','P','G') && frameSize.height == 1)
{
CV_Assert(depth == CV_8U);
ipl_depth = IPL_DEPTH_8U;
input_pix_fmt = GST_VIDEO_FORMAT_ENCODED;
caps.attach(gst_caps_new_simple("image/jpeg",
"framerate", GST_TYPE_FRACTION, int(fps_num), int(fps_denom),
@ -1556,6 +1562,8 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
}
else if (is_color)
{
CV_Assert(depth == CV_8U);
ipl_depth = IPL_DEPTH_8U;
input_pix_fmt = GST_VIDEO_FORMAT_BGR;
bufsize = frameSize.width * frameSize.height * 3;
@ -1569,8 +1577,9 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
caps.attach(gst_caps_fixate(caps.detach()));
CV_Assert(caps);
}
else
else if (!is_color && depth == CV_8U)
{
ipl_depth = IPL_DEPTH_8U;
input_pix_fmt = GST_VIDEO_FORMAT_GRAY8;
bufsize = frameSize.width * frameSize.height;
@ -1582,6 +1591,26 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
NULL));
caps.attach(gst_caps_fixate(caps.detach()));
}
else if (!is_color && depth == CV_16U)
{
ipl_depth = IPL_DEPTH_16U;
input_pix_fmt = GST_VIDEO_FORMAT_GRAY16_LE;
bufsize = frameSize.width * frameSize.height * 2;
caps.attach(gst_caps_new_simple("video/x-raw",
"format", G_TYPE_STRING, "GRAY16_LE",
"width", G_TYPE_INT, frameSize.width,
"height", G_TYPE_INT, frameSize.height,
"framerate", GST_TYPE_FRACTION, gint(fps_num), gint(fps_denom),
NULL));
caps.attach(gst_caps_fixate(caps.detach()));
}
else
{
CV_WARN("unsupported depth=" << depth <<", and is_color=" << is_color << " combination");
pipeline.release();
return false;
}
gst_app_src_set_caps(GST_APP_SRC(source.get()), caps);
gst_app_src_set_stream_type(GST_APP_SRC(source.get()), GST_APP_STREAM_TYPE_STREAM);
@ -1659,6 +1688,12 @@ bool CvVideoWriter_GStreamer::writeFrame( const IplImage * image )
return false;
}
}
else if (input_pix_fmt == GST_VIDEO_FORMAT_GRAY16_LE) {
if (image->nChannels != 1 || image->depth != IPL_DEPTH_16U) {
CV_WARN("cvWriteFrame() needs images with depth = IPL_DEPTH_16U and nChannels = 1.");
return false;
}
}
else {
CV_WARN("cvWriteFrame() needs BGR or grayscale images\n");
return false;
@ -1699,9 +1734,10 @@ Ptr<IVideoWriter> create_GStreamer_writer(const std::string& filename, int fourc
{
CvVideoWriter_GStreamer* wrt = new CvVideoWriter_GStreamer;
const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
try
{
if (wrt->open(filename, fourcc, fps, frameSize, isColor))
if (wrt->open(filename, fourcc, fps, frameSize, isColor, depth))
return makePtr<LegacyWriter>(wrt);
delete wrt;
}
@ -1921,15 +1957,40 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
}
static
CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
CV_OUT CvPluginWriter* handle)
CvResult CV_API_CALL cv_writer_open_with_params(
const char* filename, int fourcc, double fps, int width, int height,
int* params, unsigned n_params,
CV_OUT CvPluginWriter* handle)
{
CvVideoWriter_GStreamer* wrt = 0;
try
{
wrt = new CvVideoWriter_GStreamer();
CvSize sz = { width, height };
if(wrt && wrt->open(filename, fourcc, fps, sz, isColor))
bool isColor = true;
int depth = CV_8U;
if (params)
{
for (unsigned i = 0; i < n_params; ++i)
{
const int prop = params[i*2];
const int value = params[i*2 + 1];
switch (prop)
{
case VIDEOWRITER_PROP_IS_COLOR:
isColor = value != 0;
break;
case VIDEOWRITER_PROP_DEPTH:
depth = value;
break;
default:
// TODO emit message about non-recognized propert
// FUTURE: there should be mandatory and optional properties
return CV_ERROR_FAIL;
}
}
}
wrt = new CvVideoWriter_GStreamer();
if (wrt && wrt->open(filename, fourcc, fps, sz, isColor, depth))
{
*handle = (CvPluginWriter)wrt;
return CV_ERROR_OK;
@ -1943,6 +2004,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps
return CV_ERROR_FAIL;
}
static
CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
CV_OUT CvPluginWriter* handle)
{
int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor };
return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle);
}
static
CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
{
@ -1975,7 +2044,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
CvVideoWriter_GStreamer* instance = (CvVideoWriter_GStreamer*)handle;
CvSize sz = { width, height };
IplImage img;
cvInitImageHeader(&img, sz, IPL_DEPTH_8U, cn);
cvInitImageHeader(&img, sz, instance->getIplDepth(), cn);
cvSetData(&img, const_cast<unsigned char*>(data), step);
return instance->writeFrame(&img) ? CV_ERROR_OK : CV_ERROR_FAIL;
}
@ -2003,7 +2072,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
/* 9*/cv_writer_release,
/* 10*/cv_writer_get_prop,
/* 11*/cv_writer_set_prop,
/* 12*/cv_writer_write
/* 12*/cv_writer_write,
/* 13*/cv_writer_open_with_params
};
} // namespace
@ -2012,7 +2082,7 @@ const OpenCV_VideoIO_Plugin_API_preview* opencv_videoio_plugin_init_v0(int reque
{
if (requested_abi_version != 0)
return NULL;
if (requested_api_version != 0)
if (requested_api_version != 0 && requested_api_version != 1)
return NULL;
return &cv::plugin_api_v0;
}

12
modules/videoio/src/cap_interface.hpp
View File

@ -116,6 +116,18 @@ public:
}
return unusedParams;
}
std::vector<int> getIntVector() const
{
std::vector<int> vint_params;
for (const auto& param : params_)
{
vint_params.push_back(param.key);
vint_params.push_back(param.value);
}
return vint_params;
}
private:
std::vector<VideoWriterParameter> params_;
};

5
modules/videoio/src/cap_mfx_plugin.cpp
View File

@ -188,7 +188,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
{
{
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
"MediaSDK OpenCV Video I/O plugin"
},
@ -203,7 +203,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
/* 9*/cv_writer_release,
/* 10*/cv_writer_get_prop,
/* 11*/cv_writer_set_prop,
/* 12*/cv_writer_write
/* 12*/cv_writer_write,
/* 13 Writer_open_with_params*/NULL
};
} // namespace

2
modules/videoio/src/cap_msmf.cpp
View File

@ -1866,7 +1866,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char*
static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
{
{
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
"Microsoft Media Foundation OpenCV Video I/O plugin"
},

5
modules/videoio/src/cap_ueye.cpp
View File

@ -467,7 +467,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter /*handle*/, const unsigned c
const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
{
{
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
"uEye OpenCV Video I/O plugin"
},
@ -482,7 +482,8 @@ const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
/* 9*/cv_writer_release,
/* 10*/cv_writer_get_prop,
/* 11*/cv_writer_set_prop,
/* 12*/cv_writer_write
/* 12*/cv_writer_write,
/* 13 Writer_open_with_params*/NULL
};
} // namespace
} // namespace cv

34
modules/videoio/src/plugin_api.hpp
View File

@ -9,10 +9,12 @@
#include <opencv2/core/llapi/llapi.h>
// increase for backward-compatible changes, e.g. add new function
// Main API <= Plugin API -> plugin is compatible
#define API_VERSION 0 // preview
// Main API <= Plugin API -> plugin is fully compatible
// Main API > Plugin API -> plugin is not compatible, caller should use shim code to use plugins with old API
#define API_VERSION 1 // preview
// increase for incompatible changes, e.g. remove function argument
// Main ABI == Plugin ABI -> plugin is compatible
// Main ABI > Plugin ABI -> plugin is not compatible, caller should use shim code to use old ABI plugins
#define ABI_VERSION 0 // preview
#ifdef __cplusplus
@ -93,8 +95,12 @@ typedef struct OpenCV_VideoIO_Plugin_API_preview
/** @brief Try to open video writer
@param filename File name or NULL to use camera_index instead
@param camera_index Camera index (used if filename == NULL)
@param filename Destination location
@param fourcc FOURCC code
@param fps FPS
@param width frame width
@param height frame height
@param isColor true if video stream should save color frames
@param[out] handle pointer on Writer handle
@note API-CALL 8, API-Version == 0
@ -143,6 +149,26 @@ typedef struct OpenCV_VideoIO_Plugin_API_preview
*/
CvResult (CV_API_CALL *Writer_write)(CvPluginWriter handle, const unsigned char *data, int step, int width, int height, int cn);
/** @brief Try to open video writer
@param filename Destination location
@param fourcc FOURCC code
@param fps FPS
@param width frame width
@param height frame height
@param params pointer on 2*n_params array of 'key,value' pairs
@param n_params number of passed parameters
@param[out] handle pointer on Writer handle
@note API-CALL 13, API-Version == 1
*/
CvResult (CV_API_CALL* Writer_open_with_params)(
const char* filename, int fourcc, double fps, int width, int height,
int* params, unsigned n_params,
CV_OUT CvPluginWriter* handle
);
} OpenCV_VideoIO_Plugin_API_preview;
#ifdef BUILD_PLUGIN

51
modules/videoio/test/test_gstreamer.cpp
View File

@ -73,22 +73,63 @@ Param test_data[] = {
INSTANTIATE_TEST_CASE_P(videoio, videoio_gstreamer, testing::ValuesIn(test_data));
TEST(Videoio_GStreamer, unsupported_pipeline)
TEST(videoio_gstreamer, unsupported_pipeline)
{
VideoCaptureAPIs apiPref = CAP_GSTREAMER;
if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends()))
throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref));
if (!videoio_registry::hasBackend(CAP_GSTREAMER))
throw SkipTestException("GStreamer backend was not found");
// could not link videoconvert0 to matroskamux0, matroskamux0 can't handle caps video/x-raw, format=(string)RGBA
std::string pipeline = "appsrc ! videoconvert ! video/x-raw, format=(string)RGBA ! matroskamux ! filesink location=test.mkv";
Size frame_size(640, 480);
VideoWriter writer;
EXPECT_NO_THROW(writer.open(pipeline, apiPref, 0/*fourcc*/, 30/*fps*/, frame_size, true));
EXPECT_NO_THROW(writer.open(pipeline, CAP_GSTREAMER, 0/*fourcc*/, 30/*fps*/, frame_size, true));
EXPECT_FALSE(writer.isOpened());
// no frames
EXPECT_NO_THROW(writer.release());
}
TEST(videoio_gstreamer, gray16_writing)
{
if (!videoio_registry::hasBackend(CAP_GSTREAMER))
throw SkipTestException("GStreamer backend was not found");
Size frame_size(320, 240);
// generate a noise frame
Mat frame = Mat(frame_size, CV_16U);
randu(frame, 0, 65535);
// generate a temp filename, and fix path separators to how GStreamer expects them
cv::String temp_file = cv::tempfile(".raw");
std::replace(temp_file.begin(), temp_file.end(), '\\', '/');
// write noise frame to file using GStreamer
std::ostringstream writer_pipeline;
writer_pipeline << "appsrc ! filesink location=" << temp_file;
std::vector<int> params {
VIDEOWRITER_PROP_IS_COLOR, 0/*false*/,
VIDEOWRITER_PROP_DEPTH, CV_16U
};
VideoWriter writer;
ASSERT_NO_THROW(writer.open(writer_pipeline.str(), CAP_GSTREAMER, 0/*fourcc*/, 30/*fps*/, frame_size, params));
ASSERT_TRUE(writer.isOpened());
ASSERT_NO_THROW(writer.write(frame));
ASSERT_NO_THROW(writer.release());
// read noise frame back in
Mat written_frame(frame_size, CV_16U);
std::ifstream fs(temp_file, std::ios::in | std::ios::binary);
fs.read((char*)written_frame.ptr(0), frame_size.width * frame_size.height * 2);
ASSERT_TRUE(fs);
fs.close();
// compare to make sure it's identical
EXPECT_EQ(0, cv::norm(frame, written_frame, NORM_INF));
// remove temp file
EXPECT_EQ(0, remove(temp_file.c_str()));
}
} // namespace