From 2b79a6ff8f8c810b291b973eaa20b7c2950f7cdb Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 26 Jan 2022 17:01:13 +0300 Subject: [PATCH] Merge pull request #20832 from TolyaTalamanov:at/python-gstreamer-source G-API: Wrap GStreamerSource * Wrap GStreamerSource into python * Fixed test skipping when can't make Gst-src * Wrapped GStreamerPipeline class, added dummy test for it * Fix no_gst testing * Changed wrap for GStreamerPipeline::getStreamingSource() : now python-specific in-class method GStreamerPipeline::get_streaming_source() * Added accuracy tests vs OCV:VideoCapture(Gstreamer) * Add skipping when can't use VideoCapture(GSTREAMER); Add better handling of GStreamer backend unavailable; Changed video to avoid terminations * Applying comments * back to a separate get_streaming_source function, with comment Co-authored-by: OrestChura --- modules/gapi/CMakeLists.txt | 1 + .../streaming/gstreamer/gstreamerpipeline.hpp | 16 +- .../streaming/gstreamer/gstreamersource.hpp | 8 + .../gapi/misc/python/package/gapi/__init__.py | 2 + modules/gapi/misc/python/pyopencv_gapi.hpp | 3 +- .../misc/python/test/test_gapi_streaming.py | 194 +++++++++++++++++- 6 files changed, 220 insertions(+), 4 deletions(-) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index a689bc3e81..3ca898d4a1 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -55,6 +55,7 @@ file(GLOB gapi_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/gstreamer/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/onevpl/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/util/*.hpp" ) diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp index 83afc99393..c566656cb6 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamerpipeline.hpp @@ -19,12 +19,12 @@ namespace gapi { namespace wip { namespace gst { -class GAPI_EXPORTS GStreamerPipeline +class GAPI_EXPORTS_W GStreamerPipeline { public: class Priv; - explicit GStreamerPipeline(const std::string& pipeline); + GAPI_WRAP explicit GStreamerPipeline(const std::string& pipeline); IStreamSource::Ptr getStreamingSource(const std::string& appsinkName, const GStreamerSource::OutputType outputType = GStreamerSource::OutputType::MAT); @@ -40,6 +40,18 @@ protected: using GStreamerPipeline = gst::GStreamerPipeline; +// NB: Function for using from python +// FIXME: a separate function is created due to absence of wrappers for `shared_ptr<> ` +// Ideally would be to wrap the `GStreamerPipeline::getStreamingSource()` method as is +GAPI_EXPORTS_W cv::Ptr +inline get_streaming_source(cv::Ptr& pipeline, + const std::string& appsinkName, + const GStreamerSource::OutputType outputType + = GStreamerSource::OutputType::MAT) +{ + return pipeline->getStreamingSource(appsinkName, outputType); +} + } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp index b81bad31b8..1365afa1b9 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/gstreamer/gstreamersource.hpp @@ -82,6 +82,14 @@ protected: using GStreamerSource = gst::GStreamerSource; +// NB: Overload for using from python +GAPI_EXPORTS_W cv::Ptr +inline make_gst_src(const std::string& pipeline, + const GStreamerSource::OutputType outputType = + GStreamerSource::OutputType::MAT) +{ + return make_src(pipeline, outputType); +} } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index b1326712fc..6323582f5b 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -297,3 +297,5 @@ cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly cv.gapi.streaming.queue_capacity = cv.gapi_streaming_queue_capacity + +cv.gapi.wip.GStreamerPipeline = cv.gapi_wip_gst_GStreamerPipeline diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index a71366250c..b4be0048d0 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -19,6 +19,7 @@ using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; using vector_GNetParam = std::vector; using gapi_streaming_queue_capacity = cv::gapi::streaming::queue_capacity; +using GStreamerSource_OutputType = cv::gapi::wip::GStreamerSource::OutputType; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -230,7 +231,7 @@ PyObject* pyopencv_from(const cv::GArg& value) { HANDLE_CASE(BOOL, bool); HANDLE_CASE(INT, int); - HANDLE_CASE(INT64, int64_t); + HANDLE_CASE(INT64, int64_t); HANDLE_CASE(DOUBLE, double); HANDLE_CASE(FLOAT, float); HANDLE_CASE(STRING, std::string); diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index d7914c5157..d06447d791 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -34,6 +34,16 @@ try: return img + def convertNV12p2BGR(in_nv12): + shape = in_nv12.shape + y_height = shape[0] // 3 * 2 + uv_shape = (shape[0] // 3, shape[1]) + new_uv_shape = (uv_shape[0], uv_shape[1] // 2, 2) + return cv.cvtColorTwoPlane(in_nv12[:y_height, :], + in_nv12[ y_height:, :].reshape(new_uv_shape), + cv.COLOR_YUV2BGR_NV12) + + class test_gapi_streaming(NewOpenCVTests): def test_image_input(self): @@ -229,7 +239,6 @@ try: def test_gapi_streaming_meta(self): - ksize = 3 path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) # G-API @@ -350,6 +359,189 @@ try: cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + def get_gst_source(self, gstpipeline): + # NB: Skip test in case gstreamer isn't available. + try: + return cv.gapi.wip.make_gst_src(gstpipeline) + except cv.error as e: + if str(e).find('Built without GStreamer support!') == -1: + raise e + else: + raise unittest.SkipTest(str(e)) + + + def test_gst_source(self): + if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER): + raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER") + + gstpipeline = """videotestsrc is-live=true pattern=colors num-buffers=10 ! + videorate ! videoscale ! video/x-raw,width=1920,height=1080, + framerate=30/1 ! appsink""" + + g_in = cv.GMat() + g_out = cv.gapi.copy(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + + source = self.get_gst_source(gstpipeline) + + ccomp.setSource(cv.gin(source)) + ccomp.start() + + has_frame, output = ccomp.pull() + while has_frame: + self.assertTrue(output.size != 0) + has_frame, output = ccomp.pull() + + + def open_VideoCapture_gstreamer(self, gstpipeline): + try: + cap = cv.VideoCapture(gstpipeline, cv.CAP_GSTREAMER) + except Exception as e: + raise unittest.SkipTest("Backend GSTREAMER can't open the video; " + + "cause: " + str(e)) + if not cap.isOpened(): + raise unittest.SkipTest("Backend GSTREAMER can't open the video") + return cap + + + def test_gst_source_accuracy(self): + if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER): + raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER") + + path = self.find_file('highgui/video/big_buck_bunny.avi', + [os.environ['OPENCV_TEST_DATA_PATH']]) + gstpipeline = """filesrc location=""" + path + """ ! decodebin ! videoconvert ! + videoscale ! video/x-raw,format=NV12 ! appsink""" + + # G-API pipeline + g_in = cv.GMat() + g_out = cv.gapi.copy(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + + # G-API Gst-source + source = self.get_gst_source(gstpipeline) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # OpenCV Gst-source + cap = self.open_VideoCapture_gstreamer(gstpipeline) + + # Assert + max_num_frames = 10 + for _ in range(max_num_frames): + has_expected, expected = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_expected: + break + + self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected), actual, cv.NORM_INF)) + + + def get_gst_pipeline(self, gstpipeline): + # NB: Skip test in case gstreamer isn't available. + try: + return cv.gapi.wip.GStreamerPipeline(gstpipeline) + except cv.error as e: + if str(e).find('Built without GStreamer support!') == -1: + raise e + else: + raise unittest.SkipTest(str(e)) + except SystemError as e: + raise unittest.SkipTest(str(e) + ", casued by " + str(e.__cause__)) + + + def test_gst_multiple_sources(self): + if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER): + raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER") + + gstpipeline = """videotestsrc is-live=true pattern=colors num-buffers=10 ! + videorate ! videoscale ! + video/x-raw,width=1920,height=1080,framerate=30/1 ! + appsink name=sink1 + videotestsrc is-live=true pattern=colors num-buffers=10 ! + videorate ! videoscale ! + video/x-raw,width=1920,height=1080,framerate=30/1 ! + appsink name=sink2""" + + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + + pp = self.get_gst_pipeline(gstpipeline) + src1 = cv.gapi.wip.get_streaming_source(pp, "sink1") + src2 = cv.gapi.wip.get_streaming_source(pp, "sink2") + + ccomp.setSource(cv.gin(src1, src2)) + ccomp.start() + + has_frame, out = ccomp.pull() + while has_frame: + self.assertTrue(out.size != 0) + has_frame, out = ccomp.pull() + + + def test_gst_multiple_sources_accuracy(self): + if not cv.videoio_registry.hasBackend(cv.CAP_GSTREAMER): + raise unittest.SkipTest("Backend is not available/disabled: GSTREAMER") + + path = self.find_file('highgui/video/big_buck_bunny.avi', + [os.environ['OPENCV_TEST_DATA_PATH']]) + gstpipeline1 = """filesrc location=""" + path + """ ! decodebin ! videoconvert ! + videoscale ! video/x-raw,format=NV12 ! appsink""" + gstpipeline2 = """filesrc location=""" + path + """ ! decodebin ! + videoflip method=clockwise ! videoconvert ! videoscale ! + video/x-raw,format=NV12 ! appsink""" + gstpipeline_gapi = gstpipeline1 + ' name=sink1 ' + gstpipeline2 + ' name=sink2' + + # G-API pipeline + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out1 = cv.gapi.copy(g_in1) + g_out2 = cv.gapi.copy(g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out1, g_out2)) + + ccomp = c.compileStreaming() + + # G-API Gst-source + pp = self.get_gst_pipeline(gstpipeline_gapi) + + src1 = cv.gapi.wip.get_streaming_source(pp, "sink1") + src2 = cv.gapi.wip.get_streaming_source(pp, "sink2") + ccomp.setSource(cv.gin(src1, src2)) + ccomp.start() + + # OpenCV Gst-source + cap1 = self.open_VideoCapture_gstreamer(gstpipeline1) + cap2 = self.open_VideoCapture_gstreamer(gstpipeline2) + + # Assert + max_num_frames = 10 + for _ in range(max_num_frames): + has_expected1, expected1 = cap1.read() + has_expected2, expected2 = cap2.read() + has_actual, (actual1, actual2) = ccomp.pull() + + self.assertEqual(has_expected1, has_expected2) + has_expected = has_expected1 and has_expected2 + self.assertEqual(has_expected, has_actual) + + if not has_expected: + break + + self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected1), actual1, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(convertNV12p2BGR(expected2), actual2, cv.NORM_INF)) + + except unittest.SkipTest as e: