From 6b96512d46f5c3b082ef8051ee3523bddae7a359 Mon Sep 17 00:00:00 2001 From: Quentin Chateau Date: Fri, 28 Dec 2018 11:53:46 +0100 Subject: [PATCH] Merge pull request #13532 from Tytan:channel_exp_comp Channels exposure compensators (#13532) * feed compatible with single channel images * Simplified BlockGainCompensator::apply * ChannelsCompensator * BlocksChannelsCompensator * Make source level compatibility detector happy --- .../stitching/detail/exposure_compensate.hpp | 78 ++++++- modules/stitching/perf/perf_stich.cpp | 12 +- modules/stitching/src/exposure_compensate.cpp | 202 ++++++++++++++---- 3 files changed, 241 insertions(+), 51 deletions(-) diff --git a/modules/stitching/include/opencv2/stitching/detail/exposure_compensate.hpp b/modules/stitching/include/opencv2/stitching/detail/exposure_compensate.hpp index 8d684acd04..bb958a41e6 100644 --- a/modules/stitching/include/opencv2/stitching/detail/exposure_compensate.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/exposure_compensate.hpp @@ -62,7 +62,7 @@ class CV_EXPORTS_W ExposureCompensator public: virtual ~ExposureCompensator() {} - enum { NO, GAIN, GAIN_BLOCKS }; + enum { NO, GAIN, GAIN_BLOCKS, CHANNELS, CHANNELS_BLOCKS }; CV_WRAP static Ptr createDefault(int type); /** @@ -110,6 +110,7 @@ intensities, see @cite BL07 and @cite WJ10 for details. class CV_EXPORTS_W GainCompensator : public ExposureCompensator { public: + // This Constructor only exists to make source level compatibility detector happy CV_WRAP GainCompensator() : GainCompensator(1) {} CV_WRAP GainCompensator(int nr_feeds) @@ -130,16 +131,13 @@ private: int nr_feeds_; }; -/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block -intensities, see @cite UES01 for details. +/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image +intensities on each channel independantly. */ -class CV_EXPORTS_W BlocksGainCompensator : public ExposureCompensator +class CV_EXPORTS_W ChannelsCompensator : public ExposureCompensator { public: - CV_WRAP BlocksGainCompensator(int bl_width = 32, int bl_height = 32) - : BlocksGainCompensator(bl_width, bl_height, 1) {} - CV_WRAP BlocksGainCompensator(int bl_width, int bl_height, int nr_feeds) - : bl_width_(bl_width), bl_height_(bl_height), nr_feeds_(nr_feeds) {setUpdateGain(true);} + CV_WRAP ChannelsCompensator(int nr_feeds=1) : nr_feeds_(nr_feeds) {} void feed(const std::vector &corners, const std::vector &images, const std::vector > &masks) CV_OVERRIDE; CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE; @@ -147,12 +145,76 @@ public: CV_WRAP void setMatGains(std::vector& umv) CV_OVERRIDE; CV_WRAP void setNrFeeds(int nr_feeds) { nr_feeds_ = nr_feeds; } CV_WRAP int getNrFeeds() { return nr_feeds_; } + std::vector gains() const { return gains_; } private: + std::vector gains_; + int nr_feeds_; +}; + +/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image blocks. + */ +class CV_EXPORTS_W BlocksCompensator : public ExposureCompensator +{ +public: + BlocksCompensator(int bl_width=32, int bl_height=32, int nr_feeds=1) + : bl_width_(bl_width), bl_height_(bl_height), nr_feeds_(nr_feeds) {} + CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE; + CV_WRAP void getMatGains(CV_OUT std::vector& umv) CV_OVERRIDE; + CV_WRAP void setMatGains(std::vector& umv) CV_OVERRIDE; + CV_WRAP void setNrFeeds(int nr_feeds) { nr_feeds_ = nr_feeds; } + CV_WRAP int getNrFeeds() { return nr_feeds_; } + +protected: + template + void feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks); + +private: + UMat getGainMap(const GainCompensator& compensator, int bl_idx, Size bl_per_img); + UMat getGainMap(const ChannelsCompensator& compensator, int bl_idx, Size bl_per_img); + int bl_width_, bl_height_; std::vector gain_maps_; int nr_feeds_; }; + +/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block +intensities, see @cite UES01 for details. + */ +class CV_EXPORTS_W BlocksGainCompensator : public BlocksCompensator +{ +public: + // This Constructor only exists to make source level compatibility detector happy + CV_WRAP BlocksGainCompensator(int bl_width = 32, int bl_height = 32) + : BlocksGainCompensator(bl_width, bl_height, 1) {} + CV_WRAP BlocksGainCompensator(int bl_width, int bl_height, int nr_feeds) + : BlocksCompensator(bl_width, bl_height, nr_feeds) {setUpdateGain(true);} + + void feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) CV_OVERRIDE; + + // This function only exists to make source level compatibility detector happy + CV_WRAP void apply(int index, Point corner, InputOutputArray image, InputArray mask) CV_OVERRIDE { + BlocksCompensator::apply(index, corner, image, mask); } + // This function only exists to make source level compatibility detector happy + CV_WRAP void getMatGains(CV_OUT std::vector& umv) CV_OVERRIDE { BlocksCompensator::getMatGains(umv); } + // This function only exists to make source level compatibility detector happy + CV_WRAP void setMatGains(std::vector& umv) CV_OVERRIDE { BlocksCompensator::setMatGains(umv); } +}; + +/** @brief Exposure compensator which tries to remove exposure related artifacts by adjusting image block +on each channel. + */ +class CV_EXPORTS_W BlocksChannelsCompensator : public BlocksCompensator +{ +public: + CV_WRAP BlocksChannelsCompensator(int bl_width=32, int bl_height=32, int nr_feeds=1) + : BlocksCompensator(bl_width, bl_height, nr_feeds) {setUpdateGain(true);} + + void feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) CV_OVERRIDE; +}; //! @} } // namespace detail diff --git a/modules/stitching/perf/perf_stich.cpp b/modules/stitching/perf/perf_stich.cpp index 36f12f0cc3..24c000dc91 100644 --- a/modules/stitching/perf/perf_stich.cpp +++ b/modules/stitching/perf/perf_stich.cpp @@ -24,7 +24,7 @@ typedef TestBaseWithParam> stitchExposureCompMultiFeed; #endif #define TEST_EXP_COMP_BS testing::Values(32, 16, 12, 10, 8) #define TEST_EXP_COMP_NR_FEED testing::Values(1, 2, 3, 4, 5) -#define TEST_EXP_COMP_MODE testing::Values("gain", "blocks") +#define TEST_EXP_COMP_MODE testing::Values("gain", "channels", "blocks_gain", "blocks_channels") #define AFFINE_DATASETS testing::Values("s", "budapest", "newspaper", "prague") PERF_TEST_P(stitch, a123, TEST_DETECTORS) @@ -111,10 +111,14 @@ PERF_TEST_P(stitchExposureCompMultiFeed, a123, testing::Combine(TEST_EXP_COMP_MO declare.time(30 * 10).iterations(10); Ptr exp_comp; - if (mode == "blocks") - exp_comp = makePtr(block_size, block_size, nr_feeds); - else if (mode == "gain") + if (mode == "gain") exp_comp = makePtr(nr_feeds); + else if (mode == "channels") + exp_comp = makePtr(nr_feeds); + else if (mode == "blocks_gain") + exp_comp = makePtr(block_size, block_size, nr_feeds); + else if (mode == "blocks_channels") + exp_comp = makePtr(block_size, block_size, nr_feeds); while(next()) { diff --git a/modules/stitching/src/exposure_compensate.cpp b/modules/stitching/src/exposure_compensate.cpp index 97d113f2e5..5b5a50196a 100644 --- a/modules/stitching/src/exposure_compensate.cpp +++ b/modules/stitching/src/exposure_compensate.cpp @@ -56,8 +56,12 @@ Ptr ExposureCompensator::createDefault(int type) e = makePtr(); else if (type == GAIN) e = makePtr(); - if (type == GAIN_BLOCKS) + else if (type == GAIN_BLOCKS) e = makePtr(); + else if (type == CHANNELS) + e = makePtr(); + else if (type == CHANNELS_BLOCKS) + e = makePtr(); if (e.get() != nullptr) { e->setUpdateGain(true); @@ -114,12 +118,19 @@ void GainCompensator::singleFeed(const std::vector &corners, const std::v { CV_Assert(corners.size() == images.size() && images.size() == masks.size()); + if (images.size() == 0) + return; + + const int num_channels = images[0].channels(); + CV_Assert(std::all_of(images.begin(), images.end(), + [num_channels](const UMat& image) { return image.channels() == num_channels; })); + CV_Assert(num_channels == 1 || num_channels == 3); + const int num_images = static_cast(images.size()); Mat_ N(num_images, num_images); N.setTo(0); Mat_ I(num_images, num_images); I.setTo(0); Mat_ skip(num_images, 1); skip.setTo(true); - //Rect dst_roi = resultRoi(corners, images); Mat subimg1, subimg2; Mat_ submask1, submask2, intersect; @@ -154,14 +165,30 @@ void GainCompensator::singleFeed(const std::vector &corners, const std::v double Isum1 = 0, Isum2 = 0; for (int y = 0; y < roi.height; ++y) { - const Point3_* r1 = subimg1.ptr >(y); - const Point3_* r2 = subimg2.ptr >(y); - for (int x = 0; x < roi.width; ++x) + if (num_channels == 3) { - if (intersect(y, x)) + const Vec* r1 = subimg1.ptr >(y); + const Vec* r2 = subimg2.ptr >(y); + for (int x = 0; x < roi.width; ++x) { - Isum1 += std::sqrt(static_cast(sqr(r1[x].x) + sqr(r1[x].y) + sqr(r1[x].z))); - Isum2 += std::sqrt(static_cast(sqr(r2[x].x) + sqr(r2[x].y) + sqr(r2[x].z))); + if (intersect(y, x)) + { + Isum1 += norm(r1[x]); + Isum2 += norm(r2[x]); + } + } + } + else // if (num_channels == 1) + { + const uchar* r1 = subimg1.ptr(y); + const uchar* r2 = subimg2.ptr(y); + for (int x = 0; x < roi.width; ++x) + { + if (intersect(y, x)) + { + Isum1 += r1[x]; + Isum2 += r2[x]; + } } } } @@ -268,9 +295,70 @@ void GainCompensator::setMatGains(std::vector& umv) } } +void ChannelsCompensator::feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) +{ + std::array, 3> images_channels; -void BlocksGainCompensator::feed(const std::vector &corners, const std::vector &images, - const std::vector > &masks) + // Split channels of each input image + for (const UMat& image: images) + { + std::vector image_channels; + image_channels.resize(3); + split(image, image_channels); + + for (int i = 0; i < int(images_channels.size()); ++i) + images_channels[i].emplace_back(std::move(image_channels[i])); + } + + // For each channel, feed the channel of each image in a GainCompensator + gains_.clear(); + gains_.resize(images.size()); + for (int c = 0; c < 3; ++c) + { + const std::vector& channels = images_channels[c]; + + GainCompensator compensator(getNrFeeds()); + compensator.feed(corners, channels, masks); + + std::vector gains = compensator.gains(); + for (int i = 0; i < int(gains.size()); ++i) + gains_.at(i)[c] = gains[i]; + } +} + +void ChannelsCompensator::apply(int index, Point /*corner*/, InputOutputArray image, InputArray /*mask*/) +{ + CV_INSTRUMENT_REGION(); + + multiply(image, gains_.at(index), image); +} + +void ChannelsCompensator::getMatGains(std::vector& umv) +{ + umv.clear(); + for (int i = 0; i < static_cast(gains_.size()); ++i) + { + Mat m; + Mat(gains_[i]).copyTo(m); + umv.push_back(m); + } +} + +void ChannelsCompensator::setMatGains(std::vector& umv) +{ + for (int i = 0; i < static_cast(umv.size()); i++) + { + Scalar s; + umv[i].copyTo(s); + gains_.push_back(s); + } +} + + +template +void BlocksCompensator::feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) { CV_Assert(corners.size() == images.size() && images.size() == masks.size()); @@ -305,11 +393,13 @@ void BlocksGainCompensator::feed(const std::vector &corners, const std::v } } - if (getUpdateGain()) + if (getUpdateGain() || int(gain_maps_.size()) != num_images) { - GainCompensator compensator(nr_feeds_); + Compensator compensator; + compensator.setNrFeeds(getNrFeeds()); compensator.feed(block_corners, block_images, block_masks); - std::vector gains = compensator.gains(); + + gain_maps_.clear(); gain_maps_.resize(num_images); Mat_ ker(1, 3); @@ -319,50 +409,71 @@ void BlocksGainCompensator::feed(const std::vector &corners, const std::v for (int img_idx = 0; img_idx < num_images; ++img_idx) { Size bl_per_img = bl_per_imgs[img_idx]; - gain_maps_[img_idx].create(bl_per_img, CV_32F); + UMat gain_map = getGainMap(compensator, bl_idx, bl_per_img); + bl_idx += bl_per_img.width*bl_per_img.height; - { - Mat_ gain_map = gain_maps_[img_idx].getMat(ACCESS_WRITE); - for (int by = 0; by < bl_per_img.height; ++by) - for (int bx = 0; bx < bl_per_img.width; ++bx, ++bl_idx) - gain_map(by, bx) = static_cast(gains[bl_idx]); - } + sepFilter2D(gain_map, gain_map, CV_32F, ker, ker); + sepFilter2D(gain_map, gain_map, CV_32F, ker, ker); - sepFilter2D(gain_maps_[img_idx], gain_maps_[img_idx], CV_32F, ker, ker); - sepFilter2D(gain_maps_[img_idx], gain_maps_[img_idx], CV_32F, ker, ker); + gain_maps_[img_idx] = gain_map; } } } +UMat BlocksCompensator::getGainMap(const GainCompensator& compensator, int bl_idx, Size bl_per_img) +{ + std::vector gains = compensator.gains(); -void BlocksGainCompensator::apply(int index, Point /*corner*/, InputOutputArray _image, InputArray /*mask*/) + UMat u_gain_map(bl_per_img, CV_32F); + Mat_ gain_map = u_gain_map.getMat(ACCESS_WRITE); + + for (int by = 0; by < bl_per_img.height; ++by) + for (int bx = 0; bx < bl_per_img.width; ++bx, ++bl_idx) + gain_map(by, bx) = static_cast(gains[bl_idx]); + + return u_gain_map; +} + +UMat BlocksCompensator::getGainMap(const ChannelsCompensator& compensator, int bl_idx, Size bl_per_img) +{ + std::vector gains = compensator.gains(); + + UMat u_gain_map(bl_per_img, CV_32FC3); + Mat_ gain_map = u_gain_map.getMat(ACCESS_WRITE); + + for (int by = 0; by < bl_per_img.height; ++by) + for (int bx = 0; bx < bl_per_img.width; ++bx, ++bl_idx) + for (int c = 0; c < 3; ++c) + gain_map(by, bx)[c] = static_cast(gains[bl_idx][c]); + + return u_gain_map; +} + +void BlocksCompensator::apply(int index, Point /*corner*/, InputOutputArray _image, InputArray /*mask*/) { CV_INSTRUMENT_REGION(); CV_Assert(_image.type() == CV_8UC3); UMat u_gain_map; - if (gain_maps_[index].size() == _image.size()) - u_gain_map = gain_maps_[index]; + if (gain_maps_.at(index).size() == _image.size()) + u_gain_map = gain_maps_.at(index); else - resize(gain_maps_[index], u_gain_map, _image.size(), 0, 0, INTER_LINEAR); + resize(gain_maps_.at(index), u_gain_map, _image.size(), 0, 0, INTER_LINEAR); - Mat_ gain_map = u_gain_map.getMat(ACCESS_READ); - Mat image = _image.getMat(); - for (int y = 0; y < image.rows; ++y) + if (u_gain_map.channels() != 3) { - const float* gain_row = gain_map.ptr(y); - Point3_* row = image.ptr >(y); - for (int x = 0; x < image.cols; ++x) - { - row[x].x = saturate_cast(row[x].x * gain_row[x]); - row[x].y = saturate_cast(row[x].y * gain_row[x]); - row[x].z = saturate_cast(row[x].z * gain_row[x]); - } + std::vector gains_channels; + gains_channels.push_back(u_gain_map); + gains_channels.push_back(u_gain_map); + gains_channels.push_back(u_gain_map); + merge(gains_channels, u_gain_map); } + + multiply(_image, u_gain_map, _image, 1, _image.type()); } -void BlocksGainCompensator::getMatGains(std::vector& umv) +void BlocksCompensator::getMatGains(std::vector& umv) { umv.clear(); for (int i = 0; i < static_cast(gain_maps_.size()); ++i) @@ -372,7 +483,8 @@ void BlocksGainCompensator::getMatGains(std::vector& umv) umv.push_back(m); } } -void BlocksGainCompensator::setMatGains(std::vector& umv) + +void BlocksCompensator::setMatGains(std::vector& umv) { for (int i = 0; i < static_cast(umv.size()); i++) { @@ -382,6 +494,18 @@ void BlocksGainCompensator::setMatGains(std::vector& umv) } } +void BlocksGainCompensator::feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) +{ + BlocksCompensator::feed(corners, images, masks); +} + +void BlocksChannelsCompensator::feed(const std::vector &corners, const std::vector &images, + const std::vector > &masks) +{ + BlocksCompensator::feed(corners, images, masks); +} + } // namespace detail } // namespace cv