From efa0717d012d13c9d987ed58f11f832b4ff9f40f Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 4 Apr 2012 11:45:16 +0000 Subject: [PATCH] Added support of homography estimation into videostab module --- .../opencv2/videostab/global_motion.hpp | 27 ++++++++----- .../include/opencv2/videostab/inpainting.hpp | 6 +++ .../include/opencv2/videostab/stabilizer.hpp | 6 +-- modules/videostab/src/global_motion.cpp | 35 +++++++++++++++-- modules/videostab/src/inpainting.cpp | 39 +++++++++++++++---- modules/videostab/src/precomp.hpp | 1 + modules/videostab/src/stabilizer.cpp | 24 +++++++++--- samples/cpp/videostab.cpp | 10 ++--- 8 files changed, 114 insertions(+), 34 deletions(-) diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index 2bf07816d9..8e0bba0919 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -58,7 +58,9 @@ enum MotionModel TRANSLATION = 0, TRANSLATION_AND_SCALE = 1, LINEAR_SIMILARITY = 2, - AFFINE = 3 + AFFINE = 3, + HOMOGRAPHY = 4, + UNKNOWN = 5 }; CV_EXPORTS Mat estimateGlobalMotionLeastSquares( @@ -76,10 +78,11 @@ struct CV_EXPORTS RansacParams RansacParams(int size, float thresh, float eps, float prob) : size(size), thresh(thresh), eps(eps), prob(prob) {} - static RansacParams translationMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); } + static RansacParams translation2dMotionStd() { return RansacParams(2, 0.5f, 0.5f, 0.99f); } static RansacParams translationAndScale2dMotionStd() { return RansacParams(3, 0.5f, 0.5f, 0.99f); } - static RansacParams linearSimilarityMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); } + static RansacParams linearSimilarity2dMotionStd() { return RansacParams(4, 0.5f, 0.5f, 0.99f); } static RansacParams affine2dMotionStd() { return RansacParams(6, 0.5f, 0.5f, 0.99f); } + static RansacParams homography2dMotionStd() { return RansacParams(8, 0.5f, 0.5f, 0.99f); } }; CV_EXPORTS Mat estimateGlobalMotionRobust( @@ -87,14 +90,22 @@ CV_EXPORTS Mat estimateGlobalMotionRobust( int model = AFFINE, const RansacParams ¶ms = RansacParams::affine2dMotionStd(), float *rmse = 0, int *ninliers = 0); -class CV_EXPORTS IGlobalMotionEstimator +class CV_EXPORTS GlobalMotionEstimatorBase { public: - virtual ~IGlobalMotionEstimator() {} + GlobalMotionEstimatorBase() : motionModel_(UNKNOWN) {} + virtual ~GlobalMotionEstimatorBase() {} + + virtual void setMotionModel(MotionModel val) { motionModel_ = val; } + virtual MotionModel motionModel() const { return motionModel_; } + virtual Mat estimate(const Mat &frame0, const Mat &frame1) = 0; + +protected: + MotionModel motionModel_; }; -class CV_EXPORTS PyrLkRobustMotionEstimator : public IGlobalMotionEstimator +class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase { public: PyrLkRobustMotionEstimator(); @@ -105,9 +116,6 @@ public: void setOptFlowEstimator(Ptr val) { optFlowEstimator_ = val; } Ptr optFlowEstimator() const { return optFlowEstimator_; } - void setMotionModel(MotionModel val) { motionModel_ = val; } - MotionModel motionModel() const { return motionModel_; } - void setRansacParams(const RansacParams &val) { ransacParams_ = val; } RansacParams ransacParams() const { return ransacParams_; } @@ -122,7 +130,6 @@ public: private: Ptr detector_; Ptr optFlowEstimator_; - MotionModel motionModel_; RansacParams ransacParams_; std::vector status_; std::vector keypointsPrev_; diff --git a/modules/videostab/include/opencv2/videostab/inpainting.hpp b/modules/videostab/include/opencv2/videostab/inpainting.hpp index 9c18dcf1c2..25b5639456 100644 --- a/modules/videostab/include/opencv2/videostab/inpainting.hpp +++ b/modules/videostab/include/opencv2/videostab/inpainting.hpp @@ -47,6 +47,7 @@ #include "opencv2/core/core.hpp" #include "opencv2/videostab/optical_flow.hpp" #include "opencv2/videostab/fast_marching.hpp" +#include "opencv2/videostab/global_motion.hpp" #include "opencv2/photo/photo.hpp" namespace cv @@ -66,6 +67,9 @@ public: virtual void setRadius(int val) { radius_ = val; } virtual int radius() const { return radius_; } + virtual void setMotionModel(MotionModel val) { motionModel_ = val; } + virtual MotionModel motionModel() const { return motionModel_; } + virtual void setFrames(const std::vector &val) { frames_ = &val; } virtual const std::vector& frames() const { return *frames_; } @@ -82,6 +86,7 @@ public: protected: int radius_; + MotionModel motionModel_; const std::vector *frames_; const std::vector *motions_; const std::vector *stabilizedFrames_; @@ -101,6 +106,7 @@ public: bool empty() const { return inpainters_.empty(); } virtual void setRadius(int val); + virtual void setMotionModel(MotionModel val); virtual void setFrames(const std::vector &val); virtual void setMotions(const std::vector &val); virtual void setStabilizedFrames(const std::vector &val); diff --git a/modules/videostab/include/opencv2/videostab/stabilizer.hpp b/modules/videostab/include/opencv2/videostab/stabilizer.hpp index 97fa62473b..4eef2a6cc0 100644 --- a/modules/videostab/include/opencv2/videostab/stabilizer.hpp +++ b/modules/videostab/include/opencv2/videostab/stabilizer.hpp @@ -72,8 +72,8 @@ public: void setFrameSource(Ptr val) { frameSource_ = val; } Ptr frameSource() const { return frameSource_; } - void setMotionEstimator(Ptr val) { motionEstimator_ = val; } - Ptr motionEstimator() const { return motionEstimator_; } + void setMotionEstimator(Ptr val) { motionEstimator_ = val; } + Ptr motionEstimator() const { return motionEstimator_; } void setDeblurer(Ptr val) { deblurer_ = val; } Ptr deblurrer() const { return deblurer_; } @@ -104,7 +104,7 @@ protected: Ptr log_; Ptr frameSource_; - Ptr motionEstimator_; + Ptr motionEstimator_; Ptr deblurer_; Ptr inpainter_; int radius_; diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index a6c051b3c0..9d9f503e50 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -41,7 +41,6 @@ //M*/ #include "precomp.hpp" -#include "opencv2/highgui/highgui.hpp" #include "opencv2/videostab/global_motion.hpp" #include "opencv2/videostab/ring_buffer.hpp" @@ -324,8 +323,38 @@ Mat PyrLkRobustMotionEstimator::estimate(const Mat &frame0, const Mat &frame1) float rmse; int ninliers; - Mat M = estimateGlobalMotionRobust( - pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, &rmse, &ninliers); + Mat_ M; + + if (motionModel_ != HOMOGRAPHY) + M = estimateGlobalMotionRobust( + pointsPrevGood_, pointsGood_, motionModel_, ransacParams_, &rmse, &ninliers); + else + { + vector mask; + M = findHomography(pointsPrevGood_, pointsGood_, CV_RANSAC, ransacParams_.thresh, mask); + + ninliers = 0; + rmse = 0; + + Point2d p0, p1; + float x, y, z; + + for (size_t i = 0; i < pointsGood_.size(); ++i) + { + if (mask[i]) + { + p0 = pointsPrevGood_[i]; p1 = pointsGood_[i]; + x = M(0,0)*p0.x + M(0,1)*p0.y + M(0,2); + y = M(1,0)*p0.x + M(1,1)*p0.y + M(1,2); + z = M(2,0)*p0.x + M(2,1)*p0.y + M(2,2); + x /= z; y /= z; + rmse += sqr(x - p1.x) + sqr(y - p1.y); + ninliers++; + } + } + + rmse = sqrt(rmse / static_cast(ninliers)); + } if (rmse > maxRmse_ || static_cast(ninliers) / pointsGood_.size() < minInlierRatio_) M = Mat::eye(3, 3, CV_32F); diff --git a/modules/videostab/src/inpainting.cpp b/modules/videostab/src/inpainting.cpp index c465a22bb8..67109984d9 100644 --- a/modules/videostab/src/inpainting.cpp +++ b/modules/videostab/src/inpainting.cpp @@ -70,6 +70,14 @@ void InpaintingPipeline::setFrames(const vector &val) } +void InpaintingPipeline::setMotionModel(MotionModel val) +{ + for (size_t i = 0; i < inpainters_.size(); ++i) + inpainters_[i]->setMotionModel(val); + InpainterBase::setMotionModel(val); +} + + void InpaintingPipeline::setMotions(const vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) @@ -361,17 +369,34 @@ void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask) Mat motion1to0 = motions[radius_ + neighbor - idx].inv(); - frame1_ = at(neighbor, *frames_); - warpAffine( - frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(), - INTER_LINEAR, borderMode_); + // warp frame + + frame1_ = at(neighbor, *frames_); + + if (motionModel_ != HOMOGRAPHY) + warpAffine( + frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(), + INTER_LINEAR, borderMode_); + else + warpPerspective( + frame1_, transformedFrame1_, motion1to0, frame1_.size(), INTER_LINEAR, + borderMode_); + cvtColor(transformedFrame1_, transformedGrayFrame1_, CV_BGR2GRAY); - warpAffine( - mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(), - INTER_NEAREST); + // warp mask + + if (motionModel_ != HOMOGRAPHY) + warpAffine( + mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(), + INTER_NEAREST); + else + warpPerspective(mask1_, transformedMask1_, motion1to0, mask1_.size(), INTER_NEAREST); + erode(transformedMask1_, transformedMask1_, Mat()); + // update flow + optFlowEstimator_->run(grayFrame_, transformedGrayFrame1_, flowX_, flowY_, flowErrors_); calcFlowMask( diff --git a/modules/videostab/src/precomp.hpp b/modules/videostab/src/precomp.hpp index 5c74f04201..6d773b95cb 100644 --- a/modules/videostab/src/precomp.hpp +++ b/modules/videostab/src/precomp.hpp @@ -54,6 +54,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "opencv2/video/video.hpp" #include "opencv2/features2d/features2d.hpp" +#include "opencv2/calib3d/calib3d.hpp" // some aux. functions diff --git a/modules/videostab/src/stabilizer.cpp b/modules/videostab/src/stabilizer.cpp index becc47ee30..6177d3f231 100644 --- a/modules/videostab/src/stabilizer.cpp +++ b/modules/videostab/src/stabilizer.cpp @@ -71,6 +71,7 @@ void StabilizerBase::setUp(int cacheSize, const Mat &frame) doInpainting_ = dynamic_cast(inpainter) == 0; if (doInpainting_) { + inpainter_->setMotionModel(motionEstimator_->motionModel()); inpainter_->setFrames(frames_); inpainter_->setMotions(motions_); inpainter_->setStabilizedFrames(stabilizedFrames_); @@ -176,15 +177,26 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion) preProcessedFrame_ = at(curStabilizedPos_, frames_); // apply stabilization transformation - warpAffine( - preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), - stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); + + if (motionEstimator_->motionModel() != HOMOGRAPHY) + warpAffine( + preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), + stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); + else + warpPerspective( + preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), + stabilizationMotion_, frameSize_, INTER_LINEAR, borderMode_); if (doInpainting_) { - warpAffine( - frameMask_, at(curStabilizedPos_, stabilizedMasks_), - stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); + if (motionEstimator_->motionModel() != HOMOGRAPHY) + warpAffine( + frameMask_, at(curStabilizedPos_, stabilizedMasks_), + stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); + else + warpPerspective( + frameMask_, at(curStabilizedPos_, stabilizedMasks_), + stabilizationMotion_, frameSize_, INTER_NEAREST); erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_), Mat()); diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp index 13c96d89dd..74ef1bf9f8 100644 --- a/samples/cpp/videostab.cpp +++ b/samples/cpp/videostab.cpp @@ -29,7 +29,7 @@ void run(); void saveMotionsIfNecessary(); void printHelp(); -class GlobalMotionReader : public IGlobalMotionEstimator +class GlobalMotionReader : public GlobalMotionEstimatorBase { public: GlobalMotionReader(string path) @@ -256,11 +256,11 @@ int main(int argc, const char **argv) { RansacParams ransac; PyrLkRobustMotionEstimator *est = new PyrLkRobustMotionEstimator(); - Ptr est_(est); + Ptr est_(est); if (arg("model") == "transl") { est->setMotionModel(TRANSLATION); - ransac = RansacParams::translationMotionStd(); + ransac = RansacParams::translation2dMotionStd(); } else if (arg("model") == "transl_and_scale") { @@ -270,13 +270,13 @@ int main(int argc, const char **argv) else if (arg("model") == "linear_sim") { est->setMotionModel(LINEAR_SIMILARITY); - ransac = RansacParams::linearSimilarityMotionStd(); + ransac = RansacParams::linearSimilarity2dMotionStd(); } else if (arg("model") == "affine") { est->setMotionModel(AFFINE); ransac = RansacParams::affine2dMotionStd(); - } + } else throw runtime_error("unknown motion model: " + arg("model")); ransac.eps = argf("outlier-ratio");