diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index 13271cebf4..4e104df3ca 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -760,6 +760,12 @@ public: */ CV_WRAP void setEpsY(double epsY); + /** @brief use markers to improve the position of the corners of the QR code + * + * alignmentMarkers using by default + */ + CV_WRAP void setUseAlignmentMarkers(bool useAlignmentMarkers); + /** @brief Detects QR code in image and returns the quadrangle containing the code. @param img grayscale or color (BGR) image containing (or not) QR code. @param points Output vector of vertices of the minimum-area quadrangle containing the code. diff --git a/modules/objdetect/perf/perf_qrcode_pipeline.cpp b/modules/objdetect/perf/perf_qrcode_pipeline.cpp index 4255eb5a6c..6722978b9a 100644 --- a/modules/objdetect/perf/perf_qrcode_pipeline.cpp +++ b/modules/objdetect/perf/perf_qrcode_pipeline.cpp @@ -114,7 +114,6 @@ PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti) straight_barcode_sort.push_back(result[i].second); } SANITY_CHECK(decoded_info_sort); - SANITY_CHECK(straight_barcode_sort); } #endif diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 1f3ccc4041..a62b90bbe8 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -23,6 +23,7 @@ namespace cv { using std::vector; +using std::pair; static bool checkQRInputImage(InputArray img, Mat& gray) { @@ -948,6 +949,7 @@ vector QRDetect::getQuadrilateral(vector angle_list) return result_angle_list; } + struct QRCodeDetector::Impl { public: @@ -955,9 +957,13 @@ public: ~Impl() {} double epsX, epsY; + vector> alignmentMarkers; + vector updateQrCorners; + bool useAlignmentMarkers = true; }; QRCodeDetector::QRCodeDetector() : p(new Impl) {} + QRCodeDetector::~QRCodeDetector() {} void QRCodeDetector::setEpsX(double epsX) { p->epsX = epsX; } @@ -981,6 +987,7 @@ bool QRCodeDetector::detect(InputArray in, OutputArray points) const class QRDecode { public: + QRDecode(bool useAlignmentMarkers); void init(const Mat &src, const vector &points); Mat getIntermediateBarcode() { return intermediate; } Mat getStraightBarcode() { return straight; } @@ -988,10 +995,23 @@ public: std::string getDecodeInformation() { return result_info; } bool straightDecodingProcess(); bool curvedDecodingProcess(); + vector alignment_coords; + float coeff_expansion = 1.f; + vector getOriginalPoints() {return original_points;} + bool useAlignmentMarkers; protected: double getNumModules(); - bool updatePerspective(); + Mat getHomography() { + CV_TRACE_FUNCTION(); + vector perspective_points = {{0.f, 0.f}, {test_perspective_size, 0.f}, + {test_perspective_size, test_perspective_size}, + {0.f, test_perspective_size}}; + vector pts = original_points; + return findHomography(pts, perspective_points); + } + bool updatePerspective(const Mat& H); bool versionDefinition(); + void detectAlignment(); bool samplingForVersion(); bool decodingProcess(); inline double pointPosition(Point2f a, Point2f b , Point2f c); @@ -1020,6 +1040,7 @@ protected: const static int NUM_SIDES = 2; Mat original, bin_barcode, no_border_intermediate, intermediate, straight, curved_to_straight, test_image; vector original_points; + Mat homography; vector original_curved_points; vector qrcode_locations; vector > closest_points; @@ -1071,6 +1092,7 @@ float static getMinSideLen(const vector &points) { return static_cast(res); } + void QRDecode::init(const Mat &src, const vector &points) { CV_TRACE_FUNCTION(); @@ -2181,6 +2203,8 @@ bool QRDecode::straightenQRCodeInParts() pts.push_back(center_point); Mat H = findHomography(pts, perspective_points); + if (H.empty()) + return false; Mat temp_intermediate(temporary_size, CV_8UC1); warpPerspective(test_mask, temp_intermediate, H, temporary_size, INTER_NEAREST); perspective_result += temp_intermediate; @@ -2213,6 +2237,8 @@ bool QRDecode::straightenQRCodeInParts() perspective_straight_points.push_back(Point2f(perspective_curved_size * 0.5f, perspective_curved_size * 0.5f)); Mat H = findHomography(original_curved_points, perspective_straight_points); + if (H.empty()) + return false; warpPerspective(inversion, temp_result, H, temporary_size, INTER_NEAREST, BORDER_REPLICATE); no_border_intermediate = temp_result(Range(1, temp_result.rows), Range(1, temp_result.cols)); @@ -2281,7 +2307,7 @@ static inline bool checkFinderPatternByAspect(const vector &finderPattern * findPatternsVerticesPoints() may be erroneous, so they are checked. */ static inline std::pair matchPatternPoints(const vector &finderPattern, - const vector cornerPointsQR) { + const vector& cornerPointsQR) { if (!checkFinderPatternByAspect(finderPattern)) return std::make_pair(-1, -1); @@ -2299,6 +2325,7 @@ static inline std::pair matchPatternPoints(const vector &finder } } } + distanceToOrig = sqrt(distanceToOrig); // check that the distance from the QR pattern to the corners of the QR code is small const float originalQrSide = sqrt(normL2Sqr(cornerPointsQR[0] - cornerPointsQR[1]))*0.5f + @@ -2315,7 +2342,7 @@ double QRDecode::getNumModules() { double numModulesX = 0., numModulesY = 0.; bool flag = findPatternsVerticesPoints(finderPatterns); if (flag) { - vector pattern_distance(4); + double pattern_distance[4]; for (auto& pattern : finderPatterns) { auto indexes = matchPatternPoints(pattern, original_points); if (indexes == std::make_pair(-1, -1)) @@ -2336,30 +2363,38 @@ double QRDecode::getNumModules() { return (numModulesX + numModulesY)/2.; } -bool QRDecode::updatePerspective() +// use code from https://stackoverflow.com/questions/13238704/calculating-the-position-of-qr-code-alignment-patterns +static inline vector> getAlignmentCoordinates(int version) { + if (version <= 1) return {}; + int intervals = (version / 7) + 1; // Number of gaps between alignment patterns + int distance = 4 * version + 4; // Distance between first and last alignment pattern + int step = cvRound((double)distance / (double)intervals); // Round equal spacing to nearest integer + step += step & 0b1; // Round step to next even number + vector coordinates((size_t)intervals + 1ull); + coordinates[0] = 6; // First coordinate is always 6 (can't be calculated with step) + for (int i = 1; i <= intervals; i++) { + coordinates[i] = (6 + distance - step * (intervals - i)); // Start right/bottom and go left/up by step*k + } + if (version >= 7) { + return {std::make_pair(coordinates.back(), coordinates.back()), + std::make_pair(coordinates.back(), coordinates[coordinates.size()-2]), + std::make_pair(coordinates[coordinates.size()-2], coordinates.back()), + std::make_pair(coordinates[coordinates.size()-2], coordinates[coordinates.size()-2]), + std::make_pair(coordinates[0], coordinates[1]), + std::make_pair(coordinates[1], coordinates[0]), + }; + } + return {std::make_pair(coordinates.back(), coordinates.back())}; +} + + +bool QRDecode::updatePerspective(const Mat& H) { - CV_TRACE_FUNCTION(); - const Point2f centerPt = intersectionLines(original_points[0], original_points[2], - original_points[1], original_points[3]); - if (cvIsNaN(centerPt.x) || cvIsNaN(centerPt.y)) + if (H.empty()) return false; - - const Size temporary_size(cvRound(test_perspective_size), cvRound(test_perspective_size)); - - vector perspective_points; - perspective_points.push_back(Point2f(0.f, 0.f)); - perspective_points.push_back(Point2f(test_perspective_size, 0.f)); - - perspective_points.push_back(Point2f(test_perspective_size, test_perspective_size)); - perspective_points.push_back(Point2f(0.f, test_perspective_size)); - - perspective_points.push_back(Point2f(test_perspective_size * 0.5f, test_perspective_size * 0.5f)); - - vector pts = original_points; - pts.push_back(centerPt); - - Mat H = findHomography(pts, perspective_points); + homography = H; Mat temp_intermediate; + const Size temporary_size(cvRound(test_perspective_size), cvRound(test_perspective_size)); warpPerspective(bin_barcode, temp_intermediate, H, temporary_size, INTER_NEAREST); no_border_intermediate = temp_intermediate(Range(1, temp_intermediate.rows), Range(1, temp_intermediate.cols)); @@ -2455,7 +2490,7 @@ static inline std::pair getVersionByCode(double numModules, Mat qr, bool QRDecode::versionDefinition() { CV_TRACE_FUNCTION(); - CV_LOG_INFO(NULL, "QR corners: " << original_points[0] << " " << original_points[1] << " " << original_points[2] << + CV_LOG_DEBUG(NULL, "QR corners: " << original_points[0] << " " << original_points[1] << " " << original_points[2] << " " << original_points[3]); LineIterator line_iter(intermediate, Point2f(0, 0), Point2f(test_perspective_size, test_perspective_size)); Point black_point = Point(0, 0); @@ -2549,23 +2584,84 @@ bool QRDecode::versionDefinition() } if (useCode) { - CV_LOG_INFO(NULL, "Version type: useCode"); + CV_LOG_DEBUG(NULL, "Version type: useCode"); version = (uint8_t)versionByCode; } else if (useFinderPattern ) { - CV_LOG_INFO(NULL, "Version type: useFinderPattern"); + CV_LOG_DEBUG(NULL, "Version type: useFinderPattern"); version = (uint8_t)cvRound(versionByFinderPattern); } else { - CV_LOG_INFO(NULL, "Version type: useTransition"); + CV_LOG_DEBUG(NULL, "Version type: useTransition"); version = (uint8_t)versionByTransition; } version_size = 21 + (version - 1) * 4; if ( !(0 < version && version <= 40) ) { return false; } - CV_LOG_INFO(NULL, "QR version: " << (int)version); + CV_LOG_DEBUG(NULL, "QR version: " << (int)version); return true; } +void QRDecode::detectAlignment() { + vector> alignmentPositions = getAlignmentCoordinates(version); + if (alignmentPositions.size() > 0) { + vector perspective_points = {{0.f, 0.f}, {test_perspective_size, 0.f}, {0.f, test_perspective_size}}; + vector object_points = {original_points[0], original_points[1], original_points[3]}; + + // create alignment image + static uint8_t alignmentMarker[25] = { + 0, 0, 0, 0, 0, + 0, 255, 255, 255, 0, + 0, 255, 0, 255, 0, + 0, 255, 255, 255, 0, + 0, 0, 0, 0, 0 + }; + Mat alignmentMarkerMat(5, 5, CV_8UC1, alignmentMarker); + const float module_size = test_perspective_size / version_size; + Mat resizedAlignmentMarker; + resize(alignmentMarkerMat, resizedAlignmentMarker, + Size(cvRound(module_size * 5.f), cvRound(module_size * 5.f)), 0, 0, INTER_AREA); + const float module_offset = 1.9f; + const float offset = (module_size * (5 + module_offset * 2)); // 5 modules in alignment marker, 2 x module_offset modules in offset + for (const pair& alignmentPos : alignmentPositions) { + const float left_top_x = (module_size * (alignmentPos.first - 2.f - module_offset)); // add offset + const float left_top_y = (module_size * (alignmentPos.second - 2.f - module_offset)); // add offset + Mat subImage(no_border_intermediate, Rect(cvRound(left_top_x), cvRound(left_top_y), cvRound(offset), cvRound(offset))); + Mat resTemplate; + matchTemplate(subImage, resizedAlignmentMarker, resTemplate, TM_CCOEFF_NORMED); + double minVal = 0., maxVal = 0.; + Point minLoc, maxLoc, matchLoc; + minMaxLoc(resTemplate, &minVal, &maxVal, &minLoc, &maxLoc); + CV_LOG_DEBUG(NULL, "Alignment maxVal: " << maxVal); + if (maxVal > 0.65) { + const float templateOffset = static_cast(resizedAlignmentMarker.size().width) / 2.f; + Point2f alignmentCoord(Point2f(maxLoc.x + left_top_x + templateOffset, maxLoc.y + left_top_y + templateOffset)); + alignment_coords.push_back(alignmentCoord); + perspectiveTransform(alignment_coords, alignment_coords, homography.inv()); + CV_LOG_DEBUG(NULL, "Alignment coords: " << alignment_coords); + const float relativePosX = (alignmentPos.first + 0.5f) / version_size; + const float relativePosY = (alignmentPos.second + 0.5f) / version_size; + perspective_points.push_back({relativePosX * test_perspective_size, relativePosY * test_perspective_size}); + object_points.push_back(alignment_coords.back()); + } + } + if (object_points.size() > 3ull) { + double ransacReprojThreshold = 10.; + if (version == 2) { // in low version original_points[2] may be calculated more accurately using intersections method + object_points.push_back(original_points[2]); + ransacReprojThreshold = 5.; // set more strict ransacReprojThreshold + perspective_points.push_back({test_perspective_size, test_perspective_size}); + } + Mat H = findHomography(object_points, perspective_points, RANSAC, ransacReprojThreshold); + if (H.empty()) + return; + updatePerspective(H); + vector newCorner2 = {{test_perspective_size, test_perspective_size}}; + perspectiveTransform(newCorner2, newCorner2, H.inv()); + original_points[2] = newCorner2.front(); + } + } +} + bool QRDecode::samplingForVersion() { CV_TRACE_FUNCTION(); @@ -2652,8 +2748,10 @@ bool QRDecode::decodingProcess() bool QRDecode::straightDecodingProcess() { #ifdef HAVE_QUIRC - if (!updatePerspective()) { return false; } + if (!updatePerspective(getHomography())) { return false; } if (!versionDefinition()) { return false; } + if (useAlignmentMarkers) + detectAlignment(); if (!samplingForVersion()) { return false; } if (!decodingProcess()) { return false; } return true; @@ -2677,6 +2775,8 @@ bool QRDecode::curvedDecodingProcess() #endif } +QRDecode::QRDecode(bool _useAlignmentMarkers): useAlignmentMarkers(_useAlignmentMarkers) {} + std::string QRCodeDetector::decode(InputArray in, InputArray points, OutputArray straight_qrcode) { @@ -2689,7 +2789,7 @@ std::string QRCodeDetector::decode(InputArray in, InputArray points, CV_Assert(src_points.size() == 4); CV_CheckGT(contourArea(src_points), 0.0, "Invalid QR code source points"); - QRDecode qrdec; + QRDecode qrdec(p->useAlignmentMarkers); qrdec.init(inarr, src_points); bool ok = qrdec.straightDecodingProcess(); @@ -2702,7 +2802,10 @@ std::string QRCodeDetector::decode(InputArray in, InputArray points, { qrdec.getStraightBarcode().convertTo(straight_qrcode, CV_8UC1); } - + if (ok && !decoded_info.empty()) { + p->alignmentMarkers = {qrdec.alignment_coords}; + p->updateQrCorners = qrdec.getOriginalPoints(); + } return ok ? decoded_info : std::string(); } @@ -2718,7 +2821,7 @@ cv::String QRCodeDetector::decodeCurved(InputArray in, InputArray points, CV_Assert(src_points.size() == 4); CV_CheckGT(contourArea(src_points), 0.0, "Invalid QR code source points"); - QRDecode qrdec; + QRDecode qrdec(p->useAlignmentMarkers); qrdec.init(inarr, src_points); bool ok = qrdec.curvedDecodingProcess(); @@ -3753,15 +3856,15 @@ public: else if (std::min(inarr.size().width, inarr.size().height) > 512) { const int min_side = std::min(inarr.size().width, inarr.size().height); - double coeff_expansion = min_side / 512; - const int width = cvRound(inarr.size().width / coeff_expansion); - const int height = cvRound(inarr.size().height / coeff_expansion); + qrdec[i].coeff_expansion = min_side / 512.f; + const int width = cvRound(inarr.size().width / qrdec[i].coeff_expansion); + const int height = cvRound(inarr.size().height / qrdec[i].coeff_expansion); Size new_size(width, height); Mat inarr2; resize(inarr, inarr2, new_size, 0, 0, INTER_AREA); - for (size_t j = 0; j < 4; j++) + for (size_t j = 0ull; j < 4ull; j++) { - src_points[i][j] /= static_cast(coeff_expansion); + src_points[i][j] /= qrdec[i].coeff_expansion; } qrdec[i].init(inarr2, src_points[i]); ok = qrdec[i].straightDecodingProcess(); @@ -3769,6 +3872,8 @@ public: { decoded_info[i] = qrdec[i].getDecodeInformation(); straight_barcode[i] = qrdec[i].getStraightBarcode(); + for (size_t j = 0ull; j < qrdec[i].alignment_coords.size(); j++) + qrdec[i].alignment_coords[j] *= qrdec[i].coeff_expansion; } } if (decoded_info[i].empty()) @@ -3809,7 +3914,7 @@ bool QRCodeDetector::decodeMulti( } } CV_Assert(src_points.size() > 0); - vector qrdec(src_points.size()); + vector qrdec(src_points.size(), p->useAlignmentMarkers); vector straight_barcode(src_points.size()); vector info(src_points.size()); ParallelDecodeProcess parallelDecodeProcess(inarr, qrdec, info, straight_barcode, src_points); @@ -3840,6 +3945,13 @@ bool QRCodeDetector::decodeMulti( { decoded_info.push_back(info[i]); } + p->alignmentMarkers.resize(src_points.size()); + p->updateQrCorners.resize(src_points.size()*4ull); + for (size_t i = 0ull; i < src_points.size(); i++) { + p->alignmentMarkers[i] = qrdec[i].alignment_coords; + for (size_t j = 0ull; j < 4ull; j++) + p->updateQrCorners[i*4ull+j] = qrdec[i].getOriginalPoints()[j] * qrdec[i].coeff_expansion; + } if (!decoded_info.empty()) return true; else @@ -3870,7 +3982,13 @@ bool QRCodeDetector::detectAndDecodeMulti( updatePointsResult(points_, points); decoded_info.clear(); ok = decodeMulti(inarr, points, decoded_info, straight_qrcode); + updatePointsResult(points_, p->updateQrCorners); return ok; } +void QRCodeDetector::setUseAlignmentMarkers(bool useAlignmentMarkers) { + p->useAlignmentMarkers = useAlignmentMarkers; +} + + } // namespace diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index 7b8242d274..568324b7ca 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -32,6 +32,7 @@ std::string qrcode_images_multiple[] = { "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png", "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png" }; + //#define UPDATE_QRCODE_TEST_DATA #ifdef UPDATE_QRCODE_TEST_DATA @@ -501,7 +502,7 @@ TEST_P(Objdetect_QRCode_Multi, regression) { const std::string name_current_image = GetParam(); const std::string root = "qrcode/multiple/"; - const int pixels_error = 3; + const int pixels_error = 4; std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); @@ -760,6 +761,26 @@ TEST(Objdetect_QRCode_decode, decode_regression_version_25) #endif } +TEST(Objdetect_QRCode_decodeMulti, decode_9_qrcodes_version7) +{ + const std::string name_current_image = "9_qrcodes_version7.jpg"; + const std::string root = "qrcode/multiple/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + QRCodeDetector qrcode; + std::vector corners; + std::vector decoded_info; + + std::vector straight_barcode; + qrcode.detectAndDecodeMulti(src, decoded_info, corners, straight_barcode); + EXPECT_EQ(9ull, decoded_info.size()); + const string gold_info = "I love OpenCV, QR Code version = 7, error correction = level Quartile"; + for (const auto& info : decoded_info) { + EXPECT_EQ(info, gold_info); + } +} + #endif // UPDATE_QRCODE_TEST_DATA }} // namespace