Merge pull request #15356 from APrigarina:3.4_version2

QR-Code detection: accuracy improvement

* first commit

* resize improvement

* resize improvement

* dummy commit

* warnings fix

* warnings fix

* warnings fix

* test added

* resize fix

* resize fix

* tests changed

* tests changed

modules/objdetect/src/qrcode.cpp

@@ -40,9 +40,10 @@ protected:
     bool testBypassRoute(vector<Point2f> hull, int start, int finish);
     inline double getCosVectors(Point2f a, Point2f b, Point2f c);
 
-    Mat barcode, bin_barcode, straight_barcode;
+    Mat barcode, bin_barcode, resized_barcode, resized_bin_barcode, straight_barcode;
     vector<Point2f> localization_points, transformation_points;
     double eps_vertical, eps_horizontal, coeff_expansion;
+    enum resize_direction { ZOOMING, SHRINKING, UNCHANGED } purpose;
 };
 
 
@@ -50,24 +51,36 @@ void QRDetect::init(const Mat& src, double eps_vertical_, double eps_horizontal_
 {
     CV_TRACE_FUNCTION();
     CV_Assert(!src.empty());
+    barcode = src.clone();
     const double min_side = std::min(src.size().width, src.size().height);
     if (min_side < 512.0)
     {
+        purpose = ZOOMING;
         coeff_expansion = 512.0 / min_side;
         const int width  = cvRound(src.size().width  * coeff_expansion);
         const int height = cvRound(src.size().height  * coeff_expansion);
         Size new_size(width, height);
         resize(src, barcode, new_size, 0, 0, INTER_LINEAR);
     }
+    else if (min_side > 512.0)
+    {
+        purpose = SHRINKING;
+        coeff_expansion = min_side / 512.0;
+        const int width  = cvRound(src.size().width  / coeff_expansion);
+        const int height = cvRound(src.size().height  / coeff_expansion);
+        Size new_size(width, height);
+        resize(src, resized_barcode, new_size, 0, 0, INTER_AREA);
+    }
     else
     {
+        purpose = UNCHANGED;
         coeff_expansion = 1.0;
-        barcode = src;
     }
 
     eps_vertical   = eps_vertical_;
     eps_horizontal = eps_horizontal_;
     adaptiveThreshold(barcode, bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2);
+    adaptiveThreshold(resized_barcode, resized_bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2);
 
 }
 
@@ -140,78 +153,96 @@ vector<Point2f> QRDetect::separateVerticalLines(const vector<Vec3d> &list_lines)
 {
     CV_TRACE_FUNCTION();
     vector<Vec3d> result;
-    int temp_length = 0;
+    int temp_length;
+    vector<Point2f> point2f_result;
     uint8_t next_pixel;
     vector<double> test_lines;
 
 
-    for (size_t pnt = 0; pnt < list_lines.size(); pnt++)
+    for (int coeff_epsilon = 1; coeff_epsilon < 10; coeff_epsilon++)
     {
-        const int x = cvRound(list_lines[pnt][0] + list_lines[pnt][2] * 0.5);
+        result.clear();
-        const int y = cvRound(list_lines[pnt][1]);
+        temp_length = 0;
+        point2f_result.clear();
 
-        // --------------- Search vertical up-lines --------------- //
+        for (size_t pnt = 0; pnt < list_lines.size(); pnt++)
-
-        test_lines.clear();
-        uint8_t future_pixel_up = 255;
-
-        for (int j = y; j < bin_barcode.rows - 1; j++)
         {
-            next_pixel = bin_barcode.ptr<uint8_t>(j + 1)[x];
+            const int x = cvRound(list_lines[pnt][0] + list_lines[pnt][2] * 0.5);
-            temp_length++;
+            const int y = cvRound(list_lines[pnt][1]);
-            if (next_pixel == future_pixel_up)
+
+            // --------------- Search vertical up-lines --------------- //
+
+            test_lines.clear();
+            uint8_t future_pixel_up = 255;
+
+            for (int j = y; j < bin_barcode.rows - 1; j++)
             {
-                future_pixel_up = 255 - future_pixel_up;
+                next_pixel = bin_barcode.ptr<uint8_t>(j + 1)[x];
-                test_lines.push_back(temp_length);
+                temp_length++;
-                temp_length = 0;
+                if (next_pixel == future_pixel_up)
-                if (test_lines.size() == 3) { break; }
+                {
+                    future_pixel_up = 255 - future_pixel_up;
+                    test_lines.push_back(temp_length);
+                    temp_length = 0;
+                    if (test_lines.size() == 3) { break; }
+                }
+            }
+
+            // --------------- Search vertical down-lines --------------- //
+
+            uint8_t future_pixel_down = 255;
+            for (int j = y; j >= 1; j--)
+            {
+                next_pixel = bin_barcode.ptr<uint8_t>(j - 1)[x];
+                temp_length++;
+                if (next_pixel == future_pixel_down)
+                {
+                    future_pixel_down = 255 - future_pixel_down;
+                    test_lines.push_back(temp_length);
+                    temp_length = 0;
+                    if (test_lines.size() == 6) { break; }
+                }
+            }
+
+            // --------------- Compute vertical lines --------------- //
+
+            if (test_lines.size() == 6)
+            {
+                double length = 0.0, weight = 0.0;
+
+                for (size_t i = 0; i < test_lines.size(); i++) { length += test_lines[i]; }
+
+                CV_Assert(length > 0);
+                for (size_t i = 0; i < test_lines.size(); i++)
+                {
+                    if (i % 3 != 0) { weight += fabs((test_lines[i] / length) - 1.0/ 7.0); }
+                    else            { weight += fabs((test_lines[i] / length) - 3.0/14.0); }
+                }
+
+                if(weight < eps_horizontal * coeff_epsilon)
+                {
+                    result.push_back(list_lines[pnt]);
+                }
             }
         }
-
+        if (result.size() > 2)
-        // --------------- Search vertical down-lines --------------- //
-
-        uint8_t future_pixel_down = 255;
-        for (int j = y; j >= 1; j--)
         {
-            next_pixel = bin_barcode.ptr<uint8_t>(j - 1)[x];
+            for (size_t i = 0; i < result.size(); i++)
-            temp_length++;
-            if (next_pixel == future_pixel_down)
             {
-                future_pixel_down = 255 - future_pixel_down;
+                point2f_result.push_back(
-                test_lines.push_back(temp_length);
+                      Point2f(static_cast<float>(result[i][0] + result[i][2] * 0.5),
-                temp_length = 0;
+                              static_cast<float>(result[i][1])));
-                if (test_lines.size() == 6) { break; }
             }
+
+            vector<Point2f> centers;
+            Mat labels;
+            double compactness;
+            compactness = kmeans(point2f_result, 3, labels,
+                 TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1),
+                 3, KMEANS_PP_CENTERS, centers);
+            if (compactness == 0) { continue; }
+            if (compactness > 0)  { break; }
         }
-
-        // --------------- Compute vertical lines --------------- //
-
-        if (test_lines.size() == 6)
-        {
-            double length = 0.0, weight = 0.0;
-
-            for (size_t i = 0; i < test_lines.size(); i++) { length += test_lines[i]; }
-
-            CV_Assert(length > 0);
-            for (size_t i = 0; i < test_lines.size(); i++)
-            {
-                if (i % 3 != 0) { weight += fabs((test_lines[i] / length) - 1.0/ 7.0); }
-                else            { weight += fabs((test_lines[i] / length) - 3.0/14.0); }
-            }
-
-            if(weight < eps_horizontal)
-            {
-                result.push_back(list_lines[pnt]);
-            }
-        }
-    }
-
-    vector<Point2f> point2f_result;
-    for (size_t i = 0; i < result.size(); i++)
-    {
-        point2f_result.push_back(
-              Point2f(static_cast<float>(result[i][0] + result[i][2] * 0.5),
-                      static_cast<float>(result[i][1])));
     }
     return point2f_result;
 }
@@ -316,7 +347,7 @@ bool QRDetect::localization()
     vector<Vec3d> list_lines_x = searchHorizontalLines();
     if( list_lines_x.empty() ) { return false; }
     vector<Point2f> list_lines_y = separateVerticalLines(list_lines_x);
-    if( list_lines_y.size() < 3 ) { return false; }
+    if( list_lines_y.empty() ) { return false; }
 
     vector<Point2f> centers;
     Mat labels;
@@ -325,9 +356,56 @@ bool QRDetect::localization()
            3, KMEANS_PP_CENTERS, localization_points);
 
     fixationPoints(localization_points);
-    if (localization_points.size() != 3) { return false; }
 
-    if (coeff_expansion > 1.0)
+    bool suare_flag = false, local_points_flag = false;
+    double triangle_sides[3];
+    triangle_sides[0] = norm(localization_points[0] - localization_points[1]);
+    triangle_sides[1] = norm(localization_points[1] - localization_points[2]);
+    triangle_sides[2] = norm(localization_points[2] - localization_points[0]);
+
+    double triangle_perim = (triangle_sides[0] + triangle_sides[1] + triangle_sides[2]) / 2;
+
+    double square_area = sqrt((triangle_perim * (triangle_perim - triangle_sides[0])
+                                              * (triangle_perim - triangle_sides[1])
+                                              * (triangle_perim - triangle_sides[2]))) * 2;
+    double img_square_area = bin_barcode.cols * bin_barcode.rows;
+
+    if (square_area > (img_square_area * 0.2))
+    {
+        suare_flag = true;
+    }
+    if (localization_points.size() != 3)
+    {
+        local_points_flag = true;
+    }
+    if ((suare_flag || local_points_flag) && purpose == SHRINKING)
+    {
+        localization_points.clear();
+        bin_barcode = resized_bin_barcode.clone();
+        list_lines_x = searchHorizontalLines();
+        if( list_lines_x.empty() ) { return false; }
+        list_lines_y = separateVerticalLines(list_lines_x);
+        if( list_lines_y.empty() ) { return false; }
+
+        kmeans(list_lines_y, 3, labels,
+               TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1),
+               3, KMEANS_PP_CENTERS, localization_points);
+
+        fixationPoints(localization_points);
+        if (localization_points.size() != 3) { return false; }
+
+        const int width  = cvRound(bin_barcode.size().width  * coeff_expansion);
+        const int height = cvRound(bin_barcode.size().height * coeff_expansion);
+        Size new_size(width, height);
+        Mat intermediate;
+        resize(bin_barcode, intermediate, new_size, 0, 0, INTER_LINEAR);
+        bin_barcode = intermediate.clone();
+        for (size_t i = 0; i < localization_points.size(); i++)
+        {
+            localization_points[i] *= coeff_expansion;
+        }
+    }
+    if (purpose == ZOOMING)
     {
         const int width  = cvRound(bin_barcode.size().width  / coeff_expansion);
         const int height = cvRound(bin_barcode.size().height / coeff_expansion);
@@ -475,6 +553,13 @@ bool QRDetect::computeTransformationPoints()
     vector<Point2f> quadrilateral = getQuadrilateral(transformation_points);
     transformation_points = quadrilateral;
 
+    int width = bin_barcode.size().width;
+    int height = bin_barcode.size().height;
+    for (size_t i = 0; i < transformation_points.size(); i++)
+    {
+        if ((cvRound(transformation_points[i].x) > width) ||
+            (cvRound(transformation_points[i].y) > height)) { return false; }
+    }
     return true;
 }
 
@@ -835,9 +920,27 @@ protected:
 void QRDecode::init(const Mat &src, const vector<Point2f> &points)
 {
     CV_TRACE_FUNCTION();
-    original = src.clone();
+    vector<Point2f> bbox = points;
-    intermediate = Mat::zeros(src.size(), CV_8UC1);
+    double coeff_expansion;
-    original_points = points;
+    const int min_side = std::min(src.size().width, src.size().height);
+    if (min_side > 512)
+    {
+        coeff_expansion = min_side / 512;
+        const int width  = cvRound(src.size().width  / coeff_expansion);
+        const int height = cvRound(src.size().height / coeff_expansion);
+        Size new_size(width, height);
+        resize(src, original, new_size, 0, 0, INTER_AREA);
+            for (size_t i = 0; i < bbox.size(); i++)
+            {
+                bbox[i] /= static_cast<float>(coeff_expansion);
+            }
+    }
+    else
+    {
+        original = src.clone();
+    }
+    intermediate = Mat::zeros(original.size(), CV_8UC1);
+    original_points = bbox;
     version = 0;
     version_size = 0;
     test_perspective_size = 251;

modules/objdetect/test/test_qrcode.cpp

@@ -15,6 +15,12 @@ std::string qrcode_images_name[] = {
   "russian.jpg", "kanji.jpg", "link_github_ocv.jpg", "link_ocv.jpg", "link_wiki_cv.jpg"
 };
 
+std::string qrcode_images_close[] = {
+  "close_1.png", "close_2.png", "close_3.png", "close_4.png", "close_5.png"
+};
+std::string qrcode_images_monitor[] = {
+  "monitor_1.png", "monitor_2.png", "monitor_3.png", "monitor_4.png", "monitor_5.png"
+};
 // #define UPDATE_QRCODE_TEST_DATA
 #ifdef  UPDATE_QRCODE_TEST_DATA
 
@@ -51,6 +57,83 @@ TEST(Objdetect_QRCode, generate_test_data)
     file_config.release();
 }
 
+TEST(Objdetect_QRCode_Close, generate_test_data)
+{
+    const std::string root = "qrcode/close/";
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::WRITE);
+
+    file_config << "close_images" << "[";
+    size_t close_count = sizeof(qrcode_images_close) / sizeof(qrcode_images_close[0]);
+    for (size_t i = 0; i < close_count; i++)
+    {
+        file_config << "{:" << "image_name" << qrcode_images_close[i];
+        std::string image_path = findDataFile(root + qrcode_images_close[i]);
+        std::vector<Point> corners;
+        Mat src = imread(image_path, IMREAD_GRAYSCALE), barcode, straight_barcode;
+        std::string decoded_info;
+        ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+        const double min_side = std::min(src.size().width, src.size().height);
+        double coeff_expansion = 1024.0 / min_side;
+        const int width  = cvRound(src.size().width  * coeff_expansion);
+        const int height = cvRound(src.size().height  * coeff_expansion);
+        Size new_size(width, height);
+        resize(src, barcode, new_size, 0, 0, INTER_LINEAR);
+        EXPECT_TRUE(detectQRCode(barcode, corners));
+#ifdef HAVE_QUIRC
+        EXPECT_TRUE(decodeQRCode(barcode, corners, decoded_info, straight_barcode));
+#endif
+        file_config << "x" << "[:";
+        for (size_t j = 0; j < corners.size(); j++) { file_config << corners[j].x; }
+        file_config << "]";
+        file_config << "y" << "[:";
+        for (size_t j = 0; j < corners.size(); j++) { file_config << corners[j].y; }
+        file_config << "]";
+        file_config << "info" << decoded_info;
+        file_config << "}";
+    }
+    file_config << "]";
+    file_config.release();
+}
+TEST(Objdetect_QRCode_Monitor, generate_test_data)
+{
+    const std::string root = "qrcode/monitor/";
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::WRITE);
+
+    file_config << "monitor_images" << "[";
+    size_t monitor_count = sizeof(qrcode_images_monitor) / sizeof(qrcode_images_monitor[0]);
+    for (size_t i = 0; i < monitor_count; i++)
+    {
+        file_config << "{:" << "image_name" << qrcode_images_monitor[i];
+        std::string image_path = findDataFile(root + qrcode_images_monitor[i]);
+        std::vector<Point> corners;
+        Mat src = imread(image_path, IMREAD_GRAYSCALE), barcode, straight_barcode;
+        std::string decoded_info;
+        ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+        const double min_side = std::min(src.size().width, src.size().height);
+        double coeff_expansion = 1024.0 / min_side;
+        const int width  = cvRound(src.size().width  * coeff_expansion);
+        const int height = cvRound(src.size().height  * coeff_expansion);
+        Size new_size(width, height);
+        resize(src, barcode, new_size, 0, 0, INTER_LINEAR);
+        EXPECT_TRUE(detectQRCode(barcode, corners));
+#ifdef HAVE_QUIRC
+        EXPECT_TRUE(decodeQRCode(barcode, corners, decoded_info, straight_barcode));
+#endif
+        file_config << "x" << "[:";
+        for (size_t j = 0; j < corners.size(); j++) { file_config << corners[j].x; }
+        file_config << "]";
+        file_config << "y" << "[:";
+        for (size_t j = 0; j < corners.size(); j++) { file_config << corners[j].y; }
+        file_config << "]";
+        file_config << "info" << decoded_info;
+        file_config << "}";
+    }
+    file_config << "]";
+    file_config.release();
+}
+
 #else
 
 typedef testing::TestWithParam< std::string > Objdetect_QRCode;
@@ -113,9 +196,139 @@ TEST_P(Objdetect_QRCode, regression)
     }
 }
 
+typedef testing::TestWithParam< std::string > Objdetect_QRCode_Close;
+TEST_P(Objdetect_QRCode_Close, regression)
+{
+    const std::string name_current_image = GetParam();
+    const std::string root = "qrcode/close/";
+    const int pixels_error = 3;
+
+    std::string image_path = findDataFile(root + name_current_image);
+    Mat src = imread(image_path, IMREAD_GRAYSCALE), barcode, straight_barcode;
+    ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+    const double min_side = std::min(src.size().width, src.size().height);
+    double coeff_expansion = 1024.0 / min_side;
+    const int width  = cvRound(src.size().width  * coeff_expansion);
+    const int height = cvRound(src.size().height  * coeff_expansion);
+    Size new_size(width, height);
+    resize(src, barcode, new_size, 0, 0, INTER_LINEAR);
+    std::vector<Point> corners;
+    std::string decoded_info;
+    QRCodeDetector qrcode;
+#ifdef HAVE_QUIRC
+    decoded_info = qrcode.detectAndDecode(barcode, corners, straight_barcode);
+    ASSERT_FALSE(corners.empty());
+    ASSERT_FALSE(decoded_info.empty());
+#else
+    ASSERT_TRUE(qrcode.detect(src, corners));
+#endif
+
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::READ);
+    ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config;
+    {
+        FileNode images_list = file_config["close_images"];
+        size_t images_count = static_cast<size_t>(images_list.size());
+        ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config;
+
+        for (size_t index = 0; index < images_count; index++)
+        {
+            FileNode config = images_list[(int)index];
+            std::string name_test_image = config["image_name"];
+            if (name_test_image == name_current_image)
+            {
+                for (int i = 0; i < 4; i++)
+                {
+                    int x = config["x"][i];
+                    int y = config["y"][i];
+                    EXPECT_NEAR(x, corners[i].x, pixels_error);
+                    EXPECT_NEAR(y, corners[i].y, pixels_error);
+                }
+
+#ifdef HAVE_QUIRC
+                std::string original_info = config["info"];
+                EXPECT_EQ(decoded_info, original_info);
+#endif
+
+                return; // done
+            }
+        }
+        std::cerr
+            << "Not found results for '" << name_current_image
+            << "' image in config file:" << dataset_config << std::endl
+            << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data."
+            << std::endl;
+    }
+}
+
+typedef testing::TestWithParam< std::string > Objdetect_QRCode_Monitor;
+TEST_P(Objdetect_QRCode_Monitor, regression)
+{
+    const std::string name_current_image = GetParam();
+    const std::string root = "qrcode/monitor/";
+    const int pixels_error = 3;
+
+    std::string image_path = findDataFile(root + name_current_image);
+    Mat src = imread(image_path, IMREAD_GRAYSCALE), barcode, straight_barcode;
+    ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
+    const double min_side = std::min(src.size().width, src.size().height);
+    double coeff_expansion = 1024.0 / min_side;
+    const int width  = cvRound(src.size().width  * coeff_expansion);
+    const int height = cvRound(src.size().height  * coeff_expansion);
+    Size new_size(width, height);
+    resize(src, barcode, new_size, 0, 0, INTER_LINEAR);
+    std::vector<Point> corners;
+    std::string decoded_info;
+    QRCodeDetector qrcode;
+#ifdef HAVE_QUIRC
+    decoded_info = qrcode.detectAndDecode(barcode, corners, straight_barcode);
+    ASSERT_FALSE(corners.empty());
+    ASSERT_FALSE(decoded_info.empty());
+#else
+    ASSERT_TRUE(qrcode.detect(src, corners));
+#endif
+
+    const std::string dataset_config = findDataFile(root + "dataset_config.json");
+    FileStorage file_config(dataset_config, FileStorage::READ);
+    ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config;
+    {
+        FileNode images_list = file_config["monitor_images"];
+        size_t images_count = static_cast<size_t>(images_list.size());
+        ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config;
+
+        for (size_t index = 0; index < images_count; index++)
+        {
+            FileNode config = images_list[(int)index];
+            std::string name_test_image = config["image_name"];
+            if (name_test_image == name_current_image)
+            {
+                for (int i = 0; i < 4; i++)
+                {
+                    int x = config["x"][i];
+                    int y = config["y"][i];
+                    EXPECT_NEAR(x, corners[i].x, pixels_error);
+                    EXPECT_NEAR(y, corners[i].y, pixels_error);
+                }
+
+#ifdef HAVE_QUIRC
+                std::string original_info = config["info"];
+                EXPECT_EQ(decoded_info, original_info);
+#endif
+
+                return; // done
+            }
+        }
+        std::cerr
+            << "Not found results for '" << name_current_image
+            << "' image in config file:" << dataset_config << std::endl
+            << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data."
+            << std::endl;
+    }
+}
+
 INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode, testing::ValuesIn(qrcode_images_name));
-
+INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Close, testing::ValuesIn(qrcode_images_close));
-
+INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Monitor, testing::ValuesIn(qrcode_images_monitor));
 
 TEST(Objdetect_QRCode_basic, not_found_qrcode)
 {