diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 0b1d948156..82dedcb850 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -1895,13 +1895,33 @@ Rect_<_Tp>& operator -= ( Rect_<_Tp>& a, const Size_<_Tp>& b ) template static inline Rect_<_Tp>& operator &= ( Rect_<_Tp>& a, const Rect_<_Tp>& b ) { - _Tp x1 = std::max(a.x, b.x); - _Tp y1 = std::max(a.y, b.y); - a.width = std::min(a.x + a.width, b.x + b.width) - x1; - a.height = std::min(a.y + a.height, b.y + b.height) - y1; - a.x = x1; - a.y = y1; - if( a.width <= 0 || a.height <= 0 ) + if (a.empty() || b.empty()) { + a = Rect(); + return a; + } + const Rect_<_Tp>& Rx_min = (a.x < b.x) ? a : b; + const Rect_<_Tp>& Rx_max = (a.x < b.x) ? b : a; + const Rect_<_Tp>& Ry_min = (a.y < b.y) ? a : b; + const Rect_<_Tp>& Ry_max = (a.y < b.y) ? b : a; + // Looking at the formula below, we will compute Rx_min.width - (Rx_max.x - Rx_min.x) + // but we want to avoid overflows. Rx_min.width >= 0 and (Rx_max.x - Rx_min.x) >= 0 + // by definition so the difference does not overflow. The only thing that can overflow + // is (Rx_max.x - Rx_min.x). And it can only overflow if Rx_min.x < 0. + // Let us first deal with the following case. + if ((Rx_min.x < 0 && Rx_min.x + Rx_min.width < Rx_max.x) || + (Ry_min.y < 0 && Ry_min.y + Ry_min.height < Ry_max.y)) { + a = Rect(); + return a; + } + // We now know that either Rx_min.x >= 0, or + // Rx_min.x < 0 && Rx_min.x + Rx_min.width >= Rx_max.x and therefore + // Rx_min.width >= (Rx_max.x - Rx_min.x) which means (Rx_max.x - Rx_min.x) + // is inferior to a valid int and therefore does not overflow. + a.width = std::min(Rx_min.width - (Rx_max.x - Rx_min.x), Rx_max.width); + a.height = std::min(Ry_min.height - (Ry_max.y - Ry_min.y), Ry_max.height); + a.x = Rx_max.x; + a.y = Ry_max.y; + if (a.empty()) a = Rect(); return a; } diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 372aab7eb0..d9e9119230 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -784,4 +784,36 @@ TEST(Core_Check, testSize_1) } +template class Rect_Test : public testing::Test {}; + +TYPED_TEST_CASE_P(Rect_Test); + +// Reimplement C++11 std::numeric_limits<>::lowest. +template T cv_numeric_limits_lowest(); +template<> int cv_numeric_limits_lowest() { return INT_MIN; } +template<> float cv_numeric_limits_lowest() { return -FLT_MAX; } +template<> double cv_numeric_limits_lowest() { return -DBL_MAX; } + +TYPED_TEST_P(Rect_Test, Overflows) { + typedef Rect_ R; + TypeParam num_max = std::numeric_limits::max(); + TypeParam num_lowest = cv_numeric_limits_lowest(); + EXPECT_EQ(R(0, 0, 10, 10), R(0, 0, 10, 10) & R(0, 0, 10, 10)); + EXPECT_EQ(R(5, 6, 4, 3), R(0, 0, 10, 10) & R(5, 6, 4, 3)); + EXPECT_EQ(R(5, 6, 3, 2), R(0, 0, 8, 8) & R(5, 6, 4, 3)); + // Test with overflowing dimenions. + EXPECT_EQ(R(5, 0, 5, 10), R(0, 0, 10, 10) & R(5, 0, num_max, num_max)); + // Test with overflowing dimensions for floats/doubles. + EXPECT_EQ(R(num_max, 0, num_max / 4, 10), R(num_max, 0, num_max / 2, 10) & R(num_max, 0, num_max / 4, 10)); + // Test with overflowing coordinates. + EXPECT_EQ(R(), R(20, 0, 10, 10) & R(num_lowest, 0, 10, 10)); + EXPECT_EQ(R(), R(20, 0, 10, 10) & R(0, num_lowest, 10, 10)); + EXPECT_EQ(R(), R(num_lowest, 0, 10, 10) & R(0, num_lowest, 10, 10)); +} +REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows); + +typedef ::testing::Types RectTypes; +INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes); + + }} // namespace