handle huge matrices correctly (#11505)

* make sure that the matrix with more than INT_MAX elements is marked as non-continuous, and thus all the pixel-wise functions process it correctly (i.e. row-by-row, not as a single row, where integer overflow may occur when computing the total number of elements)

modules/core/include/opencv2/core/cuda.hpp

@@ -305,6 +305,9 @@ public:
     //! returns true if GpuMat data is NULL
     bool empty() const;
 
+    //! internal use method: updates the continuity flag
+    void updateContinuityFlag();
+
     /*! includes several bit-fields:
     - the magic signature
     - continuity flag

modules/core/include/opencv2/core/mat.hpp

@@ -2092,6 +2092,9 @@ public:
     static MatAllocator* getDefaultAllocator();
     static void setDefaultAllocator(MatAllocator* allocator);
 
+    //! internal use method: updates the continuity flag
+    void updateContinuityFlag();
+
     //! interaction with UMat
     UMatData* u;
 
@@ -2565,6 +2568,9 @@ public:
     //! and the standard allocator
     static MatAllocator* getStdAllocator();
 
+    //! internal use method: updates the continuity flag
+    void updateContinuityFlag();
+
     // black-box container of UMat data
     UMatData* u;
 

modules/core/include/opencv2/core/mat.inl.hpp

@@ -505,24 +505,20 @@ Mat::Mat(int _rows, int _cols, int _type, void* _data, size_t _step)
     if( _step == AUTO_STEP )
     {
         _step = minstep;
-        flags |= CONTINUOUS_FLAG;
     }
     else
     {
         CV_DbgAssert( _step >= minstep );
-
         if (_step % esz1 != 0)
         {
             CV_Error(Error::BadStep, "Step must be a multiple of esz1");
         }
-
-        if (_step == minstep || rows == 1)
-            flags |= CONTINUOUS_FLAG;
     }
     step[0] = _step;
     step[1] = esz;
     datalimit = datastart + _step * rows;
     dataend = datalimit - _step + minstep;
+    updateContinuityFlag();
 }
 
 inline
@@ -538,7 +534,6 @@ Mat::Mat(Size _sz, int _type, void* _data, size_t _step)
     if( _step == AUTO_STEP )
     {
         _step = minstep;
-        flags |= CONTINUOUS_FLAG;
     }
     else
     {
@@ -548,14 +543,12 @@ Mat::Mat(Size _sz, int _type, void* _data, size_t _step)
         {
             CV_Error(Error::BadStep, "Step must be a multiple of esz1");
         }
-
-        if (_step == minstep || rows == 1)
-            flags |= CONTINUOUS_FLAG;
     }
     step[0] = _step;
     step[1] = esz;
     datalimit = datastart + _step*rows;
     dataend = datalimit - _step + minstep;
+    updateContinuityFlag();
 }
 
 template<typename _Tp> inline