diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 0cf404e805..1895333fed 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -8,6 +8,7 @@ ocv_add_dispatched_file(convert_scale SSE2 AVX2) ocv_add_dispatched_file(count_non_zero SSE2 AVX2) ocv_add_dispatched_file(matmul SSE2 AVX2) ocv_add_dispatched_file(mean SSE2 AVX2) +ocv_add_dispatched_file(split SSE2 AVX2) ocv_add_dispatched_file(sum SSE2 AVX2) # dispatching for accuracy tests diff --git a/modules/core/src/split.cpp b/modules/core/src/split.dispatch.cpp similarity index 56% rename from modules/core/src/split.cpp rename to modules/core/src/split.dispatch.cpp index ac1b7217bb..e41eaf1fe7 100644 --- a/modules/core/src/split.cpp +++ b/modules/core/src/split.dispatch.cpp @@ -6,213 +6,44 @@ #include "precomp.hpp" #include "opencl_kernels_core.hpp" +#include "split.simd.hpp" +#include "split.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX2,...,BASELINE based on CMakeLists.txt content + namespace cv { namespace hal { -#if CV_SIMD -// see the comments for vecmerge_ in merge.cpp -template static void -vecsplit_( const T* src, T** dst, int len, int cn ) -{ - const int VECSZ = VecT::nlanes; - int i, i0 = 0; - T* dst0 = dst[0]; - T* dst1 = dst[1]; - - int r0 = (int)((size_t)(void*)dst0 % (VECSZ*sizeof(T))); - int r1 = (int)((size_t)(void*)dst1 % (VECSZ*sizeof(T))); - int r2 = cn > 2 ? (int)((size_t)(void*)dst[2] % (VECSZ*sizeof(T))) : r0; - int r3 = cn > 3 ? (int)((size_t)(void*)dst[3] % (VECSZ*sizeof(T))) : r0; - - hal::StoreMode mode = hal::STORE_ALIGNED_NOCACHE; - if( (r0|r1|r2|r3) != 0 ) - { - mode = hal::STORE_UNALIGNED; - if( r0 == r1 && r0 == r2 && r0 == r3 && r0 % sizeof(T) == 0 && len > VECSZ*2 ) - i0 = VECSZ - (r0 / sizeof(T)); - } - - if( cn == 2 ) - { - for( i = 0; i < len; i += VECSZ ) - { - if( i > len - VECSZ ) - { - i = len - VECSZ; - mode = hal::STORE_UNALIGNED; - } - VecT a, b; - v_load_deinterleave(src + i*cn, a, b); - v_store(dst0 + i, a, mode); - v_store(dst1 + i, b, mode); - if( i < i0 ) - { - i = i0 - VECSZ; - mode = hal::STORE_ALIGNED_NOCACHE; - } - } - } - else if( cn == 3 ) - { - T* dst2 = dst[2]; - for( i = 0; i < len; i += VECSZ ) - { - if( i > len - VECSZ ) - { - i = len - VECSZ; - mode = hal::STORE_UNALIGNED; - } - VecT a, b, c; - v_load_deinterleave(src + i*cn, a, b, c); - v_store(dst0 + i, a, mode); - v_store(dst1 + i, b, mode); - v_store(dst2 + i, c, mode); - if( i < i0 ) - { - i = i0 - VECSZ; - mode = hal::STORE_ALIGNED_NOCACHE; - } - } - } - else - { - CV_Assert( cn == 4 ); - T* dst2 = dst[2]; - T* dst3 = dst[3]; - for( i = 0; i < len; i += VECSZ ) - { - if( i > len - VECSZ ) - { - i = len - VECSZ; - mode = hal::STORE_UNALIGNED; - } - VecT a, b, c, d; - v_load_deinterleave(src + i*cn, a, b, c, d); - v_store(dst0 + i, a, mode); - v_store(dst1 + i, b, mode); - v_store(dst2 + i, c, mode); - v_store(dst3 + i, d, mode); - if( i < i0 ) - { - i = i0 - VECSZ; - mode = hal::STORE_ALIGNED_NOCACHE; - } - } - } - vx_cleanup(); -} -#endif - -template static void -split_( const T* src, T** dst, int len, int cn ) -{ - int k = cn % 4 ? cn % 4 : 4; - int i, j; - if( k == 1 ) - { - T* dst0 = dst[0]; - - if(cn == 1) - { - memcpy(dst0, src, len * sizeof(T)); - } - else - { - for( i = 0, j = 0 ; i < len; i++, j += cn ) - dst0[i] = src[j]; - } - } - else if( k == 2 ) - { - T *dst0 = dst[0], *dst1 = dst[1]; - i = j = 0; - - for( ; i < len; i++, j += cn ) - { - dst0[i] = src[j]; - dst1[i] = src[j+1]; - } - } - else if( k == 3 ) - { - T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2]; - i = j = 0; - - for( ; i < len; i++, j += cn ) - { - dst0[i] = src[j]; - dst1[i] = src[j+1]; - dst2[i] = src[j+2]; - } - } - else - { - T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2], *dst3 = dst[3]; - i = j = 0; - - for( ; i < len; i++, j += cn ) - { - dst0[i] = src[j]; dst1[i] = src[j+1]; - dst2[i] = src[j+2]; dst3[i] = src[j+3]; - } - } - - for( ; k < cn; k += 4 ) - { - T *dst0 = dst[k], *dst1 = dst[k+1], *dst2 = dst[k+2], *dst3 = dst[k+3]; - for( i = 0, j = k; i < len; i++, j += cn ) - { - dst0[i] = src[j]; dst1[i] = src[j+1]; - dst2[i] = src[j+2]; dst3[i] = src[j+3]; - } - } -} - void split8u(const uchar* src, uchar** dst, int len, int cn ) { + CV_INSTRUMENT_REGION(); CALL_HAL(split8u, cv_hal_split8u, src,dst, len, cn) - -#if CV_SIMD - if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 ) - vecsplit_(src, dst, len, cn); - else -#endif - split_(src, dst, len, cn); + CV_CPU_DISPATCH(split8u, (src, dst, len, cn), + CV_CPU_DISPATCH_MODES_ALL); } void split16u(const ushort* src, ushort** dst, int len, int cn ) { + CV_INSTRUMENT_REGION(); CALL_HAL(split16u, cv_hal_split16u, src,dst, len, cn) -#if CV_SIMD - if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 ) - vecsplit_(src, dst, len, cn); - else -#endif - split_(src, dst, len, cn); + CV_CPU_DISPATCH(split16u, (src, dst, len, cn), + CV_CPU_DISPATCH_MODES_ALL); } void split32s(const int* src, int** dst, int len, int cn ) { + CV_INSTRUMENT_REGION(); CALL_HAL(split32s, cv_hal_split32s, src,dst, len, cn) -#if CV_SIMD - if( len >= v_uint32::nlanes && 2 <= cn && cn <= 4 ) - vecsplit_(src, dst, len, cn); - else -#endif - split_(src, dst, len, cn); + CV_CPU_DISPATCH(split32s, (src, dst, len, cn), + CV_CPU_DISPATCH_MODES_ALL); } void split64s(const int64* src, int64** dst, int len, int cn ) { + CV_INSTRUMENT_REGION(); CALL_HAL(split64s, cv_hal_split64s, src,dst, len, cn) -#if CV_SIMD - if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 ) - vecsplit_(src, dst, len, cn); - else -#endif - split_(src, dst, len, cn); + CV_CPU_DISPATCH(split64s, (src, dst, len, cn), + CV_CPU_DISPATCH_MODES_ALL); } -}} // cv::hal:: +} // namespace cv::hal:: /****************************************************************************************\ * split & merge * @@ -233,7 +64,6 @@ static SplitFunc getSplitFunc(int depth) #ifdef HAVE_IPP -namespace cv { static bool ipp_split(const Mat& src, Mat* mv, int channels) { #ifdef HAVE_IPP_IW_LL @@ -282,10 +112,9 @@ static bool ipp_split(const Mat& src, Mat* mv, int channels) return false; #endif } -} #endif -void cv::split(const Mat& src, Mat* mv) +void split(const Mat& src, Mat* mv) { CV_INSTRUMENT_REGION(); @@ -341,8 +170,6 @@ void cv::split(const Mat& src, Mat* mv) #ifdef HAVE_OPENCL -namespace cv { - static bool ocl_split( InputArray _m, OutputArrayOfArrays _mv ) { int type = _m.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type), @@ -381,11 +208,9 @@ static bool ocl_split( InputArray _m, OutputArrayOfArrays _mv ) return k.run(2, globalsize, NULL, false); } -} - #endif -void cv::split(InputArray _m, OutputArrayOfArrays _mv) +void split(InputArray _m, OutputArrayOfArrays _mv) { CV_INSTRUMENT_REGION(); @@ -411,3 +236,5 @@ void cv::split(InputArray _m, OutputArrayOfArrays _mv) split(m, &dst[0]); } + +} // namespace diff --git a/modules/core/src/split.simd.hpp b/modules/core/src/split.simd.hpp new file mode 100644 index 0000000000..25e90c0063 --- /dev/null +++ b/modules/core/src/split.simd.hpp @@ -0,0 +1,223 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + + +#include "precomp.hpp" + +namespace cv { namespace hal { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +void split8u(const uchar* src, uchar** dst, int len, int cn); +void split16u(const ushort* src, ushort** dst, int len, int cn); +void split32s(const int* src, int** dst, int len, int cn); +void split64s(const int64* src, int64** dst, int len, int cn); + +#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +#if CV_SIMD +// see the comments for vecmerge_ in merge.cpp +template static void +vecsplit_( const T* src, T** dst, int len, int cn ) +{ + const int VECSZ = VecT::nlanes; + int i, i0 = 0; + T* dst0 = dst[0]; + T* dst1 = dst[1]; + + int r0 = (int)((size_t)(void*)dst0 % (VECSZ*sizeof(T))); + int r1 = (int)((size_t)(void*)dst1 % (VECSZ*sizeof(T))); + int r2 = cn > 2 ? (int)((size_t)(void*)dst[2] % (VECSZ*sizeof(T))) : r0; + int r3 = cn > 3 ? (int)((size_t)(void*)dst[3] % (VECSZ*sizeof(T))) : r0; + + hal::StoreMode mode = hal::STORE_ALIGNED_NOCACHE; + if( (r0|r1|r2|r3) != 0 ) + { + mode = hal::STORE_UNALIGNED; + if( r0 == r1 && r0 == r2 && r0 == r3 && r0 % sizeof(T) == 0 && len > VECSZ*2 ) + i0 = VECSZ - (r0 / sizeof(T)); + } + + if( cn == 2 ) + { + for( i = 0; i < len; i += VECSZ ) + { + if( i > len - VECSZ ) + { + i = len - VECSZ; + mode = hal::STORE_UNALIGNED; + } + VecT a, b; + v_load_deinterleave(src + i*cn, a, b); + v_store(dst0 + i, a, mode); + v_store(dst1 + i, b, mode); + if( i < i0 ) + { + i = i0 - VECSZ; + mode = hal::STORE_ALIGNED_NOCACHE; + } + } + } + else if( cn == 3 ) + { + T* dst2 = dst[2]; + for( i = 0; i < len; i += VECSZ ) + { + if( i > len - VECSZ ) + { + i = len - VECSZ; + mode = hal::STORE_UNALIGNED; + } + VecT a, b, c; + v_load_deinterleave(src + i*cn, a, b, c); + v_store(dst0 + i, a, mode); + v_store(dst1 + i, b, mode); + v_store(dst2 + i, c, mode); + if( i < i0 ) + { + i = i0 - VECSZ; + mode = hal::STORE_ALIGNED_NOCACHE; + } + } + } + else + { + CV_Assert( cn == 4 ); + T* dst2 = dst[2]; + T* dst3 = dst[3]; + for( i = 0; i < len; i += VECSZ ) + { + if( i > len - VECSZ ) + { + i = len - VECSZ; + mode = hal::STORE_UNALIGNED; + } + VecT a, b, c, d; + v_load_deinterleave(src + i*cn, a, b, c, d); + v_store(dst0 + i, a, mode); + v_store(dst1 + i, b, mode); + v_store(dst2 + i, c, mode); + v_store(dst3 + i, d, mode); + if( i < i0 ) + { + i = i0 - VECSZ; + mode = hal::STORE_ALIGNED_NOCACHE; + } + } + } + vx_cleanup(); +} +#endif + +template static void +split_( const T* src, T** dst, int len, int cn ) +{ + int k = cn % 4 ? cn % 4 : 4; + int i, j; + if( k == 1 ) + { + T* dst0 = dst[0]; + + if(cn == 1) + { + memcpy(dst0, src, len * sizeof(T)); + } + else + { + for( i = 0, j = 0 ; i < len; i++, j += cn ) + dst0[i] = src[j]; + } + } + else if( k == 2 ) + { + T *dst0 = dst[0], *dst1 = dst[1]; + i = j = 0; + + for( ; i < len; i++, j += cn ) + { + dst0[i] = src[j]; + dst1[i] = src[j+1]; + } + } + else if( k == 3 ) + { + T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2]; + i = j = 0; + + for( ; i < len; i++, j += cn ) + { + dst0[i] = src[j]; + dst1[i] = src[j+1]; + dst2[i] = src[j+2]; + } + } + else + { + T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2], *dst3 = dst[3]; + i = j = 0; + + for( ; i < len; i++, j += cn ) + { + dst0[i] = src[j]; dst1[i] = src[j+1]; + dst2[i] = src[j+2]; dst3[i] = src[j+3]; + } + } + + for( ; k < cn; k += 4 ) + { + T *dst0 = dst[k], *dst1 = dst[k+1], *dst2 = dst[k+2], *dst3 = dst[k+3]; + for( i = 0, j = k; i < len; i++, j += cn ) + { + dst0[i] = src[j]; dst1[i] = src[j+1]; + dst2[i] = src[j+2]; dst3[i] = src[j+3]; + } + } +} + +void split8u(const uchar* src, uchar** dst, int len, int cn ) +{ + CV_INSTRUMENT_REGION(); +#if CV_SIMD + if( len >= v_uint8::nlanes && 2 <= cn && cn <= 4 ) + vecsplit_(src, dst, len, cn); + else +#endif + split_(src, dst, len, cn); +} + +void split16u(const ushort* src, ushort** dst, int len, int cn ) +{ + CV_INSTRUMENT_REGION(); +#if CV_SIMD + if( len >= v_uint16::nlanes && 2 <= cn && cn <= 4 ) + vecsplit_(src, dst, len, cn); + else +#endif + split_(src, dst, len, cn); +} + +void split32s(const int* src, int** dst, int len, int cn ) +{ + CV_INSTRUMENT_REGION(); +#if CV_SIMD + if( len >= v_uint32::nlanes && 2 <= cn && cn <= 4 ) + vecsplit_(src, dst, len, cn); + else +#endif + split_(src, dst, len, cn); +} + +void split64s(const int64* src, int64** dst, int len, int cn ) +{ + CV_INSTRUMENT_REGION(); +#if CV_SIMD + if( len >= v_int64::nlanes && 2 <= cn && cn <= 4 ) + vecsplit_(src, dst, len, cn); + else +#endif + split_(src, dst, len, cn); +} + +#endif +CV_CPU_OPTIMIZATION_NAMESPACE_END +}} // namespace \ No newline at end of file