From 617df091438f94d50dce8aa32d83392103bc3a5b Mon Sep 17 00:00:00 2001 From: MYLS Date: Tue, 19 Jul 2016 15:54:38 +0800 Subject: [PATCH 01/10] Modify Base64 functions and add test and documentation Major changes: - modify the Base64 functions to compatible with `cvWriteRawData` and so on. - add a Base64 flag for FileStorage and outputs raw data in Base64 automatically. - complete all testing and documentation. --- modules/core/include/opencv2/core/core_c.h | 58 +- .../core/include/opencv2/core/persistence.hpp | 9 +- modules/core/include/opencv2/core/types_c.h | 2 + modules/core/perf/perf_io_base64.cpp | 86 +++ modules/core/src/persistence.cpp | 711 ++++++++++++------ modules/core/test/test_io.cpp | 147 ---- modules/core/test/test_io_base64.cpp | 270 +++++++ modules/ml/test/test_save_load.cpp | 6 +- 8 files changed, 911 insertions(+), 378 deletions(-) create mode 100644 modules/core/perf/perf_io_base64.cpp create mode 100644 modules/core/test/test_io_base64.cpp diff --git a/modules/core/include/opencv2/core/core_c.h b/modules/core/include/opencv2/core/core_c.h index a0ed632642..94ba1fc034 100644 --- a/modules/core/include/opencv2/core/core_c.h +++ b/modules/core/include/opencv2/core/core_c.h @@ -1976,8 +1976,19 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header, The function opens file storage for reading or writing data. In the latter case, a new file is created or an existing file is rewritten. The type of the read or written file is determined by the -filename extension: .xml for XML and .yml or .yaml for YAML. The function returns a pointer to the -CvFileStorage structure. If the file cannot be opened then the function returns NULL. +filename extension: .xml for XML and .yml or .yaml for YAML. + +At the same time, it also supports adding parameters like "example.xml?base64". +@code + CvFileStorage* fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE ); +@endcode +it's exactly the same as +@code + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64 ); +@endcode + +The function returns a pointer to the CvFileStorage structure. +If the file cannot be opened then the function returns NULL. @param filename Name of the file associated with the storage @param memstorage Memory storage used for temporary data and for : storing dynamic structures, such as CvSeq or CvGraph . If it is NULL, a temporary memory @@ -1985,6 +1996,7 @@ CvFileStorage structure. If the file cannot be opened then the function returns @param flags Can be one of the following: > - **CV_STORAGE_READ** the storage is open for reading > - **CV_STORAGE_WRITE** the storage is open for writing + (use **CV_STORAGE_WRITE | CV_STORAGE_WRITE_BASE64** to write rawdata in Base64) @param encoding */ CVAPI(CvFileStorage*) cvOpenFileStorage( const char* filename, CvMemStorage* memstorage, @@ -2162,7 +2174,7 @@ the file with multiple streams looks like this: @endcode The YAML file will look like this: @code{.yaml} - %YAML:1.0 + %YAML 1.0 # stream #1 data ... --- @@ -2187,6 +2199,46 @@ to a sequence rather than a map. CVAPI(void) cvWriteRawData( CvFileStorage* fs, const void* src, int len, const char* dt ); +/** @brief Writes multiple numbers in Base64. + +If either CV_STORAGE_WRITE_BASE64 or cv::FileStorage::WRITE_BASE64 is used, +this function will be the same as cvWriteRawData. If neither, the main +difference is that it outputs a sequence in Base64 encoding rather than +in plain text. + +This function can only be used to write a sequence with a type "binary". + +Consider the following two examples where their output is the same: +@code + std::vector rawdata(10, 0x00010203); + // without the flag CV_STORAGE_WRITE_BASE64. + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE ); + // both CV_NODE_SEQ and "binary" are necessary. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary"); + cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); +@endcode +and +@code + std::vector rawdata(10, 0x00010203); + // with the flag CV_STORAGE_WRITE_BASE64. + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64); + // parameter, typename "binary" could be omitted. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW); + cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); +@endcode + +@param fs File storage +@param src Pointer to the written array +@param len Number of the array elements to write +@param dt Specification of each array element, see @ref format_spec "format specification" +*/ +CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* _data, + int len, const char* dt ); + /** @brief Returns a unique pointer for a given name. The function returns a unique pointer for each particular file node name. This pointer can be then diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 65a1ff4c4c..75f0c32bd2 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -311,7 +311,10 @@ public: FORMAT_MASK = (7<<3), //!< mask for format flags FORMAT_AUTO = 0, //!< flag, auto format FORMAT_XML = (1<<3), //!< flag, XML format - FORMAT_YAML = (2<<3) //!< flag, YAML format + FORMAT_YAML = (2<<3), //!< flag, YAML format + + BASE64 = 64, //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64) + WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64 }; enum { @@ -354,7 +357,9 @@ public: Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively). Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify - the output file format (e.g. mydata.xml, .yml etc.). + the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters. + You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to + FileStorage::BASE64 flag. Note: it is case sensitive. @param flags Mode of operation. One of FileStorage::Mode @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it. diff --git a/modules/core/include/opencv2/core/types_c.h b/modules/core/include/opencv2/core/types_c.h index cb39587a9a..e693aa4724 100644 --- a/modules/core/include/opencv2/core/types_c.h +++ b/modules/core/include/opencv2/core/types_c.h @@ -1669,6 +1669,8 @@ typedef struct CvFileStorage CvFileStorage; #define CV_STORAGE_FORMAT_AUTO 0 #define CV_STORAGE_FORMAT_XML 8 #define CV_STORAGE_FORMAT_YAML 16 +#define CV_STORAGE_BASE64 64 +#define CV_STORAGE_WRITE_BASE64 (CV_STORAGE_BASE64 | CV_STORAGE_WRITE) /** @brief List of attributes. : diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp new file mode 100644 index 0000000000..693c1cc132 --- /dev/null +++ b/modules/core/perf/perf_io_base64.cpp @@ -0,0 +1,86 @@ +#include "perf_precomp.hpp" + +using namespace std; +using namespace cv; +using namespace perf; +using std::tr1::make_tuple; +using std::tr1::get; + +typedef std::tr1::tuple Size_MatType_Str_t; +typedef TestBaseWithParam Size_Mat_StrType; + +#define MAT_SIZES ::perf::sz1080p/*, ::perf::sz4320p*/ +#define MAT_TYPES CV_8UC1, CV_32FC1 +#define FILE_EXTENSION String(".xml"), String(".yml") + +PERF_TEST_P(Size_Mat_StrType, fs_text, + testing::Combine(testing::Values(MAT_SIZES), + testing::Values(MAT_TYPES), + testing::Values(FILE_EXTENSION)) + ) +{ + Size size = get<0>(GetParam()); + int type = get<1>(GetParam()); + String ext = get<2>(GetParam()); + + Mat src(size.height, size.width, type); + Mat dst = src.clone(); + + declare.in(src, WARMUP_RNG).out(dst); + + cv::String file_name = cv::tempfile(ext.c_str()); + cv::String key = "test_mat"; + + TEST_CYCLE_MULTIRUN(4) + { + { + FileStorage fs(file_name, cv::FileStorage::WRITE); + fs << key << src; + fs.release(); + } + { + FileStorage fs(file_name, cv::FileStorage::READ); + fs[key] >> dst; + fs.release(); + } + } + + remove(file_name.c_str()); + + SANITY_CHECK(dst, 1); +} + +PERF_TEST_P(Size_Mat_StrType, fs_base64, + testing::Combine(testing::Values(MAT_SIZES), + testing::Values(MAT_TYPES), + testing::Values(FILE_EXTENSION)) + ) +{ + Size size = get<0>(GetParam()); + int type = get<1>(GetParam()); + String ext = get<2>(GetParam()); + + Mat src(size.height, size.width, type); + Mat dst = src.clone(); + + cv::String file_name = cv::tempfile(ext.c_str()); + cv::String key = "test_mat"; + + declare.in(src, WARMUP_RNG).out(dst); + TEST_CYCLE_MULTIRUN(4) + { + { + FileStorage fs(file_name, cv::FileStorage::WRITE_BASE64); + fs << key << src; + fs.release(); + } + { + FileStorage fs(file_name, cv::FileStorage::READ); + fs[key] >> dst; + fs.release(); + } + } + + remove(file_name.c_str()); + SANITY_CHECK(dst, 1); +} diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 4d99a4a275..6eb3b0c75a 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -243,6 +243,17 @@ typedef struct CvFileStorage std::deque* outbuf; base64::Base64Writer * base64_writer; + bool is_default_using_base64; + enum Base64State { + Uncertain, + NotUse, + InUse, + } state_of_writing_base64; /**< used in WriteRawData only */ + + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; bool is_opened; } @@ -270,21 +281,23 @@ namespace base64 bool base64_valid (uint8_t const * src, size_t off, size_t cnt); bool base64_valid ( char const * src, size_t off = 0U, size_t cnt = 0U); - size_t base64_encode_buffer_size(size_t cnt); + size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); - size_t base64_decode_buffer_size(size_t cnt); + size_t base64_decode_buffer_size(size_t cnt, bool is_end_with_zero = true); + size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero = true); + size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true); /* binary */ - template inline size_t to_binary(_uint_t val, uchar * cur); - template<> inline size_t to_binary(double val, uchar * cur); - template<> inline size_t to_binary(float val, uchar * cur); + template inline size_t to_binary(_uint_t val, uchar * cur); + template<> inline size_t to_binary(double val, uchar * cur); + template<> inline size_t to_binary(float val, uchar * cur); template inline size_t to_binary(uchar const * val, uchar * cur); - template inline size_t binary_to(uchar const * cur, _uint_t & val); - template<> inline size_t binary_to(uchar const * cur, double & val); - template<> inline size_t binary_to(uchar const * cur, float & val); - template inline size_t binary_to(uchar const * cur, uchar * val); + template inline size_t binary_to(uchar const * cur, _uint_t & val); + template<> inline size_t binary_to(uchar const * cur, double & val); + template<> inline size_t binary_to(uchar const * cur, float & val); + template inline size_t binary_to(uchar const * cur, uchar * val); class MatToBinaryConvertor; class RawDataToBinaryConvertor; @@ -313,22 +326,37 @@ namespace base64 class Base64ContextEmitter; + class Base64Writer + { + public: + Base64Writer(::CvFileStorage * fs); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + + private: + void check_dt(const char* dt); + + private: + + ::CvFileStorage * file_storage; + Base64ContextEmitter * emitter; + std::string data_type_string; + }; + /* other */ - std::string make_base64_header(int byte_size, const char * dt); + std::string make_base64_header(const char * dt); - bool read_base64_header(std::string const & header, int & byte_size, std::string & dt); + bool read_base64_header(std::vector const & header, std::string & dt); void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq); /* sample */ - void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt); - void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len); - void cvEndWriteRawData_Base64(::CvFileStorage * fs); + void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt); - void cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt); - void cvWriteMat_Base64(CvFileStorage * fs, const char * name, ::cv::Mat const & mat); + void cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat); } @@ -1031,6 +1059,184 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr ) return fval; } +static std::vector analyze_file_name( std::string const & file_name ) +{ + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; + std::vector result; + + size_t beg = file_name.find_last_of(parameter_begin); + size_t end = file_name.size(); + result.push_back(file_name.substr(0U, beg)); + + if ( beg != std::string::npos ) + { + beg ++; + for ( size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1U ) + { + param_end = file_name.find_first_of( parameter_separator, param_beg ); + if ( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1U < end ) + { + result.push_back( file_name.substr( param_beg, param_end - param_beg ) ); + } + } + } + + return result; +} + +static bool is_param_exist( std::vector & const params, std::string & const param ) +{ + if ( params.size() < 2U ) + return false; + + return std::find(params.begin(), params.end(), param) != params.end(); +} + +static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64State state ) +{ + const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char * err_unable_to_switch = "Unexpected error, unable to switch to this state."; + + /* like a finite state machine */ + switch (fs->state_of_writing_base64) + { + case CvFileStorage::Base64State::Uncertain: + switch (state) + { + case CvFileStorage::Base64State::InUse: + CV_DbgAssert( fs->base64_writer == 0 ); + fs->base64_writer = new base64::Base64Writer( fs ); + break; + case CvFileStorage::Base64State::Uncertain: + break; + case CvFileStorage::Base64State::NotUse: + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + case CvFileStorage::Base64State::InUse: + switch (state) + { + case CvFileStorage::Base64State::InUse: + case CvFileStorage::Base64State::NotUse: + CV_Error( CV_StsError, err_unable_to_switch ); + break; + case CvFileStorage::Base64State::Uncertain: + delete fs->base64_writer; + fs->base64_writer = 0; + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + case CvFileStorage::Base64State::NotUse: + switch (state) + { + case CvFileStorage::Base64State::InUse: + case CvFileStorage::Base64State::NotUse: + CV_Error( CV_StsError, err_unable_to_switch ); + break; + case CvFileStorage::Base64State::Uncertain: + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + + fs->state_of_writing_base64 = state; +} + + +static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_type_to_base64 = false ) +{ + if ( fs->is_write_struct_delayed ) + { + /* save data to prevent recursive call errors */ + char * struct_key = 0; + char * type_name = 0; + int struct_flags = fs->delayed_struct_flags; + + if ( fs->delayed_struct_key != 0 ) + { + struct_key = new char[strlen(fs->delayed_struct_key) + 1U]; + strcpy(struct_key, fs->delayed_struct_key); + } + if ( fs->delayed_type_name != 0 ) + { + type_name = new char[strlen(type_name) + 1U]; + strcpy(type_name, fs->delayed_type_name); + } + + /* reset */ + delete fs->delayed_struct_key; + delete fs->delayed_type_name; + fs->delayed_struct_key = 0; + fs->delayed_struct_flags = 0; + fs->delayed_type_name = 0; + + fs->is_write_struct_delayed = false; + + /* call */ + if ( change_type_to_base64 ) + { + fs->start_write_struct( fs, struct_key, struct_flags, "binary"); + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + } + else + { + fs->start_write_struct( fs, struct_key, struct_flags, type_name); + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + + delete struct_key; + delete type_name; + } +} + + +static void make_write_struct_delayed( + CvFileStorage* fs, + const char* key, + int struct_flags, + const char* type_name ) +{ + CV_Assert( fs->is_write_struct_delayed == false ); + CV_DbgAssert( fs->delayed_struct_key == 0 ); + CV_DbgAssert( fs->delayed_struct_flags == 0 ); + CV_DbgAssert( fs->delayed_type_name == 0 ); + + fs->delayed_struct_flags = struct_flags; + + if ( key != 0 ) + { + fs->delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(fs->delayed_struct_key, key); + } + + if ( type_name != 0 ) + { + fs->delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(fs->delayed_type_name, type_name); + } + + fs->is_write_struct_delayed = true; +} + /****************************************************************************************\ * YAML Parser * @@ -1119,40 +1325,41 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN /* calc (decoded) total_byte_size from header */ std::string dt; - int total_byte_size = -1; { if (end - beg < static_cast(base64::ENCODED_HEADER_SIZE)) CV_PARSE_ERROR("Unrecognized Base64 header"); std::vector header(base64::HEADER_SIZE + 1, ' '); base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE); - std::istringstream iss(header.data()); - - if (!(iss >> total_byte_size) || total_byte_size < 0) - CV_PARSE_ERROR("Cannot parse size in Base64 header"); - if (!(iss >> dt) || dt.empty()) + if ( !base64::read_base64_header(header, dt) || dt.empty() ) CV_PARSE_ERROR("Cannot parse dt in Base64 header"); beg += base64::ENCODED_HEADER_SIZE; } - /* buffer for decoded data(exclude header) */ - std::vector buffer(total_byte_size + 4); + /* get all Base64 data */ + std::string base64_buffer; + base64_buffer.reserve( 16U * 1024U * 1024U ); + while( beg < end ) { - base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4); - - /* decoding */ - while(beg < end) - { - /* save this part [beg, end) */ - parser.read((const uchar *)beg, (const uchar *)end); - - beg = end; - - /* find next part */ - icvYMLGetMultilineStringContent(fs, beg, indent, beg, end); - } + base64_buffer.append( beg, end ); + beg = end; + icvYMLGetMultilineStringContent( fs, beg, indent, beg, end ); } + if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) ) + CV_PARSE_ERROR( "Invalid Base64 data." ); + + /* buffer for decoded data(exclude header) */ + std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); + int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + { + base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); + const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); + const uchar * buffer_end = buffer_beg + base64_buffer.size(); + parser.read( buffer_beg, buffer_end ); + parser.flush(); + } + /* save as CvSeq */ int elem_size = ::icvCalcStructSize(dt.c_str(), 0); if (total_byte_size % elem_size != 0) @@ -1160,9 +1367,10 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN int elem_cnt = total_byte_size / elem_size; node->tag = CV_NODE_NONE; - int struct_flags = CV_NODE_FLOW + CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */ + int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ; + /* after icvFSCreateCollection, node->tag == struct_flags */ icvFSCreateCollection(fs, struct_flags, node); - base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); + base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); if (fs->dummy_eof) { /* end of file */ @@ -1621,6 +1829,16 @@ icvYMLParse( CvFileStorage* fs ) static void icvYMLWrite( CvFileStorage* fs, const char* key, const char* data ) { + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; int struct_flags; @@ -2034,17 +2252,17 @@ static void icvXMLGetMultilineStringContent(CvFileStorage* fs, ptr = icvXMLSkipSpaces(fs, ptr, CV_XML_INSIDE_TAG); beg = ptr; end = ptr; - if (fs->dummy_eof) + if ( fs->dummy_eof ) return ; /* end of file */ - if (*beg == '<') + if ( *beg == '<' ) return; /* end of string */ /* find end */ - while(cv_isprint(*ptr)) /* no check for base64 string */ + while( cv_isprint(*ptr) ) /* no check for base64 string */ ++ ptr; - if (*ptr == '\0') - CV_PARSE_ERROR("Unexpected end of line"); + if ( *ptr == '\0' ) + CV_PARSE_ERROR( "Unexpected end of line" ); end = ptr; } @@ -2061,48 +2279,52 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node) /* calc (decoded) total_byte_size from header */ std::string dt; - int total_byte_size = -1; { if (end - beg < static_cast(base64::ENCODED_HEADER_SIZE)) CV_PARSE_ERROR("Unrecognized Base64 header"); std::vector header(base64::HEADER_SIZE + 1, ' '); base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE); - std::istringstream iss(header.data()); - if (!(iss >> total_byte_size) || total_byte_size < 0) - CV_PARSE_ERROR("Cannot parse size in Base64 header"); - if (!(iss >> dt) || dt.empty()) + if ( !base64::read_base64_header(header, dt) || dt.empty() ) CV_PARSE_ERROR("Cannot parse dt in Base64 header"); beg += base64::ENCODED_HEADER_SIZE; } - /* alloc buffer for all decoded data(include header) */ - std::vector buffer(total_byte_size + 4); + /* get all Base64 data */ + std::string base64_buffer; + base64_buffer.reserve( 16U * 1024U * 1024U ); + while( beg < end ) { - base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4); + base64_buffer.append( beg, end ); + beg = end; + icvXMLGetMultilineStringContent( fs, beg, beg, end ); + } + if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) ) + CV_PARSE_ERROR( "Invalid Base64 data." ); - /* decoding */ - while(beg < end) - { - /* save this part [beg, end) */ - parser.read((const uchar *)beg, (const uchar *)end); - beg = end; - /* find next part */ - icvXMLGetMultilineStringContent(fs, beg, beg, end); - } + /* alloc buffer for all decoded data(include header) */ + std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); + int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + { + base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); + const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); + const uchar * buffer_end = buffer_beg + base64_buffer.size(); + parser.read( buffer_beg, buffer_end ); + parser.flush(); } /* save as CvSeq */ int elem_size = ::icvCalcStructSize(dt.c_str(), 0); if (total_byte_size % elem_size != 0) - CV_PARSE_ERROR("Byte size not match elememt size"); + CV_PARSE_ERROR("data size not matches elememt size"); int elem_cnt = total_byte_size / elem_size; node->tag = CV_NODE_NONE; - int struct_flags = CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */ + int struct_flags = CV_NODE_SEQ; + /* after icvFSCreateCollection, node->tag == struct_flags */ icvFSCreateCollection(fs, struct_flags, node); - base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); + base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); if (fs->dummy_eof) { /* end of file */ @@ -2757,6 +2979,16 @@ icvXMLStartNextStream( CvFileStorage* fs ) static void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) { + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); + } + if( CV_NODE_IS_MAP(fs->struct_flags) || (!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) ) { @@ -2964,15 +3196,24 @@ icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment ) \****************************************************************************************/ CV_IMPL CvFileStorage* -cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, const char* encoding ) +cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const char* encoding ) { CvFileStorage* fs = 0; int default_block_size = 1 << 18; bool append = (flags & 3) == CV_STORAGE_APPEND; bool mem = (flags & CV_STORAGE_MEMORY) != 0; bool write_mode = (flags & 3) != 0; + bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0; bool isGZ = false; size_t fnamelen = 0; + const char * filename = 0; + + std::vector params = analyze_file_name( query ); + if ( !params.empty() ) + filename = params.begin()->c_str(); + + if (write_base64 == false && is_param_exist( params, std::string("base64") ) ) + write_base64 = true; if( !filename || filename[0] == '\0' ) { @@ -3073,6 +3314,16 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co fs->struct_flags = CV_NODE_EMPTY; fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 ); fs->buffer_end = fs->buffer_start + buf_size; + + fs->base64_writer = 0; + fs->is_default_using_base64 = write_base64; + fs->state_of_writing_base64 = CvFileStorage::Base64State::Uncertain; + + fs->is_write_struct_delayed = false; + fs->delayed_struct_key = 0; + fs->delayed_struct_flags = 0; + fs->delayed_type_name = 0; + if( fs->fmt == CV_STORAGE_FORMAT_XML ) { size_t file_size = fs->file ? (size_t)ftell( fs->file ) : (size_t)0; @@ -3247,7 +3498,48 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, const char* type_name, CvAttrList /*attributes*/ ) { CV_CHECK_OUTPUT_FILE_STORAGE(fs); - fs->start_write_struct( fs, key, struct_flags, type_name ); + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::NotUse ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain + && + CV_NODE_IS_SEQ(struct_flags) + && + fs->is_default_using_base64 + && + type_name == 0 + ) + { + /* Uncertain if output Base64 data */ + make_write_struct_delayed( fs, key, struct_flags, type_name ); + } + else if ( type_name && memcmp(type_name, "binary", 6) == 0 ) + { + /* Must output Base64 data */ + if ( !CV_NODE_IS_SEQ(struct_flags) ) + CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); + + fs->start_write_struct( fs, key, struct_flags, type_name ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + } + else + { + /* Won't output Base64 data */ + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + fs->start_write_struct( fs, key, struct_flags, type_name ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } } @@ -3255,6 +3547,11 @@ CV_IMPL void cvEndWriteStruct( CvFileStorage* fs ) { CV_CHECK_OUTPUT_FILE_STORAGE(fs); + check_if_write_struct_is_delayed( fs ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + fs->end_write_struct( fs ); } @@ -3432,6 +3729,17 @@ icvDecodeSimpleFormat( const char* dt ) CV_IMPL void cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) { + if (fs->is_default_using_base64 || + fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + base64::cvWriteRawDataBase64( fs, _data, len, dt ); + return; + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + const char* data0 = (const char*)_data; int offset = 0; int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; @@ -3760,15 +4068,15 @@ icvWriteFileNode( CvFileStorage* fs, const char* name, const CvFileNode* node ) break; case CV_NODE_SEQ: case CV_NODE_MAP: - fs->start_write_struct( fs, name, CV_NODE_TYPE(node->tag) + + cvStartWriteStruct( fs, name, CV_NODE_TYPE(node->tag) + (CV_NODE_SEQ_IS_SIMPLE(node->data.seq) ? CV_NODE_FLOW : 0), node->info ? node->info->type_name : 0 ); icvWriteCollection( fs, node ); - fs->end_write_struct( fs ); + cvEndWriteStruct( fs ); break; case CV_NODE_NONE: - fs->start_write_struct( fs, name, CV_NODE_SEQ, 0 ); - fs->end_write_struct( fs ); + cvStartWriteStruct( fs, name, CV_NODE_SEQ, 0 ); + cvEndWriteStruct( fs ); break; default: CV_Error( CV_StsBadFlag, "Unknown type of file node" ); @@ -6226,14 +6534,29 @@ bool base64::base64_valid(char const * src, size_t off, size_t cnt) return base64_valid(reinterpret_cast(src), off, cnt); } -size_t base64::base64_encode_buffer_size(size_t cnt) +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { - return size_t((cnt + 2U) / 3U * 4U + 1U); + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; } -size_t base64::base64_decode_buffer_size(size_t cnt) +size_t base64::base64_decode_buffer_size(size_t cnt, bool is_end_with_zero) { - return size_t(cnt / 4U * 3U + 1U); + size_t additional = static_cast(is_end_with_zero == true); + return cnt / 4U * 3U + additional; +} + +size_t base64::base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero) +{ + return base64_decode_buffer_size(cnt, reinterpret_cast(src), is_end_with_zero); +} + +size_t base64::base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero) +{ + size_t padding_cnt = 0U; + for (uchar const * ptr = src + cnt - 1U; *ptr == base64_padding; ptr--) + padding_cnt ++; + return base64_decode_buffer_size(cnt, is_end_with_zero) - padding_cnt; } /**************************************************************************** @@ -6308,13 +6631,10 @@ binary_to(uchar const * cur, uchar * val) * others ***************************************************************************/ -std::string base64::make_base64_header(int byte_size, const char * dt) +std::string base64::make_base64_header(const char * dt) { - int size = byte_size; - std::ostringstream oss; - oss << size << ' ' - << dt << ' '; + oss << dt << ' '; std::string buffer(oss.str()); CV_Assert(buffer.size() < HEADER_SIZE); @@ -6325,10 +6645,10 @@ std::string base64::make_base64_header(int byte_size, const char * dt) return buffer; } -bool base64::read_base64_header(std::string const & header, int & byte_size, std::string & dt) +bool base64::read_base64_header(std::vector const & header, std::string & dt) { - std::istringstream iss(header); - return static_cast(iss >> byte_size >> dt); + std::istringstream iss(header.data()); + return static_cast(iss >> dt); } /**************************************************************************** @@ -6351,10 +6671,9 @@ base64::Base64ContextParser::Base64ContextParser(uchar * buffer, size_t size) base64::Base64ContextParser::~Base64ContextParser() { - if (src_cur != src_beg) { - /* encode the rest binary data to base64 buffer */ + /* encode the rest binary data to base64 buffer */ + if (src_cur != src_beg) flush(); - } } base64::Base64ContextParser & base64::Base64ContextParser:: @@ -6383,9 +6702,12 @@ read(const uchar * beg, const uchar * end) bool base64::Base64ContextParser::flush() { - if (!base64_valid(src_beg, 0U, src_cur - src_beg)) + if ( !base64_valid(src_beg, 0U, src_cur - src_beg) ) return false; + if ( src_cur == src_beg ) + return true; + uchar * buffer = binary_buffer.data(); size_t len = base64_decode(src_beg, buffer, 0U, src_cur - src_beg); src_cur = src_beg; @@ -6514,7 +6836,7 @@ public: private: /* because of Base64, we must keep its length a multiple of 3 */ - static const size_t BUFFER_LEN = 51U; + static const size_t BUFFER_LEN = 48U; // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); private: @@ -6584,17 +6906,17 @@ public: { CV_DbgAssert(*this); - /* copy to dst */ + /* [1]copy to dst */ dst += to_binary_func(row_begin + x, dst); - /* move to next */ + /* [2]move to next */ x += step; if (x >= x_max) { - /* when x arrive end, reset it and increase y */ + /* when x arrive end, reset x and increase y */ x = 0U; ++ y; if (y >= y_max) { - /* when y arrive end, reset it and increase iter */ + /* when y arrive end, reset y and increase iter */ y = 0U; ++ mat_iter; if (mat_iter == mats.end()) { @@ -6935,68 +7257,55 @@ private: * Wapper ***************************************************************************/ -class base64::Base64Writer + +base64::Base64Writer::Base64Writer(::CvFileStorage * fs) + : file_storage(fs) + , emitter(new Base64ContextEmitter(fs)) + , data_type_string() { -public: + CV_CHECK_OUTPUT_FILE_STORAGE(fs); + icvFSFlush(fs); +} - Base64Writer(::CvFileStorage * fs, const char * name, int len, const char* dt) - : file_storage(fs) - , emitter(fs) - , remaining_data_length(len) - , data_type_string(dt) - { - CV_CHECK_OUTPUT_FILE_STORAGE(fs); +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); - cvStartWriteStruct(fs, name, CV_NODE_SEQ, "binary"); - icvFSFlush(fs); + RawDataToBinaryConvertor convertor( + _data, static_cast(len), data_type_string.c_str() + ); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( CV_StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; /* output header */ - - /* total byte size(before encode) */ - int size = len * ::icvCalcStructSize(dt, 0); - - std::string buffer = make_base64_header(size, dt); + std::string buffer = make_base64_header(dt); const uchar * beg = reinterpret_cast(buffer.data()); const uchar * end = beg + buffer.size(); - emitter.write(beg, end); - } + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( CV_StsBadArg, "\'dt\' does not match." ); +} - void write(const void* _data, int len) - { - CV_Assert(len >= 0); - CV_Assert(remaining_data_length >= static_cast(len)); - remaining_data_length -= static_cast(len); - - RawDataToBinaryConvertor convertor(_data, len, data_type_string); - emitter.write(convertor); - } - - template inline - void write(_to_binary_convertor_t & convertor, int data_length_of_convertor) - { - CV_Assert(data_length_of_convertor >= 0); - CV_Assert(remaining_data_length >= static_cast(data_length_of_convertor)); - remaining_data_length -= static_cast(data_length_of_convertor); - - emitter.write(convertor); - } - - ~Base64Writer() - { - CV_Assert(remaining_data_length == 0U); - emitter.flush(); - cvEndWriteStruct(file_storage); - icvFSFlush(file_storage); - } - -private: - - ::CvFileStorage * file_storage; - Base64ContextEmitter emitter; - size_t remaining_data_length; - const char* data_type_string; -}; void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq) { @@ -7009,51 +7318,23 @@ void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & se } } -void base64::cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt) +void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt) { CV_Assert(fs); CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer == 0); - fs->base64_writer = new Base64Writer(fs, name, len, dt); -} -void base64::cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len) -{ - CV_Assert(fs); - CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer != 0); - fs->base64_writer->write(_data, len); -} + check_if_write_struct_is_delayed( fs, true ); -void base64::cvEndWriteRawData_Base64(::CvFileStorage * fs) -{ - CV_Assert(fs); - CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer != 0); - delete fs->base64_writer; - fs->base64_writer = 0; -} - -void base64::cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt) -{ - cvStartWriteStruct(*fs, fs.elname.c_str(), CV_NODE_SEQ, "binary"); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) { - Base64ContextEmitter emitter(*fs); - { /* header */ - /* total byte size(before encode) */ - int size = len * ::icvCalcStructSize(dt, 0); - std::string buffer = make_base64_header(size, dt); - const uchar * beg = reinterpret_cast(buffer.data()); - const uchar * end = beg + buffer.size(); - - emitter.write(beg, end); - } - { /* body */ - RawDataToBinaryConvertor convert(_data, len, dt); - emitter.write(convert); - } + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); } - cvEndWriteStruct(*fs); + else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "Base64 should not be used at present." ); + } + + fs->base64_writer->write(_data, len, dt); } void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat) @@ -7064,31 +7345,30 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma { /* [1]output other attr */ if (mat.dims <= 2) { - cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT); + ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT); - cvWriteInt(fs, "rows", mat.rows ); - cvWriteInt(fs, "cols", mat.cols ); + ::cvWriteInt(fs, "rows", mat.rows ); + ::cvWriteInt(fs, "cols", mat.cols ); } else { - cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND); + ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND); - cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW); - cvWriteRawData(fs, mat.size.p, mat.dims, "i"); - cvEndWriteStruct(fs); + ::cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW); + ::cvWriteRawData(fs, mat.size.p, mat.dims, "i"); + ::cvEndWriteStruct(fs); } - cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 ); + ::cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 ); } { /* [2]deal with matrix's data */ - int len = static_cast(mat.total()); MatToBinaryConvertor convertor(mat); - cvStartWriteRawData_Base64(fs, "data", len, dt); - fs->base64_writer->write(convertor, len); - cvEndWriteRawData_Base64(fs); + ::cvStartWriteStruct(fs, "data", CV_NODE_SEQ, "binary"); + fs->base64_writer->write(convertor, dt); + ::cvEndWriteStruct(fs); } { /* [3]output end */ - cvEndWriteStruct(fs); + ::cvEndWriteStruct(fs); } } @@ -7096,36 +7376,21 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma * Interface ***************************************************************************/ -namespace cv +CV_IMPL void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) { - void cvWriteMat_Base64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) - { - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); - } - - void cvWriteMatND_Base64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) - { - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); - } - - void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt) - { - ::base64::cvStartWriteRawData_Base64(fs, name, len, dt); - } - - void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len) - { - ::base64::cvWriteRawData_Base64(fs, _data, len); - } - - void cvEndWriteRawData_Base64(::CvFileStorage * fs) - { - ::base64::cvEndWriteRawData_Base64(fs); - } - + ::cv::Mat holder = ::cv::cvarrToMat(mat); + ::base64::cvWriteMat_Base64(fs, name, holder); } +CV_IMPL void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) +{ + ::cv::Mat holder = ::cv::cvarrToMat(mat); + ::base64::cvWriteMat_Base64(fs, name, holder); +} + +CV_IMPL void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt) +{ + ::base64::cvWriteRawDataBase64(fs, _data, len, dt); +} /* End of file. */ diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index f2c53dc964..1f7191a669 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -593,150 +593,3 @@ TEST(Core_InputOutput, FileStorageSpaces) ASSERT_STREQ(values[i].c_str(), valuesRead[i].c_str()); } } - -TEST(Core_InputOutput, filestorage_yml_compatibility) -{ - // TODO: -} - -class CV_Base64IOTest : public cvtest::BaseTest -{ -private: - std::string file_name; - - struct data_t - { - uchar u1, u2; - int i1, i2, i3; - double d1, d2; - int i4; - }; - -public: - CV_Base64IOTest(std::string const & test_file_name) - : file_name(test_file_name) {} - ~CV_Base64IOTest() {} -protected: - void run(int) - { - try - { - std::vector rawdata; - - cv::Mat _em_out, _em_in; - cv::Mat _2d_out, _2d_in; - cv::Mat _nd_out, _nd_in; - - { /* init */ - - /* normal mat */ - _2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U)); - for (int i = 0; i < _2d_out.rows; ++i) - for (int j = 0; j < _2d_out.cols; ++j) - _2d_out.at(i, j)[1] = (i + j) % 256; - - /* 4d mat */ - const int Size[] = {4, 4, 4, 4}; - cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444)); - const cv::Range ranges[] = { - cv::Range(0, 2), - cv::Range(0, 2), - cv::Range(1, 2), - cv::Range(0, 2) }; - _nd_out = _4d(ranges); - - /* raw data */ - for (int i = 0; i < 1000; i++) { - data_t tmp; - tmp.u1 = 1; - tmp.u2 = 2; - tmp.i1 = 1; - tmp.i2 = 2; - tmp.i3 = 3; - tmp.d1 = 0.1; - tmp.d2 = 0.2; - tmp.i4 = i; - rawdata.push_back(tmp); - } - } - - { /* write */ - cv::FileStorage fs(file_name, cv::FileStorage::WRITE); - CvMat holder = _2d_out; - cv::cvWriteMat_Base64(*fs, "normal_2d_mat", &holder); - CvMatND holder_nd = _nd_out; - cv::cvWriteMatND_Base64(*fs, "normal_nd_mat", &holder_nd); - holder = _em_out; - cv::cvWriteMat_Base64(*fs, "empty_2d_mat", &holder); - - cv::cvStartWriteRawData_Base64(*fs, "rawdata", static_cast(rawdata.size()), "2u3i2di"); - for (int i = 0; i < 10; i++) - cv::cvWriteRawData_Base64(*fs, rawdata.data() + i * 100, 100); - cv::cvEndWriteRawData_Base64(*fs); - - fs.release(); - } - - { /* read */ - cv::FileStorage fs(file_name, cv::FileStorage::READ); - - /* mat */ - fs["empty_2d_mat"] >> _em_in; - fs["normal_2d_mat"] >> _2d_in; - fs["normal_nd_mat"] >> _nd_in; - - /* raw data */ - std::vector(1000).swap(rawdata); - cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), "2u3i2di"); - - fs.release(); - } - - for (int i = 0; i < 1000; i++) { - // TODO: Solve this bug in `cvReadRawData` - //EXPECT_EQ(rawdata[i].u1, 1); - //EXPECT_EQ(rawdata[i].u2, 2); - //EXPECT_EQ(rawdata[i].i1, 1); - //EXPECT_EQ(rawdata[i].i2, 2); - //EXPECT_EQ(rawdata[i].i3, 3); - //EXPECT_EQ(rawdata[i].d1, 0.1); - //EXPECT_EQ(rawdata[i].d2, 0.2); - //EXPECT_EQ(rawdata[i].i4, i); - } - - EXPECT_EQ(_em_in.rows , _em_out.rows); - EXPECT_EQ(_em_in.cols , _em_out.cols); - EXPECT_EQ(_em_in.dims , _em_out.dims); - EXPECT_EQ(_em_in.depth(), _em_out.depth()); - EXPECT_TRUE(_em_in.empty()); - - EXPECT_EQ(_2d_in.rows , _2d_in.rows); - EXPECT_EQ(_2d_in.cols , _2d_in.cols); - EXPECT_EQ(_2d_in.dims , _2d_in.dims); - EXPECT_EQ(_2d_in.depth(), _2d_in.depth()); - for(int i = 0; i < _2d_in.rows; ++i) - for (int j = 0; j < _2d_in.cols; ++j) - EXPECT_EQ(_2d_in.at(i, j), _2d_out.at(i, j)); - - EXPECT_EQ(_nd_in.rows , _nd_in.rows); - EXPECT_EQ(_nd_in.cols , _nd_in.cols); - EXPECT_EQ(_nd_in.dims , _nd_in.dims); - EXPECT_EQ(_nd_in.depth(), _nd_in.depth()); - EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0); - } - catch(...) - { - ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH); - } - } -}; - -TEST(Core_InputOutput, filestorage_yml_base64) -{ - CV_Base64IOTest test("base64_test_tmp_file.yml"); test.safe_run(); -} - -TEST(Core_InputOutput, filestorage_xml_base64) -{ - CV_Base64IOTest test("base64_test_tmp_file.xml"); test.safe_run(); -} diff --git a/modules/core/test/test_io_base64.cpp b/modules/core/test/test_io_base64.cpp new file mode 100644 index 0000000000..d619106eae --- /dev/null +++ b/modules/core/test/test_io_base64.cpp @@ -0,0 +1,270 @@ +#include "test_precomp.hpp" + +using namespace cv; +using namespace std; + +struct data_t +{ + typedef uchar u; + typedef char b; + typedef ushort w; + typedef short s; + typedef int i; + typedef float f; + typedef double d; + + u u1 ;u u2 ; i i1 ; + i i2 ;i i3 ; + d d1 ; + d d2 ; + i i4 ; + + static inline const char * signature() { return "2u3i2di"; } +}; + + +TEST(Core_InputOutput_Base64, basic) +{ + char const * filenames[] = { + "core_io_base64_basic_test.yml", + "core_io_base64_basic_test.xml", + 0 + }; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + std::vector rawdata; + + cv::Mat _em_out, _em_in; + cv::Mat _2d_out, _2d_in; + cv::Mat _nd_out, _nd_in; + cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in; + + { /* init */ + + /* a normal mat */ + _2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U)); + for (int i = 0; i < _2d_out.rows; ++i) + for (int j = 0; j < _2d_out.cols; ++j) + _2d_out.at(i, j)[1] = (i + j) % 256; + + /* a 4d mat */ + const int Size[] = {4, 4, 4, 4}; + cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444)); + const cv::Range ranges[] = { + cv::Range(0, 2), + cv::Range(0, 2), + cv::Range(1, 2), + cv::Range(0, 2) }; + _nd_out = _4d(ranges); + + /* a random mat */ + cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0)); + + /* raw data */ + for (int i = 0; i < 1000; i++) { + data_t tmp; + tmp.u1 = 1; + tmp.u2 = 2; + tmp.i1 = 1; + tmp.i2 = 2; + tmp.i3 = 3; + tmp.d1 = 0.1; + tmp.d2 = 0.2; + tmp.i4 = i; + rawdata.push_back(tmp); + } + } + + { /* write */ + cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64); + fs << "normal_2d_mat" << _2d_out; + fs << "normal_nd_mat" << _nd_out; + fs << "empty_2d_mat" << _em_out; + fs << "random_mat" << _rd_out; + + cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" ); + for (int i = 0; i < 10; i++) + cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature()); + cvEndWriteStruct( *fs ); + + fs.release(); + } + + { /* read */ + cv::FileStorage fs(name, cv::FileStorage::READ); + + /* mat */ + fs["empty_2d_mat"] >> _em_in; + fs["normal_2d_mat"] >> _2d_in; + fs["normal_nd_mat"] >> _nd_in; + fs["random_mat"] >> _rd_in; + + /* raw data */ + std::vector(1000).swap(rawdata); + cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature()); + + fs.release(); + } + + for (int i = 0; i < 1000; i++) { + // TODO: Solve this bug in `cvReadRawData` + //EXPECT_EQ(rawdata[i].u1, 1); + //EXPECT_EQ(rawdata[i].u2, 2); + //EXPECT_EQ(rawdata[i].i1, 1); + //EXPECT_EQ(rawdata[i].i2, 2); + //EXPECT_EQ(rawdata[i].i3, 3); + //EXPECT_EQ(rawdata[i].d1, 0.1); + //EXPECT_EQ(rawdata[i].d2, 0.2); + //EXPECT_EQ(rawdata[i].i4, i); + } + + EXPECT_EQ(_em_in.rows , _em_out.rows); + EXPECT_EQ(_em_in.cols , _em_out.cols); + EXPECT_EQ(_em_in.dims , _em_out.dims); + EXPECT_EQ(_em_in.depth(), _em_out.depth()); + EXPECT_TRUE(_em_in.empty()); + + EXPECT_EQ(_2d_in.rows , _2d_out.rows); + EXPECT_EQ(_2d_in.cols , _2d_out.cols); + EXPECT_EQ(_2d_in.dims , _2d_out.dims); + EXPECT_EQ(_2d_in.depth(), _2d_out.depth()); + for(int i = 0; i < _2d_out.rows; ++i) + for (int j = 0; j < _2d_out.cols; ++j) + EXPECT_EQ(_2d_in.at(i, j), _2d_out.at(i, j)); + + EXPECT_EQ(_nd_in.rows , _nd_out.rows); + EXPECT_EQ(_nd_in.cols , _nd_out.cols); + EXPECT_EQ(_nd_in.dims , _nd_out.dims); + EXPECT_EQ(_nd_in.depth(), _nd_out.depth()); + EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0); + + EXPECT_EQ(_rd_in.rows , _rd_out.rows); + EXPECT_EQ(_rd_in.cols , _rd_out.cols); + EXPECT_EQ(_rd_in.dims , _rd_out.dims); + EXPECT_EQ(_rd_in.depth(), _rd_out.depth()); + EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0); + + remove(name); + } +} + +TEST(Core_InputOutput_Base64, valid) +{ + char const * filenames[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + "core_io_base64_other_test.yml?base64", + "core_io_base64_other_test.xml?base64", + 0 + }; + char const * real_name[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + 0 + }; + + std::vector rawdata(10, static_cast(0x00010203)); + cv::String str_out = "test_string"; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + EXPECT_NO_THROW( + { + cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64); + + cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + for (int i = 0; i < 10; i++) + cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(*fs); + cvWriteString(*fs, 0, str_out.c_str(), 1); + cvEndWriteStruct(*fs); + + fs.release(); + }); + + { + cv::FileStorage fs(name, cv::FileStorage::READ); + std::vector data_in(rawdata.size()); + fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size()); + EXPECT_TRUE(fs["manydata"][0].isSeq()); + EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin())); + cv::String str_in; + fs["manydata"][1] >> str_in; + EXPECT_TRUE(fs["manydata"][1].isString()); + EXPECT_EQ(str_in, str_out); + fs.release(); + } + + EXPECT_NO_THROW( + { + cv::FileStorage fs(name, cv::FileStorage::WRITE); + + cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ); + cvWriteString(*fs, 0, str_out.c_str(), 1); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW, "binary"); + for (int i = 0; i < 10; i++) + cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(*fs); + cvEndWriteStruct(*fs); + + fs.release(); + }); + + { + cv::FileStorage fs(name, cv::FileStorage::READ); + cv::String str_in; + fs["manydata"][0] >> str_in; + EXPECT_TRUE(fs["manydata"][0].isString()); + EXPECT_EQ(str_in, str_out); + std::vector data_in(rawdata.size()); + fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size()); + EXPECT_TRUE(fs["manydata"][1].isSeq()); + EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin())); + fs.release(); + } + + remove(real_name[ptr - filenames]); + } +} + +TEST(Core_InputOutput_Base64, invalid) +{ + char const * filenames[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + 0 + }; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + EXPECT_ANY_THROW({ + cv::FileStorage fs(name, cv::FileStorage::WRITE); + cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ, "binary"); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + }); + + EXPECT_ANY_THROW({ + cv::FileStorage fs(name, cv::FileStorage::WRITE); + cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + cvWriteRawDataBase64(*fs, name, 1, "u"); + }); + + remove(name); + } +} + +TEST(Core_InputOutput_Base64, TODO_compatibility) +{ + // TODO: +} diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index d127471cc9..7c2a7c4a6c 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -65,11 +65,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx ) { get_test_error( testCaseIdx, &test_resps1 ); fname1 = tempfile(".yml.gz"); - save( fname1.c_str() ); + save( (fname1 + "?base64").c_str() ); load( fname1.c_str() ); get_test_error( testCaseIdx, &test_resps2 ); fname2 = tempfile(".yml.gz"); - save( fname2.c_str() ); + save( (fname2 + "?base64").c_str() ); } else ts->printf( cvtest::TS::LOG, "model can not be trained" ); @@ -280,7 +280,7 @@ TEST(DISABLED_ML_SVM, linear_save_load) svm1 = Algorithm::load("SVM45_X_38-1.xml"); svm2 = Algorithm::load("SVM45_X_38-2.xml"); string tname = tempfile("a.xml"); - svm2->save(tname); + svm2->save(tname + "?base64"); svm3 = Algorithm::load(tname); ASSERT_EQ(svm1->getVarCount(), svm2->getVarCount()); From 0823ec0ef04ffb39ba933aeafce78b2a1e944263 Mon Sep 17 00:00:00 2001 From: MYLS Date: Tue, 19 Jul 2016 16:11:20 +0800 Subject: [PATCH 02/10] modified performance test For faster test --- modules/core/perf/perf_io_base64.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index 693c1cc132..339d891008 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -13,6 +13,7 @@ typedef TestBaseWithParam Size_Mat_StrType; #define MAT_TYPES CV_8UC1, CV_32FC1 #define FILE_EXTENSION String(".xml"), String(".yml") + PERF_TEST_P(Size_Mat_StrType, fs_text, testing::Combine(testing::Values(MAT_SIZES), testing::Values(MAT_TYPES), @@ -31,7 +32,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, cv::String file_name = cv::tempfile(ext.c_str()); cv::String key = "test_mat"; - TEST_CYCLE_MULTIRUN(4) + TEST_CYCLE_MULTIRUN(2) { { FileStorage fs(file_name, cv::FileStorage::WRITE); @@ -47,7 +48,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, remove(file_name.c_str()); - SANITY_CHECK(dst, 1); + SANITY_CHECK(dst); } PERF_TEST_P(Size_Mat_StrType, fs_base64, @@ -67,7 +68,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, cv::String key = "test_mat"; declare.in(src, WARMUP_RNG).out(dst); - TEST_CYCLE_MULTIRUN(4) + TEST_CYCLE_MULTIRUN(2) { { FileStorage fs(file_name, cv::FileStorage::WRITE_BASE64); @@ -82,5 +83,5 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, } remove(file_name.c_str()); - SANITY_CHECK(dst, 1); + SANITY_CHECK(dst); } From 78ca5ddd45a25781f931125e2eebf168178b0d2b Mon Sep 17 00:00:00 2001 From: MYLS Date: Tue, 19 Jul 2016 19:56:57 +0800 Subject: [PATCH 03/10] solve errors and warnings --- modules/core/include/opencv2/core/core_c.h | 2 +- modules/core/perf/perf_io_base64.cpp | 4 +- modules/core/src/persistence.cpp | 146 ++++++++++++--------- modules/core/test/test_io_base64.cpp | 4 +- 4 files changed, 86 insertions(+), 70 deletions(-) diff --git a/modules/core/include/opencv2/core/core_c.h b/modules/core/include/opencv2/core/core_c.h index 94ba1fc034..ecfc62aceb 100644 --- a/modules/core/include/opencv2/core/core_c.h +++ b/modules/core/include/opencv2/core/core_c.h @@ -2236,7 +2236,7 @@ and @param len Number of the array elements to write @param dt Specification of each array element, see @ref format_spec "format specification" */ -CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* _data, +CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* src, int len, const char* dt ); /** @brief Returns a unique pointer for a given name. diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index 339d891008..cf292c5478 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -48,7 +48,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, remove(file_name.c_str()); - SANITY_CHECK(dst); + SANITY_CHECK(dst, 1); } PERF_TEST_P(Size_Mat_StrType, fs_base64, @@ -83,5 +83,5 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, } remove(file_name.c_str()); - SANITY_CHECK(dst); + SANITY_CHECK(dst, 1); } diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 6eb3b0c75a..781117323b 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -183,7 +183,17 @@ typedef struct CvXMLStackRecord } CvXMLStackRecord; -namespace base64 { class Base64Writer; } +namespace base64 +{ + class Base64Writer; + + enum Base64State + { + Uncertain, + NotUse, + InUse, + }; +} #define CV_XML_OPENING_TAG 1 #define CV_XML_CLOSING_TAG 2 @@ -244,11 +254,7 @@ typedef struct CvFileStorage base64::Base64Writer * base64_writer; bool is_default_using_base64; - enum Base64State { - Uncertain, - NotUse, - InUse, - } state_of_writing_base64; /**< used in WriteRawData only */ + base64::Base64State state_of_writing_base64; /**< used in WriteRawData only */ bool is_write_struct_delayed; char* delayed_struct_key; @@ -339,7 +345,6 @@ namespace base64 private: - ::CvFileStorage * file_storage; Base64ContextEmitter * emitter; std::string data_type_string; }; @@ -1061,10 +1066,14 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr ) static std::vector analyze_file_name( std::string const & file_name ) { + static const char not_file_name = '\n'; static const char parameter_begin = '?'; static const char parameter_separator = '&'; std::vector result; + if ( file_name.find(not_file_name, 0U) != std::string::npos ) + return result; + size_t beg = file_name.find_last_of(parameter_begin); size_t end = file_name.size(); result.push_back(file_name.substr(0U, beg)); @@ -1087,7 +1096,7 @@ static std::vector analyze_file_name( std::string const & file_name return result; } -static bool is_param_exist( std::vector & const params, std::string & const param ) +static bool is_param_exist( const std::vector & params, const std::string & param ) { if ( params.size() < 2U ) return false; @@ -1095,7 +1104,7 @@ static bool is_param_exist( std::vector & const params, std::string return std::find(params.begin(), params.end(), param) != params.end(); } -static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64State state ) +static void switch_to_Base64_state( CvFileStorage* fs, base64::Base64State state ) { const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; const char * err_unable_to_switch = "Unexpected error, unable to switch to this state."; @@ -1103,30 +1112,30 @@ static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64Stat /* like a finite state machine */ switch (fs->state_of_writing_base64) { - case CvFileStorage::Base64State::Uncertain: + case base64::Base64State::Uncertain: switch (state) { - case CvFileStorage::Base64State::InUse: + case base64::Base64State::InUse: CV_DbgAssert( fs->base64_writer == 0 ); fs->base64_writer = new base64::Base64Writer( fs ); break; - case CvFileStorage::Base64State::Uncertain: + case base64::Base64State::Uncertain: break; - case CvFileStorage::Base64State::NotUse: + case base64::Base64State::NotUse: break; default: CV_Error( CV_StsError, err_unkonwn_state ); break; } break; - case CvFileStorage::Base64State::InUse: + case base64::Base64State::InUse: switch (state) { - case CvFileStorage::Base64State::InUse: - case CvFileStorage::Base64State::NotUse: + case base64::Base64State::InUse: + case base64::Base64State::NotUse: CV_Error( CV_StsError, err_unable_to_switch ); break; - case CvFileStorage::Base64State::Uncertain: + case base64::Base64State::Uncertain: delete fs->base64_writer; fs->base64_writer = 0; break; @@ -1135,14 +1144,14 @@ static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64Stat break; } break; - case CvFileStorage::Base64State::NotUse: + case base64::Base64State::NotUse: switch (state) { - case CvFileStorage::Base64State::InUse: - case CvFileStorage::Base64State::NotUse: + case base64::Base64State::InUse: + case base64::Base64State::NotUse: CV_Error( CV_StsError, err_unable_to_switch ); break; - case CvFileStorage::Base64State::Uncertain: + case base64::Base64State::Uncertain: break; default: CV_Error( CV_StsError, err_unkonwn_state ); @@ -1191,16 +1200,16 @@ static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_typ if ( change_type_to_base64 ) { fs->start_write_struct( fs, struct_key, struct_flags, "binary"); - if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); - switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + switch_to_Base64_state( fs, base64::Base64State::InUse ); } else { fs->start_write_struct( fs, struct_key, struct_flags, type_name); - if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); - switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + switch_to_Base64_state( fs, base64::Base64State::NotUse ); } delete struct_key; @@ -1351,7 +1360,9 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN /* buffer for decoded data(exclude header) */ std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); - int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + int total_byte_size = static_cast( + base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ) + ); { base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); @@ -1830,11 +1841,11 @@ static void icvYMLWrite( CvFileStorage* fs, const char* key, const char* data ) { check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) { - switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::Base64State::NotUse ); } - else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + else if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) { CV_Error( CV_StsError, "At present, output Base64 data only." ); } @@ -2305,7 +2316,9 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node) /* alloc buffer for all decoded data(include header) */ std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); - int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + int total_byte_size = static_cast( + base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ) + ); { base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); @@ -2980,11 +2993,11 @@ static void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) { check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) { - switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::Base64State::NotUse ); } - else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + else if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) { CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); } @@ -3206,14 +3219,18 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0; bool isGZ = false; size_t fnamelen = 0; - const char * filename = 0; + const char * filename = query; - std::vector params = analyze_file_name( query ); - if ( !params.empty() ) - filename = params.begin()->c_str(); + std::vector params; + if ( !mem ) + { + params = analyze_file_name( query ); + if ( !params.empty() ) + filename = params.begin()->c_str(); - if (write_base64 == false && is_param_exist( params, std::string("base64") ) ) - write_base64 = true; + if ( write_base64 == false && is_param_exist( params, "base64" ) ) + write_base64 = true; + } if( !filename || filename[0] == '\0' ) { @@ -3317,7 +3334,7 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const fs->base64_writer = 0; fs->is_default_using_base64 = write_base64; - fs->state_of_writing_base64 = CvFileStorage::Base64State::Uncertain; + fs->state_of_writing_base64 = base64::Base64State::Uncertain; fs->is_write_struct_delayed = false; fs->delayed_struct_key = 0; @@ -3499,10 +3516,10 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, { CV_CHECK_OUTPUT_FILE_STORAGE(fs); check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::NotUse ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + if ( fs->state_of_writing_base64 == base64::Base64State::NotUse ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain + if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain && CV_NODE_IS_SEQ(struct_flags) && @@ -3519,26 +3536,26 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, /* Must output Base64 data */ if ( !CV_NODE_IS_SEQ(struct_flags) ) CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); - else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + else if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); fs->start_write_struct( fs, key, struct_flags, type_name ); - if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); - switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + switch_to_Base64_state( fs, base64::Base64State::InUse ); } else { /* Won't output Base64 data */ - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); fs->start_write_struct( fs, key, struct_flags, type_name ); - if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); - switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + switch_to_Base64_state( fs, base64::Base64State::NotUse ); } } @@ -3549,8 +3566,8 @@ cvEndWriteStruct( CvFileStorage* fs ) CV_CHECK_OUTPUT_FILE_STORAGE(fs); check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) - switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + switch_to_Base64_state( fs, base64::Base64State::Uncertain ); fs->end_write_struct( fs ); } @@ -3730,14 +3747,14 @@ CV_IMPL void cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) { if (fs->is_default_using_base64 || - fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + fs->state_of_writing_base64 == base64::Base64State::InUse ) { base64::cvWriteRawDataBase64( fs, _data, len, dt ); return; } - else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + else if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) { - switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::Base64State::NotUse ); } const char* data0 = (const char*)_data; @@ -7259,8 +7276,7 @@ private: base64::Base64Writer::Base64Writer(::CvFileStorage * fs) - : file_storage(fs) - , emitter(new Base64ContextEmitter(fs)) + : emitter(new Base64ContextEmitter(fs)) , data_type_string() { CV_CHECK_OUTPUT_FILE_STORAGE(fs); @@ -7325,11 +7341,11 @@ void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int le check_if_write_struct_is_delayed( fs, true ); - if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) { - switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + switch_to_Base64_state( fs, base64::Base64State::InUse ); } - else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::InUse ) + else if ( fs->state_of_writing_base64 != base64::Base64State::InUse ) { CV_Error( CV_StsError, "Base64 should not be used at present." ); } @@ -7376,13 +7392,13 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma * Interface ***************************************************************************/ -CV_IMPL void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) +void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) { ::cv::Mat holder = ::cv::cvarrToMat(mat); ::base64::cvWriteMat_Base64(fs, name, holder); } -CV_IMPL void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) +void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) { ::cv::Mat holder = ::cv::cvarrToMat(mat); ::base64::cvWriteMat_Base64(fs, name, holder); diff --git a/modules/core/test/test_io_base64.cpp b/modules/core/test/test_io_base64.cpp index d619106eae..35fb18c139 100644 --- a/modules/core/test/test_io_base64.cpp +++ b/modules/core/test/test_io_base64.cpp @@ -182,7 +182,7 @@ TEST(Core_InputOutput_Base64, valid) cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ); cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); for (int i = 0; i < 10; i++) - cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvWriteRawData(*fs, rawdata.data(), static_cast(rawdata.size()), "i"); cvEndWriteStruct(*fs); cvWriteString(*fs, 0, str_out.c_str(), 1); cvEndWriteStruct(*fs); @@ -211,7 +211,7 @@ TEST(Core_InputOutput_Base64, valid) cvWriteString(*fs, 0, str_out.c_str(), 1); cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW, "binary"); for (int i = 0; i < 10; i++) - cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvWriteRawData(*fs, rawdata.data(), static_cast(rawdata.size()), "i"); cvEndWriteStruct(*fs); cvEndWriteStruct(*fs); From cf2d6f6721704e4d8e9f410f77df2f832ce748b9 Mon Sep 17 00:00:00 2001 From: MYLS Date: Tue, 19 Jul 2016 21:18:41 +0800 Subject: [PATCH 04/10] solve errors and warnings Modified performance test and solve a problem caused by an enum type. --- modules/core/perf/perf_io_base64.cpp | 5 +- modules/core/src/persistence.cpp | 117 +++++++++++++-------------- 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index cf292c5478..a39c0cbce8 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -48,7 +48,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, remove(file_name.c_str()); - SANITY_CHECK(dst, 1); + EXPECT_EQ( src, dst ); } PERF_TEST_P(Size_Mat_StrType, fs_base64, @@ -83,5 +83,6 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, } remove(file_name.c_str()); - SANITY_CHECK(dst, 1); + + EXPECT_EQ( src, dst ); } diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 781117323b..e4fc0c6be7 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -187,12 +187,15 @@ namespace base64 { class Base64Writer; - enum Base64State + namespace fs { - Uncertain, - NotUse, - InUse, - }; + enum State + { + Uncertain, + NotUse, + InUse, + }; + } } #define CV_XML_OPENING_TAG 1 @@ -254,7 +257,7 @@ typedef struct CvFileStorage base64::Base64Writer * base64_writer; bool is_default_using_base64; - base64::Base64State state_of_writing_base64; /**< used in WriteRawData only */ + base64::fs::State state_of_writing_base64; /**< used in WriteRawData only */ bool is_write_struct_delayed; char* delayed_struct_key; @@ -1104,7 +1107,7 @@ static bool is_param_exist( const std::vector & params, const std:: return std::find(params.begin(), params.end(), param) != params.end(); } -static void switch_to_Base64_state( CvFileStorage* fs, base64::Base64State state ) +static void switch_to_Base64_state( CvFileStorage* fs, base64::fs::State state ) { const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; const char * err_unable_to_switch = "Unexpected error, unable to switch to this state."; @@ -1112,30 +1115,30 @@ static void switch_to_Base64_state( CvFileStorage* fs, base64::Base64State state /* like a finite state machine */ switch (fs->state_of_writing_base64) { - case base64::Base64State::Uncertain: + case base64::fs::Uncertain: switch (state) { - case base64::Base64State::InUse: + case base64::fs::InUse: CV_DbgAssert( fs->base64_writer == 0 ); fs->base64_writer = new base64::Base64Writer( fs ); break; - case base64::Base64State::Uncertain: + case base64::fs::Uncertain: break; - case base64::Base64State::NotUse: + case base64::fs::NotUse: break; default: CV_Error( CV_StsError, err_unkonwn_state ); break; } break; - case base64::Base64State::InUse: + case base64::fs::InUse: switch (state) { - case base64::Base64State::InUse: - case base64::Base64State::NotUse: + case base64::fs::InUse: + case base64::fs::NotUse: CV_Error( CV_StsError, err_unable_to_switch ); break; - case base64::Base64State::Uncertain: + case base64::fs::Uncertain: delete fs->base64_writer; fs->base64_writer = 0; break; @@ -1144,14 +1147,14 @@ static void switch_to_Base64_state( CvFileStorage* fs, base64::Base64State state break; } break; - case base64::Base64State::NotUse: + case base64::fs::NotUse: switch (state) { - case base64::Base64State::InUse: - case base64::Base64State::NotUse: + case base64::fs::InUse: + case base64::fs::NotUse: CV_Error( CV_StsError, err_unable_to_switch ); break; - case base64::Base64State::Uncertain: + case base64::fs::Uncertain: break; default: CV_Error( CV_StsError, err_unkonwn_state ); @@ -1200,16 +1203,16 @@ static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_typ if ( change_type_to_base64 ) { fs->start_write_struct( fs, struct_key, struct_flags, "binary"); - if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); - switch_to_Base64_state( fs, base64::Base64State::InUse ); + if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); + switch_to_Base64_state( fs, base64::fs::InUse ); } else { fs->start_write_struct( fs, struct_key, struct_flags, type_name); - if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); - switch_to_Base64_state( fs, base64::Base64State::NotUse ); + if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); + switch_to_Base64_state( fs, base64::fs::NotUse ); } delete struct_key; @@ -1841,11 +1844,11 @@ static void icvYMLWrite( CvFileStorage* fs, const char* key, const char* data ) { check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) { - switch_to_Base64_state( fs, base64::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::fs::NotUse ); } - else if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) + else if ( fs->state_of_writing_base64 == base64::fs::InUse ) { CV_Error( CV_StsError, "At present, output Base64 data only." ); } @@ -2993,11 +2996,11 @@ static void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) { check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) { - switch_to_Base64_state( fs, base64::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::fs::NotUse ); } - else if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) + else if ( fs->state_of_writing_base64 == base64::fs::InUse ) { CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); } @@ -3334,7 +3337,7 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const fs->base64_writer = 0; fs->is_default_using_base64 = write_base64; - fs->state_of_writing_base64 = base64::Base64State::Uncertain; + fs->state_of_writing_base64 = base64::fs::Uncertain; fs->is_write_struct_delayed = false; fs->delayed_struct_key = 0; @@ -3516,10 +3519,10 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, { CV_CHECK_OUTPUT_FILE_STORAGE(fs); check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == base64::Base64State::NotUse ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + if ( fs->state_of_writing_base64 == base64::fs::NotUse ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); - if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain + if ( fs->state_of_writing_base64 == base64::fs::Uncertain && CV_NODE_IS_SEQ(struct_flags) && @@ -3536,26 +3539,26 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, /* Must output Base64 data */ if ( !CV_NODE_IS_SEQ(struct_flags) ) CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); - else if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) + else if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); fs->start_write_struct( fs, key, struct_flags, type_name ); - if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); - switch_to_Base64_state( fs, base64::Base64State::InUse ); + if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); + switch_to_Base64_state( fs, base64::fs::InUse ); } else { /* Won't output Base64 data */ - if ( fs->state_of_writing_base64 == base64::Base64State::InUse ) + if ( fs->state_of_writing_base64 == base64::fs::InUse ) CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); fs->start_write_struct( fs, key, struct_flags, type_name ); - if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); - switch_to_Base64_state( fs, base64::Base64State::NotUse ); + if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); + switch_to_Base64_state( fs, base64::fs::NotUse ); } } @@ -3566,8 +3569,8 @@ cvEndWriteStruct( CvFileStorage* fs ) CV_CHECK_OUTPUT_FILE_STORAGE(fs); check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 != base64::Base64State::Uncertain ) - switch_to_Base64_state( fs, base64::Base64State::Uncertain ); + if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) + switch_to_Base64_state( fs, base64::fs::Uncertain ); fs->end_write_struct( fs ); } @@ -3747,14 +3750,14 @@ CV_IMPL void cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) { if (fs->is_default_using_base64 || - fs->state_of_writing_base64 == base64::Base64State::InUse ) + fs->state_of_writing_base64 == base64::fs::InUse ) { base64::cvWriteRawDataBase64( fs, _data, len, dt ); return; } - else if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) + else if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) { - switch_to_Base64_state( fs, base64::Base64State::NotUse ); + switch_to_Base64_state( fs, base64::fs::NotUse ); } const char* data0 = (const char*)_data; @@ -7341,11 +7344,11 @@ void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int le check_if_write_struct_is_delayed( fs, true ); - if ( fs->state_of_writing_base64 == base64::Base64State::Uncertain ) + if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) { - switch_to_Base64_state( fs, base64::Base64State::InUse ); + switch_to_Base64_state( fs, base64::fs::InUse ); } - else if ( fs->state_of_writing_base64 != base64::Base64State::InUse ) + else if ( fs->state_of_writing_base64 != base64::fs::InUse ) { CV_Error( CV_StsError, "Base64 should not be used at present." ); } @@ -7392,18 +7395,6 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma * Interface ***************************************************************************/ -void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) -{ - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); -} - -void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) -{ - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); -} - CV_IMPL void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt) { ::base64::cvWriteRawDataBase64(fs, _data, len, dt); From 27b924e99e1461ccd84f2d4a74cf306c370dc5ad Mon Sep 17 00:00:00 2001 From: MYLS Date: Tue, 19 Jul 2016 22:06:21 +0800 Subject: [PATCH 05/10] remove CHECK from performance test --- modules/core/perf/perf_io_base64.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index a39c0cbce8..f2982b2cb0 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -47,8 +47,6 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, } remove(file_name.c_str()); - - EXPECT_EQ( src, dst ); } PERF_TEST_P(Size_Mat_StrType, fs_base64, @@ -83,6 +81,4 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, } remove(file_name.c_str()); - - EXPECT_EQ( src, dst ); } From 8a65e73bfdad4aeb9ee8bbb9ca345356aa5de53c Mon Sep 17 00:00:00 2001 From: MYLS Date: Wed, 20 Jul 2016 20:18:16 +0800 Subject: [PATCH 06/10] add SANITY_CHECK_NOTHING() to perf_test --- modules/core/perf/perf_io_base64.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index f2982b2cb0..04a4a040a5 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -47,6 +47,7 @@ PERF_TEST_P(Size_Mat_StrType, fs_text, } remove(file_name.c_str()); + SANITY_CHECK_NOTHING(); } PERF_TEST_P(Size_Mat_StrType, fs_base64, @@ -81,4 +82,5 @@ PERF_TEST_P(Size_Mat_StrType, fs_base64, } remove(file_name.c_str()); + SANITY_CHECK_NOTHING(); } From 08911cbfaef9c9a3a9c203700a3380cb880a1f6d Mon Sep 17 00:00:00 2001 From: MYLS Date: Sat, 30 Jul 2016 00:35:41 +0800 Subject: [PATCH 07/10] change `code` to `snippet` and do some cleaning. - use `@snippet` instead of `@code` in docs. - remove some functions that were not used. --- modules/core/include/opencv2/core/core_c.h | 35 +---- modules/core/src/persistence.cpp | 159 ++------------------- samples/cpp/filestorage_base64.cpp | 71 +++++++++ 3 files changed, 86 insertions(+), 179 deletions(-) create mode 100644 samples/cpp/filestorage_base64.cpp diff --git a/modules/core/include/opencv2/core/core_c.h b/modules/core/include/opencv2/core/core_c.h index ecfc62aceb..dd1d93638e 100644 --- a/modules/core/include/opencv2/core/core_c.h +++ b/modules/core/include/opencv2/core/core_c.h @@ -1978,14 +1978,11 @@ The function opens file storage for reading or writing data. In the latter case, created or an existing file is rewritten. The type of the read or written file is determined by the filename extension: .xml for XML and .yml or .yaml for YAML. -At the same time, it also supports adding parameters like "example.xml?base64". -@code - CvFileStorage* fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE ); -@endcode -it's exactly the same as -@code - CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64 ); -@endcode +At the same time, it also supports adding parameters like "example.xml?base64". The three ways +are the same: +@snippet samples/cpp/filestorage_base64.cpp suffix_in_file_name +@snippet samples/cpp/filestorage_base64.cpp flag_write_base64 +@snippet samples/cpp/filestorage_base64.cpp flag_write_and_flag_base64 The function returns a pointer to the CvFileStorage structure. If the file cannot be opened then the function returns NULL. @@ -2209,27 +2206,9 @@ in plain text. This function can only be used to write a sequence with a type "binary". Consider the following two examples where their output is the same: -@code - std::vector rawdata(10, 0x00010203); - // without the flag CV_STORAGE_WRITE_BASE64. - CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE ); - // both CV_NODE_SEQ and "binary" are necessary. - cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary"); - cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i"); - cvEndWriteStruct(fs); - cvReleaseFileStorage( &fs ); -@endcode +@snippet samples/cpp/filestorage_base64.cpp without_base64_flag and -@code - std::vector rawdata(10, 0x00010203); - // with the flag CV_STORAGE_WRITE_BASE64. - CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64); - // parameter, typename "binary" could be omitted. - cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW); - cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i"); - cvEndWriteStruct(fs); - cvReleaseFileStorage( &fs ); -@endcode +@snippet samples/cpp/filestorage_base64.cpp with_write_base64_flag @param fs File storage @param src Pointer to the written array diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index e4fc0c6be7..9f52a96cec 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -308,7 +308,6 @@ namespace base64 template<> inline size_t binary_to(uchar const * cur, float & val); template inline size_t binary_to(uchar const * cur, uchar * val); - class MatToBinaryConvertor; class RawDataToBinaryConvertor; class BinaryToCvSeqConvertor; @@ -363,8 +362,6 @@ namespace base64 /* sample */ void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt); - - void cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat); } @@ -2306,7 +2303,7 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node) } /* get all Base64 data */ - std::string base64_buffer; + std::string base64_buffer; // not an efficient way. base64_buffer.reserve( 16U * 1024U * 1024U ); while( beg < end ) { @@ -3715,13 +3712,13 @@ icvCalcStructSize( const char* dt, int initial_size ) for ( const char * type = dt; *type != '\0'; type++ ) { switch ( *type ) { - case 'u': { if (elem_max_size < sizeof(uchar)) elem_max_size = sizeof(uchar); break; } - case 'c': { if (elem_max_size < sizeof(schar)) elem_max_size = sizeof(schar); break; } - case 'w': { if (elem_max_size < sizeof(ushort)) elem_max_size = sizeof(ushort); break; } - case 's': { if (elem_max_size < sizeof(short)) elem_max_size = sizeof(short); break; } - case 'i': { if (elem_max_size < sizeof(int)) elem_max_size = sizeof(int); break; } - case 'f': { if (elem_max_size < sizeof(float)) elem_max_size = sizeof(float); break; } - case 'd': { if (elem_max_size < sizeof(double)) elem_max_size = sizeof(double); break; } + case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } + case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } + case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; } + case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; } + case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } + case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } + case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } default: break; } } @@ -6869,111 +6866,6 @@ private: uchar * src_end; }; -class base64::MatToBinaryConvertor -{ -public: - - explicit MatToBinaryConvertor(const cv::Mat & src) - : y (0) - , y_max(0) - , x(0) - , x_max(0) - { - /* make sure each mat `mat.dims == 2` */ - if (src.dims > 2) { - const cv::Mat * arrays[] = { &src, 0 }; - cv::Mat plane; - cv::NAryMatIterator it(arrays, &plane, 1); - - CV_Assert(it.nplanes > 0U); /* make sure mats not empty */ - mats.reserve(it.nplanes); - for (size_t i = 0U; i < it.nplanes; ++i, ++it) - mats.push_back(*it.planes); - } else { - mats.push_back(src); - } - - /* set all to beginning */ - mat_iter = mats.begin(); - y_max = (mat_iter)->rows; - x_max = (mat_iter)->cols * (mat_iter)->elemSize(); - row_begin = (mat_iter)->ptr(0); - step = (mat_iter)->elemSize1(); - - /* choose a function */ - switch ((mat_iter)->depth()) - { - case CV_8U : - case CV_8S : { to_binary_func = to_binary ; break; } - case CV_16U: - case CV_16S: { to_binary_func = to_binary; break; } - case CV_32S: { to_binary_func = to_binary ; break; } - case CV_32F: { to_binary_func = to_binary ; break; } - case CV_64F: { to_binary_func = to_binary; break; } - case CV_USRTYPE1: - default: { CV_Assert(!"mat type is invalid"); break; } - }; - - /* check if empty */ - if (mats.empty() || mats.front().empty() || mats.front().data == 0) { - mat_iter = mats.end(); - CV_Assert(!(*this)); - } - - } - - inline MatToBinaryConvertor & operator >> (uchar * & dst) - { - CV_DbgAssert(*this); - - /* [1]copy to dst */ - dst += to_binary_func(row_begin + x, dst); - - /* [2]move to next */ - x += step; - if (x >= x_max) { - /* when x arrive end, reset x and increase y */ - x = 0U; - ++ y; - if (y >= y_max) { - /* when y arrive end, reset y and increase iter */ - y = 0U; - ++ mat_iter; - if (mat_iter == mats.end()) { - ;/* when iter arrive end, all done */ - } else { - /* usually x_max and y_max won't change */ - y_max = (mat_iter)->rows; - x_max = (mat_iter)->cols * (mat_iter)->elemSize(); - row_begin = (mat_iter)->ptr(static_cast(y)); - } - } else - row_begin = (mat_iter)->ptr(static_cast(y)); - } - - return *this; - } - - inline operator bool() const - { - return mat_iter != mats.end(); - } - -private: - - size_t y; - size_t y_max; - size_t x; - size_t x_max; - std::vector::iterator mat_iter; - std::vector mats; - - size_t step; - const uchar * row_begin; - - typedef size_t(*to_binary_t)(const uchar *, uchar *); - to_binary_t to_binary_func; -}; class base64::RawDataToBinaryConvertor { @@ -7356,41 +7248,6 @@ void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int le fs->base64_writer->write(_data, len, dt); } -void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat) -{ - char dt[4]; - ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt); - - { /* [1]output other attr */ - - if (mat.dims <= 2) { - ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT); - - ::cvWriteInt(fs, "rows", mat.rows ); - ::cvWriteInt(fs, "cols", mat.cols ); - } else { - ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND); - - ::cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW); - ::cvWriteRawData(fs, mat.size.p, mat.dims, "i"); - ::cvEndWriteStruct(fs); - } - ::cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 ); - } - - { /* [2]deal with matrix's data */ - MatToBinaryConvertor convertor(mat); - - ::cvStartWriteStruct(fs, "data", CV_NODE_SEQ, "binary"); - fs->base64_writer->write(convertor, dt); - ::cvEndWriteStruct(fs); - } - - { /* [3]output end */ - ::cvEndWriteStruct(fs); - } -} - /**************************************************************************** * Interface ***************************************************************************/ diff --git a/samples/cpp/filestorage_base64.cpp b/samples/cpp/filestorage_base64.cpp new file mode 100644 index 0000000000..75e26bf362 --- /dev/null +++ b/samples/cpp/filestorage_base64.cpp @@ -0,0 +1,71 @@ +#include "opencv2/core.hpp" +#include +#include + +static CvFileStorage * three_same_ways_of_write_base64() +{ + CvFileStorage * fs = 0; + cv::RNG rng; + switch ( rng.uniform( 0, 2 ) ) + { + case 0: + //! [suffix_in_file_name] + fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE ); + //! [suffix_in_file_name] + break; + case 1: + //! [flag_write_base64] + fs = cvOpenFileStorage( "example.yml" , 0, CV_STORAGE_WRITE_BASE64 ); + //! [flag_write_base64] + break; + case 2: + //! [flag_write_and_flag_base64] + fs = cvOpenFileStorage( "example.yml" , 0, CV_STORAGE_WRITE | CV_STORAGE_BASE64 ); + //! [flag_write_and_flag_base64] + break; + default: + break; + } + return fs; +} + +static void two_ways_to_write_rawdata_in_base64() +{ + std::vector rawdata(10, 0x00010203); + + { // [1] + //! [without_base64_flag] + CvFileStorage* fs = cvOpenFileStorage( "example.xml", 0, CV_STORAGE_WRITE ); + // both CV_NODE_SEQ and "binary" are necessary. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary"); + cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); + //! [without_base64_flag] + } + + { // [2] + //! [with_write_base64_flag] + CvFileStorage* fs = cvOpenFileStorage( "example.xml", 0, CV_STORAGE_WRITE_BASE64); + // parameter, typename "binary" could be omitted. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW); + cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); + //! [with_write_base64_flag] + } +} + +int main(int /* argc */, char** /* argv */) +{ + { // base64 mode + CvFileStorage * fs = three_same_ways_of_write_base64(); + cvReleaseFileStorage( &fs ); + } + + { // output rawdata by `cvWriteRawdata*` + two_ways_to_write_rawdata_in_base64(); + } + + return 0; +} From 7429ed07be2c08b2fa3902fed1e42bfca6d31a96 Mon Sep 17 00:00:00 2001 From: MYLS Date: Sat, 30 Jul 2016 02:05:15 +0800 Subject: [PATCH 08/10] solve warnings in sample file --- samples/cpp/filestorage_base64.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/cpp/filestorage_base64.cpp b/samples/cpp/filestorage_base64.cpp index 75e26bf362..dcd46544f9 100644 --- a/samples/cpp/filestorage_base64.cpp +++ b/samples/cpp/filestorage_base64.cpp @@ -38,7 +38,7 @@ static void two_ways_to_write_rawdata_in_base64() CvFileStorage* fs = cvOpenFileStorage( "example.xml", 0, CV_STORAGE_WRITE ); // both CV_NODE_SEQ and "binary" are necessary. cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary"); - cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i"); + cvWriteRawDataBase64(fs, rawdata.data(), static_cast(rawdata.size()), "i"); cvEndWriteStruct(fs); cvReleaseFileStorage( &fs ); //! [without_base64_flag] @@ -49,7 +49,7 @@ static void two_ways_to_write_rawdata_in_base64() CvFileStorage* fs = cvOpenFileStorage( "example.xml", 0, CV_STORAGE_WRITE_BASE64); // parameter, typename "binary" could be omitted. cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW); - cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i"); + cvWriteRawData(fs, rawdata.data(), static_cast(rawdata.size()), "i"); cvEndWriteStruct(fs); cvReleaseFileStorage( &fs ); //! [with_write_base64_flag] From 86238d2748c19876e10b4bc9299bc206cec0cf8a Mon Sep 17 00:00:00 2001 From: MYLS Date: Fri, 5 Aug 2016 15:17:21 +0800 Subject: [PATCH 09/10] make some changes according to the code review. - use `std::string` instead of `new char`; - reserve 1 MB for buffer; --- modules/core/src/persistence.cpp | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 1813905be7..0c062f39b4 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -1172,19 +1172,17 @@ static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_typ if ( fs->is_write_struct_delayed ) { /* save data to prevent recursive call errors */ - char * struct_key = 0; - char * type_name = 0; + std::string struct_key; + std::string type_name; int struct_flags = fs->delayed_struct_flags; - if ( fs->delayed_struct_key != 0 ) + if ( fs->delayed_struct_key != 0 && *fs->delayed_type_name != '\0' ) { - struct_key = new char[strlen(fs->delayed_struct_key) + 1U]; - strcpy(struct_key, fs->delayed_struct_key); + struct_key.assign(fs->delayed_struct_key); } - if ( fs->delayed_type_name != 0 ) + if ( fs->delayed_type_name != 0 && *fs->delayed_type_name != '\0' ) { - type_name = new char[strlen(type_name) + 1U]; - strcpy(type_name, fs->delayed_type_name); + struct_key.assign(fs->delayed_type_name); } /* reset */ @@ -1199,21 +1197,18 @@ static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_typ /* call */ if ( change_type_to_base64 ) { - fs->start_write_struct( fs, struct_key, struct_flags, "binary"); + fs->start_write_struct( fs, struct_key.c_str(), struct_flags, "binary"); if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) switch_to_Base64_state( fs, base64::fs::Uncertain ); switch_to_Base64_state( fs, base64::fs::InUse ); } else { - fs->start_write_struct( fs, struct_key, struct_flags, type_name); + fs->start_write_struct( fs, struct_key.c_str(), struct_flags, type_name.c_str()); if ( fs->state_of_writing_base64 != base64::fs::Uncertain ) switch_to_Base64_state( fs, base64::fs::Uncertain ); switch_to_Base64_state( fs, base64::fs::NotUse ); } - - delete struct_key; - delete type_name; } } @@ -1246,6 +1241,7 @@ static void make_write_struct_delayed( fs->is_write_struct_delayed = true; } +static const size_t PARSER_BASE64_BUFFER_SIZE = 1024U * 1024U / 8U; /****************************************************************************************\ * YAML Parser * @@ -1348,7 +1344,7 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN /* get all Base64 data */ std::string base64_buffer; - base64_buffer.reserve( 16U * 1024U * 1024U ); + base64_buffer.reserve( PARSER_BASE64_BUFFER_SIZE ); while( beg < end ) { base64_buffer.append( beg, end ); @@ -2304,7 +2300,7 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node) /* get all Base64 data */ std::string base64_buffer; // not an efficient way. - base64_buffer.reserve( 16U * 1024U * 1024U ); + base64_buffer.reserve( PARSER_BASE64_BUFFER_SIZE ); while( beg < end ) { base64_buffer.append( beg, end ); From 1da8a19af3c9d58533892473c1fcccad4321f2be Mon Sep 17 00:00:00 2001 From: MYLS Date: Fri, 5 Aug 2016 16:15:03 +0800 Subject: [PATCH 10/10] fixed two bugs that I caused. --- modules/core/src/persistence.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 0c062f39b4..6f9f992983 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -1176,13 +1176,13 @@ static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_typ std::string type_name; int struct_flags = fs->delayed_struct_flags; - if ( fs->delayed_struct_key != 0 && *fs->delayed_type_name != '\0' ) + if ( fs->delayed_struct_key != 0 && *fs->delayed_struct_key != '\0' ) { struct_key.assign(fs->delayed_struct_key); } if ( fs->delayed_type_name != 0 && *fs->delayed_type_name != '\0' ) { - struct_key.assign(fs->delayed_type_name); + type_name.assign(fs->delayed_type_name); } /* reset */ @@ -1949,6 +1949,9 @@ icvYMLStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, char buf[CV_FS_MAX_LEN + 1024]; const char* data = 0; + if ( type_name && *type_name == '\0' ) + type_name = 0; + struct_flags = (struct_flags & (CV_NODE_TYPE_MASK|CV_NODE_FLOW)) | CV_NODE_EMPTY; if( !CV_NODE_IS_COLLECTION(struct_flags)) CV_Error( CV_StsBadArg, @@ -2914,6 +2917,9 @@ icvXMLStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, CV_Error( CV_StsBadArg, "Some collection type: CV_NODE_SEQ or CV_NODE_MAP must be specified" ); + if ( type_name && *type_name == '\0' ) + type_name = 0; + if( type_name ) { attr[idx++] = "type_id";