diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index f776bdc5da..8ff91f5e44 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1205,6 +1205,7 @@ void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx: void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + CV_CheckEQ(node_proto.input_size(), 1, ""); layerParams.type = "ReLU6"; layerParams.set("min_value", layerParams.get("min", -FLT_MAX)); layerParams.set("max_value", layerParams.get("max", FLT_MAX)); diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 02a676da36..f2573f7562 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -18,8 +18,9 @@ #define INF_ENGINE_VER_MAJOR_LE(ver) (((INF_ENGINE_RELEASE) / 10000) <= ((ver) / 10000)) #define INF_ENGINE_VER_MAJOR_EQ(ver) (((INF_ENGINE_RELEASE) / 10000) == ((ver) / 10000)) - +#define CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND "dnn_skip_opencv_backend" #define CV_TEST_TAG_DNN_SKIP_HALIDE "dnn_skip_halide" +#define CV_TEST_TAG_DNN_SKIP_CPU "dnn_skip_cpu" #define CV_TEST_TAG_DNN_SKIP_OPENCL "dnn_skip_ocl" #define CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 "dnn_skip_ocl_fp16" #define CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER "dnn_skip_ie_nn_builder" @@ -38,6 +39,9 @@ #define CV_TEST_TAG_DNN_SKIP_IE_MYRIAD CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X #define CV_TEST_TAG_DNN_SKIP_IE_ARM_CPU "dnn_skip_ie_arm_cpu" +#define CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE "dnn_skip_onnx_conformance" +#define CV_TEST_TAG_DNN_SKIP_PARSER "dnn_skip_parser" + #ifdef HAVE_INF_ENGINE #if INF_ENGINE_VER_MAJOR_EQ(2018050000) diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index a11c6641b2..a55a8f788c 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -361,9 +361,15 @@ void initDNNTests() cvtest::addDataSearchPath(extraTestDataPath); registerGlobalSkipTag( - CV_TEST_TAG_DNN_SKIP_HALIDE, + CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, + CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 ); +#if defined(HAVE_HALIDE) + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_HALIDE + ) +#endif #if defined(INF_ENGINE_RELEASE) registerGlobalSkipTag( CV_TEST_TAG_DNN_SKIP_IE, @@ -392,6 +398,10 @@ void initDNNTests() CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16 ); #endif + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE, + CV_TEST_TAG_DNN_SKIP_PARSER + ); } } // namespace diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp new file mode 100644 index 0000000000..a1d83bd614 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -0,0 +1,1250 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + + +#include "test_precomp.hpp" +#include +#include +#include "npy_blob.hpp" +#include + +#if defined(_MSC_VER) // workaround for 32-bit MSVC compiler +#pragma optimize("", off) +#endif + + +#define CV_TEST_TAG_DNN_ERROR_PARSER "dnn_error_parser" +#define CV_TEST_TAG_DNN_ERROR_NET_SETUP "dnn_error_net_setup" +#define CV_TEST_TAG_DNN_ERROR_FORWARD "dnn_error_forward" +#define CV_TEST_TAG_DNN_LAYER_FALLBACK "dnn_layer_fallback" +#define CV_TEST_TAG_DNN_NO_ACCURACY_CHECK "dnn_no_accuracy_check" + + +namespace opencv_test { + +struct TestCase +{ + const char* name; + uint32_t inputs; + uint32_t outputs; +}; + +static const TestCase testConformanceConfig[] = { + {"test_abs", 1, 1}, + {"test_acos", 1, 1}, + {"test_acos_example", 1, 1}, + {"test_acosh", 1, 1}, + {"test_acosh_example", 1, 1}, + {"test_adagrad", 5, 2}, + {"test_adagrad_multiple", 8, 4}, + {"test_adam", 6, 3}, + {"test_adam_multiple", 10, 6}, + {"test_add", 2, 1}, + {"test_add_bcast", 2, 1}, + {"test_add_uint8", 2, 1}, + {"test_and2d", 2, 1}, + {"test_and3d", 2, 1}, + {"test_and4d", 2, 1}, + {"test_and_bcast3v1d", 2, 1}, + {"test_and_bcast3v2d", 2, 1}, + {"test_and_bcast4v2d", 2, 1}, + {"test_and_bcast4v3d", 2, 1}, + {"test_and_bcast4v4d", 2, 1}, + {"test_argmax_default_axis_example", 1, 1}, + {"test_argmax_default_axis_example_select_last_index", 1, 1}, + {"test_argmax_default_axis_random", 1, 1}, + {"test_argmax_default_axis_random_select_last_index", 1, 1}, + {"test_argmax_keepdims_example", 1, 1}, + {"test_argmax_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_keepdims_random", 1, 1}, + {"test_argmax_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_example", 1, 1}, + {"test_argmax_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_random", 1, 1}, + {"test_argmax_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_example", 1, 1}, + {"test_argmax_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_random", 1, 1}, + {"test_argmax_no_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_default_axis_example", 1, 1}, + {"test_argmin_default_axis_example_select_last_index", 1, 1}, + {"test_argmin_default_axis_random", 1, 1}, + {"test_argmin_default_axis_random_select_last_index", 1, 1}, + {"test_argmin_keepdims_example", 1, 1}, + {"test_argmin_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_keepdims_random", 1, 1}, + {"test_argmin_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_example", 1, 1}, + {"test_argmin_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_random", 1, 1}, + {"test_argmin_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_example", 1, 1}, + {"test_argmin_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_random", 1, 1}, + {"test_argmin_no_keepdims_random_select_last_index", 1, 1}, + {"test_asin", 1, 1}, + {"test_asin_example", 1, 1}, + {"test_asinh", 1, 1}, + {"test_asinh_example", 1, 1}, + {"test_atan", 1, 1}, + {"test_atan_example", 1, 1}, + {"test_atanh", 1, 1}, + {"test_atanh_example", 1, 1}, + {"test_averagepool_1d_default", 1, 1}, + {"test_averagepool_2d_ceil", 1, 1}, + {"test_averagepool_2d_default", 1, 1}, + {"test_averagepool_2d_pads", 1, 1}, + {"test_averagepool_2d_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_pads", 1, 1}, + {"test_averagepool_2d_precomputed_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_same_upper", 1, 1}, + {"test_averagepool_2d_precomputed_strides", 1, 1}, + {"test_averagepool_2d_same_lower", 1, 1}, + {"test_averagepool_2d_same_upper", 1, 1}, + {"test_averagepool_2d_strides", 1, 1}, + {"test_averagepool_3d_default", 1, 1}, + {"test_basic_conv_with_padding", 2, 1}, + {"test_basic_conv_without_padding", 2, 1}, + {"test_basic_convinteger", 3, 1}, + {"test_batchnorm_epsilon", 5, 1}, + {"test_batchnorm_epsilon_training_mode", 5, 3}, + {"test_batchnorm_example", 5, 1}, + {"test_batchnorm_example_training_mode", 5, 3}, + {"test_bernoulli", 1, 1}, + {"test_bernoulli_double", 1, 1}, + {"test_bernoulli_double_expanded", 1, 1}, + {"test_bernoulli_expanded", 1, 1}, + {"test_bernoulli_seed", 1, 1}, + {"test_bernoulli_seed_expanded", 1, 1}, + {"test_bitshift_left_uint16", 2, 1}, + {"test_bitshift_left_uint32", 2, 1}, + {"test_bitshift_left_uint64", 2, 1}, + {"test_bitshift_left_uint8", 2, 1}, + {"test_bitshift_right_uint16", 2, 1}, + {"test_bitshift_right_uint32", 2, 1}, + {"test_bitshift_right_uint64", 2, 1}, + {"test_bitshift_right_uint8", 2, 1}, + {"test_cast_BFLOAT16_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT16_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT16_to_FLOAT", 1, 1}, + {"test_cast_FLOAT_to_BFLOAT16", 1, 1}, + {"test_cast_FLOAT_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT_to_STRING", 1, 1}, + {"test_cast_STRING_to_FLOAT", 1, 1}, + {"test_castlike_BFLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_BFLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_STRING", 2, 1}, + {"test_castlike_FLOAT_to_STRING_expanded", 2, 1}, + {"test_castlike_STRING_to_FLOAT", 2, 1}, + {"test_castlike_STRING_to_FLOAT_expanded", 2, 1}, + {"test_ceil", 1, 1}, + {"test_ceil_example", 1, 1}, + {"test_celu", 1, 1}, + {"test_celu_expanded", 1, 1}, + {"test_clip", 3, 1}, + {"test_clip_default_inbounds", 1, 1}, + {"test_clip_default_int8_inbounds", 1, 1}, + {"test_clip_default_int8_max", 2, 1}, + {"test_clip_default_int8_min", 2, 1}, + {"test_clip_default_max", 2, 1}, + {"test_clip_default_min", 2, 1}, + {"test_clip_example", 3, 1}, + {"test_clip_inbounds", 3, 1}, + {"test_clip_outbounds", 3, 1}, + {"test_clip_splitbounds", 3, 1}, + {"test_compress_0", 2, 1}, + {"test_compress_1", 2, 1}, + {"test_compress_default_axis", 2, 1}, + {"test_compress_negative_axis", 2, 1}, + {"test_concat_1d_axis_0", 2, 1}, + {"test_concat_1d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_0", 2, 1}, + {"test_concat_2d_axis_1", 2, 1}, + {"test_concat_2d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_0", 2, 1}, + {"test_concat_3d_axis_1", 2, 1}, + {"test_concat_3d_axis_2", 2, 1}, + {"test_concat_3d_axis_negative_1", 2, 1}, + {"test_concat_3d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_negative_3", 2, 1}, + {"test_constant", 0, 1}, + {"test_constant_pad", 3, 1}, + {"test_constantofshape_float_ones", 1, 1}, + {"test_constantofshape_int_shape_zero", 1, 1}, + {"test_constantofshape_int_zeros", 1, 1}, + {"test_conv_with_autopad_same", 2, 1}, + {"test_conv_with_strides_and_asymmetric_padding", 2, 1}, + {"test_conv_with_strides_no_padding", 2, 1}, + {"test_conv_with_strides_padding", 2, 1}, + {"test_convinteger_with_padding", 3, 1}, + {"test_convinteger_without_padding", 3, 1}, + {"test_convtranspose", 2, 1}, + {"test_convtranspose_1d", 2, 1}, + {"test_convtranspose_3d", 2, 1}, + {"test_convtranspose_autopad_same", 2, 1}, + {"test_convtranspose_dilations", 2, 1}, + {"test_convtranspose_kernel_shape", 2, 1}, + {"test_convtranspose_output_shape", 2, 1}, + {"test_convtranspose_pad", 2, 1}, + {"test_convtranspose_pads", 2, 1}, + {"test_convtranspose_with_kernel", 2, 1}, + {"test_cos", 1, 1}, + {"test_cos_example", 1, 1}, + {"test_cosh", 1, 1}, + {"test_cosh_example", 1, 1}, + {"test_cumsum_1d", 2, 1}, + {"test_cumsum_1d_exclusive", 2, 1}, + {"test_cumsum_1d_reverse", 2, 1}, + {"test_cumsum_1d_reverse_exclusive", 2, 1}, + {"test_cumsum_2d_axis_0", 2, 1}, + {"test_cumsum_2d_axis_1", 2, 1}, + {"test_cumsum_2d_negative_axis", 2, 1}, + {"test_depthtospace_crd_mode", 1, 1}, + {"test_depthtospace_crd_mode_example", 1, 1}, + {"test_depthtospace_dcr_mode", 1, 1}, + {"test_depthtospace_example", 1, 1}, + {"test_dequantizelinear", 3, 1}, + {"test_dequantizelinear_axis", 3, 1}, + {"test_det_2d", 1, 1}, + {"test_det_nd", 1, 1}, + {"test_div", 2, 1}, + {"test_div_bcast", 2, 1}, + {"test_div_example", 2, 1}, + {"test_div_uint8", 2, 1}, + {"test_dropout_default", 1, 1}, + {"test_dropout_default_mask", 1, 2}, + {"test_dropout_default_mask_ratio", 2, 2}, + {"test_dropout_default_old", 1, 1}, + {"test_dropout_default_ratio", 2, 1}, + {"test_dropout_random_old", 1, 1}, + {"test_dynamicquantizelinear", 1, 3}, + {"test_dynamicquantizelinear_expanded", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted_expanded", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted_expanded", 1, 3}, + {"test_edge_pad", 2, 1}, + {"test_einsum_batch_diagonal", 1, 1}, + {"test_einsum_batch_matmul", 2, 1}, + {"test_einsum_inner_prod", 2, 1}, + {"test_einsum_sum", 1, 1}, + {"test_einsum_transpose", 1, 1}, + {"test_elu", 1, 1}, + {"test_elu_default", 1, 1}, + {"test_elu_example", 1, 1}, + {"test_equal", 2, 1}, + {"test_equal_bcast", 2, 1}, + {"test_erf", 1, 1}, + {"test_exp", 1, 1}, + {"test_exp_example", 1, 1}, + {"test_expand_dim_changed", 2, 1}, + {"test_expand_dim_unchanged", 2, 1}, + {"test_eyelike_populate_off_main_diagonal", 1, 1}, + {"test_eyelike_with_dtype", 1, 1}, + {"test_eyelike_without_dtype", 1, 1}, + {"test_flatten_axis0", 1, 1}, + {"test_flatten_axis1", 1, 1}, + {"test_flatten_axis2", 1, 1}, + {"test_flatten_axis3", 1, 1}, + {"test_flatten_default_axis", 1, 1}, + {"test_flatten_negative_axis1", 1, 1}, + {"test_flatten_negative_axis2", 1, 1}, + {"test_flatten_negative_axis3", 1, 1}, + {"test_flatten_negative_axis4", 1, 1}, + {"test_floor", 1, 1}, + {"test_floor_example", 1, 1}, + {"test_gather_0", 2, 1}, + {"test_gather_1", 2, 1}, + {"test_gather_2d_indices", 2, 1}, + {"test_gather_elements_0", 2, 1}, + {"test_gather_elements_1", 2, 1}, + {"test_gather_elements_negative_indices", 2, 1}, + {"test_gather_negative_indices", 2, 1}, + {"test_gathernd_example_float32", 2, 1}, + {"test_gathernd_example_int32", 2, 1}, + {"test_gathernd_example_int32_batch_dim1", 2, 1}, + {"test_gemm_all_attributes", 3, 1}, + {"test_gemm_alpha", 3, 1}, + {"test_gemm_beta", 3, 1}, + {"test_gemm_default_matrix_bias", 3, 1}, + {"test_gemm_default_no_bias", 2, 1}, + {"test_gemm_default_scalar_bias", 3, 1}, + {"test_gemm_default_single_elem_vector_bias", 3, 1}, + {"test_gemm_default_vector_bias", 3, 1}, + {"test_gemm_default_zero_bias", 3, 1}, + {"test_gemm_transposeA", 3, 1}, + {"test_gemm_transposeB", 3, 1}, + {"test_globalaveragepool", 1, 1}, + {"test_globalaveragepool_precomputed", 1, 1}, + {"test_globalmaxpool", 1, 1}, + {"test_globalmaxpool_precomputed", 1, 1}, + {"test_greater", 2, 1}, + {"test_greater_bcast", 2, 1}, + {"test_greater_equal", 2, 1}, + {"test_greater_equal_bcast", 2, 1}, + {"test_greater_equal_bcast_expanded", 2, 1}, + {"test_greater_equal_expanded", 2, 1}, + {"test_gridsample", 2, 1}, + {"test_gridsample_aligncorners_true", 2, 1}, + {"test_gridsample_bicubic", 2, 1}, + {"test_gridsample_bilinear", 2, 1}, + {"test_gridsample_border_padding", 2, 1}, + {"test_gridsample_nearest", 2, 1}, + {"test_gridsample_reflection_padding", 2, 1}, + {"test_gridsample_zeros_padding", 2, 1}, + {"test_gru_batchwise", 3, 2}, + {"test_gru_defaults", 3, 1}, + {"test_gru_seq_length", 4, 1}, + {"test_gru_with_initial_bias", 4, 1}, + {"test_hardmax_axis_0", 1, 1}, + {"test_hardmax_axis_1", 1, 1}, + {"test_hardmax_axis_2", 1, 1}, + {"test_hardmax_default_axis", 1, 1}, + {"test_hardmax_example", 1, 1}, + {"test_hardmax_negative_axis", 1, 1}, + {"test_hardmax_one_hot", 1, 1}, + {"test_hardsigmoid", 1, 1}, + {"test_hardsigmoid_default", 1, 1}, + {"test_hardsigmoid_example", 1, 1}, + {"test_hardswish", 1, 1}, + {"test_hardswish_expanded", 1, 1}, + {"test_identity", 1, 1}, + {"test_identity_opt", 1, 1}, + {"test_identity_sequence", 1, 1}, + {"test_if", 1, 1}, + {"test_if_opt", 1, 1}, + {"test_if_seq", 1, 1}, + {"test_instancenorm_epsilon", 3, 1}, + {"test_instancenorm_example", 3, 1}, + {"test_isinf", 1, 1}, + {"test_isinf_negative", 1, 1}, + {"test_isinf_positive", 1, 1}, + {"test_isnan", 1, 1}, + {"test_leakyrelu", 1, 1}, + {"test_leakyrelu_default", 1, 1}, + {"test_leakyrelu_example", 1, 1}, + {"test_less", 2, 1}, + {"test_less_bcast", 2, 1}, + {"test_less_equal", 2, 1}, + {"test_less_equal_bcast", 2, 1}, + {"test_less_equal_bcast_expanded", 2, 1}, + {"test_less_equal_expanded", 2, 1}, + {"test_log", 1, 1}, + {"test_log_example", 1, 1}, + {"test_logsoftmax_axis_0", 1, 1}, + {"test_logsoftmax_axis_0_expanded", 1, 1}, + {"test_logsoftmax_axis_1", 1, 1}, + {"test_logsoftmax_axis_1_expanded", 1, 1}, + {"test_logsoftmax_axis_2", 1, 1}, + {"test_logsoftmax_axis_2_expanded", 1, 1}, + {"test_logsoftmax_default_axis", 1, 1}, + {"test_logsoftmax_default_axis_expanded", 1, 1}, + {"test_logsoftmax_example_1", 1, 1}, + {"test_logsoftmax_example_1_expanded", 1, 1}, + {"test_logsoftmax_large_number", 1, 1}, + {"test_logsoftmax_large_number_expanded", 1, 1}, + {"test_logsoftmax_negative_axis", 1, 1}, + {"test_logsoftmax_negative_axis_expanded", 1, 1}, + {"test_loop11", 3, 2}, + {"test_loop13_seq", 3, 1}, + {"test_loop16_seq_none", 3, 1}, + {"test_lrn", 1, 1}, + {"test_lrn_default", 1, 1}, + {"test_lstm_batchwise", 3, 2}, + {"test_lstm_defaults", 3, 1}, + {"test_lstm_with_initial_bias", 4, 1}, + {"test_lstm_with_peepholes", 8, 1}, + {"test_matmul_2d", 2, 1}, + {"test_matmul_3d", 2, 1}, + {"test_matmul_4d", 2, 1}, + {"test_matmulinteger", 4, 1}, + {"test_max_example", 3, 1}, + {"test_max_float16", 2, 1}, + {"test_max_float32", 2, 1}, + {"test_max_float64", 2, 1}, + {"test_max_int16", 2, 1}, + {"test_max_int32", 2, 1}, + {"test_max_int64", 2, 1}, + {"test_max_int8", 2, 1}, + {"test_max_one_input", 1, 1}, + {"test_max_two_inputs", 2, 1}, + {"test_max_uint16", 2, 1}, + {"test_max_uint32", 2, 1}, + {"test_max_uint64", 2, 1}, + {"test_max_uint8", 2, 1}, + {"test_maxpool_1d_default", 1, 1}, + {"test_maxpool_2d_ceil", 1, 1}, + {"test_maxpool_2d_default", 1, 1}, + {"test_maxpool_2d_dilations", 1, 1}, + {"test_maxpool_2d_pads", 1, 1}, + {"test_maxpool_2d_precomputed_pads", 1, 1}, + {"test_maxpool_2d_precomputed_same_upper", 1, 1}, + {"test_maxpool_2d_precomputed_strides", 1, 1}, + {"test_maxpool_2d_same_lower", 1, 1}, + {"test_maxpool_2d_same_upper", 1, 1}, + {"test_maxpool_2d_strides", 1, 1}, + {"test_maxpool_2d_uint8", 1, 1}, + {"test_maxpool_3d_default", 1, 1}, + {"test_maxpool_with_argmax_2d_precomputed_pads", 1, 2}, + {"test_maxpool_with_argmax_2d_precomputed_strides", 1, 2}, + {"test_maxunpool_export_with_output_shape", 3, 1}, + {"test_maxunpool_export_without_output_shape", 2, 1}, + {"test_mean_example", 3, 1}, + {"test_mean_one_input", 1, 1}, + {"test_mean_two_inputs", 2, 1}, + {"test_min_example", 3, 1}, + {"test_min_float16", 2, 1}, + {"test_min_float32", 2, 1}, + {"test_min_float64", 2, 1}, + {"test_min_int16", 2, 1}, + {"test_min_int32", 2, 1}, + {"test_min_int64", 2, 1}, + {"test_min_int8", 2, 1}, + {"test_min_one_input", 1, 1}, + {"test_min_two_inputs", 2, 1}, + {"test_min_uint16", 2, 1}, + {"test_min_uint32", 2, 1}, + {"test_min_uint64", 2, 1}, + {"test_min_uint8", 2, 1}, + {"test_mod_broadcast", 2, 1}, + {"test_mod_int64_fmod", 2, 1}, + {"test_mod_mixed_sign_float16", 2, 1}, + {"test_mod_mixed_sign_float32", 2, 1}, + {"test_mod_mixed_sign_float64", 2, 1}, + {"test_mod_mixed_sign_int16", 2, 1}, + {"test_mod_mixed_sign_int32", 2, 1}, + {"test_mod_mixed_sign_int64", 2, 1}, + {"test_mod_mixed_sign_int8", 2, 1}, + {"test_mod_uint16", 2, 1}, + {"test_mod_uint32", 2, 1}, + {"test_mod_uint64", 2, 1}, + {"test_mod_uint8", 2, 1}, + {"test_momentum", 5, 2}, + {"test_momentum_multiple", 8, 4}, + {"test_mul", 2, 1}, + {"test_mul_bcast", 2, 1}, + {"test_mul_example", 2, 1}, + {"test_mul_uint8", 2, 1}, + {"test_mvn", 1, 1}, + {"test_mvn_expanded", 1, 1}, + {"test_neg", 1, 1}, + {"test_neg_example", 1, 1}, + {"test_nesterov_momentum", 5, 2}, + {"test_nllloss_NC", 2, 1}, + {"test_nllloss_NC_expanded", 2, 1}, + {"test_nllloss_NCd1", 2, 1}, + {"test_nllloss_NCd1_expanded", 2, 1}, + {"test_nllloss_NCd1_ii", 2, 1}, + {"test_nllloss_NCd1_ii_expanded", 2, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_nllloss_NCd1_weight", 3, 1}, + {"test_nllloss_NCd1_weight_expanded", 3, 1}, + {"test_nllloss_NCd1_weight_ii", 3, 1}, + {"test_nllloss_NCd1_weight_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2", 2, 1}, + {"test_nllloss_NCd1d2_expanded", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum_expanded", 2, 1}, + {"test_nllloss_NCd1d2_with_weight", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_nonmaxsuppression_center_point_box_format", 5, 1}, + {"test_nonmaxsuppression_flipped_coordinates", 5, 1}, + {"test_nonmaxsuppression_identical_boxes", 5, 1}, + {"test_nonmaxsuppression_limit_output_size", 5, 1}, + {"test_nonmaxsuppression_single_box", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU_and_scores", 5, 1}, + {"test_nonmaxsuppression_two_batches", 5, 1}, + {"test_nonmaxsuppression_two_classes", 5, 1}, + {"test_nonzero_example", 1, 1}, + {"test_not_2d", 1, 1}, + {"test_not_3d", 1, 1}, + {"test_not_4d", 1, 1}, + {"test_onehot_negative_indices", 3, 1}, + {"test_onehot_with_axis", 3, 1}, + {"test_onehot_with_negative_axis", 3, 1}, + {"test_onehot_without_axis", 3, 1}, + {"test_optional_get_element", 1, 1}, + {"test_optional_get_element_sequence", 1, 1}, + {"test_optional_has_element", 1, 1}, + {"test_optional_has_element_empty", 1, 1}, + {"test_or2d", 2, 1}, + {"test_or3d", 2, 1}, + {"test_or4d", 2, 1}, + {"test_or_bcast3v1d", 2, 1}, + {"test_or_bcast3v2d", 2, 1}, + {"test_or_bcast4v2d", 2, 1}, + {"test_or_bcast4v3d", 2, 1}, + {"test_or_bcast4v4d", 2, 1}, + {"test_pow", 2, 1}, + {"test_pow_bcast_array", 2, 1}, + {"test_pow_bcast_scalar", 2, 1}, + {"test_pow_example", 2, 1}, + {"test_pow_types_float", 2, 1}, + {"test_pow_types_float32_int32", 2, 1}, + {"test_pow_types_float32_int64", 2, 1}, + {"test_pow_types_float32_uint32", 2, 1}, + {"test_pow_types_float32_uint64", 2, 1}, + {"test_pow_types_int", 2, 1}, + {"test_pow_types_int32_float32", 2, 1}, + {"test_pow_types_int32_int32", 2, 1}, + {"test_pow_types_int64_float32", 2, 1}, + {"test_pow_types_int64_int64", 2, 1}, + {"test_prelu_broadcast", 2, 1}, + {"test_prelu_example", 2, 1}, + {"test_qlinearconv", 8, 1}, + {"test_qlinearmatmul_2D", 8, 1}, + {"test_qlinearmatmul_3D", 8, 1}, + {"test_quantizelinear", 3, 1}, + {"test_quantizelinear_axis", 3, 1}, + {"test_range_float_type_positive_delta", 3, 1}, + {"test_range_float_type_positive_delta_expanded", 3, 1}, + {"test_range_int32_type_negative_delta", 3, 1}, + {"test_range_int32_type_negative_delta_expanded", 3, 1}, + {"test_reciprocal", 1, 1}, + {"test_reciprocal_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l1_do_not_keepdims_example", 1, 1}, + {"test_reduce_l1_do_not_keepdims_random", 1, 1}, + {"test_reduce_l1_keep_dims_example", 1, 1}, + {"test_reduce_l1_keep_dims_random", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l2_do_not_keepdims_example", 1, 1}, + {"test_reduce_l2_do_not_keepdims_random", 1, 1}, + {"test_reduce_l2_keep_dims_example", 1, 1}, + {"test_reduce_l2_keep_dims_random", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_log_sum", 1, 1}, + {"test_reduce_log_sum_asc_axes", 1, 1}, + {"test_reduce_log_sum_default", 1, 1}, + {"test_reduce_log_sum_desc_axes", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_negative_axes", 1, 1}, + {"test_reduce_max_default_axes_keepdim_example", 1, 1}, + {"test_reduce_max_default_axes_keepdims_random", 1, 1}, + {"test_reduce_max_do_not_keepdims_example", 1, 1}, + {"test_reduce_max_do_not_keepdims_random", 1, 1}, + {"test_reduce_max_keepdims_example", 1, 1}, + {"test_reduce_max_keepdims_random", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_do_not_keepdims_example", 1, 1}, + {"test_reduce_mean_do_not_keepdims_random", 1, 1}, + {"test_reduce_mean_keepdims_example", 1, 1}, + {"test_reduce_mean_keepdims_random", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_min_default_axes_keepdims_example", 1, 1}, + {"test_reduce_min_default_axes_keepdims_random", 1, 1}, + {"test_reduce_min_do_not_keepdims_example", 1, 1}, + {"test_reduce_min_do_not_keepdims_random", 1, 1}, + {"test_reduce_min_keepdims_example", 1, 1}, + {"test_reduce_min_keepdims_random", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_do_not_keepdims_example", 1, 1}, + {"test_reduce_prod_do_not_keepdims_random", 1, 1}, + {"test_reduce_prod_keepdims_example", 1, 1}, + {"test_reduce_prod_keepdims_random", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_default_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_default_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_do_not_keepdims_example", 2, 1}, + {"test_reduce_sum_do_not_keepdims_random", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_example", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_random", 2, 1}, + {"test_reduce_sum_keepdims_example", 2, 1}, + {"test_reduce_sum_keepdims_random", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_square_default_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_default_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_example", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_random", 1, 1}, + {"test_reduce_sum_square_keepdims_example", 1, 1}, + {"test_reduce_sum_square_keepdims_random", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_random", 1, 1}, + {"test_reflect_pad", 2, 1}, + {"test_relu", 1, 1}, + {"test_reshape_allowzero_reordered", 2, 1}, + {"test_reshape_extended_dims", 2, 1}, + {"test_reshape_negative_dim", 2, 1}, + {"test_reshape_negative_extended_dims", 2, 1}, + {"test_reshape_one_dim", 2, 1}, + {"test_reshape_reduced_dims", 2, 1}, + {"test_reshape_reordered_all_dims", 2, 1}, + {"test_reshape_reordered_last_dims", 2, 1}, + {"test_reshape_zero_and_negative_dim", 2, 1}, + {"test_reshape_zero_dim", 2, 1}, + {"test_resize_downsample_scales_cubic", 2, 1}, + {"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_downsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_downsample_scales_linear", 2, 1}, + {"test_resize_downsample_scales_linear_align_corners", 2, 1}, + {"test_resize_downsample_scales_nearest", 2, 1}, + {"test_resize_downsample_sizes_cubic", 2, 1}, + {"test_resize_downsample_sizes_linear_pytorch_half_pixel", 2, 1}, + {"test_resize_downsample_sizes_nearest", 2, 1}, + {"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", 2, 1}, + {"test_resize_tf_crop_and_resize", 3, 1}, + {"test_resize_upsample_scales_cubic", 2, 1}, + {"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_upsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_upsample_scales_cubic_asymmetric", 2, 1}, + {"test_resize_upsample_scales_linear", 2, 1}, + {"test_resize_upsample_scales_linear_align_corners", 2, 1}, + {"test_resize_upsample_scales_nearest", 2, 1}, + {"test_resize_upsample_sizes_cubic", 2, 1}, + {"test_resize_upsample_sizes_nearest", 2, 1}, + {"test_resize_upsample_sizes_nearest_ceil_half_pixel", 2, 1}, + {"test_resize_upsample_sizes_nearest_floor_align_corners", 2, 1}, + {"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", 2, 1}, + {"test_reversesequence_batch", 2, 1}, + {"test_reversesequence_time", 2, 1}, + {"test_rnn_seq_length", 4, 1}, + {"test_roialign_aligned_false", 3, 1}, + {"test_roialign_aligned_true", 3, 1}, + {"test_round", 1, 1}, + {"test_scan9_sum", 2, 2}, + {"test_scan_sum", 2, 2}, + {"test_scatter_elements_with_axis", 3, 1}, + {"test_scatter_elements_with_duplicate_indices", 3, 1}, + {"test_scatter_elements_with_negative_indices", 3, 1}, + {"test_scatter_elements_without_axis", 3, 1}, + {"test_scatter_with_axis", 3, 1}, + {"test_scatter_without_axis", 3, 1}, + {"test_scatternd", 3, 1}, + {"test_scatternd_add", 3, 1}, + {"test_scatternd_multiply", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob", 3, 2}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", 2, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", 2, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", 3, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", 2, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", 2, 2}, + {"test_sce_mean", 2, 1}, + {"test_sce_mean_3d", 2, 1}, + {"test_sce_mean_3d_expanded", 2, 1}, + {"test_sce_mean_3d_log_prob", 2, 2}, + {"test_sce_mean_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_expanded", 2, 1}, + {"test_sce_mean_log_prob", 2, 2}, + {"test_sce_mean_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii", 2, 1}, + {"test_sce_mean_no_weight_ii_3d", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_4d", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_4d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_log_prob_expanded", 2, 2}, + {"test_sce_mean_weight", 3, 1}, + {"test_sce_mean_weight_expanded", 3, 1}, + {"test_sce_mean_weight_ii", 3, 1}, + {"test_sce_mean_weight_ii_3d", 3, 1}, + {"test_sce_mean_weight_ii_3d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_3d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_3d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_4d", 3, 1}, + {"test_sce_mean_weight_ii_4d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_4d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_4d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_expanded", 3, 1}, + {"test_sce_mean_weight_ii_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_log_prob", 3, 2}, + {"test_sce_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_none", 2, 1}, + {"test_sce_none_expanded", 2, 1}, + {"test_sce_none_log_prob", 2, 2}, + {"test_sce_none_log_prob_expanded", 2, 2}, + {"test_sce_none_weights", 3, 1}, + {"test_sce_none_weights_expanded", 3, 1}, + {"test_sce_none_weights_log_prob", 3, 2}, + {"test_sce_none_weights_log_prob_expanded", 3, 2}, + {"test_sce_sum", 2, 1}, + {"test_sce_sum_expanded", 2, 1}, + {"test_sce_sum_log_prob", 2, 2}, + {"test_sce_sum_log_prob_expanded", 2, 2}, + {"test_selu", 1, 1}, + {"test_selu_default", 1, 1}, + {"test_selu_example", 1, 1}, + {"test_sequence_insert_at_back", 2, 1}, + {"test_sequence_insert_at_front", 3, 1}, + {"test_shape", 1, 1}, + {"test_shape_clip_end", 1, 1}, + {"test_shape_clip_start", 1, 1}, + {"test_shape_end_1", 1, 1}, + {"test_shape_end_negative_1", 1, 1}, + {"test_shape_example", 1, 1}, + {"test_shape_start_1", 1, 1}, + {"test_shape_start_1_end_2", 1, 1}, + {"test_shape_start_1_end_negative_1", 1, 1}, + {"test_shape_start_negative_1", 1, 1}, + {"test_shrink_hard", 1, 1}, + {"test_shrink_soft", 1, 1}, + {"test_sigmoid", 1, 1}, + {"test_sigmoid_example", 1, 1}, + {"test_sign", 1, 1}, + {"test_simple_rnn_batchwise", 3, 2}, + {"test_simple_rnn_defaults", 3, 1}, + {"test_simple_rnn_with_initial_bias", 4, 1}, + {"test_sin", 1, 1}, + {"test_sin_example", 1, 1}, + {"test_sinh", 1, 1}, + {"test_sinh_example", 1, 1}, + {"test_size", 1, 1}, + {"test_size_example", 1, 1}, + {"test_slice", 5, 1}, + {"test_slice_default_axes", 3, 1}, + {"test_slice_default_steps", 4, 1}, + {"test_slice_end_out_of_bounds", 5, 1}, + {"test_slice_neg", 5, 1}, + {"test_slice_neg_steps", 5, 1}, + {"test_slice_negative_axes", 4, 1}, + {"test_slice_start_out_of_bounds", 5, 1}, + {"test_softmax_axis_0", 1, 1}, + {"test_softmax_axis_0_expanded", 1, 1}, + {"test_softmax_axis_1", 1, 1}, + {"test_softmax_axis_1_expanded", 1, 1}, + {"test_softmax_axis_2", 1, 1}, + {"test_softmax_axis_2_expanded", 1, 1}, + {"test_softmax_default_axis", 1, 1}, + {"test_softmax_default_axis_expanded", 1, 1}, + {"test_softmax_example", 1, 1}, + {"test_softmax_example_expanded", 1, 1}, + {"test_softmax_large_number", 1, 1}, + {"test_softmax_large_number_expanded", 1, 1}, + {"test_softmax_negative_axis", 1, 1}, + {"test_softmax_negative_axis_expanded", 1, 1}, + {"test_softplus", 1, 1}, + {"test_softplus_example", 1, 1}, + {"test_softsign", 1, 1}, + {"test_softsign_example", 1, 1}, + {"test_spacetodepth", 1, 1}, + {"test_spacetodepth_example", 1, 1}, + {"test_split_equal_parts_1d", 1, 3}, + {"test_split_equal_parts_2d", 1, 2}, + {"test_split_equal_parts_default_axis", 1, 3}, + {"test_split_variable_parts_1d", 2, 2}, + {"test_split_variable_parts_2d", 2, 2}, + {"test_split_variable_parts_default_axis", 2, 2}, + {"test_split_zero_size_splits", 2, 3}, + {"test_sqrt", 1, 1}, + {"test_sqrt_example", 1, 1}, + {"test_squeeze", 2, 1}, + {"test_squeeze_negative_axes", 2, 1}, + {"test_strnormalizer_export_monday_casesensintive_lower", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_nochangecase", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_upper", 1, 1}, + {"test_strnormalizer_export_monday_empty_output", 1, 1}, + {"test_strnormalizer_export_monday_insensintive_upper_twodim", 1, 1}, + {"test_strnormalizer_nostopwords_nochangecase", 1, 1}, + {"test_sub", 2, 1}, + {"test_sub_bcast", 2, 1}, + {"test_sub_example", 2, 1}, + {"test_sub_uint8", 2, 1}, + {"test_sum_example", 3, 1}, + {"test_sum_one_input", 1, 1}, + {"test_sum_two_inputs", 2, 1}, + {"test_tan", 1, 1}, + {"test_tan_example", 1, 1}, + {"test_tanh", 1, 1}, + {"test_tanh_example", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_only_bigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_levelempty", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_uniandbigrams_skip5", 1, 1}, + {"test_thresholdedrelu", 1, 1}, + {"test_thresholdedrelu_default", 1, 1}, + {"test_thresholdedrelu_example", 1, 1}, + {"test_tile", 2, 1}, + {"test_tile_precomputed", 2, 1}, + {"test_top_k", 2, 2}, + {"test_top_k_negative_axis", 2, 2}, + {"test_top_k_smallest", 2, 2}, + {"test_training_dropout", 3, 1}, + {"test_training_dropout_default", 3, 1}, + {"test_training_dropout_default_mask", 3, 2}, + {"test_training_dropout_mask", 3, 2}, + {"test_training_dropout_zero_ratio", 3, 1}, + {"test_training_dropout_zero_ratio_mask", 3, 2}, + {"test_transpose_all_permutations_0", 1, 1}, + {"test_transpose_all_permutations_1", 1, 1}, + {"test_transpose_all_permutations_2", 1, 1}, + {"test_transpose_all_permutations_3", 1, 1}, + {"test_transpose_all_permutations_4", 1, 1}, + {"test_transpose_all_permutations_5", 1, 1}, + {"test_transpose_default", 1, 1}, + {"test_tril", 1, 1}, + {"test_tril_neg", 2, 1}, + {"test_tril_one_row_neg", 1, 1}, + {"test_tril_out_neg", 2, 1}, + {"test_tril_out_pos", 2, 1}, + {"test_tril_pos", 2, 1}, + {"test_tril_square", 1, 1}, + {"test_tril_square_neg", 2, 1}, + {"test_tril_zero", 2, 1}, + {"test_triu", 1, 1}, + {"test_triu_neg", 2, 1}, + {"test_triu_one_row", 2, 1}, + {"test_triu_out_neg_out", 2, 1}, + {"test_triu_out_pos", 2, 1}, + {"test_triu_pos", 2, 1}, + {"test_triu_square", 1, 1}, + {"test_triu_square_neg", 2, 1}, + {"test_triu_zero", 2, 1}, + {"test_unique_not_sorted_without_axis", 1, 4}, + {"test_unique_sorted_with_axis", 1, 4}, + {"test_unique_sorted_with_axis_3d", 1, 4}, + {"test_unique_sorted_with_negative_axis", 1, 4}, + {"test_unique_sorted_without_axis", 1, 4}, + {"test_unsqueeze_axis_0", 2, 1}, + {"test_unsqueeze_axis_1", 2, 1}, + {"test_unsqueeze_axis_2", 2, 1}, + {"test_unsqueeze_axis_3", 1, 1}, + {"test_unsqueeze_negative_axes", 2, 1}, + {"test_unsqueeze_three_axes", 2, 1}, + {"test_unsqueeze_two_axes", 2, 1}, + {"test_unsqueeze_unsorted_axes", 2, 1}, + {"test_upsample_nearest", 2, 1}, + {"test_where_example", 3, 1}, + {"test_where_long_example", 3, 1}, + {"test_xor2d", 2, 1}, + {"test_xor3d", 2, 1}, + {"test_xor4d", 2, 1}, + {"test_xor_bcast3v1d", 2, 1}, + {"test_xor_bcast3v2d", 2, 1}, + {"test_xor_bcast4v2d", 2, 1}, + {"test_xor_bcast4v3d", 2, 1}, + {"test_xor_bcast4v4d", 2, 1}, +}; + + +struct TestCaseInput +{ + std::vector input_paths; + std::vector output_paths; + std::string model_path; + std::string name; +}; + +std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case) +{ + return os << test_case.name; +} + +typedef tuple > ONNXConfParams; + +std::string printOnnxConfParams(const testing::TestParamInfo& params) +{ + TestCaseInput test_case = get<0>(params.param); + Backend backend = get<0>(get<1>(params.param)); + Target target = get<1>(get<1>(params.param)); + + std::stringstream ss; + ss << test_case.name << "_"; + PrintTo(backend, &ss); + ss << "_"; + PrintTo(target, &ss); + + return ss.str(); +} + +template +static std::string _tf(TString filename, bool required = true) +{ + return findDataFile(std::string("dnn/onnx/") + filename, required); +} + +std::vector readTestCases() +{ + std::vector ret; + for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i) + { + const TestCase& test_case = testConformanceConfig[i]; + + TestCaseInput input; + + std::string prefix = cv::format("conformance/node/%s", test_case.name); + input.name = test_case.name; + input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str())); + + for (int i = 0; i < test_case.inputs; ++i) + { + input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i))); + } + + for (int i = 0; i < test_case.outputs; ++i) + { + input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i))); + } + + ret.push_back(input); + } + + return ret; +} + +class Test_ONNX_conformance : public TestWithParam +{ +public: + TestCaseInput test_case; + Backend backend; + Target target; + + double default_l1; + double default_lInf; + + static std::set parser_deny_list; + static std::set global_deny_list; + static std::set opencl_fp16_deny_list; + static std::set opencl_deny_list; + static std::set cpu_deny_list; + + Test_ONNX_conformance() + { + test_case = get<0>(GetParam()); + backend = get<0>(get<1>(GetParam())); + target = get<1>(get<1>(GetParam())); + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + default_l1 = 4e-3; + default_lInf = 2e-2; + } + else + { + default_l1 = 1e-5; + default_lInf = 1e-4; + } + } + + bool checkFallbacks(Net& net) const + { + // Check if all the layers are supported with current backend and target. + // Some layers might be fused so their timings equal to zero. + std::vector timings; + net.getPerfProfile(timings); + std::vector names = net.getLayerNames(); + CV_CheckEQ(names.size(), timings.size(), "DNN critical error"); + + bool hasFallbacks = false; + for (int i = 0; i < names.size(); ++i) + { + Ptr l = net.getLayer(net.getLayerId(names[i])); + bool fused = timings[i] == 0.; + if ((!l->supportBackend(backend) || l->preferableTarget != target) && !fused) + { + hasFallbacks = true; + std::cout << "FALLBACK: Layer [" << l->type << "]:[" << l->name << "] is expected to has backend implementation" << endl; + } + } + return hasFallbacks; + } + + static void initDenyList(std::set& deny_set, const char* const deny_list[], const size_t n) + { + for (size_t i = 0; i < n; ++i) + { + deny_set.insert(deny_list[i]); + } + } + + static void SetUpTestCase() + { + const char* const parser[] = { + #include "test_onnx_conformance_layer_parser_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(parser_deny_list, parser, sizeof(parser)/sizeof(parser[0])); + + const char* const global[] = { + #include "test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(global_deny_list, global, sizeof(global)/sizeof(global[0])); + + const char* const opencl_fp16[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_fp16_deny_list, opencl_fp16, sizeof(opencl_fp16)/sizeof(opencl_fp16[0])); + + const char* const opencl[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_deny_list, opencl, sizeof(opencl)/sizeof(opencl[0])); + + const char* const cpu[] = { + #include "test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0])); + } + + void checkFilterLists() const + { + const std::string& name = test_case.name; + if(parser_deny_list.find(name) != parser_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + + if (backend == DNN_BACKEND_OPENCV) + { + if(global_deny_list.find(name) != global_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#if 0 //def HAVE_HALIDE + else if (backend == DNN_BACKEND_HALIDE) + { + #include "test_onnx_conformance_layer_filter__halide.inl.hpp" + } +#endif +#if 0 //def HAVE_INF_ENGINE + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp" + } +#endif +#if 0 //def HAVE_VULKAN + else if (backend == DNN_BACKEND_VKCOM) + { + #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + } +#endif +#if 0 //def HAVE_CUDA + else if (backend == DNN_BACKEND_CUDA) + { + #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + } +#endif + else + { + std::ostringstream ss; + ss << "No test filter available for backend "; + PrintTo(backend, &ss); + ss << ". Run test by default"; + std::cout << ss.str() << std::endl; + } + } +}; + +std::set Test_ONNX_conformance::parser_deny_list; +std::set Test_ONNX_conformance::global_deny_list; +std::set Test_ONNX_conformance::opencl_fp16_deny_list; +std::set Test_ONNX_conformance::opencl_deny_list; +std::set Test_ONNX_conformance::cpu_deny_list; + +TEST_P(Test_ONNX_conformance, Layer_Test) +{ + std::string name = test_case.name; + ASSERT_FALSE(name.empty()); + + bool checkLayersFallbacks = true; + bool checkAccuracy = true; + + checkFilterLists(); + + std::vector inputs; + std::vector ref_outputs; + + Net net; + try + { + //cout << "Read ONNX inputs..." << endl; + std::transform(test_case.input_paths.begin(), test_case.input_paths.end(), + std::back_inserter(inputs), readTensorFromONNX); + + //cout << "Read ONNX reference outputs..." << endl; + std::transform(test_case.output_paths.begin(), test_case.output_paths.end(), + std::back_inserter(ref_outputs), readTensorFromONNX); + + //cout << "Parse model..." << endl; + net = readNetFromONNX(test_case.model_path); + if (net.empty()) + { + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + } + } + catch (...) + { + cout << "Exception during ONNX model parse / loading input / loading reference data!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + throw; + } + ASSERT_FALSE(net.empty()); + + std::vector inputNames; + for (int i = 0; i < inputs.size(); ++i) + inputNames.push_back(cv::format("%d", i)); + net.setInputsNames(inputNames); + + try + { + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < inputs.size(); ++i) + { + net.setInput(inputs[i], inputNames[i]); + } + } + catch (...) + { + cout << "Exception during network configuration!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_NET_SETUP); + throw; + } + + std::vector layerNames = net.getUnconnectedOutLayersNames(); + std::vector< std::vector > outputs_; + try + { + net.forward(outputs_, layerNames); + } + catch (...) + { + cout << "Exception during net.forward() call!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_FORWARD); + throw; + } + ASSERT_GE(outputs_.size(), 1); + const std::vector& outputs = outputs_[0]; + + if (checkLayersFallbacks && checkFallbacks(net)) + { + applyTestTag(CV_TEST_TAG_DNN_LAYER_FALLBACK); + } + + if (checkAccuracy) + { + try + { + if (ref_outputs.size() == 1) + { + // probably we found random unconnected layers. + normAssert(ref_outputs[0], outputs[0], "", default_l1, default_lInf); + } + else + { + ASSERT_EQ(outputs.size(), ref_outputs.size()); + for (size_t i = 0; i < ref_outputs.size(); ++i) + { + normAssert(ref_outputs[i], outputs[i], "", default_l1, default_lInf); + } + } + } + catch (...) + { + cout << "Exception during accuracy check!" << endl; + throw; + } + } + else + { + applyTestTag(CV_TEST_TAG_DNN_NO_ACCURACY_CHECK); + } + + if (!HasFailure()) + cout << "Test passed!" << endl; +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance, + testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()), + printOnnxConfParams); + +}; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp new file mode 100644 index 0000000000..cff1e93aa0 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -0,0 +1,23 @@ +"test_add_bcast", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_same_lower", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_logsoftmax_default_axis", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_softmax_default_axis", +"test_sub_bcast", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp new file mode 100644 index 0000000000..573d847985 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp @@ -0,0 +1,21 @@ +"test_averagepool_3d_default", +"test_dropout_default_ratio", +"test_globalmaxpool", +"test_globalmaxpool_precomputed", +"test_logsoftmax_large_number", +"test_logsoftmax_large_number_expanded", +"test_maxpool_1d_default", +"test_maxpool_2d_ceil", +"test_maxpool_2d_default", +"test_maxpool_2d_pads", +"test_maxpool_2d_precomputed_pads", +"test_maxpool_2d_precomputed_same_upper", +"test_maxpool_2d_precomputed_strides", +"test_maxpool_2d_same_upper", +"test_maxpool_2d_strides", +"test_maxpool_3d_default", +"test_softmax_large_number", +"test_softmax_large_number_expanded", +"test_split_equal_parts_1d", +"test_split_equal_parts_2d", +"test_split_equal_parts_default_axis", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp new file mode 100644 index 0000000000..9a7a21f393 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp @@ -0,0 +1,2 @@ +"test_averagepool_3d_default", +"test_maxpool_3d_default", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp new file mode 100644 index 0000000000..5f1d99d4bb --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -0,0 +1,717 @@ +// The file is autogenerated +// Update note: execute /testdata/dnn/onnx/generate_conformance_list.py +"test_abs", +"test_acos", +"test_acos_example", +"test_acosh", +"test_acosh_example", +"test_adagrad", +"test_adagrad_multiple", +"test_adam", +"test_adam_multiple", +"test_add_uint8", +"test_and2d", +"test_and3d", +"test_and4d", +"test_and_bcast3v1d", +"test_and_bcast3v2d", +"test_and_bcast4v2d", +"test_and_bcast4v3d", +"test_and_bcast4v4d", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +"test_asin", +"test_asin_example", +"test_asinh", +"test_asinh_example", +"test_atan", +"test_atan_example", +"test_atanh", +"test_atanh_example", +"test_basic_convinteger", +"test_batchnorm_epsilon", +"test_batchnorm_epsilon_training_mode", +"test_batchnorm_example", +"test_batchnorm_example_training_mode", +"test_bernoulli", +"test_bernoulli_double", +"test_bernoulli_double_expanded", +"test_bernoulli_expanded", +"test_bernoulli_seed", +"test_bernoulli_seed_expanded", +"test_bitshift_left_uint16", +"test_bitshift_left_uint32", +"test_bitshift_left_uint64", +"test_bitshift_left_uint8", +"test_bitshift_right_uint16", +"test_bitshift_right_uint32", +"test_bitshift_right_uint64", +"test_bitshift_right_uint8", +"test_cast_BFLOAT16_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT16", +"test_cast_FLOAT16_to_DOUBLE", +"test_cast_FLOAT16_to_FLOAT", +"test_cast_FLOAT_to_BFLOAT16", +"test_cast_FLOAT_to_DOUBLE", +"test_cast_FLOAT_to_FLOAT16", +"test_castlike_BFLOAT16_to_FLOAT", +"test_castlike_BFLOAT16_to_FLOAT_expanded", +"test_castlike_DOUBLE_to_FLOAT", +"test_castlike_DOUBLE_to_FLOAT16", +"test_castlike_DOUBLE_to_FLOAT16_expanded", +"test_castlike_DOUBLE_to_FLOAT_expanded", +"test_castlike_FLOAT16_to_DOUBLE", +"test_castlike_FLOAT16_to_DOUBLE_expanded", +"test_castlike_FLOAT16_to_FLOAT", +"test_castlike_FLOAT16_to_FLOAT_expanded", +"test_castlike_FLOAT_to_BFLOAT16", +"test_castlike_FLOAT_to_BFLOAT16_expanded", +"test_castlike_FLOAT_to_DOUBLE", +"test_castlike_FLOAT_to_DOUBLE_expanded", +"test_castlike_FLOAT_to_FLOAT16", +"test_castlike_FLOAT_to_FLOAT16_expanded", +"test_castlike_FLOAT_to_STRING", +"test_castlike_STRING_to_FLOAT", +"test_ceil", +"test_ceil_example", +"test_celu", +"test_clip", +"test_clip_default_inbounds", +"test_clip_default_int8_inbounds", +"test_clip_default_int8_max", +"test_clip_default_int8_min", +"test_clip_default_max", +"test_clip_default_min", +"test_clip_example", +"test_clip_inbounds", +"test_clip_outbounds", +"test_clip_splitbounds", +"test_compress_0", +"test_compress_1", +"test_compress_default_axis", +"test_compress_negative_axis", +"test_constant", +"test_constant_pad", +"test_constantofshape_float_ones", +"test_constantofshape_int_shape_zero", +"test_constantofshape_int_zeros", +"test_convinteger_with_padding", +"test_convinteger_without_padding", +"test_convtranspose", +"test_convtranspose_1d", +"test_convtranspose_3d", +"test_convtranspose_autopad_same", +"test_convtranspose_dilations", +"test_convtranspose_kernel_shape", +"test_convtranspose_output_shape", +"test_convtranspose_pad", +"test_convtranspose_pads", +"test_convtranspose_with_kernel", +"test_cos", +"test_cos_example", +"test_cosh", +"test_cosh_example", +"test_cumsum_1d", +"test_cumsum_1d_exclusive", +"test_cumsum_1d_reverse", +"test_cumsum_1d_reverse_exclusive", +"test_cumsum_2d_axis_0", +"test_cumsum_2d_axis_1", +"test_cumsum_2d_negative_axis", +"test_depthtospace_crd_mode", +"test_depthtospace_crd_mode_example", +"test_depthtospace_dcr_mode", +"test_depthtospace_example", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_det_2d", +"test_det_nd", +"test_div_example", +"test_div_uint8", +"test_dropout_default_mask", +"test_dropout_default_mask_ratio", +"test_dynamicquantizelinear", +"test_dynamicquantizelinear_expanded", +"test_dynamicquantizelinear_max_adjusted", +"test_dynamicquantizelinear_max_adjusted_expanded", +"test_dynamicquantizelinear_min_adjusted", +"test_dynamicquantizelinear_min_adjusted_expanded", +"test_edge_pad", +"test_einsum_batch_diagonal", +"test_einsum_batch_matmul", +"test_einsum_inner_prod", +"test_einsum_sum", +"test_einsum_transpose", +"test_equal", +"test_equal_bcast", +"test_erf", +"test_expand_dim_changed", +"test_expand_dim_unchanged", +"test_eyelike_populate_off_main_diagonal", +"test_eyelike_with_dtype", +"test_eyelike_without_dtype", +"test_floor", +"test_floor_example", +"test_gather_0", +"test_gather_1", +"test_gather_2d_indices", +"test_gather_elements_0", +"test_gather_elements_1", +"test_gather_elements_negative_indices", +"test_gather_negative_indices", +"test_gathernd_example_float32", +"test_gathernd_example_int32", +"test_gathernd_example_int32_batch_dim1", +"test_gemm_all_attributes", +"test_gemm_alpha", +"test_gemm_beta", +"test_gemm_default_matrix_bias", +"test_gemm_default_no_bias", +"test_gemm_default_scalar_bias", +"test_gemm_default_single_elem_vector_bias", +"test_gemm_default_vector_bias", +"test_gemm_default_zero_bias", +"test_gemm_transposeA", +"test_gemm_transposeB", +"test_greater", +"test_greater_bcast", +"test_greater_equal", +"test_greater_equal_bcast", +"test_greater_equal_bcast_expanded", +"test_greater_equal_expanded", +"test_gridsample", +"test_gridsample_aligncorners_true", +"test_gridsample_bicubic", +"test_gridsample_bilinear", +"test_gridsample_border_padding", +"test_gridsample_nearest", +"test_gridsample_reflection_padding", +"test_gridsample_zeros_padding", +"test_gru_batchwise", +"test_gru_defaults", +"test_gru_seq_length", +"test_gru_with_initial_bias", +"test_hardmax_axis_0", +"test_hardmax_axis_1", +"test_hardmax_axis_2", +"test_hardmax_default_axis", +"test_hardmax_example", +"test_hardmax_negative_axis", +"test_hardmax_one_hot", +"test_hardsigmoid", +"test_hardsigmoid_default", +"test_hardsigmoid_example", +"test_hardswish", +"test_hardswish_expanded", +"test_identity_opt", +"test_identity_sequence", +"test_if", +"test_if_opt", +"test_if_seq", +"test_instancenorm_epsilon", +"test_instancenorm_example", +"test_isinf", +"test_isinf_negative", +"test_isinf_positive", +"test_isnan", +"test_less", +"test_less_bcast", +"test_less_equal", +"test_less_equal_bcast", +"test_less_equal_bcast_expanded", +"test_less_equal_expanded", +"test_log", +"test_log_example", +"test_loop11", +"test_loop13_seq", +"test_loop16_seq_none", +"test_lstm_batchwise", +"test_lstm_defaults", +"test_lstm_with_initial_bias", +"test_lstm_with_peepholes", +"test_matmulinteger", +"test_max_example", +"test_max_float16", +"test_max_float32", +"test_max_float64", +"test_max_int16", +"test_max_int32", +"test_max_int64", +"test_max_int8", +"test_max_one_input", +"test_max_two_inputs", +"test_max_uint16", +"test_max_uint32", +"test_max_uint64", +"test_max_uint8", +"test_maxpool_2d_uint8", +"test_maxunpool_export_with_output_shape", +"test_maxunpool_export_without_output_shape", +"test_mean_example", +"test_mean_one_input", +"test_mean_two_inputs", +"test_min_example", +"test_min_float16", +"test_min_float32", +"test_min_float64", +"test_min_int16", +"test_min_int32", +"test_min_int64", +"test_min_int8", +"test_min_one_input", +"test_min_two_inputs", +"test_min_uint16", +"test_min_uint32", +"test_min_uint64", +"test_min_uint8", +"test_mod_broadcast", +"test_mod_int64_fmod", +"test_mod_mixed_sign_float16", +"test_mod_mixed_sign_float32", +"test_mod_mixed_sign_float64", +"test_mod_mixed_sign_int16", +"test_mod_mixed_sign_int32", +"test_mod_mixed_sign_int64", +"test_mod_mixed_sign_int8", +"test_mod_uint16", +"test_mod_uint32", +"test_mod_uint64", +"test_mod_uint8", +"test_momentum", +"test_momentum_multiple", +"test_mul_example", +"test_mul_uint8", +"test_mvn", +"test_mvn_expanded", +"test_nesterov_momentum", +"test_nllloss_NC", +"test_nllloss_NC_expanded", +"test_nllloss_NCd1", +"test_nllloss_NCd1_expanded", +"test_nllloss_NCd1_ii", +"test_nllloss_NCd1_ii_expanded", +"test_nllloss_NCd1_mean_weight_negative_ii", +"test_nllloss_NCd1_mean_weight_negative_ii_expanded", +"test_nllloss_NCd1_weight", +"test_nllloss_NCd1_weight_expanded", +"test_nllloss_NCd1_weight_ii", +"test_nllloss_NCd1_weight_ii_expanded", +"test_nllloss_NCd1d2", +"test_nllloss_NCd1d2_expanded", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", +"test_nllloss_NCd1d2_reduction_mean", +"test_nllloss_NCd1d2_reduction_mean_expanded", +"test_nllloss_NCd1d2_reduction_sum", +"test_nllloss_NCd1d2_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight", +"test_nllloss_NCd1d2_with_weight_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_mean", +"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum", +"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_nllloss_NCd1d2d3_sum_weight_high_ii", +"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", +"test_nllloss_NCd1d2d3d4d5_mean_weight", +"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", +"test_nllloss_NCd1d2d3d4d5_none_no_weight", +"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", +"test_nonmaxsuppression_center_point_box_format", +"test_nonmaxsuppression_flipped_coordinates", +"test_nonmaxsuppression_identical_boxes", +"test_nonmaxsuppression_limit_output_size", +"test_nonmaxsuppression_single_box", +"test_nonmaxsuppression_suppress_by_IOU", +"test_nonmaxsuppression_suppress_by_IOU_and_scores", +"test_nonmaxsuppression_two_batches", +"test_nonmaxsuppression_two_classes", +"test_nonzero_example", +"test_not_2d", +"test_not_3d", +"test_not_4d", +"test_onehot_negative_indices", +"test_onehot_with_axis", +"test_onehot_with_negative_axis", +"test_onehot_without_axis", +"test_optional_get_element", +"test_optional_get_element_sequence", +"test_optional_has_element", +"test_optional_has_element_empty", +"test_or2d", +"test_or3d", +"test_or4d", +"test_or_bcast3v1d", +"test_or_bcast3v2d", +"test_or_bcast4v2d", +"test_or_bcast4v3d", +"test_or_bcast4v4d", +"test_pow", +"test_pow_bcast_array", +"test_pow_bcast_scalar", +"test_pow_example", +"test_pow_types_float", +"test_pow_types_float32_int32", +"test_pow_types_float32_int64", +"test_pow_types_float32_uint32", +"test_pow_types_float32_uint64", +"test_pow_types_int", +"test_pow_types_int32_float32", +"test_pow_types_int32_int32", +"test_pow_types_int64_float32", +"test_pow_types_int64_int64", +"test_prelu_broadcast", +"test_prelu_example", +"test_qlinearconv", +"test_qlinearmatmul_2D", +"test_qlinearmatmul_3D", +"test_quantizelinear", +"test_quantizelinear_axis", +"test_range_float_type_positive_delta", +"test_range_float_type_positive_delta_expanded", +"test_range_int32_type_negative_delta", +"test_range_int32_type_negative_delta_expanded", +"test_reciprocal", +"test_reciprocal_example", +"test_reduce_l1_default_axes_keepdims_example", +"test_reduce_l1_default_axes_keepdims_random", +"test_reduce_l1_do_not_keepdims_example", +"test_reduce_l1_do_not_keepdims_random", +"test_reduce_l1_keep_dims_example", +"test_reduce_l1_keep_dims_random", +"test_reduce_l1_negative_axes_keep_dims_example", +"test_reduce_l1_negative_axes_keep_dims_random", +"test_reduce_l2_default_axes_keepdims_example", +"test_reduce_l2_default_axes_keepdims_random", +"test_reduce_l2_do_not_keepdims_example", +"test_reduce_l2_do_not_keepdims_random", +"test_reduce_l2_keep_dims_example", +"test_reduce_l2_keep_dims_random", +"test_reduce_l2_negative_axes_keep_dims_example", +"test_reduce_l2_negative_axes_keep_dims_random", +"test_reduce_log_sum", +"test_reduce_log_sum_asc_axes", +"test_reduce_log_sum_default", +"test_reduce_log_sum_desc_axes", +"test_reduce_log_sum_exp_default_axes_keepdims_example", +"test_reduce_log_sum_exp_default_axes_keepdims_random", +"test_reduce_log_sum_exp_do_not_keepdims_example", +"test_reduce_log_sum_exp_do_not_keepdims_random", +"test_reduce_log_sum_exp_keepdims_example", +"test_reduce_log_sum_exp_keepdims_random", +"test_reduce_log_sum_exp_negative_axes_keepdims_example", +"test_reduce_log_sum_exp_negative_axes_keepdims_random", +"test_reduce_log_sum_negative_axes", +"test_reduce_max_default_axes_keepdim_example", +"test_reduce_max_default_axes_keepdims_random", +"test_reduce_mean_default_axes_keepdims_example", +"test_reduce_mean_default_axes_keepdims_random", +"test_reduce_min_default_axes_keepdims_example", +"test_reduce_min_default_axes_keepdims_random", +"test_reduce_min_do_not_keepdims_example", +"test_reduce_min_do_not_keepdims_random", +"test_reduce_min_keepdims_example", +"test_reduce_min_keepdims_random", +"test_reduce_min_negative_axes_keepdims_example", +"test_reduce_min_negative_axes_keepdims_random", +"test_reduce_prod_default_axes_keepdims_example", +"test_reduce_prod_default_axes_keepdims_random", +"test_reduce_prod_do_not_keepdims_example", +"test_reduce_prod_do_not_keepdims_random", +"test_reduce_prod_keepdims_example", +"test_reduce_prod_keepdims_random", +"test_reduce_prod_negative_axes_keepdims_example", +"test_reduce_prod_negative_axes_keepdims_random", +"test_reduce_sum_default_axes_keepdims_example", +"test_reduce_sum_default_axes_keepdims_random", +"test_reduce_sum_do_not_keepdims_example", +"test_reduce_sum_do_not_keepdims_random", +"test_reduce_sum_empty_axes_input_noop_example", +"test_reduce_sum_empty_axes_input_noop_random", +"test_reduce_sum_keepdims_example", +"test_reduce_sum_keepdims_random", +"test_reduce_sum_negative_axes_keepdims_example", +"test_reduce_sum_negative_axes_keepdims_random", +"test_reduce_sum_square_default_axes_keepdims_example", +"test_reduce_sum_square_default_axes_keepdims_random", +"test_reduce_sum_square_do_not_keepdims_example", +"test_reduce_sum_square_do_not_keepdims_random", +"test_reduce_sum_square_keepdims_example", +"test_reduce_sum_square_keepdims_random", +"test_reduce_sum_square_negative_axes_keepdims_example", +"test_reduce_sum_square_negative_axes_keepdims_random", +"test_reflect_pad", +"test_reshape_allowzero_reordered", +"test_reshape_extended_dims", +"test_reshape_negative_dim", +"test_reshape_negative_extended_dims", +"test_reshape_one_dim", +"test_reshape_reduced_dims", +"test_reshape_reordered_all_dims", +"test_reshape_reordered_last_dims", +"test_reshape_zero_and_negative_dim", +"test_reshape_zero_dim", +"test_resize_downsample_scales_cubic", +"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_downsample_scales_cubic_align_corners", +"test_resize_downsample_scales_linear", +"test_resize_downsample_scales_linear_align_corners", +"test_resize_downsample_scales_nearest", +"test_resize_downsample_sizes_cubic", +"test_resize_downsample_sizes_linear_pytorch_half_pixel", +"test_resize_downsample_sizes_nearest", +"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", +"test_resize_tf_crop_and_resize", +"test_resize_upsample_scales_cubic", +"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_upsample_scales_cubic_align_corners", +"test_resize_upsample_scales_cubic_asymmetric", +"test_resize_upsample_scales_linear", +"test_resize_upsample_scales_linear_align_corners", +"test_resize_upsample_scales_nearest", +"test_resize_upsample_sizes_cubic", +"test_resize_upsample_sizes_nearest", +"test_resize_upsample_sizes_nearest_ceil_half_pixel", +"test_resize_upsample_sizes_nearest_floor_align_corners", +"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", +"test_reversesequence_batch", +"test_reversesequence_time", +"test_rnn_seq_length", +"test_roialign_aligned_false", +"test_roialign_aligned_true", +"test_round", +"test_scan9_sum", +"test_scan_sum", +"test_scatter_elements_with_axis", +"test_scatter_elements_with_duplicate_indices", +"test_scatter_elements_with_negative_indices", +"test_scatter_elements_without_axis", +"test_scatter_with_axis", +"test_scatter_without_axis", +"test_scatternd", +"test_scatternd_add", +"test_scatternd_multiply", +"test_sce_NCd1_mean_weight_negative_ii", +"test_sce_NCd1_mean_weight_negative_ii_expanded", +"test_sce_NCd1_mean_weight_negative_ii_log_prob", +"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii", +"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight", +"test_sce_NCd1d2d3d4d5_mean_weight_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight", +"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", +"test_sce_mean", +"test_sce_mean_3d", +"test_sce_mean_3d_expanded", +"test_sce_mean_3d_log_prob", +"test_sce_mean_3d_log_prob_expanded", +"test_sce_mean_expanded", +"test_sce_mean_log_prob", +"test_sce_mean_log_prob_expanded", +"test_sce_mean_no_weight_ii", +"test_sce_mean_no_weight_ii_3d", +"test_sce_mean_no_weight_ii_3d_expanded", +"test_sce_mean_no_weight_ii_3d_log_prob", +"test_sce_mean_no_weight_ii_3d_log_prob_expanded", +"test_sce_mean_no_weight_ii_4d", +"test_sce_mean_no_weight_ii_4d_expanded", +"test_sce_mean_no_weight_ii_4d_log_prob", +"test_sce_mean_no_weight_ii_4d_log_prob_expanded", +"test_sce_mean_no_weight_ii_expanded", +"test_sce_mean_no_weight_ii_log_prob", +"test_sce_mean_no_weight_ii_log_prob_expanded", +"test_sce_mean_weight", +"test_sce_mean_weight_expanded", +"test_sce_mean_weight_ii", +"test_sce_mean_weight_ii_3d", +"test_sce_mean_weight_ii_3d_expanded", +"test_sce_mean_weight_ii_3d_log_prob", +"test_sce_mean_weight_ii_3d_log_prob_expanded", +"test_sce_mean_weight_ii_4d", +"test_sce_mean_weight_ii_4d_expanded", +"test_sce_mean_weight_ii_4d_log_prob", +"test_sce_mean_weight_ii_4d_log_prob_expanded", +"test_sce_mean_weight_ii_expanded", +"test_sce_mean_weight_ii_log_prob", +"test_sce_mean_weight_ii_log_prob_expanded", +"test_sce_mean_weight_log_prob", +"test_sce_mean_weight_log_prob_expanded", +"test_sce_none", +"test_sce_none_expanded", +"test_sce_none_log_prob", +"test_sce_none_log_prob_expanded", +"test_sce_none_weights", +"test_sce_none_weights_expanded", +"test_sce_none_weights_log_prob", +"test_sce_none_weights_log_prob_expanded", +"test_sce_sum", +"test_sce_sum_expanded", +"test_sce_sum_log_prob", +"test_sce_sum_log_prob_expanded", +"test_selu", +"test_selu_default", +"test_selu_example", +"test_sequence_insert_at_back", +"test_sequence_insert_at_front", +"test_shape", +"test_shape_clip_end", +"test_shape_clip_start", +"test_shape_end_1", +"test_shape_end_negative_1", +"test_shape_example", +"test_shape_start_1", +"test_shape_start_1_end_2", +"test_shape_start_1_end_negative_1", +"test_shape_start_negative_1", +"test_shrink_hard", +"test_shrink_soft", +"test_sign", +"test_simple_rnn_batchwise", +"test_simple_rnn_defaults", +"test_simple_rnn_with_initial_bias", +"test_sin", +"test_sin_example", +"test_sinh", +"test_sinh_example", +"test_size", +"test_size_example", +"test_slice", +"test_slice_default_axes", +"test_slice_default_steps", +"test_slice_end_out_of_bounds", +"test_slice_neg", +"test_slice_neg_steps", +"test_slice_negative_axes", +"test_slice_start_out_of_bounds", +"test_softplus", +"test_softplus_example", +"test_softsign", +"test_softsign_example", +"test_spacetodepth", +"test_spacetodepth_example", +"test_split_variable_parts_1d", +"test_split_variable_parts_2d", +"test_split_variable_parts_default_axis", +"test_split_zero_size_splits", +"test_sqrt", +"test_sqrt_example", +"test_squeeze", +"test_squeeze_negative_axes", +"test_strnormalizer_export_monday_casesensintive_lower", +"test_strnormalizer_export_monday_casesensintive_nochangecase", +"test_strnormalizer_export_monday_casesensintive_upper", +"test_strnormalizer_export_monday_empty_output", +"test_strnormalizer_export_monday_insensintive_upper_twodim", +"test_strnormalizer_nostopwords_nochangecase", +"test_sub_example", +"test_sub_uint8", +"test_sum_example", +"test_sum_two_inputs", +"test_tan", +"test_tan_example", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", +"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", +"test_tfidfvectorizer_tf_only_bigrams_skip0", +"test_tfidfvectorizer_tf_onlybigrams_levelempty", +"test_tfidfvectorizer_tf_onlybigrams_skip5", +"test_tfidfvectorizer_tf_uniandbigrams_skip5", +"test_thresholdedrelu", +"test_thresholdedrelu_default", +"test_thresholdedrelu_example", +"test_tile", +"test_tile_precomputed", +"test_top_k", +"test_top_k_negative_axis", +"test_top_k_smallest", +"test_training_dropout", +"test_training_dropout_default", +"test_training_dropout_default_mask", +"test_training_dropout_mask", +"test_training_dropout_zero_ratio", +"test_training_dropout_zero_ratio_mask", +"test_tril", +"test_tril_neg", +"test_tril_one_row_neg", +"test_tril_out_neg", +"test_tril_out_pos", +"test_tril_pos", +"test_tril_square", +"test_tril_square_neg", +"test_tril_zero", +"test_triu", +"test_triu_neg", +"test_triu_one_row", +"test_triu_out_neg_out", +"test_triu_out_pos", +"test_triu_pos", +"test_triu_square", +"test_triu_square_neg", +"test_triu_zero", +"test_unique_not_sorted_without_axis", +"test_unique_sorted_with_axis", +"test_unique_sorted_with_axis_3d", +"test_unique_sorted_with_negative_axis", +"test_unique_sorted_without_axis", +"test_unsqueeze_axis_0", +"test_unsqueeze_axis_1", +"test_unsqueeze_axis_2", +"test_unsqueeze_negative_axes", +"test_unsqueeze_three_axes", +"test_unsqueeze_two_axes", +"test_unsqueeze_unsorted_axes", +"test_where_example", +"test_where_long_example", +"test_xor2d", +"test_xor3d", +"test_xor4d", +"test_xor_bcast3v1d", +"test_xor_bcast3v2d", +"test_xor_bcast4v2d", +"test_xor_bcast4v3d", +"test_xor_bcast4v4d", diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 2e9718be3e..f52a051c8f 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -182,9 +182,12 @@ class TestInfo(object): return 1 return 0 + def __lt__(self, other): + return self.__cmp__(other) == -1 + # This is a Sequence for compatibility with old scripts, # which treat parseLogFile's return value as a list. -class TestRunInfo(collections.Sequence): +class TestRunInfo(object): def __init__(self, properties, tests): self.properties = properties self.tests = tests