From 1b8fba8e260ef6251c0168c23052773b8e9d1466 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Wed, 13 Jul 2022 13:46:16 +0800 Subject: [PATCH 1/3] support ReduceSum with two input and dynamic shape batch size in ReduceLayer. --- modules/dnn/src/onnx/onnx_importer.cpp | 54 +++++++++++++++++-------- modules/dnn/test/test_onnx_importer.cpp | 2 + 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index ebbda98d51..7390d0307a 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1180,32 +1180,43 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node layerParams.set("reduce", reduceType); bool keepdims = layerParams.get("keepdims", 1) == 1; - if (layer_type == "ReduceSum" && node_proto.input_size() == 2) - { - // TODO support the opset 13 of ReduceSum. - // in opset 13, the ReduceSum has two input, it takes axes as input instead of attribute - // details:https://github.com/onnx/onnx/issues/3420#issuecomment-844295687 - CV_Error(Error::StsNotImplemented, "Unsupported " + layer_type + " operation of opset 13, please try to " - "re-export the onnx model with opset 11."); - } - MatShape inpShape = outShapes[node_proto.input(0)]; std::vector shouldDelete(inpShape.size(), false); - if (layerParams.has("axes")) + if (layer_type == "ReduceSum" && node_proto.input_size() == 2) { - DictValue axes = layerParams.get("axes"); - for (int i = 0; i < axes.size(); i++) + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; + Mat axesMat = getBlob(node_proto, 1); + int axesNum = axesMat.total(); + for (int i = 0; i < axesNum; i++) + { + int axis = normalize_axis(static_cast(axesMat.at(i)), inpShape.size()); + shouldDelete[axis] = true; + } } + else + // in opset 13, the ReduceSum has two input, it takes axes as input instead of attribute + // details:https://github.com/onnx/onnx/issues/3420#issuecomment-844295687 + CV_Error(Error::StsNotImplemented, "Non-constant axis values in ReduceSum are not supported."); } else { - for (int i = 0; i < inpShape.size(); i++) + if (layerParams.has("axes")) { - shouldDelete[i] = true; + DictValue axes = layerParams.get("axes"); + for (int i = 0; i < axes.size(); i++) + { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + } + else + { + for (int i = 0; i < inpShape.size(); i++) + { + shouldDelete[i] = true; + } } } @@ -1291,6 +1302,17 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node layerParams.type = (depth == CV_8S) ? "ReshapeInt8" : "Reshape"; layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); + // Set batchsize dim as dynamic to be compatible with batch size >= 2. + if (targetShape[0] == 1 && targetShape.size() > 1) + { + std::vector dynamicAxes = {0}; // The index of batchsize dim is 0. + std::vector inputIndices = {0}; + + layerParams.set("has_dynamic_shapes", true); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + node_proto.set_input(0, node_proto.output(0)); node_proto.set_output(0, output_name); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 578e0442b2..5f94f9884d 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -411,6 +411,8 @@ TEST_P(Test_ONNX_layers, ReduceMean) TEST_P(Test_ONNX_layers, ReduceSum) { testONNXModels("reduce_sum"); + testONNXModels("reduce_sum_axis"); + testONNXModels("reduce_sum_axis_dynamic_batch"); } TEST_P(Test_ONNX_layers, ReduceMax) From 98c33c605dfb09bd4b974cc35db87c5077e677b3 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Wed, 20 Jul 2022 19:02:16 +0800 Subject: [PATCH 2/3] batchsize dynamic is set to index 0. --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7390d0307a..e90581eeb5 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1303,7 +1303,7 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); // Set batchsize dim as dynamic to be compatible with batch size >= 2. - if (targetShape[0] == 1 && targetShape.size() > 1) + if (targetShape.size() > 1) { std::vector dynamicAxes = {0}; // The index of batchsize dim is 0. std::vector inputIndices = {0}; From d4640f464797d8b1abafc4d0c657534940db3453 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Tue, 2 Aug 2022 10:32:31 +0800 Subject: [PATCH 3/3] support ReduceLayer without reshape layer. --- .../dnn/include/opencv2/dnn/all_layers.hpp | 3 +- modules/dnn/src/int8layers/reduce_layer.cpp | 29 ++++++++++++++++--- modules/dnn/src/layers/reduce_layer.cpp | 28 +++++++++++++++--- modules/dnn/src/onnx/onnx_importer.cpp | 28 +++--------------- modules/dnn/test/test_onnx_importer.cpp | 1 - 5 files changed, 55 insertions(+), 34 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 5c86da2be4..26fa66ebe5 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -334,7 +334,8 @@ CV__DNN_INLINE_NS_BEGIN { public: int reduceType; - std::vector reduceDims; + // reduceDims contains the dimensions that need to be reduced, targetDims is the target output dimension. + std::vector reduceDims, targetDims; static Ptr create(const LayerParams& params); }; diff --git a/modules/dnn/src/int8layers/reduce_layer.cpp b/modules/dnn/src/int8layers/reduce_layer.cpp index 935bdc0659..9ffb4897a0 100644 --- a/modules/dnn/src/int8layers/reduce_layer.cpp +++ b/modules/dnn/src/int8layers/reduce_layer.cpp @@ -38,6 +38,15 @@ public: { reduceDims[i] = tempDims.get(i); } + + CV_Assert(params.has("target_dims")); + tempDims = params.get("target_dims"); + n = tempDims.size(); + targetDims.resize(n); + for (i = 0; i < n; i++) + { + targetDims[i] = tempDims.get(i); + } } virtual bool supportBackend(int backendId) CV_OVERRIDE @@ -161,18 +170,30 @@ public: std::vector &internals) const CV_OVERRIDE { CV_Assert(inputs.size() > 0); - CV_Assert(reduceDims.size() != 0 && inputs[0].size() >= reduceDims.size()); + CV_Assert( reduceDims.size() !=0 && targetDims.size() != 0 && inputs[0].size() >= reduceDims.size()); - std::vector outShape; + // outShapeTmp can save the right number of `total(outShapeTmp)`. And the outShape is used as the final output shape. + std::vector outShapeTmp, outShape; + outShape.assign(targetDims.begin(), targetDims.end()); if (inputs[0].size() == reduceDims.size()) - outShape.push_back(1); + outShapeTmp.push_back(1); else { for (int i = 0; i < inputs[0].size() - reduceDims.size(); i++) { - outShape.push_back(inputs[0][i]); + outShapeTmp.push_back(inputs[0][i]); } } + + // Support dynamic shape of Batch size. + // Note that: when there are multiple dynamic inputs, we will give an error. + if (total(outShape) != total(outShapeTmp)) + { + if (outShape[0] != outShapeTmp[0]) + outShape[0] = outShapeTmp[0]; + } + + CV_Assert(total(outShape) == total(outShapeTmp)); outputs.assign(1, outShape); return false; diff --git a/modules/dnn/src/layers/reduce_layer.cpp b/modules/dnn/src/layers/reduce_layer.cpp index 47aec237c7..c1f74f1cc1 100644 --- a/modules/dnn/src/layers/reduce_layer.cpp +++ b/modules/dnn/src/layers/reduce_layer.cpp @@ -61,6 +61,15 @@ public: { reduceDims[i] = tempDims.get(i); } + + CV_Assert(params.has("target_dims")); + tempDims = params.get("target_dims"); + n = tempDims.size(); + targetDims.resize(n); + for (i = 0; i < n; i++) + { + targetDims[i] = tempDims.get(i); + } } virtual bool supportBackend(int backendId) CV_OVERRIDE @@ -325,18 +334,29 @@ public: std::vector &internals) const CV_OVERRIDE { CV_Assert(inputs.size() > 0); - CV_Assert(reduceDims.size() != 0 && inputs[0].size() >= reduceDims.size()); + CV_Assert( reduceDims.size() !=0 && targetDims.size() != 0 && inputs[0].size() >= reduceDims.size()); - std::vector outShape; + // outShapeTmp can save the right number of `total(outShapeTmp)`. And the outShape is used as the final output shape. + std::vector outShapeTmp, outShape; + outShape.assign(targetDims.begin(), targetDims.end()); if (inputs[0].size() == reduceDims.size()) - outShape.push_back(1); + outShapeTmp.push_back(1); else { for (int i = 0; i < inputs[0].size() - reduceDims.size(); i++) { - outShape.push_back(inputs[0][i]); + outShapeTmp.push_back(inputs[0][i]); } } + + // Support dynamic shape of Batch size. + // Note that: when there are multiple dynamic inputs, we will give an error. + if (total(outShape) != total(outShapeTmp) && outShape[0] != outShapeTmp[0]) + { + outShape[0] = outShapeTmp[0]; + } + + CV_Assert(total(outShape) == total(outShapeTmp)); outputs.assign(1, outShape); return false; diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index e90581eeb5..25d4ed94a1 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1191,7 +1191,7 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node int axesNum = axesMat.total(); for (int i = 0; i < axesNum; i++) { - int axis = normalize_axis(static_cast(axesMat.at(i)), inpShape.size()); + int axis = normalize_axis(axesMat.at(i), inpShape.size()); shouldDelete[axis] = true; } } @@ -1220,7 +1220,7 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node } } - MatShape targetShape; + std::vector targetShape; for (int i = 0; i < inpShape.size(); ++i) { if (!shouldDelete[i]) @@ -1290,30 +1290,10 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node } } - LayerParams reduceLp = layerParams; - reduceLp.name = layerParams.name + "/reduce"; - CV_Assert(layer_id.find(reduceLp.name) == layer_id.end()); - reduceLp.set("deleted_dims", DictValue::arrayInt(&deletedDims[0], deletedDims.size())); + layerParams.set("deleted_dims", DictValue::arrayInt(&deletedDims[0], deletedDims.size())); + layerParams.set("target_dims", DictValue::arrayInt(&targetShape[0], targetShape.size())); node_proto.set_input(0, inputString); - node_proto.set_output(0, reduceLp.name); - addLayer(reduceLp, node_proto); - - layerParams.type = (depth == CV_8S) ? "ReshapeInt8" : "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - - // Set batchsize dim as dynamic to be compatible with batch size >= 2. - if (targetShape.size() > 1) - { - std::vector dynamicAxes = {0}; // The index of batchsize dim is 0. - std::vector inputIndices = {0}; - - layerParams.set("has_dynamic_shapes", true); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - - node_proto.set_input(0, node_proto.output(0)); node_proto.set_output(0, output_name); addLayer(layerParams, node_proto); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 5f94f9884d..6a0de29c38 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -411,7 +411,6 @@ TEST_P(Test_ONNX_layers, ReduceMean) TEST_P(Test_ONNX_layers, ReduceSum) { testONNXModels("reduce_sum"); - testONNXModels("reduce_sum_axis"); testONNXModels("reduce_sum_axis_dynamic_batch"); }