diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 65b8205f1d..28bcde4827 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -247,8 +247,6 @@ std::vector getAvailableTargets(Backend be) namespace { - typedef std::vector ShapesVec; - struct LayerShapes { ShapesVec in, out, internal; @@ -2986,20 +2984,24 @@ struct Net::Impl : public detail::NetImplBase void getLayerShapesRecursively(int id, LayersShapesMap& inOutShapes) { - std::vector& inputLayerIds = layers[id].inputBlobsId; + CV_CheckGE(id, 0, ""); + CV_CheckLT(id, (int)layers.size(), ""); + LayerData& layerData = layers[id]; + std::vector& inputLayerIds = layerData.inputBlobsId; + LayerShapes& layerShapes = inOutShapes[id]; - if (id == 0 && inOutShapes[id].in[0].empty()) + if (id == 0 && layerShapes.in[0].empty()) { - if (!layers[0].outputBlobs.empty()) + if (!layerData.outputBlobs.empty()) { ShapesVec shapes; - for (int i = 0; i < layers[0].outputBlobs.size(); i++) + for (int i = 0; i < layerData.outputBlobs.size(); i++) { - Mat& inp = layers[0].outputBlobs[i]; - CV_Assert(inp.total()); + Mat& inp = layerData.outputBlobs[i]; + CV_Assert(!inp.empty()); shapes.push_back(shape(inp)); } - inOutShapes[0].in = shapes; + layerShapes.in = shapes; } else { @@ -3015,17 +3017,17 @@ struct Net::Impl : public detail::NetImplBase } if (none) { - inOutShapes[0].out.clear(); + layerShapes.out.clear(); return; } else { - inOutShapes[0].in = inputShapes; + layerShapes.in = inputShapes; } } } - if (inOutShapes[id].in.empty()) + if (layerShapes.in.empty()) { for(int i = 0; i < inputLayerIds.size(); i++) { @@ -3038,14 +3040,14 @@ struct Net::Impl : public detail::NetImplBase getLayerShapesRecursively(layerId, inOutShapes); } const MatShape& shape = inOutShapes[layerId].out[inputLayerIds[i].oid]; - inOutShapes[id].in.push_back(shape); + layerShapes.in.push_back(shape); } } - const ShapesVec& is = inOutShapes[id].in; - ShapesVec& os = inOutShapes[id].out; - ShapesVec& ints = inOutShapes[id].internal; - int requiredOutputs = layers[id].requiredOutputs.size(); - Ptr l = layers[id].getLayerInstance(); + const ShapesVec& is = layerShapes.in; + ShapesVec& os = layerShapes.out; + ShapesVec& ints = layerShapes.internal; + int requiredOutputs = layerData.requiredOutputs.size(); + Ptr l = layerData.getLayerInstance(); CV_Assert(l); bool layerSupportInPlace = false; try @@ -3073,13 +3075,38 @@ struct Net::Impl : public detail::NetImplBase CV_LOG_ERROR(NULL, "Exception message: " << e.what()); throw; } - inOutShapes[id].supportInPlace = layerSupportInPlace; + layerShapes.supportInPlace = layerSupportInPlace; - for (int i = 0; i < ints.size(); i++) - CV_Assert(total(ints[i]) > 0); + try + { + for (int i = 0; i < ints.size(); i++) + CV_CheckGT(total(ints[i]), 0, ""); - for (int i = 0; i < os.size(); i++) - CV_Assert(total(os[i]) > 0); + for (int i = 0; i < os.size(); i++) + CV_CheckGT(total(os[i]), 0, ""); + } + catch (const cv::Exception& e) + { + CV_LOG_ERROR(NULL, "OPENCV/DNN: [" << l->type << "]:(" << l->name << "): getMemoryShapes() post validation failed." << + " inputs=" << is.size() << + " outputs=" << os.size() << "/" << requiredOutputs << + " blobs=" << l->blobs.size() << + " inplace=" << layerSupportInPlace); + for (size_t i = 0; i < is.size(); ++i) + { + CV_LOG_ERROR(NULL, " input[" << i << "] = " << toString(is[i])); + } + for (size_t i = 0; i < os.size(); ++i) + { + CV_LOG_ERROR(NULL, " output[" << i << "] = " << toString(os[i])); + } + for (size_t i = 0; i < l->blobs.size(); ++i) + { + CV_LOG_ERROR(NULL, " blobs[" << i << "] = " << typeToString(l->blobs[i].type()) << " " << toString(shape(l->blobs[i]))); + } + CV_LOG_ERROR(NULL, "Exception message: " << e.what()); + throw; + } } void getLayersShapes(const ShapesVec& netInputShapes, @@ -3107,42 +3134,57 @@ struct Net::Impl : public detail::NetImplBase void updateLayersShapes() { - CV_Assert(!layers[0].outputBlobs.empty()); + CV_LOG_DEBUG(NULL, "updateLayersShapes() with layers.size=" << layers.size()); + CV_Assert(netInputLayer); + DataLayer& inputLayer = *netInputLayer; + LayerData& inputLayerData = layers[0]; + CV_Assert(inputLayerData.layerInstance.get() == &inputLayer); + CV_Assert(!inputLayerData.outputBlobs.empty()); ShapesVec inputShapes; - for(int i = 0; i < layers[0].outputBlobs.size(); i++) + for(int i = 0; i < inputLayerData.outputBlobs.size(); i++) { - Mat& inp = layers[0].outputBlobs[i]; - CV_Assert(inp.total()); - if (preferableBackend == DNN_BACKEND_OPENCV && + Mat& inp = inputLayerData.outputBlobs[i]; + CV_Assert(!inp.empty()); + if (preferableBackend == DNN_BACKEND_OPENCV && // FIXIT: wrong place for output allocation preferableTarget == DNN_TARGET_OPENCL_FP16) { - layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); + inp.create(inp.dims, inp.size, CV_16S); } inputShapes.push_back(shape(inp)); } + CV_LOG_DEBUG(NULL, toString(inputShapes, "Network input shapes")); LayersShapesMap layersShapes; layersShapes[0].in = inputShapes; for (MapIdToLayerData::iterator it = layers.begin(); it != layers.end(); it++) { int layerId = it->first; - std::vector& inputLayerIds = it->second.inputBlobsId; - if (layersShapes[layerId].in.empty()) + LayerData& layerData = it->second; + std::vector& inputLayerIds = layerData.inputBlobsId; + LayerShapes& layerShapes = layersShapes[layerId]; + CV_LOG_DEBUG(NULL, "layer " << layerId << ": [" << layerData.type << "]:(" << layerData.name << ") with inputs.size=" << inputLayerIds.size()); + if (layerShapes.in.empty()) { for(int i = 0; i < inputLayerIds.size(); i++) { - int inputLayerId = inputLayerIds[i].lid; + const LayerPin& inputPin = inputLayerIds[i]; + int inputLayerId = inputPin.lid; + CV_LOG_DEBUG(NULL, " input[" << i << "] " << inputLayerId << ":" << inputPin.oid << " as [" << layers[inputLayerId].type << "]:(" << layers[inputLayerId].name << ")"); LayersShapesMap::iterator inputIt = layersShapes.find(inputLayerId); - if(inputIt == layersShapes.end() || inputIt->second.out.empty()) + if (inputIt == layersShapes.end() || inputIt->second.out.empty()) { getLayerShapesRecursively(inputLayerId, layersShapes); } - const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid]; - layersShapes[layerId].in.push_back(shape); + const MatShape& shape = layersShapes[inputLayerId].out[inputPin.oid]; + layerShapes.in.push_back(shape); } - it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in); + layerData.layerInstance->updateMemoryShapes(layerShapes.in); } + CV_LOG_DEBUG(NULL, "Layer " << layerId << ": " << toString(layerShapes.in, "input shapes")); + CV_LOG_IF_DEBUG(NULL, !layerShapes.out.empty(), "Layer " << layerId << ": " << toString(layerShapes.out, "output shapes")); + CV_LOG_IF_DEBUG(NULL, !layerShapes.internal.empty(), "Layer " << layerId << ": " << toString(layerShapes.internal, "internal shapes")); } + CV_LOG_DEBUG(NULL, "updateLayersShapes() - DONE"); } LayerPin getLatestLayerPin(const std::vector& pins) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index 0f3feda91b..7360031801 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -29,6 +29,43 @@ struct NetImplBase } // namespace detail + +typedef std::vector ShapesVec; + +static inline std::string toString(const ShapesVec& shapes, const std::string& name = std::string()) +{ + std::ostringstream ss; + if (!name.empty()) + ss << name << ' '; + ss << '['; + for(size_t i = 0, n = shapes.size(); i < n; ++i) + ss << ' ' << toString(shapes[i]); + ss << " ]"; + return ss.str(); +} + +static inline std::string toString(const Mat& blob, const std::string& name = std::string()) +{ + std::ostringstream ss; + if (!name.empty()) + ss << name << ' '; + if (blob.empty()) + { + ss << ""; + } + else if (blob.dims == 1) + { + Mat blob_ = blob; + blob_.dims = 2; // hack + ss << blob_.t(); + } + else + { + ss << blob.reshape(1, 1); + } + return ss.str(); +} + CV__DNN_EXPERIMENTAL_NS_END }} // namespace diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.hpp b/modules/dnn/src/onnx/onnx_graph_simplifier.hpp index b4497adb75..dd4948d729 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.hpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.hpp @@ -8,8 +8,6 @@ #ifndef __OPENCV_DNN_ONNX_SIMPLIFIER_HPP__ #define __OPENCV_DNN_ONNX_SIMPLIFIER_HPP__ -#include "../precomp.hpp" - #if defined(__GNUC__) && __GNUC__ >= 5 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsuggest-override" diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7bedf9543f..21bd6cc065 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -65,6 +65,7 @@ public: ONNXImporter(Net& net, const char *onnxFile) : dstNet(net), dispatch(buildDispatchMap()) + , onnx_opset(0) { hasDynamicShapes = false; CV_Assert(onnxFile); @@ -86,6 +87,7 @@ public: ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) : dstNet(net), dispatch(buildDispatchMap()) + , onnx_opset(0) { hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); @@ -178,6 +180,9 @@ private: const DispatchMap dispatch; static const DispatchMap buildDispatchMap(); + + int onnx_opset; // OperatorSetIdProto for 'onnx' domain + void parseOperatorSet(); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -489,10 +494,45 @@ void ONNXImporter::addNegation(const LayerParams& layerParams, opencv_onnx::Node void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { + CV_LOG_DEBUG(NULL, "DNN/ONNX: add constant '" << name << "' shape=" << toString(shape(blob)) << ": " << toString(blob)); constBlobs.insert(std::make_pair(name, blob)); outShapes.insert(std::make_pair(name, shape(blob))); } +void ONNXImporter::parseOperatorSet() +{ + int ir_version = model_proto.has_ir_version() ? static_cast(model_proto.ir_version()) : -1; + if (ir_version < 3) + return; + + int opset_size = model_proto.opset_import_size(); + if (opset_size <= 0) + { + CV_LOG_INFO(NULL, "DNN/ONNX: missing opset information") + return; + } + + for (int i = 0; i < opset_size; ++i) + { + const ::opencv_onnx::OperatorSetIdProto& opset_entry = model_proto.opset_import(i); + const std::string& domain = opset_entry.has_domain() ? opset_entry.domain() : std::string(); + int version = opset_entry.has_version() ? opset_entry.version() : -1; + if (domain.empty() || domain == "ai.onnx") + { + // ONNX opset covered by specification: https://github.com/onnx/onnx/blob/master/docs/Operators.md + onnx_opset = std::max(onnx_opset, version); + } + else + { + // OpenCV don't know other opsets + // will fail later on unsupported node processing + CV_LOG_WARNING(NULL, "DNN/ONNX: unsupported opset[" << i << "]: domain='" << domain << "' version=" << version); + } + } + + CV_LOG_INFO(NULL, "DNN/ONNX: ONNX opset version = " << onnx_opset); +} + void ONNXImporter::populateNet() { CV_Assert(model_proto.has_graph()); @@ -513,6 +553,8 @@ void ONNXImporter::populateNet() << ", outputs = " << graph_proto.output_size() ); + parseOperatorSet(); + simplifySubgraphs(graph_proto); const int layersSize = graph_proto.node_size(); @@ -539,7 +581,8 @@ void ONNXImporter::populateNet() if (!tensorShape.dim(j).dim_param().empty() && !(j == 0 && inpShape.size() >= 3)) hasDynamicShapes = true; } - if (!inpShape.empty() && !hasDynamicShapes) + CV_LOG_DEBUG(NULL, "DNN/ONNX: input[" << i << "] shape=" << toString(inpShape)); + if (!inpShape.empty() && !hasDynamicShapes) // FIXIT result is not reliable for models with multiple inputs { inpShape[0] = std::max(inpShape[0], 1); // It's OK to have undetermined batch size } @@ -573,6 +616,15 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); const std::string& layer_type = node_proto.op_type(); + const std::string& layer_type_domain = node_proto.has_domain() ? node_proto.domain() : std::string(); + if (!layer_type_domain.empty() && layer_type_domain != "ai.onnx") + { + CV_LOG_WARNING(NULL, "DNN/ONNX: can't handle node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s@%s]:(%s)", layer_type.c_str(), layer_type_domain.c_str(), name.c_str()) + ); + CV_Error(Error::StsNotImplemented, cv::format("ONNX: unsupported domain: %s", layer_type_domain.c_str())); + } + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -2094,11 +2146,22 @@ void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeP CV_Assert(shapeIt != outShapes.end()); const MatShape& inpShape = shapeIt->second; - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; + int dims = static_cast(inpShape.size()); + Mat shapeMat(dims, 1, CV_32S); + bool isDynamicShape = false; + for (int j = 0; j < dims; ++j) + { + int sz = inpShape[j]; + isDynamicShape |= (sz == 0); + shapeMat.at(j) = sz; + } + shapeMat.dims = 1; // FIXIT Mat 1D + if (isDynamicShape) + { + CV_LOG_ERROR(NULL, "DNN/ONNX(Shape): dynamic 'zero' shapes are not supported, input " << toString(inpShape, node_proto.input(0))); + CV_Assert(!isDynamicShape); // not supported + } addConstant(layerParams.name, shapeMat); } @@ -2296,6 +2359,7 @@ void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } +// https://github.com/onnx/onnx/blob/master/docs/Operators.md#Resize void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { for (int i = 1; i < node_proto.input_size(); i++) @@ -2316,30 +2380,38 @@ void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::Node if (layerParams.get("mode") == "linear" && framework_name == "pytorch") layerParams.set("mode", "opencv_linear"); - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; + // opset-10: input = [X, scales] + // opset-11: input = [X, roi, scales] or [x, roi, scales, sizes] + int scalesInputId = node_proto.input_size() == 2 ? 1 : 2; - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) + Mat scales = getBlob(node_proto, scalesInputId); + if (!scales.empty()) { + CV_CheckEQ(scales.total(), (size_t)4, "HCHW layout is expected"); layerParams.set("zoom_factor_y", scales.at(2)); layerParams.set("zoom_factor_x", scales.at(3)); } - else + else if (node_proto.input_size() >= 4) // opset-11 { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) + const std::string& inputSizes = node_proto.input(3); + if (constBlobs.find(inputSizes) != constBlobs.end()) { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); + Mat shapes = getBlob(inputSizes); + CV_CheckEQ(shapes.total(), (size_t)4, "HCHW layout is expected"); CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); if (shapes.depth() == CV_32F) shapes.convertTo(shapes, CV_32S); layerParams.set("width", shapes.at(3)); layerParams.set("height", shapes.at(2)); } + else + { + CV_Error(Error::StsNotImplemented, cv::format("ONNX/Resize: doesn't support dynamic non-constant 'sizes' input: %s", inputSizes.c_str())); + } + } + else + { + CV_Error(Error::StsNotImplemented, "ONNX/Resize: can't find neither 'scale' nor destination sizes parameters"); } replaceLayerParam(layerParams, "mode", "interpolation"); addLayer(layerParams, node_proto); diff --git a/modules/dnn/src/precomp.hpp b/modules/dnn/src/precomp.hpp index 62f8714af1..76ebc14f7d 100644 --- a/modules/dnn/src/precomp.hpp +++ b/modules/dnn/src/precomp.hpp @@ -60,5 +60,6 @@ #include #include #include +#include #include "dnn_common.hpp" diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp index 4f3dfa870d..39e1fefaf3 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp @@ -8,8 +8,6 @@ #ifndef __OPENCV_DNN_TF_SIMPLIFIER_HPP__ #define __OPENCV_DNN_TF_SIMPLIFIER_HPP__ -#include "../precomp.hpp" - #ifdef HAVE_PROTOBUF #include "tf_io.hpp"