third_party/opencv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #21357 from rogday:model_diag

Browse Source
This commit is contained in:
Alexander Alekhin 2022-01-18 15:50:11 +00:00
commit 6ffa2b01e1
6 changed files with 113 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/dnn/include/opencv2/dnn/layer.hpp
View File

@ -66,6 +66,9 @@ public:
//! Unregisters registered layer with specified type name. Thread-safe.
static void unregisterLayer(const String &type);
//! Check if layer is registered.
static bool isLayerRegistered(const std::string& type);
/** @brief Creates instance of registered layer.
* @param type type name of creating layer.
* @param params parameters which will be used for layer initialization.

11
modules/dnn/src/debug_utils.cpp
View File

@ -37,11 +37,8 @@ void skipModelImport(bool skip)
void detail::LayerHandler::addMissing(const std::string& name, const std::string& type)
{
cv::AutoLock lock(getLayerFactoryMutex());
auto& registeredLayers = getLayerFactoryImpl();
// If we didn't add it, but can create it, it's custom and not missing.
if (layers.find(type) == layers.end() && registeredLayers.find(type) != registeredLayers.end())
if (!contains(type) && LayerFactory::isLayerRegistered(type))
{
return;
}
@ -51,17 +48,17 @@ void detail::LayerHandler::addMissing(const std::string& name, const std::string
bool detail::LayerHandler::contains(const std::string& type) const
{
return layers.find(type) != layers.end();
return layers.count(type) != 0;
}
void detail::LayerHandler::printMissing()
void detail::LayerHandler::printMissing() const
{
if (layers.empty())
{
return;
}
std::stringstream ss;
std::ostringstream ss;
ss << "DNN: Not supported types:\n";
for (const auto& type_names : layers)
{

7
modules/dnn/src/dnn.cpp
View File

@ -6092,6 +6092,13 @@ void LayerFactory::unregisterLayer(const String &type)
}
}
bool LayerFactory::isLayerRegistered(const std::string& type)
{
cv::AutoLock lock(getLayerFactoryMutex());
auto& registeredLayers = getLayerFactoryImpl();
return registeredLayers.find(type) != registeredLayers.end();
}
Ptr<Layer> LayerFactory::createLayerInstance(const String &type, LayerParams& params)
{
CV_TRACE_FUNCTION();

4
modules/dnn/src/dnn_common.hpp
View File

@ -5,8 +5,8 @@
#ifndef __OPENCV_DNN_COMMON_HPP__
#define __OPENCV_DNN_COMMON_HPP__
#include <unordered_set>
#include <unordered_map>
#include <unordered_set>
#include <opencv2/dnn.hpp>
@ -59,7 +59,7 @@ class LayerHandler
public:
void addMissing(const std::string& name, const std::string& type);
bool contains(const std::string& type) const;
void printMissing();
void printMissing() const;
protected:
LayerParams getNotImplementedParams(const std::string& name, const std::string& op);

151
modules/dnn/src/onnx/onnx_importer.cpp
View File

@ -48,6 +48,8 @@ CV__DNN_INLINE_NS_BEGIN
extern bool DNN_DIAGNOSTICS_RUN;
class ONNXLayerHandler;
class ONNXImporter
{
opencv_onnx::ModelProto model_proto;
@ -80,7 +82,7 @@ public:
void populateNet();
protected:
std::unique_ptr<detail::LayerHandler> missingLayerHandler;
std::unique_ptr<ONNXLayerHandler> layerHandler;
Net& dstNet;
opencv_onnx::GraphProto graph_proto;
@ -98,11 +100,14 @@ protected:
void handleNode(const opencv_onnx::NodeProto& node_proto);
private:
friend class ONNXLayerHandler;
typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
typedef std::map<std::string, ONNXImporterNodeParser> DispatchMap;
typedef std::map<std::string, DispatchMap> DomainDispatchMap;
DomainDispatchMap domain_dispatch_map;
std::string getLayerTypeDomain(const opencv_onnx::NodeProto& node_proto);
const DispatchMap& getDispatchMap(const opencv_onnx::NodeProto& node_proto);
void buildDispatchMap_ONNX_AI(int opset_version);
void buildDispatchMap_COM_MICROSOFT(int opset_version);
@ -156,6 +161,7 @@ private:
void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseSimpleLayers (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
// Domain: com.microsoft
// URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md
@ -178,9 +184,38 @@ private:
const std::string str_domain_ai_onnx = "ai.onnx";
};
class ONNXLayerHandler : public detail::LayerHandler
{
public:
explicit ONNXLayerHandler(ONNXImporter* importer_);
void fillRegistry(const opencv_onnx::GraphProto& net);
protected:
ONNXImporter* importer;
};
ONNXLayerHandler::ONNXLayerHandler(ONNXImporter* importer_) : importer(importer_){}
void ONNXLayerHandler::fillRegistry(const opencv_onnx::GraphProto &net)
{
int layersSize = net.node_size();
for (int li = 0; li < layersSize; li++) {
const opencv_onnx::NodeProto &node_proto = net.node(li);
const std::string& name = node_proto.output(0);
const std::string& type = node_proto.op_type();
const std::string& layer_type_domain = importer->getLayerTypeDomain(node_proto);
const auto& dispatch = importer->getDispatchMap(node_proto);
if (dispatch.find(type) == dispatch.end())
{
addMissing(name, cv::format("%s.%s", layer_type_domain.c_str(), type.c_str()));
}
}
printMissing();
}
ONNXImporter::ONNXImporter(Net& net, const char *onnxFile)
: missingLayerHandler(DNN_DIAGNOSTICS_RUN ? new detail::LayerHandler() : nullptr)
: layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr)
, dstNet(net)
, onnx_opset(0)
{
@ -203,7 +238,7 @@ ONNXImporter::ONNXImporter(Net& net, const char *onnxFile)
}
ONNXImporter::ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer)
: missingLayerHandler(DNN_DIAGNOSTICS_RUN ? new detail::LayerHandler() : nullptr)
: layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr)
, dstNet(net)
, onnx_opset(0)
{
@ -795,6 +830,7 @@ void ONNXImporter::populateNet()
if (DNN_DIAGNOSTICS_RUN) {
CV_LOG_INFO(NULL, "DNN/ONNX: start diagnostic run!");
layerHandler->fillRegistry(graph_proto);
}
for(int li = 0; li < layersSize; li++)
@ -806,54 +842,48 @@ void ONNXImporter::populateNet()
CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN ? "DNN/ONNX: diagnostic run completed!" : "DNN/ONNX: import completed!"));
}
std::string ONNXImporter::getLayerTypeDomain(const opencv_onnx::NodeProto& node_proto)
{
if (!node_proto.has_domain())
return str_domain_ai_onnx;
const std::string& domain = node_proto.domain();
if (domain.empty())
return str_domain_ai_onnx;
return domain;
}
const ONNXImporter::DispatchMap& ONNXImporter::getDispatchMap(const opencv_onnx::NodeProto& node_proto)
{
static DispatchMap empty_map;
const std::string& layer_type_domain = getLayerTypeDomain(node_proto);
auto it = domain_dispatch_map.find(layer_type_domain);
if (it == domain_dispatch_map.end())
{
return empty_map;
}
return it->second;
}
void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto)
{
CV_Assert(node_proto.output_size() >= 1);
const std::string& name = node_proto.output(0);
const std::string& layer_type = node_proto.op_type();
const std::string& layer_type_domain = [&]()
const std::string& layer_type_domain = getLayerTypeDomain(node_proto);
const auto& dispatch = getDispatchMap(node_proto);
CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and "
<< node_proto.output_size() << " outputs: "
<< cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str())
<< cv::format(" from %sdomain='", onnx_opset_map.count(layer_type_domain) == 1 ? "" : "undeclared ")
<< layer_type_domain << "'"
);
if (dispatch.empty())
{
if (!node_proto.has_domain())
return str_domain_ai_onnx;
const std::string& domain = node_proto.domain();
if (domain.empty())
return str_domain_ai_onnx;
return domain;
}();
const auto& dispatch = [&]()
{
if (layer_type_domain != str_domain_ai_onnx)
{
if (onnx_opset_map.find(layer_type_domain) == onnx_opset_map.end())
{
CV_LOG_WARNING(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: "
<< cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str())
<< " from undeclared domain='" << layer_type_domain << "'"
);
}
else
{
CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: "
<< cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str())
<< " from domain='" << layer_type_domain << "'"
);
}
auto it = domain_dispatch_map.find(layer_type_domain);
if (it == domain_dispatch_map.end())
{
CV_LOG_WARNING(NULL, "DNN/ONNX: missing dispatch map for domain='" << layer_type_domain << "'");
return DispatchMap();
}
return it->second;
}
else
{
CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: "
<< cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str())
);
return domain_dispatch_map[str_domain_ai_onnx];
}
}();
CV_LOG_WARNING(NULL, "DNN/ONNX: missing dispatch map for domain='" << layer_type_domain << "'");
}
LayerParams layerParams;
try
@ -2882,6 +2912,15 @@ void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::Node
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseSimpleLayers(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
for (int j = 0; j < node_proto.input_size(); j++) {
if (layer_id.find(node_proto.input(j)) == layer_id.end())
layerParams.blobs.push_back(getBlob(node_proto, j));
}
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
const std::string& name = layerParams.name;
@ -2897,20 +2936,11 @@ void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx:
}
}
CV_LOG_INFO(NULL, "DNN/ONNX: unknown node type, try using custom handler for node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: "
CV_LOG_IF_INFO(NULL, !LayerFactory::isLayerRegistered(layer_type), "DNN/ONNX: unknown node type, try using custom handler for node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: "
<< cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str())
);
if (missingLayerHandler)
{
missingLayerHandler->addMissing(layerParams.name, layerParams.type);
}
for (int j = 0; j < node_proto.input_size(); j++) {
if (layer_id.find(node_proto.input(j)) == layer_id.end())
layerParams.blobs.push_back(getBlob(node_proto, j));
}
addLayer(layerParams, node_proto);
parseSimpleLayers(layerParams, node_proto);
}
void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
@ -3360,6 +3390,15 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput;
dispatch["CumSum"] = &ONNXImporter::parseCumSum;
std::vector<std::string> simpleLayers{"Acos", "Acosh", "Asin", "Asinh", "Atan", "Atanh", "Ceil", "Celu", "Cos",
"Cosh", "Dropout", "Erf", "Exp", "Floor", "HardSigmoid", "HardSwish",
"Identity", "Log", "Round", "Selu", "Sigmoid", "Sin", "Sinh", "Softmax",
"Softplus", "Softsign", "Sqrt", "Tan", "ThresholdedRelu"};
for (const auto& name : simpleLayers)
{
dispatch[name] = &ONNXImporter::parseSimpleLayers;
}
// ai.onnx: opset 10+
dispatch["QuantizeLinear"] = dispatch["DequantizeLinear"] = &ONNXImporter::parseQuantDequant;
dispatch["QLinearConv"] = &ONNXImporter::parseQConv;

6
modules/dnn/src/tensorflow/tf_importer.cpp
View File

@ -3090,10 +3090,8 @@ void TFImporter::populateNet()
{
const tensorflow::NodeDef& layer = net.node(li);
const std::string name = layer.name();
const std::string type = layer.op();
const int ninputs = layer.input_size();
CV_LOG_DEBUG(NULL, "DNN/TF: (" << li << "/" << layersSize << ") Parse layer " << name << " @ " << type << " with " << ninputs << " inputs");
CV_LOG_DEBUG(NULL, "DNN/TF: processing node (" << li << "/" << layersSize << ") with " << layer.input_size() << " inputs: "
<< cv::format("[%s]:(%s)", layer.op().c_str(), layer.name().c_str()));
parseNode(layer);
}