From acedb4a5792b2b7fcd9a88ce2cd3784b1a344fff Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Mon, 16 Oct 2017 19:16:52 +0300 Subject: [PATCH 1/3] dnn: make NMS function public --- modules/dnn/include/opencv2/dnn/dnn.hpp | 13 ++ modules/dnn/include/opencv2/dnn/nms.inl.hpp | 114 ++++++++++++++++++ .../dnn/src/layers/detection_output_layer.cpp | 72 ++--------- modules/dnn/src/nms.cpp | 24 ++++ 4 files changed, 161 insertions(+), 62 deletions(-) create mode 100644 modules/dnn/include/opencv2/dnn/nms.inl.hpp create mode 100644 modules/dnn/src/nms.cpp diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index ff222d7232..d276abf099 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -734,6 +734,19 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN */ CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst); + /** @brief + * @param bboxes + * @param scores + * @param score_threshold + * @param nms_threshold + * @param eta + * @param top_k + * @param indices + */ + CV_EXPORTS_W void NMSBoxes(const std::vector& bboxes, const std::vector& scores, + const float score_threshold, const float nms_threshold, + const float eta, const int top_k, CV_OUT std::vector& indices); + //! @} CV__DNN_EXPERIMENTAL_NS_END diff --git a/modules/dnn/include/opencv2/dnn/nms.inl.hpp b/modules/dnn/include/opencv2/dnn/nms.inl.hpp new file mode 100644 index 0000000000..26183a083a --- /dev/null +++ b/modules/dnn/include/opencv2/dnn/nms.inl.hpp @@ -0,0 +1,114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#ifndef OPENCV_DNN_NMS_INL_HPP +#define OPENCV_DNN_NMS_INL_HPP + +#include + +namespace cv { +namespace dnn { + +namespace +{ + +template +static inline bool SortScorePairDescend(const std::pair& pair1, + const std::pair& pair2) +{ + return pair1.first > pair2.first; +} + +} // namespace + +// Get max scores with corresponding indices. +// scores: a set of scores. +// threshold: only consider scores higher than the threshold. +// top_k: if -1, keep all; otherwise, keep at most top_k. +// score_index_vec: store the sorted (score, index) pair. +inline void GetMaxScoreIndex(const std::vector& scores, const float threshold, const int top_k, + std::vector >& score_index_vec) +{ + CV_DbgAssert(score_index_vec.empty()); + // Generate index score pairs. + for (size_t i = 0; i < scores.size(); ++i) + { + if (scores[i] > threshold) + { + score_index_vec.push_back(std::make_pair(scores[i], i)); + } + } + + // Sort the score pair according to the scores in descending order + std::stable_sort(score_index_vec.begin(), score_index_vec.end(), + SortScorePairDescend); + + // Keep top_k scores if needed. + if (top_k > -1 && top_k < (int)score_index_vec.size()) + { + score_index_vec.resize(top_k); + } +} + +template +struct NMSOverlap +{ + float operator() (const BoxType& a, const BoxType& b); +}; + +template <> +inline float NMSOverlap::operator() (const Rect& a, const Rect& b) +{ + float rectIntersectionArea = (float)(a & b).area(); + return rectIntersectionArea / (a.area() + b.area() - rectIntersectionArea); +} + +// Do non maximum suppression given bboxes and scores. +// Inspired by Piotr Dollar's NMS implementation in EdgeBox. +// https://goo.gl/jV3JYS +// bboxes: a set of bounding boxes. +// scores: a set of corresponding confidences. +// score_threshold: a threshold used to filter detection results. +// nms_threshold: a threshold used in non maximum suppression. +// top_k: if not -1, keep at most top_k picked indices. +// indices: the kept indices of bboxes after nms. +template +inline void NMSFast_(const std::vector& bboxes, + const std::vector& scores, const float score_threshold, + const float nms_threshold, const float eta, const int top_k, + std::vector& indices, NMSOverlap computeOverlap) +{ + CV_Assert(bboxes.size() == scores.size()); + + // Get top_k scores (with corresponding indices). + std::vector > score_index_vec; + GetMaxScoreIndex(scores, score_threshold, top_k, score_index_vec); + + // Do nms. + float adaptive_threshold = nms_threshold; + indices.clear(); + while (score_index_vec.size() != 0) { + const int idx = score_index_vec.front().second; + bool keep = true; + for (int k = 0; k < (int)indices.size() && keep; ++k) { + const int kept_idx = indices[k]; + float overlap = computeOverlap(bboxes[idx], bboxes[kept_idx]); + keep = overlap <= adaptive_threshold; + } + if (keep) + indices.push_back(idx); + score_index_vec.erase(score_index_vec.begin()); + if (keep && eta < 1 && adaptive_threshold > 0.5) { + adaptive_threshold *= eta; + } + } +} + +}// dnn +}// cv + +#endif diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index 505b9c7b74..1312a81b13 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -45,6 +45,7 @@ #include #include #include +#include namespace cv { @@ -619,73 +620,14 @@ public: } } - // Do non maximum suppression given bboxes and scores. - // Inspired by Piotr Dollar's NMS implementation in EdgeBox. - // https://goo.gl/jV3JYS - // bboxes: a set of bounding boxes. - // scores: a set of corresponding confidences. - // score_threshold: a threshold used to filter detection results. - // nms_threshold: a threshold used in non maximum suppression. - // top_k: if not -1, keep at most top_k picked indices. - // indices: the kept indices of bboxes after nms. + + static void ApplyNMSFast(const std::vector& bboxes, const std::vector& scores, const float score_threshold, const float nms_threshold, const float eta, const int top_k, std::vector& indices) { - CV_Assert(bboxes.size() == scores.size()); - - // Get top_k scores (with corresponding indices). - std::vector > score_index_vec; - GetMaxScoreIndex(scores, score_threshold, top_k, score_index_vec); - - // Do nms. - float adaptive_threshold = nms_threshold; - indices.clear(); - while (score_index_vec.size() != 0) { - const int idx = score_index_vec.front().second; - bool keep = true; - for (int k = 0; k < (int)indices.size() && keep; ++k) { - const int kept_idx = indices[k]; - float overlap = JaccardOverlap(bboxes[idx], bboxes[kept_idx]); - keep = overlap <= adaptive_threshold; - } - if (keep) - indices.push_back(idx); - score_index_vec.erase(score_index_vec.begin()); - if (keep && eta < 1 && adaptive_threshold > 0.5) { - adaptive_threshold *= eta; - } - } - } - - // Get max scores with corresponding indices. - // scores: a set of scores. - // threshold: only consider scores higher than the threshold. - // top_k: if -1, keep all; otherwise, keep at most top_k. - // score_index_vec: store the sorted (score, index) pair. - static void GetMaxScoreIndex(const std::vector& scores, const float threshold, const int top_k, - std::vector >& score_index_vec) - { - CV_DbgAssert(score_index_vec.empty()); - // Generate index score pairs. - for (size_t i = 0; i < scores.size(); ++i) - { - if (scores[i] > threshold) - { - score_index_vec.push_back(std::make_pair(scores[i], i)); - } - } - - // Sort the score pair according to the scores in descending order - std::stable_sort(score_index_vec.begin(), score_index_vec.end(), - util::SortScorePairDescend); - - // Keep top_k scores if needed. - if (top_k > -1 && top_k < (int)score_index_vec.size()) - { - score_index_vec.resize(top_k); - } + NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, NMSOverlap()); } // Compute the jaccard (intersection over union IoU) overlap between two bboxes. @@ -733,6 +675,12 @@ public: } }; +template <> +float NMSOverlap::operator() (const caffe::NormalizedBBox& a, const caffe::NormalizedBBox& b) +{ + return DetectionOutputLayerImpl::JaccardOverlap(a, b); +} + const std::string DetectionOutputLayerImpl::_layerName = std::string("DetectionOutput"); Ptr DetectionOutputLayer::create(const LayerParams ¶ms) diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp new file mode 100644 index 0000000000..5f433dba22 --- /dev/null +++ b/modules/dnn/src/nms.cpp @@ -0,0 +1,24 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "precomp.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +void NMSBoxes(const std::vector& bboxes, const std::vector& scores, + const float score_threshold, const float nms_threshold, + const float eta, const int top_k, std::vector& indices) +{ + NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, NMSOverlap()); +} + +}// dnn +}// cv From c704942b8ae217f196e1220d293b17f1d16a9d7c Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 17 Oct 2017 11:24:50 +0300 Subject: [PATCH 2/3] dnn: add a documentation for NMS, fix missing experimantal namespace --- modules/dnn/include/opencv2/dnn/dnn.hpp | 20 +++++++++------- modules/dnn/include/opencv2/dnn/nms.inl.hpp | 24 ++++--------------- .../dnn/src/layers/detection_output_layer.cpp | 18 ++++---------- modules/dnn/src/nms.cpp | 13 ++++++++-- 4 files changed, 32 insertions(+), 43 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index d276abf099..7f1a2b3630 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -734,18 +734,20 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN */ CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst); - /** @brief - * @param bboxes - * @param scores - * @param score_threshold - * @param nms_threshold - * @param eta - * @param top_k - * @param indices + /** @brief Performs non maximum suppression given boxes and corresponding scores. + + * @param bboxes a set of bounding boxes to apply NMS. + * @param scores a set of corresponding confidences. + * @param score_threshold a threshold used to filter boxes by score. + * @param nms_threshold a threshold used in non maximum suppression. + * @param indices the kept indices of bboxes after NMS. + * @param eta a coefficient in adaptive threshold formula: \f$nms\_threshold_{i+1}=eta\cdot nms\_threshold_i\f$. + * @param top_k if `>0`, keep at most @p top_k picked indices. */ CV_EXPORTS_W void NMSBoxes(const std::vector& bboxes, const std::vector& scores, const float score_threshold, const float nms_threshold, - const float eta, const int top_k, CV_OUT std::vector& indices); + CV_OUT std::vector& indices, + const float eta = 1.f, const int top_k = 0); //! @} diff --git a/modules/dnn/include/opencv2/dnn/nms.inl.hpp b/modules/dnn/include/opencv2/dnn/nms.inl.hpp index 26183a083a..89e3adfcf5 100644 --- a/modules/dnn/include/opencv2/dnn/nms.inl.hpp +++ b/modules/dnn/include/opencv2/dnn/nms.inl.hpp @@ -48,25 +48,12 @@ inline void GetMaxScoreIndex(const std::vector& scores, const float thres SortScorePairDescend); // Keep top_k scores if needed. - if (top_k > -1 && top_k < (int)score_index_vec.size()) + if (top_k > 0 && top_k < (int)score_index_vec.size()) { score_index_vec.resize(top_k); } } -template -struct NMSOverlap -{ - float operator() (const BoxType& a, const BoxType& b); -}; - -template <> -inline float NMSOverlap::operator() (const Rect& a, const Rect& b) -{ - float rectIntersectionArea = (float)(a & b).area(); - return rectIntersectionArea / (a.area() + b.area() - rectIntersectionArea); -} - // Do non maximum suppression given bboxes and scores. // Inspired by Piotr Dollar's NMS implementation in EdgeBox. // https://goo.gl/jV3JYS @@ -74,13 +61,13 @@ inline float NMSOverlap::operator() (const Rect& a, const Rect& b) // scores: a set of corresponding confidences. // score_threshold: a threshold used to filter detection results. // nms_threshold: a threshold used in non maximum suppression. -// top_k: if not -1, keep at most top_k picked indices. +// top_k: if not > 0, keep at most top_k picked indices. // indices: the kept indices of bboxes after nms. template inline void NMSFast_(const std::vector& bboxes, const std::vector& scores, const float score_threshold, const float nms_threshold, const float eta, const int top_k, - std::vector& indices, NMSOverlap computeOverlap) + std::vector& indices, float (*computeOverlap)(const BoxType&, const BoxType&)) { CV_Assert(bboxes.size() == scores.size()); @@ -91,8 +78,8 @@ inline void NMSFast_(const std::vector& bboxes, // Do nms. float adaptive_threshold = nms_threshold; indices.clear(); - while (score_index_vec.size() != 0) { - const int idx = score_index_vec.front().second; + for (size_t i = 0; i < score_index_vec.size(); ++i) { + const int idx = score_index_vec[i].second; bool keep = true; for (int k = 0; k < (int)indices.size() && keep; ++k) { const int kept_idx = indices[k]; @@ -101,7 +88,6 @@ inline void NMSFast_(const std::vector& bboxes, } if (keep) indices.push_back(idx); - score_index_vec.erase(score_index_vec.begin()); if (keep && eta < 1 && adaptive_threshold > 0.5) { adaptive_threshold *= eta; } diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index 1312a81b13..970984b341 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -62,6 +62,8 @@ static inline bool SortScorePairDescend(const std::pair& pair1, return pair1.first > pair2.first; } +static inline float caffe_box_overlap(const caffe::NormalizedBBox& a, const caffe::NormalizedBBox& b); + } // namespace class DetectionOutputLayerImpl : public DetectionOutputLayer @@ -309,7 +311,8 @@ public: LabelBBox::const_iterator label_bboxes = decodeBBoxes.find(label); if (label_bboxes == decodeBBoxes.end()) CV_ErrorNoReturn_(cv::Error::StsError, ("Could not find location predictions for label %d", label)); - ApplyNMSFast(label_bboxes->second, scores, _confidenceThreshold, _nmsThreshold, 1.0, _topK, indices[c]); + NMSFast_(label_bboxes->second, scores, _confidenceThreshold, _nmsThreshold, 1.0, _topK, + indices[c], util::caffe_box_overlap); numDetections += indices[c].size(); } if (_keepTopK > -1 && numDetections > (size_t)_keepTopK) @@ -620,16 +623,6 @@ public: } } - - - static void ApplyNMSFast(const std::vector& bboxes, - const std::vector& scores, const float score_threshold, - const float nms_threshold, const float eta, const int top_k, - std::vector& indices) - { - NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, NMSOverlap()); - } - // Compute the jaccard (intersection over union IoU) overlap between two bboxes. template static float JaccardOverlap(const caffe::NormalizedBBox& bbox1, @@ -675,8 +668,7 @@ public: } }; -template <> -float NMSOverlap::operator() (const caffe::NormalizedBBox& a, const caffe::NormalizedBBox& b) +float util::caffe_box_overlap(const caffe::NormalizedBBox& a, const caffe::NormalizedBBox& b) { return DetectionOutputLayerImpl::JaccardOverlap(a, b); } diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp index 5f433dba22..af9e9c855c 100644 --- a/modules/dnn/src/nms.cpp +++ b/modules/dnn/src/nms.cpp @@ -12,13 +12,22 @@ namespace cv { namespace dnn { +CV__DNN_EXPERIMENTAL_NS_BEGIN + +static inline float rectOverlap(const Rect& a, const Rect& b) +{ + return 1.f - static_cast(jaccardDistance(a, b)); +} void NMSBoxes(const std::vector& bboxes, const std::vector& scores, const float score_threshold, const float nms_threshold, - const float eta, const int top_k, std::vector& indices) + std::vector& indices, const float eta, const int top_k) { - NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, NMSOverlap()); + CV_Assert(bboxes.size() == scores.size(), score_threshold >= 0, + nms_threshold >= 0, eta > 0); + NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, rectOverlap); } +CV__DNN_EXPERIMENTAL_NS_END }// dnn }// cv From 7e3e9144de2037bc80b5b90928a5811a7e740775 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 17 Oct 2017 16:43:04 +0300 Subject: [PATCH 3/3] dnn: add an accuracy test for NMS --- .../dnn/src/layers/detection_output_layer.cpp | 2 +- modules/dnn/src/nms.cpp | 2 +- .../{include/opencv2/dnn => src}/nms.inl.hpp | 0 modules/dnn/test/test_nms.cpp | 41 +++++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) rename modules/dnn/{include/opencv2/dnn => src}/nms.inl.hpp (100%) create mode 100644 modules/dnn/test/test_nms.cpp diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index 970984b341..2e381b2e1d 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -45,7 +45,7 @@ #include #include #include -#include +#include "../nms.inl.hpp" namespace cv { diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp index af9e9c855c..f56191f9e0 100644 --- a/modules/dnn/src/nms.cpp +++ b/modules/dnn/src/nms.cpp @@ -6,7 +6,7 @@ // Third party copyrights are property of their respective owners. #include "precomp.hpp" -#include +#include namespace cv { diff --git a/modules/dnn/include/opencv2/dnn/nms.inl.hpp b/modules/dnn/src/nms.inl.hpp similarity index 100% rename from modules/dnn/include/opencv2/dnn/nms.inl.hpp rename to modules/dnn/src/nms.inl.hpp diff --git a/modules/dnn/test/test_nms.cpp b/modules/dnn/test/test_nms.cpp new file mode 100644 index 0000000000..1359a77111 --- /dev/null +++ b/modules/dnn/test/test_nms.cpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "test_precomp.hpp" + +namespace cvtest +{ + +TEST(NMS, Accuracy) +{ + //reference results obtained using tf.image.non_max_suppression with iou_threshold=0.5 + std::string dataPath = findDataFile("dnn/nms_reference.yml"); + FileStorage fs(dataPath, FileStorage::READ); + + std::vector bboxes; + std::vector scores; + std::vector ref_indices; + + fs["boxes"] >> bboxes; + fs["probs"] >> scores; + fs["output"] >> ref_indices; + + const float nms_thresh = .5f; + const float score_thresh = .01f; + std::vector indices; + cv::dnn::NMSBoxes(bboxes, scores, score_thresh, nms_thresh, indices); + + ASSERT_EQ(ref_indices.size(), indices.size()); + + std::sort(indices.begin(), indices.end()); + std::sort(ref_indices.begin(), ref_indices.end()); + + for(size_t i = 0; i < indices.size(); i++) + ASSERT_EQ(indices[i], ref_indices[i]); +} + +}//cvtest