Merge pull request #17570 from HannibalAPE:text_det_recog_demo
[GSoC] High Level API and Samples for Scene Text Detection and Recognition * APIs and samples for scene text detection and recognition * update APIs and tutorial for Text Detection and Recognition * API updates: (1) put decodeType into struct Voc (2) optimize the post-processing of DB * sample update: (1) add transformation into scene_text_spotting.cpp (2) modify text_detection.cpp with API update * update tutorial * simplify text recognition API update tutorial * update impl usage in recognize() and detect() * dnn: refactoring public API of TextRecognitionModel/TextDetectionModel * update provided models update opencv.bib * dnn: adjust text rectangle angle * remove points ordering operation in model.cpp * update gts of DB test in test_model.cpp * dnn: ensure to keep text rectangle angle - avoid 90/180 degree turns * dnn(text): use quadrangle result in TextDetectionModel API * dnn: update Text Detection API (1) keep points' order consistent with (bl, tl, tr, br) in unclip (2) update contourScore with boundingRect
This commit is contained in:
Wenqing Zhang
@@ -0,0 +1,36 @@
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
a
|
||||
b
|
||||
c
|
||||
d
|
||||
e
|
||||
f
|
||||
g
|
||||
h
|
||||
i
|
||||
j
|
||||
k
|
||||
l
|
||||
m
|
||||
n
|
||||
o
|
||||
p
|
||||
q
|
||||
r
|
||||
s
|
||||
t
|
||||
u
|
||||
v
|
||||
w
|
||||
x
|
||||
y
|
||||
z
|
||||
@@ -0,0 +1,94 @@
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
a
|
||||
b
|
||||
c
|
||||
d
|
||||
e
|
||||
f
|
||||
g
|
||||
h
|
||||
i
|
||||
j
|
||||
k
|
||||
l
|
||||
m
|
||||
n
|
||||
o
|
||||
p
|
||||
q
|
||||
r
|
||||
s
|
||||
t
|
||||
u
|
||||
v
|
||||
w
|
||||
x
|
||||
y
|
||||
z
|
||||
A
|
||||
B
|
||||
C
|
||||
D
|
||||
E
|
||||
F
|
||||
G
|
||||
H
|
||||
I
|
||||
J
|
||||
K
|
||||
L
|
||||
M
|
||||
N
|
||||
O
|
||||
P
|
||||
Q
|
||||
R
|
||||
S
|
||||
T
|
||||
U
|
||||
V
|
||||
W
|
||||
X
|
||||
Y
|
||||
Z
|
||||
!
|
||||
"
|
||||
#
|
||||
$
|
||||
%
|
||||
&
|
||||
'
|
||||
(
|
||||
)
|
||||
*
|
||||
+
|
||||
,
|
||||
-
|
||||
.
|
||||
/
|
||||
:
|
||||
;
|
||||
<
|
||||
=
|
||||
>
|
||||
?
|
||||
@
|
||||
[
|
||||
\
|
||||
]
|
||||
^
|
||||
_
|
||||
`
|
||||
{
|
||||
|
|
||||
}
|
||||
~
|
||||
@@ -0,0 +1,151 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <regex>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/dnn/dnn.hpp>
|
||||
|
||||
using namespace cv;
|
||||
using namespace cv::dnn;
|
||||
|
||||
std::string keys =
|
||||
"{ help h | | Print help message. }"
|
||||
"{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }"
|
||||
"{ modelPath mp | | Path to a binary .onnx file contains trained DB detector model. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ inputHeight ih |736| image height of the model input. It should be multiple by 32.}"
|
||||
"{ inputWidth iw |736| image width of the model input. It should be multiple by 32.}"
|
||||
"{ binaryThreshold bt |0.3| Confidence threshold of the binary map. }"
|
||||
"{ polygonThreshold pt |0.5| Confidence threshold of polygons. }"
|
||||
"{ maxCandidate max |200| Max candidates of polygons. }"
|
||||
"{ unclipRatio ratio |2.0| unclip ratio. }"
|
||||
"{ evaluate e |false| false: predict with input images; true: evaluate on benchmarks. }"
|
||||
"{ evalDataPath edp | | Path to benchmarks for evaluation. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}";
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
// Parse arguments
|
||||
CommandLineParser parser(argc, argv, keys);
|
||||
parser.about("Use this script to run the official PyTorch implementation (https://github.com/MhLiao/DB) of "
|
||||
"Real-time Scene Text Detection with Differentiable Binarization (https://arxiv.org/abs/1911.08947)\n"
|
||||
"The current version of this script is a variant of the original network without deformable convolution");
|
||||
if (argc == 1 || parser.has("help"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
float binThresh = parser.get<float>("binaryThreshold");
|
||||
float polyThresh = parser.get<float>("polygonThreshold");
|
||||
uint maxCandidates = parser.get<uint>("maxCandidate");
|
||||
String modelPath = parser.get<String>("modelPath");
|
||||
double unclipRatio = parser.get<double>("unclipRatio");
|
||||
int height = parser.get<int>("inputHeight");
|
||||
int width = parser.get<int>("inputWidth");
|
||||
|
||||
if (!parser.check())
|
||||
{
|
||||
parser.printErrors();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Load the network
|
||||
CV_Assert(!modelPath.empty());
|
||||
TextDetectionModel_DB detector(modelPath);
|
||||
detector.setBinaryThreshold(binThresh)
|
||||
.setPolygonThreshold(polyThresh)
|
||||
.setUnclipRatio(unclipRatio)
|
||||
.setMaxCandidates(maxCandidates);
|
||||
|
||||
double scale = 1.0 / 255.0;
|
||||
Size inputSize = Size(width, height);
|
||||
Scalar mean = Scalar(122.67891434, 116.66876762, 104.00698793);
|
||||
detector.setInputParams(scale, inputSize, mean);
|
||||
|
||||
// Create a window
|
||||
static const std::string winName = "TextDetectionModel";
|
||||
|
||||
if (parser.get<bool>("evaluate")) {
|
||||
// for evaluation
|
||||
String evalDataPath = parser.get<String>("evalDataPath");
|
||||
CV_Assert(!evalDataPath.empty());
|
||||
String testListPath = evalDataPath + "/test_list.txt";
|
||||
std::ifstream testList;
|
||||
testList.open(testListPath);
|
||||
CV_Assert(testList.is_open());
|
||||
|
||||
// Create a window for showing groundtruth
|
||||
static const std::string winNameGT = "GT";
|
||||
|
||||
String testImgPath;
|
||||
while (std::getline(testList, testImgPath)) {
|
||||
String imgPath = evalDataPath + "/test_images/" + testImgPath;
|
||||
std::cout << "Image Path: " << imgPath << std::endl;
|
||||
|
||||
Mat frame = imread(samples::findFile(imgPath), IMREAD_COLOR);
|
||||
CV_Assert(!frame.empty());
|
||||
Mat src = frame.clone();
|
||||
|
||||
// Inference
|
||||
std::vector<std::vector<Point>> results;
|
||||
detector.detect(frame, results);
|
||||
|
||||
polylines(frame, results, true, Scalar(0, 255, 0), 2);
|
||||
imshow(winName, frame);
|
||||
|
||||
// load groundtruth
|
||||
String imgName = testImgPath.substr(0, testImgPath.length() - 4);
|
||||
String gtPath = evalDataPath + "/test_gts/" + imgName + ".txt";
|
||||
// std::cout << gtPath << std::endl;
|
||||
std::ifstream gtFile;
|
||||
gtFile.open(gtPath);
|
||||
CV_Assert(gtFile.is_open());
|
||||
|
||||
std::vector<std::vector<Point>> gts;
|
||||
String gtLine;
|
||||
while (std::getline(gtFile, gtLine)) {
|
||||
size_t splitLoc = gtLine.find_last_of(',');
|
||||
String text = gtLine.substr(splitLoc+1);
|
||||
if ( text == "###\r" || text == "1") {
|
||||
// ignore difficult instances
|
||||
continue;
|
||||
}
|
||||
gtLine = gtLine.substr(0, splitLoc);
|
||||
|
||||
std::regex delimiter(",");
|
||||
std::vector<String> v(std::sregex_token_iterator(gtLine.begin(), gtLine.end(), delimiter, -1),
|
||||
std::sregex_token_iterator());
|
||||
std::vector<int> loc;
|
||||
std::vector<Point> pts;
|
||||
for (auto && s : v) {
|
||||
loc.push_back(atoi(s.c_str()));
|
||||
}
|
||||
for (size_t i = 0; i < loc.size() / 2; i++) {
|
||||
pts.push_back(Point(loc[2 * i], loc[2 * i + 1]));
|
||||
}
|
||||
gts.push_back(pts);
|
||||
}
|
||||
polylines(src, gts, true, Scalar(0, 255, 0), 2);
|
||||
imshow(winNameGT, src);
|
||||
|
||||
waitKey();
|
||||
}
|
||||
} else {
|
||||
// Open an image file
|
||||
CV_Assert(parser.has("inputImage"));
|
||||
Mat frame = imread(samples::findFile(parser.get<String>("inputImage")));
|
||||
CV_Assert(!frame.empty());
|
||||
|
||||
// Detect
|
||||
std::vector<std::vector<Point>> results;
|
||||
detector.detect(frame, results);
|
||||
|
||||
polylines(frame, results, true, Scalar(0, 255, 0), 2);
|
||||
imshow(winName, frame);
|
||||
waitKey();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/dnn/dnn.hpp>
|
||||
|
||||
using namespace cv;
|
||||
using namespace cv::dnn;
|
||||
|
||||
String keys =
|
||||
"{ help h | | Print help message. }"
|
||||
"{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }"
|
||||
"{ modelPath mp | | Path to a binary .onnx file contains trained CRNN text recognition model. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }"
|
||||
"{ evaluate e |false| false: predict with input images; true: evaluate on benchmarks. }"
|
||||
"{ evalDataPath edp | | Path to benchmarks for evaluation. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ vocabularyPath vp | alphabet_36.txt | Path to recognition vocabulary. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}";
|
||||
|
||||
String convertForEval(String &input);
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
// Parse arguments
|
||||
CommandLineParser parser(argc, argv, keys);
|
||||
parser.about("Use this script to run the PyTorch implementation of "
|
||||
"An End-to-End Trainable Neural Network for Image-based SequenceRecognition and Its Application to Scene Text Recognition "
|
||||
"(https://arxiv.org/abs/1507.05717)");
|
||||
if (argc == 1 || parser.has("help"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
String modelPath = parser.get<String>("modelPath");
|
||||
String vocPath = parser.get<String>("vocabularyPath");
|
||||
int imreadRGB = parser.get<int>("RGBInput");
|
||||
|
||||
if (!parser.check())
|
||||
{
|
||||
parser.printErrors();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Load the network
|
||||
CV_Assert(!modelPath.empty());
|
||||
TextRecognitionModel recognizer(modelPath);
|
||||
|
||||
// Load vocabulary
|
||||
CV_Assert(!vocPath.empty());
|
||||
std::ifstream vocFile;
|
||||
vocFile.open(samples::findFile(vocPath));
|
||||
CV_Assert(vocFile.is_open());
|
||||
String vocLine;
|
||||
std::vector<String> vocabulary;
|
||||
while (std::getline(vocFile, vocLine)) {
|
||||
vocabulary.push_back(vocLine);
|
||||
}
|
||||
recognizer.setVocabulary(vocabulary);
|
||||
recognizer.setDecodeType("CTC-greedy");
|
||||
|
||||
// Set parameters
|
||||
double scale = 1.0 / 127.5;
|
||||
Scalar mean = Scalar(127.5, 127.5, 127.5);
|
||||
Size inputSize = Size(100, 32);
|
||||
recognizer.setInputParams(scale, inputSize, mean);
|
||||
|
||||
if (parser.get<bool>("evaluate"))
|
||||
{
|
||||
// For evaluation
|
||||
String evalDataPath = parser.get<String>("evalDataPath");
|
||||
CV_Assert(!evalDataPath.empty());
|
||||
String gtPath = evalDataPath + "/test_gts.txt";
|
||||
std::ifstream evalGts;
|
||||
evalGts.open(gtPath);
|
||||
CV_Assert(evalGts.is_open());
|
||||
|
||||
String gtLine;
|
||||
int cntRight=0, cntAll=0;
|
||||
TickMeter timer;
|
||||
timer.reset();
|
||||
|
||||
while (std::getline(evalGts, gtLine)) {
|
||||
size_t splitLoc = gtLine.find_first_of(' ');
|
||||
String imgPath = evalDataPath + '/' + gtLine.substr(0, splitLoc);
|
||||
String gt = gtLine.substr(splitLoc+1);
|
||||
|
||||
// Inference
|
||||
Mat frame = imread(samples::findFile(imgPath), imreadRGB);
|
||||
CV_Assert(!frame.empty());
|
||||
timer.start();
|
||||
std::string recognitionResult = recognizer.recognize(frame);
|
||||
timer.stop();
|
||||
|
||||
if (gt == convertForEval(recognitionResult))
|
||||
cntRight++;
|
||||
|
||||
cntAll++;
|
||||
}
|
||||
std::cout << "Accuracy(%): " << (double)(cntRight) / (double)(cntAll) << std::endl;
|
||||
std::cout << "Average Inference Time(ms): " << timer.getTimeMilli() / (double)(cntAll) << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create a window
|
||||
static const std::string winName = "Input Cropped Image";
|
||||
|
||||
// Open an image file
|
||||
CV_Assert(parser.has("inputImage"));
|
||||
Mat frame = imread(samples::findFile(parser.get<String>("inputImage")), imreadRGB);
|
||||
CV_Assert(!frame.empty());
|
||||
|
||||
// Recognition
|
||||
std::string recognitionResult = recognizer.recognize(frame);
|
||||
|
||||
imshow(winName, frame);
|
||||
std::cout << "Predition: '" << recognitionResult << "'" << std::endl;
|
||||
waitKey();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Convert the predictions to lower case, and remove other characters.
|
||||
// Only for Evaluation
|
||||
String convertForEval(String & input)
|
||||
{
|
||||
String output;
|
||||
for (uint i = 0; i < input.length(); i++){
|
||||
char ch = input[i];
|
||||
if ((int)ch >= 97 && (int)ch <= 122) {
|
||||
output.push_back(ch);
|
||||
} else if ((int)ch >= 65 && (int)ch <= 90) {
|
||||
output.push_back((char)(ch + 32));
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/dnn/dnn.hpp>
|
||||
|
||||
using namespace cv;
|
||||
using namespace cv::dnn;
|
||||
|
||||
std::string keys =
|
||||
"{ help h | | Print help message. }"
|
||||
"{ inputImage i | | Path to an input image. Skip this argument to capture frames from a camera. }"
|
||||
"{ detModelPath dmp | | Path to a binary .onnx model for detection. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ recModelPath rmp | | Path to a binary .onnx model for recognition. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ inputHeight ih |736| image height of the model input. It should be multiple by 32.}"
|
||||
"{ inputWidth iw |736| image width of the model input. It should be multiple by 32.}"
|
||||
"{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }"
|
||||
"{ binaryThreshold bt |0.3| Confidence threshold of the binary map. }"
|
||||
"{ polygonThreshold pt |0.5| Confidence threshold of polygons. }"
|
||||
"{ maxCandidate max |200| Max candidates of polygons. }"
|
||||
"{ unclipRatio ratio |2.0| unclip ratio. }"
|
||||
"{ vocabularyPath vp | alphabet_36.txt | Path to benchmarks for evaluation. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}";
|
||||
|
||||
void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result);
|
||||
bool sortPts(const Point& p1, const Point& p2);
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
// Parse arguments
|
||||
CommandLineParser parser(argc, argv, keys);
|
||||
parser.about("Use this script to run an end-to-end inference sample of textDetectionModel and textRecognitionModel APIs\n"
|
||||
"Use -h for more information");
|
||||
if (argc == 1 || parser.has("help"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
float binThresh = parser.get<float>("binaryThreshold");
|
||||
float polyThresh = parser.get<float>("polygonThreshold");
|
||||
uint maxCandidates = parser.get<uint>("maxCandidate");
|
||||
String detModelPath = parser.get<String>("detModelPath");
|
||||
String recModelPath = parser.get<String>("recModelPath");
|
||||
String vocPath = parser.get<String>("vocabularyPath");
|
||||
double unclipRatio = parser.get<double>("unclipRatio");
|
||||
int height = parser.get<int>("inputHeight");
|
||||
int width = parser.get<int>("inputWidth");
|
||||
int imreadRGB = parser.get<int>("RGBInput");
|
||||
|
||||
if (!parser.check())
|
||||
{
|
||||
parser.printErrors();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Load networks
|
||||
CV_Assert(!detModelPath.empty());
|
||||
TextDetectionModel_DB detector(detModelPath);
|
||||
detector.setBinaryThreshold(binThresh)
|
||||
.setPolygonThreshold(polyThresh)
|
||||
.setUnclipRatio(unclipRatio)
|
||||
.setMaxCandidates(maxCandidates);
|
||||
|
||||
CV_Assert(!recModelPath.empty());
|
||||
TextRecognitionModel recognizer(recModelPath);
|
||||
|
||||
// Load vocabulary
|
||||
CV_Assert(!vocPath.empty());
|
||||
std::ifstream vocFile;
|
||||
vocFile.open(samples::findFile(vocPath));
|
||||
CV_Assert(vocFile.is_open());
|
||||
String vocLine;
|
||||
std::vector<String> vocabulary;
|
||||
while (std::getline(vocFile, vocLine)) {
|
||||
vocabulary.push_back(vocLine);
|
||||
}
|
||||
recognizer.setVocabulary(vocabulary);
|
||||
recognizer.setDecodeType("CTC-greedy");
|
||||
|
||||
// Parameters for Detection
|
||||
double detScale = 1.0 / 255.0;
|
||||
Size detInputSize = Size(width, height);
|
||||
Scalar detMean = Scalar(122.67891434, 116.66876762, 104.00698793);
|
||||
detector.setInputParams(detScale, detInputSize, detMean);
|
||||
|
||||
// Parameters for Recognition
|
||||
double recScale = 1.0 / 127.5;
|
||||
Scalar recMean = Scalar(127.5);
|
||||
Size recInputSize = Size(100, 32);
|
||||
recognizer.setInputParams(recScale, recInputSize, recMean);
|
||||
|
||||
// Create a window
|
||||
static const std::string winName = "Text_Spotting";
|
||||
|
||||
// Input data
|
||||
Mat frame = imread(samples::findFile(parser.get<String>("inputImage")));
|
||||
std::cout << frame.size << std::endl;
|
||||
|
||||
// Inference
|
||||
std::vector< std::vector<Point> > detResults;
|
||||
detector.detect(frame, detResults);
|
||||
|
||||
if (detResults.size() > 0) {
|
||||
// Text Recognition
|
||||
Mat recInput;
|
||||
if (!imreadRGB) {
|
||||
cvtColor(frame, recInput, cv::COLOR_BGR2GRAY);
|
||||
} else {
|
||||
recInput = frame;
|
||||
}
|
||||
std::vector< std::vector<Point> > contours;
|
||||
for (uint i = 0; i < detResults.size(); i++)
|
||||
{
|
||||
const auto& quadrangle = detResults[i];
|
||||
CV_CheckEQ(quadrangle.size(), (size_t)4, "");
|
||||
|
||||
contours.emplace_back(quadrangle);
|
||||
|
||||
std::vector<Point2f> quadrangle_2f;
|
||||
for (int j = 0; j < 4; j++)
|
||||
quadrangle_2f.emplace_back(quadrangle[j]);
|
||||
|
||||
// Transform and Crop
|
||||
Mat cropped;
|
||||
fourPointsTransform(recInput, &quadrangle_2f[0], cropped);
|
||||
|
||||
std::string recognitionResult = recognizer.recognize(cropped);
|
||||
std::cout << i << ": '" << recognitionResult << "'" << std::endl;
|
||||
|
||||
putText(frame, recognitionResult, quadrangle[3], FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 0, 255), 2);
|
||||
}
|
||||
polylines(frame, contours, true, Scalar(0, 255, 0), 2);
|
||||
} else {
|
||||
std::cout << "No Text Detected." << std::endl;
|
||||
}
|
||||
imshow(winName, frame);
|
||||
waitKey();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result)
|
||||
{
|
||||
const Size outputSize = Size(100, 32);
|
||||
|
||||
Point2f targetVertices[4] = {
|
||||
Point(0, outputSize.height - 1),
|
||||
Point(0, 0),
|
||||
Point(outputSize.width - 1, 0),
|
||||
Point(outputSize.width - 1, outputSize.height - 1)
|
||||
};
|
||||
Mat rotationMatrix = getPerspectiveTransform(vertices, targetVertices);
|
||||
|
||||
warpPerspective(frame, result, rotationMatrix, outputSize);
|
||||
|
||||
#if 0
|
||||
imshow("roi", result);
|
||||
waitKey();
|
||||
#endif
|
||||
}
|
||||
|
||||
bool sortPts(const Point& p1, const Point& p2)
|
||||
{
|
||||
return p1.x < p2.x;
|
||||
}
|
||||
+92
-177
@@ -2,22 +2,23 @@
|
||||
Text detection model: https://github.com/argman/EAST
|
||||
Download link: https://www.dropbox.com/s/r2ingd0l3zt8hxs/frozen_east_text_detection.tar.gz?dl=1
|
||||
|
||||
CRNN Text recognition model taken from here: https://github.com/meijieru/crnn.pytorch
|
||||
Text recognition models can be downloaded directly here:
|
||||
Download link: https://drive.google.com/drive/folders/1cTbQ3nuZG-EKWak6emD_s8_hHXWz7lAr?usp=sharing
|
||||
and doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown
|
||||
|
||||
How to convert from pb to onnx:
|
||||
Using classes from here: https://github.com/meijieru/crnn.pytorch/blob/master/models/crnn.py
|
||||
|
||||
More converted onnx text recognition models can be downloaded directly here:
|
||||
Download link: https://drive.google.com/drive/folders/1cTbQ3nuZG-EKWak6emD_s8_hHXWz7lAr?usp=sharing
|
||||
And these models taken from here:https://github.com/clovaai/deep-text-recognition-benchmark
|
||||
|
||||
import torch
|
||||
from models.crnn import CRNN
|
||||
|
||||
model = CRNN(32, 1, 37, 256)
|
||||
model.load_state_dict(torch.load('crnn.pth'))
|
||||
dummy_input = torch.randn(1, 1, 32, 100)
|
||||
torch.onnx.export(model, dummy_input, "crnn.onnx", verbose=True)
|
||||
|
||||
For more information, please refer to doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown and doc/tutorials/dnn/dnn_OCR/dnn_OCR.markdown
|
||||
*/
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
@@ -27,21 +28,20 @@ using namespace cv;
|
||||
using namespace cv::dnn;
|
||||
|
||||
const char* keys =
|
||||
"{ help h | | Print help message. }"
|
||||
"{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera.}"
|
||||
"{ model m | | Path to a binary .pb file contains trained detector network.}"
|
||||
"{ ocr | | Path to a binary .pb or .onnx file contains trained recognition network.}"
|
||||
"{ width | 320 | Preprocess input image by resizing to a specific width. It should be multiple by 32. }"
|
||||
"{ height | 320 | Preprocess input image by resizing to a specific height. It should be multiple by 32. }"
|
||||
"{ thr | 0.5 | Confidence threshold. }"
|
||||
"{ nms | 0.4 | Non-maximum suppression threshold. }";
|
||||
"{ help h | | Print help message. }"
|
||||
"{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera.}"
|
||||
"{ detModel dmp | | Path to a binary .pb file contains trained detector network.}"
|
||||
"{ width | 320 | Preprocess input image by resizing to a specific width. It should be multiple by 32. }"
|
||||
"{ height | 320 | Preprocess input image by resizing to a specific height. It should be multiple by 32. }"
|
||||
"{ thr | 0.5 | Confidence threshold. }"
|
||||
"{ nms | 0.4 | Non-maximum suppression threshold. }"
|
||||
"{ recModel rmp | | Path to a binary .onnx file contains trained CRNN text recognition model. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}"
|
||||
"{ RGBInput rgb |0| 0: imread with flags=IMREAD_GRAYSCALE; 1: imread with flags=IMREAD_COLOR. }"
|
||||
"{ vocabularyPath vp | alphabet_36.txt | Path to benchmarks for evaluation. "
|
||||
"Download links are provided in doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown}";
|
||||
|
||||
void decodeBoundingBoxes(const Mat& scores, const Mat& geometry, float scoreThresh,
|
||||
std::vector<RotatedRect>& detections, std::vector<float>& confidences);
|
||||
|
||||
void fourPointsTransform(const Mat& frame, Point2f vertices[4], Mat& result);
|
||||
|
||||
void decodeText(const Mat& scores, std::string& text);
|
||||
void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result);
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
@@ -57,10 +57,12 @@ int main(int argc, char** argv)
|
||||
|
||||
float confThreshold = parser.get<float>("thr");
|
||||
float nmsThreshold = parser.get<float>("nms");
|
||||
int inpWidth = parser.get<int>("width");
|
||||
int inpHeight = parser.get<int>("height");
|
||||
String modelDecoder = parser.get<String>("model");
|
||||
String modelRecognition = parser.get<String>("ocr");
|
||||
int width = parser.get<int>("width");
|
||||
int height = parser.get<int>("height");
|
||||
int imreadRGB = parser.get<int>("RGBInput");
|
||||
String detModelPath = parser.get<String>("detModel");
|
||||
String recModelPath = parser.get<String>("recModel");
|
||||
String vocPath = parser.get<String>("vocabularyPath");
|
||||
|
||||
if (!parser.check())
|
||||
{
|
||||
@@ -68,14 +70,39 @@ int main(int argc, char** argv)
|
||||
return 1;
|
||||
}
|
||||
|
||||
CV_Assert(!modelDecoder.empty());
|
||||
|
||||
// Load networks.
|
||||
Net detector = readNet(modelDecoder);
|
||||
Net recognizer;
|
||||
CV_Assert(!detModelPath.empty() && !recModelPath.empty());
|
||||
TextDetectionModel_EAST detector(detModelPath);
|
||||
detector.setConfidenceThreshold(confThreshold)
|
||||
.setNMSThreshold(nmsThreshold);
|
||||
|
||||
if (!modelRecognition.empty())
|
||||
recognizer = readNet(modelRecognition);
|
||||
TextRecognitionModel recognizer(recModelPath);
|
||||
|
||||
// Load vocabulary
|
||||
CV_Assert(!vocPath.empty());
|
||||
std::ifstream vocFile;
|
||||
vocFile.open(samples::findFile(vocPath));
|
||||
CV_Assert(vocFile.is_open());
|
||||
String vocLine;
|
||||
std::vector<String> vocabulary;
|
||||
while (std::getline(vocFile, vocLine)) {
|
||||
vocabulary.push_back(vocLine);
|
||||
}
|
||||
recognizer.setVocabulary(vocabulary);
|
||||
recognizer.setDecodeType("CTC-greedy");
|
||||
|
||||
// Parameters for Recognition
|
||||
double recScale = 1.0 / 127.5;
|
||||
Scalar recMean = Scalar(127.5, 127.5, 127.5);
|
||||
Size recInputSize = Size(100, 32);
|
||||
recognizer.setInputParams(recScale, recInputSize, recMean);
|
||||
|
||||
// Parameters for Detection
|
||||
double detScale = 1.0;
|
||||
Size detInputSize = Size(width, height);
|
||||
Scalar detMean = Scalar(123.68, 116.78, 103.94);
|
||||
bool swapRB = true;
|
||||
detector.setInputParams(detScale, detInputSize, detMean, swapRB);
|
||||
|
||||
// Open a video file or an image file or a camera stream.
|
||||
VideoCapture cap;
|
||||
@@ -83,15 +110,8 @@ int main(int argc, char** argv)
|
||||
CV_Assert(openSuccess);
|
||||
|
||||
static const std::string kWinName = "EAST: An Efficient and Accurate Scene Text Detector";
|
||||
namedWindow(kWinName, WINDOW_NORMAL);
|
||||
|
||||
std::vector<Mat> outs;
|
||||
std::vector<String> outNames(2);
|
||||
outNames[0] = "feature_fusion/Conv_7/Sigmoid";
|
||||
outNames[1] = "feature_fusion/concat_3";
|
||||
|
||||
Mat frame, blob;
|
||||
TickMeter tickMeter;
|
||||
Mat frame;
|
||||
while (waitKey(1) < 0)
|
||||
{
|
||||
cap >> frame;
|
||||
@@ -101,162 +121,57 @@ int main(int argc, char** argv)
|
||||
break;
|
||||
}
|
||||
|
||||
blobFromImage(frame, blob, 1.0, Size(inpWidth, inpHeight), Scalar(123.68, 116.78, 103.94), true, false);
|
||||
detector.setInput(blob);
|
||||
tickMeter.start();
|
||||
detector.forward(outs, outNames);
|
||||
tickMeter.stop();
|
||||
std::cout << frame.size << std::endl;
|
||||
|
||||
Mat scores = outs[0];
|
||||
Mat geometry = outs[1];
|
||||
// Detection
|
||||
std::vector< std::vector<Point> > detResults;
|
||||
detector.detect(frame, detResults);
|
||||
|
||||
// Decode predicted bounding boxes.
|
||||
std::vector<RotatedRect> boxes;
|
||||
std::vector<float> confidences;
|
||||
decodeBoundingBoxes(scores, geometry, confThreshold, boxes, confidences);
|
||||
|
||||
// Apply non-maximum suppression procedure.
|
||||
std::vector<int> indices;
|
||||
NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
|
||||
|
||||
Point2f ratio((float)frame.cols / inpWidth, (float)frame.rows / inpHeight);
|
||||
|
||||
// Render text.
|
||||
for (size_t i = 0; i < indices.size(); ++i)
|
||||
{
|
||||
RotatedRect& box = boxes[indices[i]];
|
||||
|
||||
Point2f vertices[4];
|
||||
box.points(vertices);
|
||||
|
||||
for (int j = 0; j < 4; ++j)
|
||||
{
|
||||
vertices[j].x *= ratio.x;
|
||||
vertices[j].y *= ratio.y;
|
||||
if (detResults.size() > 0) {
|
||||
// Text Recognition
|
||||
Mat recInput;
|
||||
if (!imreadRGB) {
|
||||
cvtColor(frame, recInput, cv::COLOR_BGR2GRAY);
|
||||
} else {
|
||||
recInput = frame;
|
||||
}
|
||||
|
||||
if (!modelRecognition.empty())
|
||||
std::vector< std::vector<Point> > contours;
|
||||
for (uint i = 0; i < detResults.size(); i++)
|
||||
{
|
||||
const auto& quadrangle = detResults[i];
|
||||
CV_CheckEQ(quadrangle.size(), (size_t)4, "");
|
||||
|
||||
contours.emplace_back(quadrangle);
|
||||
|
||||
std::vector<Point2f> quadrangle_2f;
|
||||
for (int j = 0; j < 4; j++)
|
||||
quadrangle_2f.emplace_back(quadrangle[j]);
|
||||
|
||||
Mat cropped;
|
||||
fourPointsTransform(frame, vertices, cropped);
|
||||
fourPointsTransform(recInput, &quadrangle_2f[0], cropped);
|
||||
|
||||
cvtColor(cropped, cropped, cv::COLOR_BGR2GRAY);
|
||||
std::string recognitionResult = recognizer.recognize(cropped);
|
||||
std::cout << i << ": '" << recognitionResult << "'" << std::endl;
|
||||
|
||||
Mat blobCrop = blobFromImage(cropped, 1.0/127.5, Size(), Scalar::all(127.5));
|
||||
recognizer.setInput(blobCrop);
|
||||
|
||||
tickMeter.start();
|
||||
Mat result = recognizer.forward();
|
||||
tickMeter.stop();
|
||||
|
||||
std::string wordRecognized = "";
|
||||
decodeText(result, wordRecognized);
|
||||
putText(frame, wordRecognized, vertices[1], FONT_HERSHEY_SIMPLEX, 1.5, Scalar(0, 0, 255));
|
||||
putText(frame, recognitionResult, quadrangle[3], FONT_HERSHEY_SIMPLEX, 1.5, Scalar(0, 0, 255), 2);
|
||||
}
|
||||
|
||||
for (int j = 0; j < 4; ++j)
|
||||
line(frame, vertices[j], vertices[(j + 1) % 4], Scalar(0, 255, 0), 1);
|
||||
polylines(frame, contours, true, Scalar(0, 255, 0), 2);
|
||||
}
|
||||
|
||||
// Put efficiency information.
|
||||
std::string label = format("Inference time: %.2f ms", tickMeter.getTimeMilli());
|
||||
putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0));
|
||||
|
||||
imshow(kWinName, frame);
|
||||
|
||||
tickMeter.reset();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void decodeBoundingBoxes(const Mat& scores, const Mat& geometry, float scoreThresh,
|
||||
std::vector<RotatedRect>& detections, std::vector<float>& confidences)
|
||||
{
|
||||
detections.clear();
|
||||
CV_Assert(scores.dims == 4); CV_Assert(geometry.dims == 4); CV_Assert(scores.size[0] == 1);
|
||||
CV_Assert(geometry.size[0] == 1); CV_Assert(scores.size[1] == 1); CV_Assert(geometry.size[1] == 5);
|
||||
CV_Assert(scores.size[2] == geometry.size[2]); CV_Assert(scores.size[3] == geometry.size[3]);
|
||||
|
||||
const int height = scores.size[2];
|
||||
const int width = scores.size[3];
|
||||
for (int y = 0; y < height; ++y)
|
||||
{
|
||||
const float* scoresData = scores.ptr<float>(0, 0, y);
|
||||
const float* x0_data = geometry.ptr<float>(0, 0, y);
|
||||
const float* x1_data = geometry.ptr<float>(0, 1, y);
|
||||
const float* x2_data = geometry.ptr<float>(0, 2, y);
|
||||
const float* x3_data = geometry.ptr<float>(0, 3, y);
|
||||
const float* anglesData = geometry.ptr<float>(0, 4, y);
|
||||
for (int x = 0; x < width; ++x)
|
||||
{
|
||||
float score = scoresData[x];
|
||||
if (score < scoreThresh)
|
||||
continue;
|
||||
|
||||
// Decode a prediction.
|
||||
// Multiple by 4 because feature maps are 4 time less than input image.
|
||||
float offsetX = x * 4.0f, offsetY = y * 4.0f;
|
||||
float angle = anglesData[x];
|
||||
float cosA = std::cos(angle);
|
||||
float sinA = std::sin(angle);
|
||||
float h = x0_data[x] + x2_data[x];
|
||||
float w = x1_data[x] + x3_data[x];
|
||||
|
||||
Point2f offset(offsetX + cosA * x1_data[x] + sinA * x2_data[x],
|
||||
offsetY - sinA * x1_data[x] + cosA * x2_data[x]);
|
||||
Point2f p1 = Point2f(-sinA * h, -cosA * h) + offset;
|
||||
Point2f p3 = Point2f(-cosA * w, sinA * w) + offset;
|
||||
RotatedRect r(0.5f * (p1 + p3), Size2f(w, h), -angle * 180.0f / (float)CV_PI);
|
||||
detections.push_back(r);
|
||||
confidences.push_back(score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void fourPointsTransform(const Mat& frame, Point2f vertices[4], Mat& result)
|
||||
void fourPointsTransform(const Mat& frame, const Point2f vertices[], Mat& result)
|
||||
{
|
||||
const Size outputSize = Size(100, 32);
|
||||
|
||||
Point2f targetVertices[4] = {Point(0, outputSize.height - 1),
|
||||
Point(0, 0), Point(outputSize.width - 1, 0),
|
||||
Point(outputSize.width - 1, outputSize.height - 1),
|
||||
};
|
||||
Point2f targetVertices[4] = {
|
||||
Point(0, outputSize.height - 1),
|
||||
Point(0, 0), Point(outputSize.width - 1, 0),
|
||||
Point(outputSize.width - 1, outputSize.height - 1)
|
||||
};
|
||||
Mat rotationMatrix = getPerspectiveTransform(vertices, targetVertices);
|
||||
|
||||
warpPerspective(frame, result, rotationMatrix, outputSize);
|
||||
}
|
||||
|
||||
void decodeText(const Mat& scores, std::string& text)
|
||||
{
|
||||
static const std::string alphabet = "0123456789abcdefghijklmnopqrstuvwxyz";
|
||||
Mat scoresMat = scores.reshape(1, scores.size[0]);
|
||||
|
||||
std::vector<char> elements;
|
||||
elements.reserve(scores.size[0]);
|
||||
|
||||
for (int rowIndex = 0; rowIndex < scoresMat.rows; ++rowIndex)
|
||||
{
|
||||
Point p;
|
||||
minMaxLoc(scoresMat.row(rowIndex), 0, 0, 0, &p);
|
||||
if (p.x > 0 && static_cast<size_t>(p.x) <= alphabet.size())
|
||||
{
|
||||
elements.push_back(alphabet[p.x - 1]);
|
||||
}
|
||||
else
|
||||
{
|
||||
elements.push_back('-');
|
||||
}
|
||||
}
|
||||
|
||||
if (elements.size() > 0 && elements[0] != '-')
|
||||
text += elements[0];
|
||||
|
||||
for (size_t elementIndex = 1; elementIndex < elements.size(); ++elementIndex)
|
||||
{
|
||||
if (elementIndex > 0 && elements[elementIndex] != '-' &&
|
||||
elements[elementIndex - 1] != elements[elementIndex])
|
||||
{
|
||||
text += elements[elementIndex];
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user