diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 04fc165..2cdaaa4 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -27,6 +27,7 @@ set(lpr_source_files textdetection/platemask.cpp textdetection/textcontours.cpp textdetection/textline.cpp + textdetection/linefinder.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index d44bfff..6d9ca4f 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -18,6 +18,7 @@ */ #include "characteranalysis.h" +#include "linefinder.h" using namespace cv; using namespace std; @@ -101,8 +102,6 @@ void CharacterAnalysis::analyze() int bestFitIndex = -1; for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - //vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); - //charSegments.push_back(goodIndices); int segmentCount = allTextContours[i].getGoodIndicesCount(); @@ -125,31 +124,16 @@ void CharacterAnalysis::analyze() if (this->config->debugCharAnalysis) { - Mat img_contours(bestThreshold.size(), CV_8U); - bestThreshold.copyTo(img_contours); - cvtColor(img_contours, img_contours, CV_GRAY2RGB); - - vector > allowedContours; - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestContours.goodIndices[i]) - allowedContours.push_back(bestContours.contours[i]); - } - - drawContours(img_contours, bestContours.contours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 - - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 + Mat img_contours = bestContours.drawDebugImage(bestThreshold); displayImage(config, "Matching Contours", img_contours); } - vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); + LineFinder lf(pipeline_data); + lf.findLines(pipeline_data->crop_gray, bestContours); + + vector linePolygon; + //vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); if (linePolygon.size() > 0) { @@ -370,7 +354,7 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - + int goodIndices = textContours.getGoodIndicesCount(); if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; @@ -381,8 +365,8 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - vector lines = getBestVotedLines(img, textContours); - this->filterBetweenLines(img, textContours, lines); + //vector lines = getBestVotedLines(img, textContours); + //this->filterBetweenLines(img, textContours, lines); int segmentCount = textContours.getGoodIndicesCount(); @@ -415,11 +399,13 @@ void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeigh float minWidth = mr.height * 0.2; //Crop image - //Mat auxRoi(img, mr); + + //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl; if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) { float charAspect= (float)mr.width/(float)mr.height; + //cout << " -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl; if (abs(charAspect - idealAspect) < aspecttolerance) textContours.goodIndices[i] = true; } @@ -545,7 +531,7 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, // Create a white mask for the area inside the polygon Mat outerMask = Mat::zeros(img.size(), CV_8U); - Mat innerArea(img.size(), CV_8U); + fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); // For each contour, determine if enough of it is between the lines to qualify @@ -556,35 +542,14 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, textContours.goodIndices[i] = false; // Set it to not included unless it proves - innerArea.setTo(Scalar(0,0,0)); + float percentInsideMask = getContourAreaPercentInsideMask(outerMask, + textContours.contours, + textContours.hierarchy, + (int) i); - drawContours(innerArea, textContours.contours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - textContours.hierarchy, - 0 - ); - - bitwise_and(innerArea, outerMask, innerArea); - - vector > tempContours; - findContours(innerArea, tempContours, - CV_RETR_EXTERNAL, // retrieve the external contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours ); - - double totalArea = contourArea(textContours.contours[i]); - double areaBetweenLines = 0; - - for (uint tempContourIdx = 0; tempContourIdx < tempContours.size(); tempContourIdx++) - { - areaBetweenLines += contourArea(tempContours[tempContourIdx]); - } - - - if (areaBetweenLines / totalArea < MIN_AREA_PERCENT_WITHIN_LINES) + + if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) { // Not enough area is inside the lines. continue; diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index fd26ea8..1eb8eab 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -26,6 +26,7 @@ #include "pipeline_data.h" #include "textcontours.h" #include "platemask.h" +#include "linefinder.h" class CharacterAnalysis { @@ -40,7 +41,6 @@ class CharacterAnalysis TextContours bestContours; bool thresholdsInverted; - bool isTwoLine; std::vector allTextContours; @@ -68,7 +68,7 @@ class CharacterAnalysis bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - + }; #endif // OPENALPR_CHARACTERANALYSIS_H diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp new file mode 100644 index 0000000..db30e43 --- /dev/null +++ b/src/openalpr/textdetection/linefinder.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include + +#include "linefinder.h" +#include "utility.h" +#include "pipeline_data.h" + +using namespace std; +using namespace cv; + +LineFinder::LineFinder(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; +} + +LineFinder::~LineFinder() { +} + +vector LineFinder::findLines(Mat image, const TextContours contours) +{ + vector linesFound; + + + cvtColor(image, image, CV_GRAY2BGR); + vector boxes = this->getBoundingBoxes(contours); + + vector tops = this->getCharTops(boxes); + vector bottoms = this->getCharBottoms(boxes); + + for (uint i = 0; i < tops.size(); i++) + { + circle(image, tops[i], 1, Scalar(255, 0, 0), 2); + circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2); + } + + drawAndWait(&image); + + vector bestLine = getBestLine(contours, tops, bottoms); + + if (pipeline_data->isMultiline) + { + // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. + } + + for (uint i = 0; i < contours.goodIndices.size(); i++) + { + + } + + + return linesFound; +} + + +vector LineFinder::getBoundingBoxes(const TextContours contours) { + + vector boxes; + for (uint i = 0; i < contours.goodIndices.size(); i++) + { + if (contours.goodIndices[i] == false) + continue; + + Rect bRect = cv::boundingRect( Mat(contours.contours[i]) ); + + boxes.push_back(bRect); + } + + return boxes; +} + + +vector LineFinder::getCharTops(vector boxes) { + + vector tops; + for (uint i = 0; i < boxes.size(); i++) + { + int x = boxes[i].x + (boxes[i].width / 2); + int y = boxes[i].y; + + tops.push_back(Point(x, y)); + } + + return tops; +} + +vector LineFinder::getCharBottoms(vector boxes) { + + vector bottoms; + for (uint i = 0; i < boxes.size(); i++) + { + int x = boxes[i].x + (boxes[i].width / 2); + int y = boxes[i].y + boxes[i].height; + + bottoms.push_back(Point(x, y)); + } + + return bottoms; +} + + + + + +// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width +vector LineFinder::getBestLine(const TextContours contours, vector tops, vector bottoms) +{ + + + vector bestStripe; + + // Find the best fit line segment that is parallel with the most char segments + if (tops.size() <= 1) + { + // Maybe do something about this later, for now let's just ignore + } + else + { + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < tops.size() - 1; i++) + { + for (uint k = i+1; k < tops.size(); k++) + { + + Point topLeft, topRight; + Point bottomLeft, bottomRight; + if (tops[i].x < tops[k].x) + { + topLeft = tops[i]; + topRight = tops[k]; + bottomLeft = bottoms[i]; + bottomRight = bottoms[k]; + } + else + { + topLeft = tops[k]; + topRight = tops[i]; + bottomLeft = bottoms[k]; + bottomRight = bottoms[i]; + } + + topLines.push_back(LineSegment(topLeft, topRight)); + bottomLines.push_back(LineSegment(bottomLeft, bottomRight)); + + } + } + + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < tops.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(tops[charidx].x); + float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x); + + float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD; + float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD; + float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD; + float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (true) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + drawAndWait(&tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); + } + + return bestStripe; +} diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h new file mode 100644 index 0000000..aa4db39 --- /dev/null +++ b/src/openalpr/textdetection/linefinder.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +// This class finds lines of text given an array of contours + +#ifndef OPENALPR_LINEFINDER_H +#define OPENALPR_LINEFINDER_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "textcontours.h" +#include "textline.h" +#include "pipeline_data.h" + + +class LineFinder { +public: + LineFinder(PipelineData* pipeline_data); + virtual ~LineFinder(); + + std::vector findLines(cv::Mat image, const TextContours contours); +private: + PipelineData* pipeline_data; + + std::vector getBoundingBoxes(const TextContours contours); + std::vector getCharTops(std::vector boxes); + std::vector getCharBottoms(std::vector boxes); + + std::vector getBestLine(const TextContours contours, std::vector tops, std::vector bottoms); +}; + +#endif /* OPENALPR_LINEFINDER_H */ + diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp index 01a2272..2b7dd2d 100644 --- a/src/openalpr/textdetection/textcontours.cpp +++ b/src/openalpr/textdetection/textcontours.cpp @@ -50,6 +50,9 @@ void TextContours::load(cv::Mat threshold) { for (uint i = 0; i < contours.size(); i++) goodIndices.push_back(true); + + this->width = threshold.cols; + this->height = threshold.rows; } @@ -95,4 +98,41 @@ void TextContours::setIndices(std::vector newIndices) { assert("Invalid set operation on indices"); } -} \ No newline at end of file +} + +Mat TextContours::drawDebugImage() { + + Mat img_contours = Mat::zeros(Size(width, height), CV_8U); + + return drawDebugImage(img_contours); +} + +Mat TextContours::drawDebugImage(Mat baseImage) { + Mat img_contours(baseImage.size(), CV_8U); + baseImage.copyTo(img_contours); + + cvtColor(img_contours, img_contours, CV_GRAY2RGB); + + vector > allowedContours; + for (uint i = 0; i < this->contours.size(); i++) + { + if (this->goodIndices[i]) + allowedContours.push_back(this->contours[i]); + } + + drawContours(img_contours, this->contours, + -1, // draw all contours + cv::Scalar(255,0,0), // in blue + 1); // with a thickness of 1 + + drawContours(img_contours, allowedContours, + -1, // draw all contours + cv::Scalar(0,255,0), // in green + 1); // with a thickness of 1 + + + return img_contours; +} + + + diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h index 428a767..cf6d6bb 100644 --- a/src/openalpr/textdetection/textcontours.h +++ b/src/openalpr/textdetection/textcontours.h @@ -31,6 +31,9 @@ public: void load(cv::Mat threshold); + int width; + int height; + std::vector goodIndices; std::vector > contours; std::vector hierarchy; @@ -41,6 +44,9 @@ public: std::vector getIndicesCopy(); void setIndices(std::vector newIndices); + cv::Mat drawDebugImage(); + cv::Mat drawDebugImage(cv::Mat baseImage); + private: