From 2ac6337342286faeac002f4c5d695d602c6e71d1 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sat, 18 Oct 2014 20:13:30 -0400 Subject: [PATCH] Added logic to LineFinder to find both lines for multiline plates --- .../textdetection/characteranalysis.cpp | 60 ++++--- .../textdetection/characteranalysis.h | 2 +- src/openalpr/textdetection/linefinder.cpp | 158 ++++++++---------- src/openalpr/textdetection/linefinder.h | 19 ++- src/openalpr/textdetection/textline.cpp | 45 ++++- src/openalpr/textdetection/textline.h | 4 + 6 files changed, 164 insertions(+), 124 deletions(-) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 6d9ca4f..f849b72 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -130,18 +130,17 @@ void CharacterAnalysis::analyze() } LineFinder lf(pipeline_data); - lf.findLines(pipeline_data->crop_gray, bestContours); + vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); - vector linePolygon; - //vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); - if (linePolygon.size() > 0) + for (uint i = 0; i < linePolygons.size(); i++) { + vector linePolygon = linePolygons[i]; + + cout << "Polygon: " << linePolygon[0] << " - " << linePolygon[1] << " - " << linePolygon[2] << " - " << linePolygon[3] << endl; LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); - - filterBetweenLines(bestThreshold, bestContours, linePolygon); vector textArea = getCharArea(topLine, bottomLine); @@ -149,7 +148,17 @@ void CharacterAnalysis::analyze() pipeline_data->textLines.push_back(textLine); } + + filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines); + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + cout << "Test1" << endl; + Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold); + + cout << "Test2" << endl; + drawAndWait(&debugImage); + } this->thresholdsInverted = isPlateInverted(); } @@ -510,29 +519,22 @@ void CharacterAnalysis::filterByParentContour( TextContours& textContours) } -void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector outerPolygon ) +void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector textLines ) { static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; - if (outerPolygon.size() == 0) + if (textLines.size() == 0) return; vector validPoints; - // Figure out the line height - LineSegment topLine(outerPolygon[0].x, outerPolygon[0].y, outerPolygon[1].x, outerPolygon[1].y); - LineSegment bottomLine(outerPolygon[3].x, outerPolygon[3].y, outerPolygon[2].x, outerPolygon[2].y); - - float x = ((float) img.cols) / 2; - Point midpoint = Point(x, bottomLine.getPointAt(x)); - Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); - float lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); // Create a white mask for the area inside the polygon Mat outerMask = Mat::zeros(img.size(), CV_8U); - fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); + for (uint i = 0; i < textLines.size(); i++) + fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255)); // For each contour, determine if enough of it is between the lines to qualify for (uint i = 0; i < textContours.size(); i++) @@ -579,19 +581,23 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, } // Get the absolute distance from the top and bottom lines - Point closestTopPoint = topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); - Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); - float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); - - float maxDistance = lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; - - if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + for (uint i = 0; i < textLines.size(); i++) { - textContours.goodIndices[i] = true; - } + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); + float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); + + float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + + if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + { + textContours.goodIndices[i] = true; + } + } + } } diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 1eb8eab..441444c 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -64,7 +64,7 @@ class CharacterAnalysis std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); std::vector getBestVotedLines(cv::Mat img, TextContours textContours); - void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector outerPolygon ); + void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index 401537a..cf6ea87 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -33,35 +33,51 @@ LineFinder::LineFinder(PipelineData* pipeline_data) { LineFinder::~LineFinder() { } -vector LineFinder::findLines(Mat image, const TextContours contours) +vector > LineFinder::findLines(Mat image, const TextContours contours) { - vector linesFound; + vector > linesFound; cvtColor(image, image, CV_GRAY2BGR); - vector boxes = this->getBoundingBoxes(contours); - vector tops = this->getCharTops(boxes); - vector bottoms = this->getCharBottoms(boxes); + vector charPoints; - for (uint i = 0; i < tops.size(); i++) + for (uint i = 0; i < contours.contours.size(); i++) { - circle(image, tops[i], 1, Scalar(255, 0, 0), 2); - circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2); + if (contours.goodIndices[i] == false) + continue; + + charPoints.push_back( CharPointInfo(contours.contours[i], i) ); } - drawAndWait(&image); + vector bestLine = getBestLine(contours, charPoints); - vector bestLine = getBestLine(contours, tops, bottoms); + if (bestLine.size() > 0) + linesFound.push_back(bestLine); if (pipeline_data->isMultiline) { // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. - } - - for (uint i = 0; i < contours.goodIndices.size(); i++) - { + // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it. + vector remainingPoints; + for (uint i = 0; i < charPoints.size(); i++) + { + Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U); + fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255)); + + float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); + + if (percentInside < .85) + { + remainingPoints.push_back(charPoints[i]); + } + } + + vector nextBestLine = getBestLine(contours, remainingPoints); + + if (nextBestLine.size() > 0) + linesFound.push_back(nextBestLine); } @@ -69,64 +85,15 @@ vector LineFinder::findLines(Mat image, const TextContours contours) } -vector LineFinder::getBoundingBoxes(const TextContours contours) { - - vector boxes; - for (uint i = 0; i < contours.goodIndices.size(); i++) - { - if (contours.goodIndices[i] == false) - continue; - - Rect bRect = cv::boundingRect( Mat(contours.contours[i]) ); - - boxes.push_back(bRect); - } - - return boxes; -} - - -vector LineFinder::getCharTops(vector boxes) { - - vector tops; - for (uint i = 0; i < boxes.size(); i++) - { - int x = boxes[i].x + (boxes[i].width / 2); - int y = boxes[i].y; - - tops.push_back(Point(x, y)); - } - - return tops; -} - -vector LineFinder::getCharBottoms(vector boxes) { - - vector bottoms; - for (uint i = 0; i < boxes.size(); i++) - { - int x = boxes[i].x + (boxes[i].width / 2); - int y = boxes[i].y + boxes[i].height; - - bottoms.push_back(Point(x, y)); - } - - return bottoms; -} - - - - - // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector LineFinder::getBestLine(const TextContours contours, vector tops, vector bottoms) +vector LineFinder::getBestLine(const TextContours contours, vector charPoints) { vector bestStripe; // Find the best fit line segment that is parallel with the most char segments - if (tops.size() <= 1) + if (charPoints.size() <= 1) { // Maybe do something about this later, for now let's just ignore } @@ -135,31 +102,26 @@ vector LineFinder::getBestLine(const TextContours contours, vector vector topLines; vector bottomLines; // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < tops.size() - 1; i++) + for (uint i = 0; i < charPoints.size() - 1; i++) { - for (uint k = i+1; k < tops.size(); k++) + for (uint k = i+1; k < charPoints.size(); k++) { - Point topLeft, topRight; - Point bottomLeft, bottomRight; - if (tops[i].x < tops[k].x) + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) { - topLeft = tops[i]; - topRight = tops[k]; - bottomLeft = bottoms[i]; - bottomRight = bottoms[k]; + leftCPIndex = i; + rightCPIndex = k; } else { - topLeft = tops[k]; - topRight = tops[i]; - bottomLeft = bottoms[k]; - bottomRight = bottoms[i]; + leftCPIndex = k; + rightCPIndex = i; } - LineSegment top(topLeft, topRight); - LineSegment bottom(bottomLeft, bottomRight); + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); // Only allow lines that have a sane angle if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && @@ -184,15 +146,15 @@ vector LineFinder::getBestLine(const TextContours contours, vector float SCORING_MAX_THRESHOLD = 1.03; int curScore = 0; - for (uint charidx = 0; charidx < tops.size(); charidx++) + for (uint charidx = 0; charidx < charPoints.size(); charidx++) { - float topYPos = topLines[i].getPointAt(tops[charidx].x); - float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x); + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); - float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD; - float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD; - float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD; - float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD; + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; if ( (topYPos >= minTop && topYPos <= maxTop) && (botYPos >= minBot && botYPos <= maxBot)) { @@ -202,7 +164,7 @@ vector LineFinder::getBestLine(const TextContours contours, vector //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; //drawAndWait(&tempImg); } - + // Tie goes to the one with longer line segments if ((curScore > bestScore) || (curScore == bestScore && topLines[i].length > bestScoreDistance)) @@ -240,3 +202,23 @@ vector LineFinder::getBestLine(const TextContours contours, vector return bestStripe; } + +CharPointInfo::CharPointInfo(vector contour, int index) { + + + this->contourIndex = index; + + this->boundingBox = cv::boundingRect( Mat(contour) ); + + + int x = boundingBox.x + (boundingBox.width / 2); + int y = boundingBox.y; + + this->top = Point(x, y); + + x = boundingBox.x + (boundingBox.width / 2); + y = boundingBox.y + boundingBox.height; + + this->bottom = Point(x,y); + +} diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h index aa4db39..7b7398f 100644 --- a/src/openalpr/textdetection/linefinder.h +++ b/src/openalpr/textdetection/linefinder.h @@ -27,21 +27,28 @@ #include "textline.h" #include "pipeline_data.h" +class CharPointInfo +{ +public: + CharPointInfo(std::vector contour, int index); + + cv::Rect boundingBox; + cv::Point top; + cv::Point bottom; + int contourIndex; + +}; class LineFinder { public: LineFinder(PipelineData* pipeline_data); virtual ~LineFinder(); - std::vector findLines(cv::Mat image, const TextContours contours); + std::vector > findLines(cv::Mat image, const TextContours contours); private: PipelineData* pipeline_data; - std::vector getBoundingBoxes(const TextContours contours); - std::vector getCharTops(std::vector boxes); - std::vector getCharBottoms(std::vector boxes); - - std::vector getBestLine(const TextContours contours, std::vector tops, std::vector bottoms); + std::vector getBestLine(const TextContours contours, std::vector charPoints); }; #endif /* OPENALPR_LINEFINDER_H */ diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index 0e70d4d..a41ce8c 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -18,13 +18,20 @@ */ +#include + #include "textline.h" +using namespace cv; + TextLine::TextLine(std::vector textArea, std::vector linePolygon) { if (textArea.size() > 0) { - this->textArea = textArea; - this->linePolygon = linePolygon; + for (uint i = 0; i < textArea.size(); i++) + this->textArea.push_back(textArea[i]); + + for (uint i = 0; i < linePolygon.size(); i++) + this->linePolygon.push_back(linePolygon[i]); this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); @@ -33,9 +40,43 @@ TextLine::TextLine(std::vector textArea, std::vector lineP this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); + + + float x = ((float) linePolygon[1].x) / 2; + Point midpoint = Point(x, bottomLine.getPointAt(x)); + Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); + this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); } } TextLine::~TextLine() { } + +cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { + cv::Mat debugImage(baseImage.size(), baseImage.type()); + + baseImage.copyTo(debugImage); + + cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR); + + + fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); + + drawAndWait(&debugImage); + fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); + + drawAndWait(&debugImage); + line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); + line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); + + drawAndWait(&debugImage); + line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); + + drawAndWait(&debugImage); + + return debugImage; +} diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index cf5fc86..5403beb 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -22,6 +22,7 @@ #define OPENALPR_TEXTLINE_H #include "utility.h" +#include "opencv2/imgproc/imgproc.hpp" class TextLine { public: @@ -38,6 +39,9 @@ public: LineSegment charBoxLeft; LineSegment charBoxRight; + float lineHeight; + + cv::Mat drawDebugImage(cv::Mat baseImage); private: };