From d2bfebe443c58090cfaf29ce62e36b530d494c7c Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 15:54:59 -0400 Subject: [PATCH] Added multiline support in plate corner detector --- src/openalpr/platecorners.cpp | 218 +++++++++++++---- src/openalpr/platecorners.h | 38 ++- .../segmentation/charactersegmenter.cpp | 1 + .../textdetection/characteranalysis.cpp | 205 ++-------------- src/openalpr/textdetection/linefinder.cpp | 231 ++++++++++-------- 5 files changed, 369 insertions(+), 324 deletions(-) diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index d4e1d0c..76fda4d 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -22,7 +22,8 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) +PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) : + tlc(pipelineData) { this->pipelineData = pipelineData; @@ -35,12 +36,7 @@ PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* this->bestHorizontalScore = 9999999999999; this->bestVerticalScore = 9999999999999; - Point topPoint = pipelineData->textLines[0].topLine.midpoint(); - Point bottomPoint = pipelineData->textLines[0].bottomLine.closestPointOnSegmentTo(topPoint); - this->charHeight = distanceBetweenPoints(topPoint, bottomPoint); - - this->charAngle = angleBetweenPoints(pipelineData->textLines[0].textArea[0], pipelineData->textLines[0].textArea[1]); } PlateCorners::~PlateCorners() @@ -86,8 +82,12 @@ vector PlateCorners::findPlateCorners() Mat imgCorners = Mat(inputImage.size(), inputImage.type()); inputImage.copyTo(imgCorners); - for (int i = 0; i < 4; i++) - circle(imgCorners, pipelineData->textLines[0].textArea[i], 2, Scalar(0, 0, 0)); + + for (uint linenum = 0; linenum < pipelineData->textLines.size(); linenum++) + { + for (int i = 0; i < 4; i++) + circle(imgCorners, pipelineData->textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + } line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); @@ -128,8 +128,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) LineSegment left; LineSegment right; + float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; - float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -137,12 +138,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (v1 == NO_LINE && v2 == NO_LINE) { //return; - Point centerTop = pipelineData->textLines[0].charBoxTop.midpoint(); - Point centerBottom = pipelineData->textLines[0].charBoxBottom.midpoint(); - LineSegment centerLine = LineSegment(centerBottom.x, centerBottom.y, centerTop.x, centerTop.y); - left = centerLine.getParallelLine(idealPixelWidth / 2); - right = centerLine.getParallelLine(-1 * idealPixelWidth / 2 ); + left = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2); + right = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; confidenceDiff += 2; @@ -172,12 +170,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is to the left of our license plate letters - if (left.isPointBelowLine(pipelineData->textLines[0].charBoxLeft.midpoint()) == false) - return; - - // Make sure this line is to the right of our license plate letters - if (right.isPointBelowLine(pipelineData->textLines[0].charBoxRight.midpoint())) + // Make sure that the left and right lines are to the left and right of our text + // area + if (tlc.isLeftOfText(left) < 1 || tlc.isLeftOfText(right) > -1) return; ///////////////////////////////////////////////////////////////////////// @@ -202,7 +197,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) // Score angle difference from detected character box ///////////////////////////////////////////////////////////////////////// - float perpendicularCharAngle = charAngle - 90; + float perpendicularCharAngle = tlc.charAngle - 90; float charanglediff = abs(perpendicularCharAngle - left.angle) + abs(perpendicularCharAngle - right.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -211,8 +206,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) // SCORE the shape wrt character position and height relative to position ////////////////////////////////////////////////////////////////////////// - Point leftMidLinePoint = left.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxLeft.midpoint()); - Point rightMidLinePoint = right.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxRight.midpoint()); + Point leftMidLinePoint = left.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); + Point rightMidLinePoint = right.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); @@ -224,7 +219,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealwidth " << idealPixelWidth << endl; cout << "xx xx Score: v1,v2= " << v1 << "," << v2 << endl; cout << "xx xx Score: Left= " << left.str() << endl; @@ -278,7 +273,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) LineSegment bottom; float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; - float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -286,12 +281,10 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (h1 == NO_LINE && h2 == NO_LINE) { // return; - Point centerLeft = pipelineData->textLines[0].charBoxLeft.midpoint(); - Point centerRight = pipelineData->textLines[0].charBoxRight.midpoint(); - LineSegment centerLine = LineSegment(centerLeft.x, centerLeft.y, centerRight.x, centerRight.y); - top = centerLine.getParallelLine(idealPixelHeight / 2); - bottom = centerLine.getParallelLine(-1 * idealPixelHeight / 2 ); + + top = tlc.centerHorizontalLine.getParallelLine(idealPixelHeight / 2); + bottom = tlc.centerHorizontalLine.getParallelLine(-1 * idealPixelHeight / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; confidenceDiff += 2; @@ -321,14 +314,11 @@ void PlateCorners::scoreHorizontals(int h1, int h2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is above our license plate letters - if (top.isPointBelowLine(pipelineData->textLines[0].charBoxTop.midpoint()) == false) + // Make sure that the top and bottom lines are above and below + // the text area + if (tlc.isAboveText(top) < 1 || tlc.isAboveText(bottom) > -1) return; - - // Make sure this line is below our license plate letters - if (bottom.isPointBelowLine(pipelineData->textLines[0].charBoxBottom.midpoint())) - return; - + // We now have 4 possible lines. Let's put them to the test and score them... ///////////////////////////////////////////////////////////////////////// @@ -351,7 +341,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // Get the height difference - float heightRatio = charHeight / plateHeightPx; + float heightRatio = tlc.charHeight / plateHeightPx; float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); //if (leftRatio < MIN_CHAR_HEIGHT_RATIO || leftRatio > MAX_CHAR_HEIGHT_RATIO || rightRatio < MIN_CHAR_HEIGHT_RATIO || rightRatio > MAX_CHAR_HEIGHT_RATIO) float heightRatioDiff = abs(heightRatio - idealHeightRatio); @@ -372,7 +362,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle ////////////////////////////////////////////////////////////////////////// - Point charAreaMidPoint = pipelineData->textLines[0].charBoxLeft.midpoint(); + Point charAreaMidPoint = tlc.centerVerticalLine.midpoint(); Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); @@ -394,7 +384,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE: the shape for angles matching the character region ////////////////////////////////////////////////////////////// - float charanglediff = abs(charAngle - top.angle) + abs(charAngle - bottom.angle); + float charanglediff = abs(tlc.charAngle - top.angle) + abs(tlc.charAngle - bottom.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -407,7 +397,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealHeight " << idealPixelHeight << endl; cout << "xx xx Score: h1,h2= " << h1 << "," << h2 << endl; cout << "xx xx Score: Top= " << top.str() << endl; @@ -447,3 +437,149 @@ void PlateCorners::scoreHorizontals(int h1, int h2) bestBottom = LineSegment(bottom.p1.x, bottom.p1.y, bottom.p2.x, bottom.p2.y); } } + +TextLineCollection::TextLineCollection(PipelineData* pipelineData) { + + this->pipelineData = pipelineData; + + charHeight = 0; + charAngle = 0; + for (uint i = 0; i < pipelineData->textLines.size(); i++) + { + charHeight += pipelineData->textLines[i].lineHeight; + charAngle += pipelineData->textLines[i].angle; + + } + charHeight = charHeight / pipelineData->textLines.size(); + charAngle = charAngle / pipelineData->textLines.size(); + + this->topCharArea = pipelineData->textLines[0].charBoxTop; + this->bottomCharArea = pipelineData->textLines[0].charBoxBottom; + for (uint i = 1; i < pipelineData->textLines.size(); i++) + { + + if (this->topCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxTop.midpoint()) == false) + this->topCharArea = pipelineData->textLines[i].charBoxTop; + + if (this->bottomCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxBottom.midpoint())) + this->bottomCharArea = pipelineData->textLines[i].charBoxBottom; + + } + + longerSegment = this->bottomCharArea; + shorterSegment = this->topCharArea; + if (this->topCharArea.length > this->bottomCharArea.length) + { + longerSegment = this->topCharArea; + shorterSegment = this->bottomCharArea; + } + + findCenterHorizontal(); + findCenterVertical(); + // Center Vertical Line + + Mat debugImage = Mat::zeros(pipelineData->crop_gray.size(), CV_8U); + line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); + line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); + + drawAndWait(&debugImage); +} + +// Returns 1 for above, 0 for within, and -1 for below +int TextLineCollection::isAboveText(LineSegment line) { + // Test four points (left and right corner of top and bottom line) + + Point topLeft = line.closestPointOnSegmentTo(topCharArea.p1); + Point topRight = line.closestPointOnSegmentTo(topCharArea.p2); + + bool lineIsBelowTop = topCharArea.isPointBelowLine(topLeft) || topCharArea.isPointBelowLine(topRight); + + if (!lineIsBelowTop) + return 1; + + Point bottomLeft = line.closestPointOnSegmentTo(bottomCharArea.p1); + Point bottomRight = line.closestPointOnSegmentTo(bottomCharArea.p2); + + bool lineIsBelowBottom = bottomCharArea.isPointBelowLine(bottomLeft) && + bottomCharArea.isPointBelowLine(bottomRight); + + if (lineIsBelowBottom) + return -1; + + return 0; + +} + +// Returns 1 for left, 0 for within, and -1 for to the right +int TextLineCollection::isLeftOfText(LineSegment line) { + + LineSegment leftSide = LineSegment(bottomCharArea.p1, topCharArea.p1); + + Point topLeft = line.closestPointOnSegmentTo(leftSide.p2); + Point bottomLeft = line.closestPointOnSegmentTo(leftSide.p1); + + bool lineIsAboveLeft = (!leftSide.isPointBelowLine(topLeft)) && (!leftSide.isPointBelowLine(bottomLeft)); + + if (lineIsAboveLeft) + return 1; + + LineSegment rightSide = LineSegment(bottomCharArea.p2, topCharArea.p2); + + Point topRight = line.closestPointOnSegmentTo(rightSide.p2); + Point bottomRight = line.closestPointOnSegmentTo(rightSide.p1); + + + bool lineIsBelowRight = rightSide.isPointBelowLine(topRight) && rightSide.isPointBelowLine(bottomRight); + + if (lineIsBelowRight) + return -1; + + return 0; +} + +void TextLineCollection::findCenterHorizontal() { + // To find the center horizontal line: + // Find the longer of the lines (if multiline) + // Get the nearest point on the bottom-most line for the + // left and right + + + + Point leftP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p1); + Point leftP2 = longerSegment.p1; + LineSegment left = LineSegment(leftP1, leftP2); + + Point leftMidpoint = left.midpoint(); + + + + Point rightP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p2); + Point rightP2 = longerSegment.p2; + LineSegment right = LineSegment(rightP1, rightP2); + + Point rightMidpoint = right.midpoint(); + + this->centerHorizontalLine = LineSegment(leftMidpoint, rightMidpoint); + +} + +void TextLineCollection::findCenterVertical() { + // To find the center vertical line: + // Choose the longest line (if multiline) + // Get the midpoint + // Draw a line up/down using the closest point on the bottom line + + + Point p1 = longerSegment.midpoint(); + + Point p2 = shorterSegment.closestPointOnSegmentTo(p1); + + // Draw bottom to top + if (p1.y < p2.y) + this->centerVerticalLine = LineSegment(p1, p2); + else + this->centerVerticalLine = LineSegment(p2, p1); +} + + + diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index d6a91ac..5fc06df 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -43,11 +43,43 @@ #define SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT 0.05 +class TextLineCollection +{ +public: + TextLineCollection(PipelineData* pipelineData); + + int isLeftOfText(LineSegment line); + int isAboveText(LineSegment line); + + LineSegment centerHorizontalLine; + LineSegment centerVerticalLine; + + float charHeight; + float charAngle; + + + +private: + PipelineData* pipelineData; + + LineSegment topCharArea; + LineSegment bottomCharArea; + + LineSegment longerSegment; + LineSegment shorterSegment; + + cv::Mat textMask; + + void findCenterHorizontal(); + void findCenterVertical(); +}; + class PlateCorners { public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData); + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) ; + virtual ~PlateCorners(); std::vector findPlateCorners(); @@ -58,9 +90,9 @@ class PlateCorners PipelineData* pipelineData; cv::Mat inputImage; - float charHeight; - float charAngle; + TextLineCollection tlc; + float bestHorizontalScore; float bestVerticalScore; LineSegment bestTop; diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 1b83628..3ca2e5c 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -40,6 +40,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); + cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; if (pipeline_data->plate_inverted) bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index f849b72..6d2c0a7 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "characteranalysis.h" #include "linefinder.h" @@ -128,7 +130,7 @@ void CharacterAnalysis::analyze() displayImage(config, "Matching Contours", img_contours); } - + LineFinder lf(pipeline_data); vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); @@ -149,6 +151,8 @@ void CharacterAnalysis::analyze() pipeline_data->textLines.push_back(textLine); } + cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; + filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines); for (uint i = 0; i < pipeline_data->textLines.size(); i++) @@ -160,7 +164,10 @@ void CharacterAnalysis::analyze() drawAndWait(&debugImage); } + cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; + this->thresholdsInverted = isPlateInverted(); + cout << "Plate inverted: " << this->thresholdsInverted << endl; } @@ -189,161 +196,6 @@ Mat CharacterAnalysis::getCharacterMask() return charMask; } -// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector CharacterAnalysis::getBestVotedLines(Mat img, TextContours textContours) -{ - //if (this->debug) - // cout << "CharacterAnalysis::getBestVotedLines" << endl; - - vector bestStripe; - - vector charRegions; - - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i]) - charRegions.push_back(boundingRect(textContours.contours[i])); - } - - // Find the best fit line segment that is parallel with the most char segments - if (charRegions.size() <= 1) - { - // Maybe do something about this later, for now let's just ignore - } - else - { - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charRegions.size() - 1; i++) - { - for (uint k = i+1; k < charRegions.size(); k++) - { - //Mat tempImg; - //result.copyTo(tempImg); - - Rect* leftRect; - Rect* rightRect; - if (charRegions[i].x < charRegions[k].x) - { - leftRect = &charRegions[i]; - rightRect = &charRegions[k]; - } - else - { - leftRect = &charRegions[k]; - rightRect = &charRegions[i]; - } - - //rectangle(tempImg, *leftRect, Scalar(0, 255, 0), 2); - //rectangle(tempImg, *rightRect, Scalar(255, 255, 255), 2); - - int x1, y1, x2, y2; - - if (leftRect->y > rightRect->y) // Rising line, use the top left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - else // falling line, use the top right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - y1 = leftRect->y; - y2 = rightRect->y; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - topLines.push_back(LineSegment(x1, y1, x2, y2)); - - if (leftRect->y > rightRect->y) // Rising line, use the bottom right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - else // falling line, use the bottom left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - y1 = leftRect->y + leftRect->height; - y2 = rightRect->y + leftRect->height; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - bottomLines.push_back(LineSegment(x1, y1, x2, y2)); - - } - } - - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charRegions.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charRegions[charidx].x); - float botYPos = bottomLines[i].getPointAt(charRegions[charidx].x); - - float minTop = charRegions[charidx].y * SCORING_MIN_THRESHOLD; - float maxTop = charRegions[charidx].y * SCORING_MAX_THRESHOLD; - float minBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MIN_THRESHOLD; - float maxBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (this->config->debugCharAnalysis) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - //Mat tempImg; - //result.copyTo(tempImg); - //cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - //cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - //displayImage(config, "Lines", tempImg); - } - - //winningLines.push_back(topLines[bestScoreIndex]); - //winningLines.push_back(bottomLines[bestScoreIndex]); - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(img.cols, topLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomRight = Point(img.cols, bottomLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); - } - - return bestStripe; -} void CharacterAnalysis::filter(Mat img, TextContours& textContours) { @@ -374,8 +226,6 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - //vector lines = getBestVotedLines(img, textContours); - //this->filterBetweenLines(img, textContours, lines); int segmentCount = textContours.getGoodIndicesCount(); @@ -554,48 +404,43 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) { // Not enough area is inside the lines. + if (config->debugCharAnalysis) + cout << "Rejecting due to insufficient area" << endl; continue; } + textContours.goodIndices[i] = true; // now check to make sure that the top and bottom of the contour are near enough to the lines // First get the high and low point for the contour // Remember that origin is top-left, so the top Y values are actually closer to 0. - int highPointIndex = 0; - int highPointValue = 999999999; - int lowPointIndex = 0; - int lowPointValue = 0; - for (uint cidx = 0; cidx < textContours.contours[i].size(); cidx++) - { - if (textContours.contours[i][cidx].y < highPointValue) - { - highPointIndex = cidx; - highPointValue = textContours.contours[i][cidx].y; - } - if (textContours.contours[i][cidx].y > lowPointValue) - { - lowPointIndex = cidx; - lowPointValue = textContours.contours[i][cidx].y; - } - } + Rect brect = boundingRect(textContours.contours[i]); + int xmiddle = brect.x + (brect.width / 2); + Point topMiddle = Point(xmiddle, brect.y); + Point botMiddle = Point(xmiddle, brect.y+brect.height); // Get the absolute distance from the top and bottom lines for (uint i = 0; i < textLines.size(); i++) { - Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); - Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(topMiddle); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(botMiddle); - float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); + float absTopDistance = distanceBetweenPoints(closestTopPoint, topMiddle); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, botMiddle); float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + cout << "Distances: " << absTopDistance << " : " << maxDistance << " - " << absBottomDistance << " : " << maxDistance << endl; if (absTopDistance < maxDistance && absBottomDistance < maxDistance) { textContours.goodIndices[i] = true; } + else if (config->debugCharAnalysis) + { + cout << "Rejecting due to top/bottom points that are out of range" << endl; + } } } @@ -663,6 +508,8 @@ void CharacterAnalysis::filterByOuterMask(TextContours& textContours) bool CharacterAnalysis::isPlateInverted() { Mat charMask = getCharacterMask(); + + drawAndWait(&charMask); Scalar meanVal = mean(bestThreshold, charMask)[0]; diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index cf6ea87..6e81abc 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -35,8 +35,9 @@ LineFinder::~LineFinder() { vector > LineFinder::findLines(Mat image, const TextContours contours) { - vector > linesFound; + const float MIN_AREA_TO_IGNORE = 0.65; + vector > linesFound; cvtColor(image, image, CV_GRAY2BGR); @@ -68,7 +69,7 @@ vector > LineFinder::findLines(Mat image, const TextContours conto float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); - if (percentInside < .85) + if (percentInside < MIN_AREA_TO_IGNORE) { remainingPoints.push_back(charPoints[i]); } @@ -88,118 +89,146 @@ vector > LineFinder::findLines(Mat image, const TextContours conto // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width vector LineFinder::getBestLine(const TextContours contours, vector charPoints) { - - vector bestStripe; - + // Find the best fit line segment that is parallel with the most char segments if (charPoints.size() <= 1) { // Maybe do something about this later, for now let's just ignore + return bestStripe; } - else + + + vector charheights; + for (uint i = 0; i < charPoints.size(); i++) + charheights.push_back(charPoints[i].boundingBox.height); + float medianCharHeight = median(charheights.data(), charheights.size()); + + + + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < charPoints.size() - 1; i++) { - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charPoints.size() - 1; i++) + for (uint k = i+1; k < charPoints.size(); k++) { - for (uint k = i+1; k < charPoints.size(); k++) - { - - int leftCPIndex, rightCPIndex; - if (charPoints[i].top.x < charPoints[k].top.x) - { - leftCPIndex = i; - rightCPIndex = k; - } - else - { - leftCPIndex = k; - rightCPIndex = i; - } - - - LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); - LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); - - // Only allow lines that have a sane angle - if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && - abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) - { - topLines.push_back(top); - bottomLines.push_back(bottom); - } - + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) + { + leftCPIndex = i; + rightCPIndex = k; + } + else + { + leftCPIndex = k; + rightCPIndex = i; + } + + + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); + + + // Only allow lines that have a sane angle +// if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && +// abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) +// { +// topLines.push_back(top); +// bottomLines.push_back(bottom); +// } + + LineSegment parallelBot = top.getParallelLine(medianCharHeight * -1); + LineSegment parallelTop = bottom.getParallelLine(medianCharHeight); + + // Only allow lines that have a sane angle + if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(parallelBot.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(top); + bottomLines.push_back(parallelBot); + } + + // Only allow lines that have a sane angle + if (abs(parallelTop.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(parallelTop); + bottomLines.push_back(bottom); } } - - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charPoints.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); - float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); - - float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; - float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; - float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; - float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (true) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); - cvtColor(tempImg, tempImg, CV_GRAY2BGR); - - cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - drawAndWait(&tempImg); - } - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); } + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < charPoints.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); + + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (bestScore < 0) + return bestStripe; + + if (true) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + drawAndWait(&tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); + + return bestStripe; }