From 5b1e7429ed5e0a172e888d387e8f7bfcd2da02c0 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 2 Jul 2014 18:26:08 -0400 Subject: [PATCH] Added confidence to plate lines -- using them in plate corner detection tweaked plate corner detection defaults --- src/openalpr/platecorners.cpp | 78 +++++++++++++++++++++++++++-------- src/openalpr/platecorners.h | 9 ++-- src/openalpr/platelines.cpp | 28 +++++++++---- src/openalpr/platelines.h | 12 ++++-- 4 files changed, 94 insertions(+), 33 deletions(-) diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index bdec822..e497192 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -132,6 +132,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) float charHeightToPlateWidthRatio = config->plateWidthMM / config->charHeightMM; float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + if (v1 == NO_LINE && v2 == NO_LINE) { //return; @@ -142,25 +145,33 @@ void PlateCorners::scoreVerticals(int v1, int v2) left = centerLine.getParallelLine(idealPixelWidth / 2); right = centerLine.getParallelLine(-1 * idealPixelWidth / 2 ); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; + confidenceDiff += 2; } else if (v1 != NO_LINE && v2 != NO_LINE) { - left = this->plateLines->verticalLines[v1]; - right = this->plateLines->verticalLines[v2]; + left = this->plateLines->verticalLines[v1].line; + right = this->plateLines->verticalLines[v2].line; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); } else if (v1 == NO_LINE && v2 != NO_LINE) { - right = this->plateLines->verticalLines[v2]; + right = this->plateLines->verticalLines[v2].line; left = right.getParallelLine(idealPixelWidth); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); } else if (v1 != NO_LINE && v2 == NO_LINE) { - left = this->plateLines->verticalLines[v1]; + left = this->plateLines->verticalLines[v1].line; right = left.getParallelLine(-1 * idealPixelWidth); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); } + + score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + score += missingSegmentPenalty; // Make sure this line is to the left of our license plate letters if (left.isPointBelowLine(charRegion->getCharBoxLeft().midpoint()) == false) @@ -197,7 +208,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); - score += plateDistance * SCORING_VERTICALDISTANCE_WEIGHT; + score += plateDistance * SCORING_DISTANCE_WEIGHT_VERTICAL; if (score < this->bestVerticalScore) { @@ -211,7 +222,13 @@ void PlateCorners::scoreVerticals(int v1, int v2) cout << "xx xx Score: Left= " << left.str() << endl; cout << "xx xx Score: Right= " << right.str() << endl; + cout << "Vertical breakdown Score:" << endl; + + cout << " -- Missing Segment Score: " << missingSegmentPenalty << " -- Weight (1.0)" << endl; + scorecomponent = missingSegmentPenalty ; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Boxiness Score: " << verticalAngleDiff << " -- Weight (" << SCORING_BOXINESS_WEIGHT << ")" << endl; scorecomponent = verticalAngleDiff * SCORING_BOXINESS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; @@ -220,10 +237,14 @@ void PlateCorners::scoreVerticals(int v1, int v2) scorecomponent = distanceFromEdge * SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; - cout << " -- Distance Score: " << plateDistance << " -- Weight (" << SCORING_VERTICALDISTANCE_WEIGHT << ")" << endl; - scorecomponent = plateDistance * SCORING_VERTICALDISTANCE_WEIGHT; + cout << " -- Distance Score: " << plateDistance << " -- Weight (" << SCORING_DISTANCE_WEIGHT_VERTICAL << ")" << endl; + scorecomponent = plateDistance * SCORING_DISTANCE_WEIGHT_VERTICAL; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; - + + cout << " -- Plate line confidence Score: " << confidenceDiff << " -- Weight (" << SCORING_LINE_CONFIDENCE_WEIGHT << ")" << endl; + scorecomponent = confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Score: " << score << endl; } @@ -247,6 +268,9 @@ void PlateCorners::scoreHorizontals(int h1, int h2) float charHeightToPlateHeightRatio = config->plateHeightMM / config->charHeightMM; float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + if (h1 == NO_LINE && h2 == NO_LINE) { // return; @@ -257,25 +281,33 @@ void PlateCorners::scoreHorizontals(int h1, int h2) top = centerLine.getParallelLine(idealPixelHeight / 2); bottom = centerLine.getParallelLine(-1 * idealPixelHeight / 2 ); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; + confidenceDiff += 2; } else if (h1 != NO_LINE && h2 != NO_LINE) { - top = this->plateLines->horizontalLines[h1]; - bottom = this->plateLines->horizontalLines[h2]; + top = this->plateLines->horizontalLines[h1].line; + bottom = this->plateLines->horizontalLines[h2].line; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); } else if (h1 == NO_LINE && h2 != NO_LINE) { - bottom = this->plateLines->horizontalLines[h2]; + bottom = this->plateLines->horizontalLines[h2].line; top = bottom.getParallelLine(idealPixelHeight); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); } else if (h1 != NO_LINE && h2 == NO_LINE) { - top = this->plateLines->horizontalLines[h1]; + top = this->plateLines->horizontalLines[h1].line; bottom = top.getParallelLine(-1 * idealPixelHeight); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); } + + score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + score += missingSegmentPenalty; // Make sure this line is above our license plate letters if (top.isPointBelowLine(charRegion->getCharBoxTop().midpoint()) == false) @@ -369,7 +401,13 @@ void PlateCorners::scoreHorizontals(int h1, int h2) cout << "xx xx Score: Top= " << top.str() << endl; cout << "xx xx Score: Bottom= " << bottom.str() << endl; + cout << "Horizontal breakdown Score:" << endl; + + cout << " -- Missing Segment Score: " << missingSegmentPenalty << " -- Weight (1.0)" << endl; + scorecomponent = missingSegmentPenalty ; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Boxiness Score: " << horizontalAngleDiff << " -- Weight (" << SCORING_BOXINESS_WEIGHT << ")" << endl; scorecomponent = horizontalAngleDiff * SCORING_BOXINESS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; @@ -386,6 +424,10 @@ void PlateCorners::scoreHorizontals(int h1, int h2) scorecomponent = charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Plate line confidence Score: " << confidenceDiff << " -- Weight (" << SCORING_LINE_CONFIDENCE_WEIGHT << ")" << endl; + scorecomponent = confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Score: " << score << endl; } this->bestHorizontalScore = score; diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index fe9d32e..c0bf693 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -30,13 +30,16 @@ #define NO_LINE -1 #define SCORING_MISSING_SEGMENT_PENALTY_VERTICAL 10 -#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 15 +#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 1 #define SCORING_BOXINESS_WEIGHT 0.8 #define SCORING_PLATEHEIGHT_WEIGHT 2.2 -#define SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT 0.05 +#define SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT 0.10 #define SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT 1.1 -#define SCORING_VERTICALDISTANCE_WEIGHT 0.1 + +#define SCORING_DISTANCE_WEIGHT_VERTICAL 0.04 + +#define SCORING_LINE_CONFIDENCE_WEIGHT 18.0 #define SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT 0.05 diff --git a/src/openalpr/platelines.cpp b/src/openalpr/platelines.cpp index df53cab..a61e23d 100644 --- a/src/openalpr/platelines.cpp +++ b/src/openalpr/platelines.cpp @@ -22,6 +22,9 @@ using namespace cv; using namespace std; +const float MIN_CONFIDENCE = 0.3; + + PlateLines::PlateLines(Config* config) { this->config = config; @@ -72,15 +75,15 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float Mat mask = Mat::zeros(inputImage.size(), CV_8U); fillPoly(mask, polygons, Scalar(255,255,255)); - dilate(mask, mask, element); + dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); bitwise_not(mask, mask); // AND canny edges with the character mask bitwise_and(edges, mask, edges); - vector hlines = this->getLines(edges, sensitivity, false); - vector vlines = this->getLines(edges, sensitivity, true); + vector hlines = this->getLines(edges, sensitivity, false); + vector vlines = this->getLines(edges, sensitivity, true); for (int i = 0; i < hlines.size(); i++) this->horizontalLines.push_back(hlines[i]); for (int i = 0; i < vlines.size(); i++) @@ -98,12 +101,12 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float for( size_t i = 0; i < this->horizontalLines.size(); i++ ) { - line( debugImgHoriz, this->horizontalLines[i].p1, this->horizontalLines[i].p2, Scalar(0,0,255), 1, CV_AA); + line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } for( size_t i = 0; i < this->verticalLines.size(); i++ ) { - line( debugImgVert, this->verticalLines[i].p1, this->verticalLines[i].p2, Scalar(0,0,255), 1, CV_AA); + line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } vector images; @@ -126,7 +129,7 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float -vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) +vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) { if (this->debug) cout << "PlateLines::getLines" << endl; @@ -135,7 +138,7 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, static int VERTICAL_SENSITIVITY = config->plateLinesSensitivityVertical; vector allLines; - vector filteredLines; + vector filteredLines; int sensitivity; if (vertical) @@ -176,7 +179,11 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, LineSegment bottom(0, edges.rows, edges.cols, edges.rows); Point p1 = line.intersection(bottom); Point p2 = line.intersection(top); - filteredLines.push_back(LineSegment(p1.x, p1.y, p2.x, p2.y)); + + PlateLine plateLine; + plateLine.line = LineSegment(p1.x, p1.y, p2.x, p2.y); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); } } else @@ -196,7 +203,10 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, int newY1 = line.getPointAt(0); int newY2 = line.getPointAt(edges.cols); - filteredLines.push_back(LineSegment(0, newY1, edges.cols, newY2)); + PlateLine plateLine; + plateLine.line = LineSegment(0, newY1, edges.cols, newY2); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); } } } diff --git a/src/openalpr/platelines.h b/src/openalpr/platelines.h index c0fab97..76b9b33 100644 --- a/src/openalpr/platelines.h +++ b/src/openalpr/platelines.h @@ -27,6 +27,11 @@ #include "config.h" #include "characterregion.h" +struct PlateLine +{ + LineSegment line; + float confidence; +}; class PlateLines { @@ -37,18 +42,19 @@ class PlateLines void processImage(cv::Mat img, CharacterRegion* charRegion, float sensitivity=1.0); - std::vector horizontalLines; - std::vector verticalLines; + std::vector horizontalLines; + std::vector verticalLines; std::vector winningCorners; private: + Config* config; bool debug; cv::Mat customGrayscaleConversion(cv::Mat src); void findLines(cv::Mat inputImage); - std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); + std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); }; #endif // OPENALPR_PLATELINES_H