Added logic to LineFinder to find both lines for multiline plates

src/openalpr/textdetection/characteranalysis.cpp

@@ -130,18 +130,17 @@ void CharacterAnalysis::analyze()
   }
 
   LineFinder lf(pipeline_data);
-  lf.findLines(pipeline_data->crop_gray, bestContours);
+  vector<vector<Point> > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours);
   
-  vector<Point> linePolygon;
-  //vector<Point> linePolygon =  getBestVotedLines(pipeline_data->crop_gray, bestContours);
 
-  if (linePolygon.size() > 0)
+  for (uint i = 0; i < linePolygons.size(); i++)
   {
+    vector<Point> linePolygon = linePolygons[i];
+   
+    cout << "Polygon: " << linePolygon[0] << " - " << linePolygon[1] << " - " << linePolygon[2] << " - " << linePolygon[3] << endl;
     
     LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y);
     LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y);
-
-    filterBetweenLines(bestThreshold, bestContours, linePolygon);
     
     vector<Point> textArea = getCharArea(topLine, bottomLine);
 
@@ -149,7 +148,17 @@ void CharacterAnalysis::analyze()
     
     pipeline_data->textLines.push_back(textLine);
   }
+  
+  filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines);
 
+  for (uint i = 0; i < pipeline_data->textLines.size(); i++)
+  {
+    cout << "Test1" << endl;
+    Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold);
+    
+    cout << "Test2" << endl;
+    drawAndWait(&debugImage);
+  }
   
   this->thresholdsInverted = isPlateInverted();
 }
@@ -510,29 +519,22 @@ void CharacterAnalysis::filterByParentContour( TextContours& textContours)
 
 }
 
-void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector<Point> outerPolygon )
+void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector<TextLine> textLines )
 {
   static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88;
   static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15;
 
-  if (outerPolygon.size() == 0)
+  if (textLines.size() == 0)
     return;
 
   vector<Point> validPoints;
 
-  // Figure out the line height
-  LineSegment topLine(outerPolygon[0].x, outerPolygon[0].y, outerPolygon[1].x, outerPolygon[1].y);
-  LineSegment bottomLine(outerPolygon[3].x, outerPolygon[3].y, outerPolygon[2].x, outerPolygon[2].y);
-
-  float x = ((float) img.cols) / 2;
-  Point midpoint = Point(x, bottomLine.getPointAt(x));
-  Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint);
-  float lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint);
 
   // Create a white mask for the area inside the polygon
   Mat outerMask = Mat::zeros(img.size(), CV_8U);
 
-  fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255));
+  for (uint i = 0; i < textLines.size(); i++)
+    fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255));
 
   // For each contour, determine if enough of it is between the lines to qualify
   for (uint i = 0; i < textContours.size(); i++)
@@ -579,19 +581,23 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours,
     }
     
     // Get the absolute distance from the top and bottom lines
-    Point closestTopPoint = topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]);
-    Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]);
     
-    float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]);
+    for (uint i = 0; i < textLines.size(); i++)
-    float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]);
-    
-    float maxDistance = lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES;
-     
-    if (absTopDistance < maxDistance && absBottomDistance < maxDistance)
     {
-      textContours.goodIndices[i] = true;
+      Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]);
-    }
+      Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]);
 
+      float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]);
+      float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]);
+
+      float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES;
+
+      if (absTopDistance < maxDistance && absBottomDistance < maxDistance)
+      {
+        textContours.goodIndices[i] = true;
+      }
+    }
+    
   }
 
 }

src/openalpr/textdetection/characteranalysis.h

@@ -64,7 +64,7 @@ class CharacterAnalysis
 
     std::vector<cv::Point> getCharArea(LineSegment topLine, LineSegment bottomLine);
     std::vector<cv::Point> getBestVotedLines(cv::Mat img, TextContours textContours);
-    void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector<cv::Point> outerPolygon );
+    void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector<TextLine> textLines );
 
     bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx);
 

src/openalpr/textdetection/linefinder.cpp

@@ -33,35 +33,51 @@ LineFinder::LineFinder(PipelineData* pipeline_data) {
 LineFinder::~LineFinder() {
 }
 
-vector<TextLine> LineFinder::findLines(Mat image, const TextContours contours)
+vector<vector<Point> > LineFinder::findLines(Mat image, const TextContours contours)
 {
-  vector<TextLine> linesFound;
+  vector<vector<Point> > linesFound;
   
   
   cvtColor(image, image, CV_GRAY2BGR);
-  vector<Rect> boxes = this->getBoundingBoxes(contours);
   
-  vector<Point> tops = this->getCharTops(boxes);
+  vector<CharPointInfo> charPoints;
-  vector<Point> bottoms = this->getCharBottoms(boxes);
   
-  for (uint i = 0; i < tops.size(); i++)
+  for (uint i = 0; i < contours.contours.size(); i++)
   {
-    circle(image, tops[i], 1, Scalar(255, 0, 0), 2);
+    if (contours.goodIndices[i] == false)
-    circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2);
+      continue;
+    
+    charPoints.push_back( CharPointInfo(contours.contours[i], i) );
   }
   
-  drawAndWait(&image);
+  vector<Point> bestLine = getBestLine(contours, charPoints);
   
-  vector<Point> bestLine = getBestLine(contours, tops, bottoms);
+  if (bestLine.size() > 0)
+    linesFound.push_back(bestLine);
   
   if (pipeline_data->isMultiline)
   {
     // we have a two-line plate.  Find the next best line, removing the tops/bottoms from before.
-  }
+    // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it.
-  
-  for (uint i = 0; i < contours.goodIndices.size(); i++)
-  {
     
+    vector<CharPointInfo> remainingPoints;
+    for (uint i = 0; i < charPoints.size(); i++)
+    {
+      Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U);
+      fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255));
+      
+      float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex);
+      
+      if (percentInside < .85)
+      {
+        remainingPoints.push_back(charPoints[i]);
+      }
+    }
+    
+    vector<Point> nextBestLine = getBestLine(contours, remainingPoints);
+    
+    if (nextBestLine.size() > 0)
+      linesFound.push_back(nextBestLine);
   }
   
   
@@ -69,64 +85,15 @@ vector<TextLine> LineFinder::findLines(Mat image, const TextContours contours)
 }
 
 
-vector<Rect> LineFinder::getBoundingBoxes(const TextContours contours) {
-  
-  vector<Rect> boxes;
-  for (uint i = 0; i < contours.goodIndices.size(); i++)
-  {
-    if (contours.goodIndices[i] == false)
-      continue;
-    
-    Rect bRect = cv::boundingRect( Mat(contours.contours[i]) );
-    
-    boxes.push_back(bRect);
-  }
-  
-  return boxes;
-}
-
-
-vector<Point> LineFinder::getCharTops(vector<Rect> boxes) {
-  
-  vector<Point> tops;
-  for (uint i = 0; i < boxes.size(); i++)
-  {
-    int x = boxes[i].x + (boxes[i].width / 2);
-    int y = boxes[i].y;
-    
-    tops.push_back(Point(x, y));
-  }
-
-  return tops;
-}
-
-vector<Point> LineFinder::getCharBottoms(vector<Rect> boxes) {
-  
-  vector<Point> bottoms;
-  for (uint i = 0; i < boxes.size(); i++)
-  {
-    int x = boxes[i].x + (boxes[i].width / 2);
-    int y = boxes[i].y + boxes[i].height;
-    
-    bottoms.push_back(Point(x, y));
-  }
-
-  return bottoms;
-}
-
-
-
-
-
 // Returns a polygon "stripe" across the width of the character region.  The lines are voted and the polygon starts at 0 and extends to image width
-vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point> tops, vector<Point> bottoms)
+vector<Point> LineFinder::getBestLine(const TextContours contours, vector<CharPointInfo> charPoints)
 {
 
 
   vector<Point> bestStripe;
 
   // Find the best fit line segment that is parallel with the most char segments
-  if (tops.size() <= 1)
+  if (charPoints.size() <= 1)
   {
     // Maybe do something about this later, for now let's just ignore
   }
@@ -135,31 +102,26 @@ vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point>
     vector<LineSegment> topLines;
     vector<LineSegment> bottomLines;
     // Iterate through each possible char and find all possible lines for the top and bottom of each char segment
-    for (uint i = 0; i < tops.size() - 1; i++)
+    for (uint i = 0; i < charPoints.size() - 1; i++)
     {
-      for (uint k = i+1; k < tops.size(); k++)
+      for (uint k = i+1; k < charPoints.size(); k++)
       {
         
-        Point topLeft, topRight;
+        int leftCPIndex, rightCPIndex;
-        Point bottomLeft, bottomRight;
+        if (charPoints[i].top.x < charPoints[k].top.x)
-        if (tops[i].x < tops[k].x)
         {
-          topLeft = tops[i];
+          leftCPIndex = i;
-          topRight = tops[k];
+          rightCPIndex = k;
-          bottomLeft = bottoms[i];
-          bottomRight = bottoms[k];
         }
         else
         {
-          topLeft = tops[k];
+          leftCPIndex = k;
-          topRight = tops[i];
+          rightCPIndex = i;
-          bottomLeft = bottoms[k];
-          bottomRight = bottoms[i];
         }
         
         
-        LineSegment top(topLeft, topRight);
+        LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top);
-        LineSegment bottom(bottomLeft, bottomRight);
+        LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom);
         
         // Only allow lines that have a sane angle
         if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees &&
@@ -184,15 +146,15 @@ vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point>
       float SCORING_MAX_THRESHOLD = 1.03;
 
       int curScore = 0;
-      for (uint charidx = 0; charidx < tops.size(); charidx++)
+      for (uint charidx = 0; charidx < charPoints.size(); charidx++)
       {
-        float topYPos = topLines[i].getPointAt(tops[charidx].x);
+        float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x);
-        float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x);
+        float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x);
 
-        float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD;
+        float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD;
-        float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD;
+        float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD;
-        float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD;
+        float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD;
-        float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD;
+        float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD;
         if ( (topYPos >= minTop && topYPos <= maxTop) &&
              (botYPos >= minBot && botYPos <= maxBot))
         {
@@ -202,7 +164,7 @@ vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point>
         //cout << "Slope: " << topslope << " yPos: " << topYPos << endl;
         //drawAndWait(&tempImg);
       }
-
+      
       // Tie goes to the one with longer line segments
       if ((curScore > bestScore) ||
           (curScore == bestScore && topLines[i].length > bestScoreDistance))
@@ -240,3 +202,23 @@ vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point>
 
   return bestStripe;
 }
+
+CharPointInfo::CharPointInfo(vector<Point> contour, int index) {
+
+    
+  this->contourIndex = index;
+  
+  this->boundingBox = cv::boundingRect( Mat(contour) );
+      
+ 
+  int x = boundingBox.x + (boundingBox.width / 2);
+  int y = boundingBox.y;
+
+  this->top = Point(x, y);
+ 
+  x = boundingBox.x + (boundingBox.width / 2);
+  y = boundingBox.y + boundingBox.height;
+
+  this->bottom = Point(x,y);
+  
+}

src/openalpr/textdetection/linefinder.h

@@ -27,21 +27,28 @@
 #include "textline.h"
 #include "pipeline_data.h"
 
+class CharPointInfo
+{
+public:
+  CharPointInfo(std::vector<cv::Point> contour, int index);
+  
+  cv::Rect boundingBox;
+  cv::Point top;
+  cv::Point bottom;
+  int contourIndex;
+  
+};
 
 class LineFinder {
 public:
   LineFinder(PipelineData* pipeline_data);
   virtual ~LineFinder();
   
-  std::vector<TextLine> findLines(cv::Mat image, const TextContours contours);
+  std::vector<std::vector<cv::Point> > findLines(cv::Mat image, const TextContours contours);
 private:
   PipelineData* pipeline_data;
           
-  std::vector<cv::Rect> getBoundingBoxes(const TextContours contours);
+  std::vector<cv::Point> getBestLine(const TextContours contours, std::vector<CharPointInfo> charPoints);
-  std::vector<cv::Point> getCharTops(std::vector<cv::Rect> boxes);
-  std::vector<cv::Point> getCharBottoms(std::vector<cv::Rect> boxes);
-  
-  std::vector<cv::Point> getBestLine(const TextContours contours, std::vector<cv::Point> tops, std::vector<cv::Point> bottoms);
 };
 
 #endif	/* OPENALPR_LINEFINDER_H */

src/openalpr/textdetection/textline.cpp

@@ -18,13 +18,20 @@
 */
 
 
+#include <opencv2/imgproc/imgproc.hpp>
+
 #include "textline.h"
 
+using namespace cv;
+
 TextLine::TextLine(std::vector<cv::Point> textArea, std::vector<cv::Point> linePolygon) {
   if (textArea.size() > 0)
   {
-    this->textArea = textArea;
+    for (uint i = 0; i < textArea.size(); i++)
-    this->linePolygon = linePolygon;
+      this->textArea.push_back(textArea[i]);
+    
+    for (uint i = 0; i < linePolygon.size(); i++)
+      this->linePolygon.push_back(linePolygon[i]);
     
     this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y);
     this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y);
@@ -33,9 +40,43 @@ TextLine::TextLine(std::vector<cv::Point> textArea, std::vector<cv::Point> lineP
     this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y);
     this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y);
     this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y);
+    
+ 
+    float x = ((float) linePolygon[1].x) / 2;
+    Point midpoint = Point(x, bottomLine.getPointAt(x));
+    Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint);
+    this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint);
   }
 }
 
 
 TextLine::~TextLine() {
 }
+
+cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) {
+  cv::Mat debugImage(baseImage.size(), baseImage.type());
+  
+  baseImage.copyTo(debugImage);
+  
+  cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR);
+  
+  
+  fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165));
+
+  drawAndWait(&debugImage);
+  fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0));
+  
+  drawAndWait(&debugImage);
+  line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1);
+  line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1);
+  
+  drawAndWait(&debugImage);
+  line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1);
+  line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1);
+  line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1);
+  line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1);
+  
+  drawAndWait(&debugImage);
+  
+  return debugImage;
+}

src/openalpr/textdetection/textline.h

@@ -22,6 +22,7 @@
 #define	OPENALPR_TEXTLINE_H
 
 #include "utility.h"
+#include "opencv2/imgproc/imgproc.hpp"
 
 class TextLine {
 public:
@@ -38,6 +39,9 @@ public:
   LineSegment charBoxLeft;
   LineSegment charBoxRight;
   
+  float lineHeight;
+
+  cv::Mat drawDebugImage(cv::Mat baseImage);
 private:
 
 };