Added LineFinder class to help find text lines in plate images

src/openalpr/textdetection/characteranalysis.cpp

@@ -18,6 +18,7 @@
 */
 
 #include "characteranalysis.h"
+#include "linefinder.h"
 
 using namespace cv;
 using namespace std;
@@ -101,8 +102,6 @@ void CharacterAnalysis::analyze()
   int bestFitIndex = -1;
   for (uint i = 0; i < pipeline_data->thresholds.size(); i++)
   {
-    //vector<bool> goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]);
-    //charSegments.push_back(goodIndices);
 
     int segmentCount = allTextContours[i].getGoodIndicesCount();
 
@@ -125,31 +124,16 @@ void CharacterAnalysis::analyze()
 
   if (this->config->debugCharAnalysis)
   {
-    Mat img_contours(bestThreshold.size(), CV_8U);
-    bestThreshold.copyTo(img_contours);
-    cvtColor(img_contours, img_contours, CV_GRAY2RGB);
-
-    vector<vector<Point> > allowedContours;
-    for (uint i = 0; i < bestContours.size(); i++)
-    {
-      if (bestContours.goodIndices[i])
-        allowedContours.push_back(bestContours.contours[i]);
-    }
-
-    drawContours(img_contours, bestContours.contours,
-                 -1, // draw all contours
-                 cv::Scalar(255,0,0), // in blue
-                 1); // with a thickness of 1
-
-    drawContours(img_contours, allowedContours,
-                 -1, // draw all contours
-                 cv::Scalar(0,255,0), // in green
-                 1); // with a thickness of 1
+    Mat img_contours = bestContours.drawDebugImage(bestThreshold);
 
     displayImage(config, "Matching Contours", img_contours);
   }
 
-  vector<Point> linePolygon =  getBestVotedLines(pipeline_data->crop_gray, bestContours);
+  LineFinder lf(pipeline_data);
+  lf.findLines(pipeline_data->crop_gray, bestContours);
+  
+  vector<Point> linePolygon;
+  //vector<Point> linePolygon =  getBestVotedLines(pipeline_data->crop_gray, bestContours);
 
   if (linePolygon.size() > 0)
   {
@@ -370,7 +354,7 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours)
     for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true;
 
     this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP));
-
+    
     int goodIndices = textContours.getGoodIndicesCount();
     if ( goodIndices == 0 || goodIndices <= bestFitScore)	// Don't bother doing more filtering if we already lost...
       continue;
@@ -381,8 +365,8 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours)
     if ( goodIndices == 0 || goodIndices <= bestFitScore)	// Don't bother doing more filtering if we already lost...
       continue;
     
-    vector<Point> lines = getBestVotedLines(img, textContours);
-    this->filterBetweenLines(img, textContours, lines);
+    //vector<Point> lines = getBestVotedLines(img, textContours);
+    //this->filterBetweenLines(img, textContours, lines);
 
     int segmentCount = textContours.getGoodIndicesCount();
 
@@ -415,11 +399,13 @@ void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeigh
 
     float minWidth = mr.height * 0.2;
     //Crop image
-    //Mat auxRoi(img, mr);
+    
+    //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl;
     if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth)
     {
       float charAspect= (float)mr.width/(float)mr.height;
 
+      //cout << "  -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl;
       if (abs(charAspect - idealAspect) < aspecttolerance)
         textContours.goodIndices[i] = true;
     }
@@ -545,7 +531,7 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours,
 
   // Create a white mask for the area inside the polygon
   Mat outerMask = Mat::zeros(img.size(), CV_8U);
-  Mat innerArea(img.size(), CV_8U);
+
   fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255));
 
   // For each contour, determine if enough of it is between the lines to qualify
@@ -556,35 +542,14 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours,
     
     textContours.goodIndices[i] = false;  // Set it to not included unless it proves 
 
-    innerArea.setTo(Scalar(0,0,0));
+    float percentInsideMask = getContourAreaPercentInsideMask(outerMask, 
+            textContours.contours,
+            textContours.hierarchy, 
+            (int) i);
     
-    drawContours(innerArea, textContours.contours,
-                 i, // draw this contour
-                 cv::Scalar(255,255,255), // in
-                 CV_FILLED,
-                 8,
-                 textContours.hierarchy,
-                 0
-                );
-
-    bitwise_and(innerArea, outerMask, innerArea);
-
-    vector<vector<Point> > tempContours;
-    findContours(innerArea, tempContours,
-                 CV_RETR_EXTERNAL, // retrieve the external contours
-                 CV_CHAIN_APPROX_SIMPLE  ); // all pixels of each contours );
-
-    double totalArea = contourArea(textContours.contours[i]);
-    double areaBetweenLines = 0;
-
-    for (uint tempContourIdx = 0; tempContourIdx < tempContours.size(); tempContourIdx++)
-    {
-      areaBetweenLines += contourArea(tempContours[tempContourIdx]);
-    }
-
-
     
-    if (areaBetweenLines / totalArea < MIN_AREA_PERCENT_WITHIN_LINES)
+
+    if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES)
     {
       // Not enough area is inside the lines.
       continue;

src/openalpr/textdetection/characteranalysis.h

@@ -26,6 +26,7 @@
 #include "pipeline_data.h"
 #include "textcontours.h"
 #include "platemask.h"
+#include "linefinder.h"
 
 class CharacterAnalysis
 {
@@ -40,7 +41,6 @@ class CharacterAnalysis
     TextContours bestContours;
 
     bool thresholdsInverted;
-    bool isTwoLine;
 
     std::vector<TextContours> allTextContours;
 
@@ -68,7 +68,7 @@ class CharacterAnalysis
 
     bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx);
 
-
+    
 };
 
 #endif // OPENALPR_CHARACTERANALYSIS_H

src/openalpr/textdetection/linefinder.cpp

@@ -0,0 +1,232 @@
+/*
+ * Copyright (c) 2014 New Designs Unlimited, LLC
+ * Opensource Automated License Plate Recognition [http://www.openalpr.com]
+ *
+ * This file is part of OpenAlpr.
+ *
+ * OpenAlpr is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * version 3 as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <opencv2/core/core.hpp>
+
+#include "linefinder.h"
+#include "utility.h"
+#include "pipeline_data.h"
+
+using namespace std;
+using namespace cv;
+
+LineFinder::LineFinder(PipelineData* pipeline_data) {
+  this->pipeline_data = pipeline_data;
+}
+
+LineFinder::~LineFinder() {
+}
+
+vector<TextLine> LineFinder::findLines(Mat image, const TextContours contours)
+{
+  vector<TextLine> linesFound;
+  
+  
+  cvtColor(image, image, CV_GRAY2BGR);
+  vector<Rect> boxes = this->getBoundingBoxes(contours);
+  
+  vector<Point> tops = this->getCharTops(boxes);
+  vector<Point> bottoms = this->getCharBottoms(boxes);
+  
+  for (uint i = 0; i < tops.size(); i++)
+  {
+    circle(image, tops[i], 1, Scalar(255, 0, 0), 2);
+    circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2);
+  }
+  
+  drawAndWait(&image);
+  
+  vector<Point> bestLine = getBestLine(contours, tops, bottoms);
+  
+  if (pipeline_data->isMultiline)
+  {
+    // we have a two-line plate.  Find the next best line, removing the tops/bottoms from before.
+  }
+  
+  for (uint i = 0; i < contours.goodIndices.size(); i++)
+  {
+    
+  }
+  
+  
+  return linesFound;
+}
+
+
+vector<Rect> LineFinder::getBoundingBoxes(const TextContours contours) {
+  
+  vector<Rect> boxes;
+  for (uint i = 0; i < contours.goodIndices.size(); i++)
+  {
+    if (contours.goodIndices[i] == false)
+      continue;
+    
+    Rect bRect = cv::boundingRect( Mat(contours.contours[i]) );
+    
+    boxes.push_back(bRect);
+  }
+  
+  return boxes;
+}
+
+
+vector<Point> LineFinder::getCharTops(vector<Rect> boxes) {
+  
+  vector<Point> tops;
+  for (uint i = 0; i < boxes.size(); i++)
+  {
+    int x = boxes[i].x + (boxes[i].width / 2);
+    int y = boxes[i].y;
+    
+    tops.push_back(Point(x, y));
+  }
+
+  return tops;
+}
+
+vector<Point> LineFinder::getCharBottoms(vector<Rect> boxes) {
+  
+  vector<Point> bottoms;
+  for (uint i = 0; i < boxes.size(); i++)
+  {
+    int x = boxes[i].x + (boxes[i].width / 2);
+    int y = boxes[i].y + boxes[i].height;
+    
+    bottoms.push_back(Point(x, y));
+  }
+
+  return bottoms;
+}
+
+
+
+
+
+// Returns a polygon "stripe" across the width of the character region.  The lines are voted and the polygon starts at 0 and extends to image width
+vector<Point> LineFinder::getBestLine(const TextContours contours, vector<Point> tops, vector<Point> bottoms)
+{
+
+
+  vector<Point> bestStripe;
+
+  // Find the best fit line segment that is parallel with the most char segments
+  if (tops.size() <= 1)
+  {
+    // Maybe do something about this later, for now let's just ignore
+  }
+  else
+  {
+    vector<LineSegment> topLines;
+    vector<LineSegment> bottomLines;
+    // Iterate through each possible char and find all possible lines for the top and bottom of each char segment
+    for (uint i = 0; i < tops.size() - 1; i++)
+    {
+      for (uint k = i+1; k < tops.size(); k++)
+      {
+        
+        Point topLeft, topRight;
+        Point bottomLeft, bottomRight;
+        if (tops[i].x < tops[k].x)
+        {
+          topLeft = tops[i];
+          topRight = tops[k];
+          bottomLeft = bottoms[i];
+          bottomRight = bottoms[k];
+        }
+        else
+        {
+          topLeft = tops[k];
+          topRight = tops[i];
+          bottomLeft = bottoms[k];
+          bottomRight = bottoms[i];
+        }
+        
+        topLines.push_back(LineSegment(topLeft, topRight));
+        bottomLines.push_back(LineSegment(bottomLeft, bottomRight));
+
+      }
+    }
+
+    int bestScoreIndex = 0;
+    int bestScore = -1;
+    int bestScoreDistance = -1; // Line segment distance is used as a tie breaker
+
+    // Now, among all possible lines, find the one that is the best fit
+    for (uint i = 0; i < topLines.size(); i++)
+    {
+      float SCORING_MIN_THRESHOLD = 0.97;
+      float SCORING_MAX_THRESHOLD = 1.03;
+
+      int curScore = 0;
+      for (uint charidx = 0; charidx < tops.size(); charidx++)
+      {
+        float topYPos = topLines[i].getPointAt(tops[charidx].x);
+        float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x);
+
+        float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD;
+        float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD;
+        float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD;
+        float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD;
+        if ( (topYPos >= minTop && topYPos <= maxTop) &&
+             (botYPos >= minBot && botYPos <= maxBot))
+        {
+          curScore++;
+        }
+
+        //cout << "Slope: " << topslope << " yPos: " << topYPos << endl;
+        //drawAndWait(&tempImg);
+      }
+
+      // Tie goes to the one with longer line segments
+      if ((curScore > bestScore) ||
+          (curScore == bestScore && topLines[i].length > bestScoreDistance))
+      {
+        bestScore = curScore;
+        bestScoreIndex = i;
+        // Just use x distance for now
+        bestScoreDistance = topLines[i].length;
+      }
+    }
+
+    if (true)
+    {
+      cout << "The winning score is: " << bestScore << endl;
+      // Draw the winning line segment
+      Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U);
+      cvtColor(tempImg, tempImg, CV_GRAY2BGR);
+
+      cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2);
+      cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2);
+
+      drawAndWait(&tempImg);
+    }
+
+    Point topLeft 		= Point(0, topLines[bestScoreIndex].getPointAt(0) );
+    Point topRight 		= Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width));
+    Point bottomRight 	= Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width));
+    Point bottomLeft 	= Point(0, bottomLines[bestScoreIndex].getPointAt(0));
+
+    bestStripe.push_back(topLeft);
+    bestStripe.push_back(topRight);
+    bestStripe.push_back(bottomRight);
+    bestStripe.push_back(bottomLeft);
+  }
+
+  return bestStripe;
+}

src/openalpr/textdetection/linefinder.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2014 New Designs Unlimited, LLC
+ * Opensource Automated License Plate Recognition [http://www.openalpr.com]
+ *
+ * This file is part of OpenAlpr.
+ *
+ * OpenAlpr is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * version 3 as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// This class finds lines of text given an array of contours
+
+#ifndef OPENALPR_LINEFINDER_H
+#define	OPENALPR_LINEFINDER_H
+
+#include "opencv2/imgproc/imgproc.hpp"
+#include "textcontours.h"
+#include "textline.h"
+#include "pipeline_data.h"
+
+
+class LineFinder {
+public:
+  LineFinder(PipelineData* pipeline_data);
+  virtual ~LineFinder();
+  
+  std::vector<TextLine> findLines(cv::Mat image, const TextContours contours);
+private:
+  PipelineData* pipeline_data;
+          
+  std::vector<cv::Rect> getBoundingBoxes(const TextContours contours);
+  std::vector<cv::Point> getCharTops(std::vector<cv::Rect> boxes);
+  std::vector<cv::Point> getCharBottoms(std::vector<cv::Rect> boxes);
+  
+  std::vector<cv::Point> getBestLine(const TextContours contours, std::vector<cv::Point> tops, std::vector<cv::Point> bottoms);
+};
+
+#endif	/* OPENALPR_LINEFINDER_H */
+

src/openalpr/textdetection/textcontours.cpp

@@ -50,6 +50,9 @@ void TextContours::load(cv::Mat threshold) {
 
   for (uint i = 0; i < contours.size(); i++)
     goodIndices.push_back(true);
+  
+  this->width = threshold.cols;
+  this->height = threshold.rows;
 }
 
 
@@ -95,4 +98,41 @@ void TextContours::setIndices(std::vector<bool> newIndices)
   {
     assert("Invalid set operation on indices");
   }
-}
+}
+
+Mat TextContours::drawDebugImage() {
+
+    Mat img_contours = Mat::zeros(Size(width, height), CV_8U);
+    
+    return drawDebugImage(img_contours);
+}
+
+Mat TextContours::drawDebugImage(Mat baseImage) {
+    Mat img_contours(baseImage.size(), CV_8U);
+    baseImage.copyTo(img_contours);
+    
+    cvtColor(img_contours, img_contours, CV_GRAY2RGB);
+
+    vector<vector<Point> > allowedContours;
+    for (uint i = 0; i < this->contours.size(); i++)
+    {
+      if (this->goodIndices[i])
+        allowedContours.push_back(this->contours[i]);
+    }
+
+    drawContours(img_contours, this->contours,
+                 -1, // draw all contours
+                 cv::Scalar(255,0,0), // in blue
+                 1); // with a thickness of 1
+
+    drawContours(img_contours, allowedContours,
+                 -1, // draw all contours
+                 cv::Scalar(0,255,0), // in green
+                 1); // with a thickness of 1
+
+    
+    return img_contours;
+}
+
+
+

src/openalpr/textdetection/textcontours.h

@@ -31,6 +31,9 @@ public:
   
   void load(cv::Mat threshold);
   
+  int width;
+  int height;
+  
   std::vector<bool> goodIndices;
   std::vector<std::vector<cv::Point> > contours;
   std::vector<cv::Vec4i> hierarchy;
@@ -41,6 +44,9 @@ public:
   std::vector<bool> getIndicesCopy();
   void setIndices(std::vector<bool> newIndices);
   
+  cv::Mat drawDebugImage();
+  cv::Mat drawDebugImage(cv::Mat baseImage);
+  
 private: