Added multiline context for postprocessing

Pattern matching can now apply line by line patterns for greater accuracy

src/main.cpp

@@ -337,7 +337,11 @@ bool detectandshow( Alpr* alpr, cv::Mat frame, std::string region, bool writeJso
       
       for (int k = 0; k < results.plates[i].topNPlates.size(); k++)
       {
-        std::cout << "    - " << results.plates[i].topNPlates[k].characters << "\t confidence: " << results.plates[i].topNPlates[k].overall_confidence;
+        // Replace the multiline newline character with a dash
+        std::string no_newline = results.plates[i].topNPlates[k].characters;
+        std::replace(no_newline.begin(), no_newline.end(), '\n','-');
+        
+        std::cout << "    - " << no_newline << "\t confidence: " << results.plates[i].topNPlates[k].overall_confidence;
         if (templatePattern.size() > 0 || results.plates[i].regionConfidence > 0)
           std::cout << "\t pattern_match: " << results.plates[i].topNPlates[k].matches_template;
         

src/openalpr/alpr_impl.cpp

@@ -347,9 +347,11 @@ namespace alpr
           for (unsigned int c_idx = 0; c_idx < ppResults[pp].letter_details.size(); c_idx++)
           {
             AlprChar character_details;
-            character_details.character = ppResults[pp].letter_details[c_idx].letter;
+            Letter l = ppResults[pp].letter_details[c_idx];
-            character_details.confidence = ppResults[pp].letter_details[c_idx].totalscore;
+            
-            cv::Rect char_rect = pipeline_data.charRegions[ppResults[pp].letter_details[c_idx].charposition];
+            character_details.character = l.letter;
+            character_details.confidence = l.totalscore;
+            cv::Rect char_rect = pipeline_data.charRegionsFlat[l.charposition];
             std::vector<AlprCoordinate> charpoints = getCharacterPoints(char_rect, charTransformMatrix );
             for (int cpt = 0; cpt < 4; cpt++)
               character_details.corners[cpt] = charpoints[cpt];

src/openalpr/ocr.cpp

@@ -61,8 +61,15 @@ namespace alpr
     postProcessor.clear();
 
     // Don't waste time on OCR processing if it is impossible to get sufficient characters
-    if (pipeline_data->charRegions.size() < config->postProcessMinCharacters)
+    int total_char_spaces = 0;
+    for (unsigned int i = 0; i < pipeline_data->charRegions.size(); i++)
+      total_char_spaces += pipeline_data->charRegions[i].size();
+    if (total_char_spaces < config->postProcessMinCharacters)
+    {
+      pipeline_data->disqualify_reason = "Insufficient character boxes detected.  No OCR performed.";
+      pipeline_data->disqualified = true;
       return;
+    }
 
     for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++)
     {
@@ -72,62 +79,68 @@ namespace alpr
                           pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height, 
                           pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1());
 
-      for (unsigned int j = 0; j < pipeline_data->charRegions.size(); j++)
+      int absolute_charpos = 0;
+      for (unsigned int line_idx = 0; line_idx < pipeline_data->charRegions.size(); line_idx++)
       {
-        Rect expandedRegion = expandRect( pipeline_data->charRegions[j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ;
+        for (unsigned int j = 0; j < pipeline_data->charRegions[line_idx].size(); j++)
-
-        tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height);
-        tesseract.Recognize(NULL);
-
-        tesseract::ResultIterator* ri = tesseract.GetIterator();
-        tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL;
-        do
         {
-          const char* symbol = ri->GetUTF8Text(level);
+          Rect expandedRegion = expandRect( pipeline_data->charRegions[line_idx][j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ;
-          float conf = ri->Confidence(level);
-          
-          bool dontcare;
-          int fontindex = 0;
-          int pointsize = 0;
-          const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex);
 
-          // Ignore NULL pointers, spaces, and characters that are way too small to be valid
+          tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height);
-          if(symbol != 0 && symbol[0] != SPACE_CHAR_CODE && pointsize >= config->ocrMinFontSize)
+          tesseract.Recognize(NULL);
+
+          tesseract::ResultIterator* ri = tesseract.GetIterator();
+          tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL;
+          do
           {
-            postProcessor.addLetter(string(symbol), j, conf);
+            const char* symbol = ri->GetUTF8Text(level);
+            float conf = ri->Confidence(level);
+
+            bool dontcare;
+            int fontindex = 0;
+            int pointsize = 0;
+            const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex);
+
+            // Ignore NULL pointers, spaces, and characters that are way too small to be valid
+            if(symbol != 0 && symbol[0] != SPACE_CHAR_CODE && pointsize >= config->ocrMinFontSize)
+            {
+              postProcessor.addLetter(string(symbol), line_idx, absolute_charpos, conf);
+
+              if (this->config->debugOcr)
+                printf("charpos%d line%d: threshold %d:  symbol %s, conf: %f font: %s (index %d) size %dpx", absolute_charpos, line_idx, i, symbol, conf, fontName, fontindex, pointsize);
+
+              bool indent = false;
+              tesseract::ChoiceIterator ci(*ri);
+              do
+              {
+                const char* choice = ci.GetUTF8Text();
+
+                postProcessor.addLetter(string(choice), line_idx, absolute_charpos, ci.Confidence());
+
+                if (this->config->debugOcr)
+                {
+                  if (indent) printf("\t\t ");
+                  printf("\t- ");
+                  printf("%s conf: %f\n", choice, ci.Confidence());
+                }
+
+                indent = true;
+              }
+              while(ci.Next());
+              
+            }
 
             if (this->config->debugOcr)
-              printf("charpos%d: threshold %d:  symbol %s, conf: %f font: %s (index %d) size %dpx", j, i, symbol, conf, fontName, fontindex, pointsize);
+              printf("---------------------------------------------\n");
 
-            bool indent = false;
+            delete[] symbol;
-            tesseract::ChoiceIterator ci(*ri);
-            do
-            {
-              const char* choice = ci.GetUTF8Text();
-
-              postProcessor.addLetter(string(choice), j, ci.Confidence());
-
-              //letterScores.addScore(*choice, j, ci.Confidence() - MIN_CONFIDENCE);
-              if (this->config->debugOcr)
-              {
-                if (indent) printf("\t\t ");
-                printf("\t- ");
-                printf("%s conf: %f\n", choice, ci.Confidence());
-              }
-
-              indent = true;
-            }
-            while(ci.Next());
           }
+          while((ri->Next(level)));
 
-          if (this->config->debugOcr)
+          delete ri;
-            printf("---------------------------------------------\n");
+          
-
+          absolute_charpos++;
-          delete[] symbol;
         }
-        while((ri->Next(level)));
-
-        delete ri;
       }
     }
 

src/openalpr/pipeline_data.h

@@ -58,7 +58,12 @@ namespace alpr
       
       ScoreKeeper confidence_weights;
 
-      std::vector<cv::Rect> charRegions;
+      // Boxes around characters in cropped image
+      // Each row in a multiline plate is an entry in the vector
+      std::vector<std::vector<cv::Rect> > charRegions;
+      
+      // Same data, just not broken down by line
+      std::vector<cv::Rect> charRegionsFlat;
 
 
 

src/openalpr/postprocess/postprocess.cpp

@@ -72,17 +72,17 @@ namespace alpr
     }
   }
 
-  void PostProcess::addLetter(string letter, int charposition, float score)
+  void PostProcess::addLetter(string letter, int line_index, int charposition, float score)
   {
     if (score < config->postProcessMinConfidence)
       return;
 
-    insertLetter(letter, charposition, score);
+    insertLetter(letter, line_index, charposition, score);
 
     if (score < config->postProcessConfidenceSkipLevel)
     {
       float adjustedScore = abs(config->postProcessConfidenceSkipLevel - score) + config->postProcessMinConfidence;
-      insertLetter(SKIP_CHAR, charposition, adjustedScore );
+      insertLetter(SKIP_CHAR, line_index, charposition, adjustedScore );
     }
 
     //if (letter == '0')
@@ -91,7 +91,7 @@ namespace alpr
     //}
   }
 
-  void PostProcess::insertLetter(string letter, int charposition, float score)
+  void PostProcess::insertLetter(string letter, int line_index, int charposition, float score)
   {
     score = score - config->postProcessMinConfidence;
 
@@ -108,6 +108,7 @@ namespace alpr
     for (int i = 0; i < letters[charposition].size(); i++)
     {
       if (letters[charposition][i].letter == letter &&
+          letters[charposition][i].line_index == line_index &&
           letters[charposition][i].charposition == charposition)
       {
         existingIndex = i;
@@ -118,6 +119,7 @@ namespace alpr
     if (existingIndex == -1)
     {
       Letter newLetter;
+      newLetter.line_index = line_index;
       newLetter.charposition = charposition;
       newLetter.letter = letter;
       newLetter.occurences = 1;
@@ -179,7 +181,7 @@ namespace alpr
       for (int i = 0; i < letters.size(); i++)
       {
         for (int j = 0; j < letters[i].size(); j++)
-          cout << "PostProcess Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl;
+          cout << "PostProcess Line " << letters[i][j].line_index << " Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl;
       }
     }
 
@@ -340,6 +342,7 @@ namespace alpr
     possibility.matchesTemplate = false;
     int plate_char_length = 0;
 
+    int last_line = 0;
     for (int i = 0; i < letters.size(); i++)
     {
       if (letters[i].size() == 0)
@@ -347,6 +350,13 @@ namespace alpr
 
       Letter letter = letters[i][letterIndices[i]];
 
+      // Add a "\n" on new lines
+      if (letter.line_index != last_line)
+      {
+        possibility.letters = possibility.letters + "\n";
+      }
+      last_line = letter.line_index;
+      
       if (letter.letter != SKIP_CHAR)
       {
         possibility.letters = possibility.letters + letter.letter;

src/openalpr/postprocess/postprocess.h

@@ -40,6 +40,7 @@ namespace alpr
   struct Letter
   {
     std::string letter;
+    int line_index;
     int charposition;
     float totalscore;
     int occurences;
@@ -62,7 +63,7 @@ namespace alpr
       PostProcess(Config* config);
       ~PostProcess();
 
-      void addLetter(std::string letter, int charposition, float score);
+      void addLetter(std::string letter, int line_index, int charposition, float score);
 
       void clear();
       void analyze(std::string templateregion, int topn);
@@ -82,7 +83,7 @@ namespace alpr
       void findAllPermutations(std::string templateregion, int topn);
       bool analyzePermutation(std::vector<int> letterIndices, std::string templateregion, int topn);
 
-      void insertLetter(std::string letter, int charPosition, float score);
+      void insertLetter(std::string letter, int line_index, int charPosition, float score);
 
       std::map<std::string, std::vector<RegexRule*> > rules;
 

src/openalpr/segmentation/charactersegmenter.cpp

@@ -158,8 +158,9 @@ namespace alpr
 
       candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes);
 
-      for (unsigned int cbox = 0; cbox < candidateBoxes.size(); cbox++)
+      pipeline_data->charRegions.push_back(candidateBoxes);
-        pipeline_data->charRegions.push_back(candidateBoxes[cbox]);
+      for (unsigned int cboxidx = 0; cboxidx < candidateBoxes.size(); cboxidx++)
+        pipeline_data->charRegionsFlat.push_back(candidateBoxes[cboxidx]);
 
       if (config->debugTiming)
       {
@@ -181,7 +182,13 @@ namespace alpr
       }
     }
 
-    cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions);
+    vector<Rect> all_regions_combined;
+    for (unsigned int lidx = 0; lidx < pipeline_data->charRegions.size(); lidx++)
+    {
+      for (unsigned int boxidx = 0; boxidx < pipeline_data->charRegions[lidx].size(); boxidx++)
+        all_regions_combined.push_back(pipeline_data->charRegions[lidx][boxidx]);
+    }
+    cleanCharRegions(pipeline_data->thresholds, all_regions_combined);
 
     if (config->debugTiming)
     {