Added "analysis_count" config setting

config/openalpr.conf.in

@@ -49,6 +49,10 @@ must_match_pattern = 0
 ; Bypasses plate detection.  If this is set to 1, the library assumes that each region provided is a likely plate area.
 skip_detection = 0
 
+; OpenALPR can scan the same image multiple times with different randomization.  Setting this to a value larger than
+; 1 may increase accuracy, but will increase processing time linearly (e.g., analysis_count = 3 is 3x slower)
+analysis_count = 1
+
 ; OpenALPR detects high-contrast plate crops and uses an alternative edge detection technique.  Setting this to 0.0 
 ; would classify  ALL images as high-contrast, setting it to 1.0 would classify no images as high-contrast. 
 contrast_detection_threshold = 0.3

src/openalpr/alpr_impl.cpp

@@ -125,22 +125,33 @@ namespace alpr
 
     // Iterate through each country provided (typically just one)
     // and aggregate the results if necessary
-    ResultAggregator aggregator;
+    ResultAggregator country_aggregator(MERGE_PICK_BEST, topN, config);
     for (unsigned int i = 0; i < config->loaded_countries.size(); i++)
     {
       if (config->debugGeneral)
         cout << "Analyzing: " << config->loaded_countries[i] << endl;
 
       config->setCountry(config->loaded_countries[i]);
-      AlprFullDetails sub_results = analyzeSingleCountry(img, grayImg, warpedRegionsOfInterest);
       
+      // Reapply analysis for each multiple analysis value set in the config,
+      // make a minor imperceptible tweak to the input image each time
+      ResultAggregator iter_aggregator(MERGE_COMBINE, topN, config);
+      for (unsigned int iteration = 0; iteration < 2; iteration++)
+      {
+        Mat iteration_image = iter_aggregator.applyImperceptibleChange(grayImg, iteration);
+        //drawAndWait(iteration_image);
+        AlprFullDetails iter_results = analyzeSingleCountry(img, iteration_image, warpedRegionsOfInterest);
+        iter_aggregator.addResults(iter_results);
+      }
+      
+      AlprFullDetails sub_results = iter_aggregator.getAggregateResults();
       sub_results.results.epoch_time = start_time;
       sub_results.results.img_width = img.cols;
       sub_results.results.img_height = img.rows;
       
-      aggregator.addResults(sub_results);
+      country_aggregator.addResults(sub_results);
     }
-    response = aggregator.getAggregateResults();
+    response = country_aggregator.getAggregateResults();
 
     timespec endTime;
     getTimeMonotonic(&endTime);

src/openalpr/config.cpp

@@ -189,6 +189,8 @@ namespace alpr
     
     skipDetection = getBoolean(ini, "", "skip_detection", false);
     
+    analysis_count = getInt(ini, "", "analysis_count", 1);
+    
     prewarp = getString(ini, "", "prewarp", "");
             
     maxPlateAngleDegrees = getInt(ini, "", "max_plate_angle_degrees", 15);

src/openalpr/config.h

@@ -61,6 +61,8 @@ namespace alpr
       
       bool skipDetection;
 
+      int analysis_count;
+      
       bool auto_invert;
       bool always_invert;
 

src/openalpr/prewarp.cpp

@@ -75,6 +75,7 @@ namespace alpr
       {
         stringstream ss(prewarp_config.substr(first_comma + 1, prewarp_config.length()));
 
+        float w, h, rotationx, rotationy, rotationz, panX, panY,  stretchX, dist;
         ss >> w;
         ss.ignore();
         ss >> h;
@@ -93,7 +94,7 @@ namespace alpr
         ss.ignore();  // Ignore comma
         ss >> panY;
         
-        this->valid = true;
+        setTransform(w, h, rotationx, rotationy, rotationz, panX, panY, stretchX, dist);
       }
 
     }
@@ -110,6 +111,20 @@ namespace alpr
   PreWarp::~PreWarp() {
   }
   
+  void PreWarp::setTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist)
+  {
+    this->w = w;
+    this->h = h;
+    this->rotationx = rotationx;
+    this->rotationy = rotationy;
+    this->rotationz = rotationz;
+    this->panX = panX;
+    this->panY = panY;
+    this->stretchX = stretchX;
+    this->dist = dist;
+    
+    this->valid = true;
+  }
   
   cv::Mat PreWarp::warpImage(Mat image) {
     if (!this->valid)
@@ -129,7 +144,7 @@ namespace alpr
     float py = panY / height_ratio;
 
 
-    transform = findTransform(image.cols, image.rows, rx, ry, rotationz, px, py, stretchX, dist);
+    transform = getTransform(image.cols, image.rows, rx, ry, rotationz, px, py, stretchX, dist);
     
     
     Mat warped_image;
@@ -208,7 +223,7 @@ namespace alpr
     }
   }
   
-  cv::Mat PreWarp::findTransform(float w, float h, 
+  cv::Mat PreWarp::getTransform(float w, float h, 
           float rotationx, float rotationy, float rotationz, 
           float panX, float panY, float stretchX, float dist) {
 

src/openalpr/prewarp.h

@@ -42,15 +42,18 @@ namespace alpr
     std::vector<cv::Rect> projectRects(std::vector<cv::Rect> rects, int maxWidth, int maxHeight, bool inverse);
     void projectPlateRegions(std::vector<PlateRegion>& plateRegions, int maxWidth, int maxHeight, bool inverse);
 
+    void setTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist);
+    
     bool valid;
     
   private:
     Config* config;
     cv::Mat transform;
     
+    cv::Mat getTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist);
+    
     float w, h, rotationx, rotationy, rotationz, stretchX, dist, panX, panY;
     
-    cv::Mat findTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist);
   };
 
 }

src/openalpr/result_aggregator.cpp

@@ -25,13 +25,16 @@ using namespace cv;
 namespace alpr
 {
 
-  ResultAggregator::ResultAggregator()
+  ResultAggregator::ResultAggregator(ResultMergeStrategy merge_strategy, int topn, Config* config)
   {
-
+    this->prewarp = new PreWarp(config);
+    this->merge_strategy = merge_strategy;
+    this->topn = topn;
+    this->config = config;
   }
 
   ResultAggregator::~ResultAggregator() {
-
+    delete prewarp;
   }
 
 
@@ -40,6 +43,45 @@ namespace alpr
     all_results.push_back(full_results);
   }
   
+
+  cv::Mat ResultAggregator::applyImperceptibleChange(cv::Mat image, int index) {
+    
+    const float WIDTH_HEIGHT = 600;
+    const float NO_MOVE_WIDTH_DIST = 1.0;
+    const float NO_PAN_VAL = 0;
+    float step = 0.000035;
+
+    // Don't warp the first indexed image
+    if (index == 0)
+      return image;
+    
+    // Use 3 bits to figure out which one is on.  Multiply by the modulus of 8
+    // 000, 001, 010, 011, 100, 101, 110, 111
+    // if 101, then x_rotation and z_rotation are on.
+    if (index % 8 == 0)
+    {
+      // Do something special for the 0s, so they don't repeat
+      index--;
+      step = step / 1.5;
+    }
+    int multiplier = (index / 8) + 1;
+    int bitwise_on = index % 8;
+    float x_rotation = ((bitwise_on & 1) == 1) * multiplier * step;
+    float y_rotation = ((bitwise_on & 2) == 2) * multiplier * step;
+    float z_rotation = ((bitwise_on & 4) == 4) * multiplier * step;
+    
+    //cout << "Iteration: " << index << ": " << x_rotation << ", " << y_rotation << ", " << z_rotation << endl;
+    
+    prewarp->setTransform(WIDTH_HEIGHT, WIDTH_HEIGHT, x_rotation, y_rotation, z_rotation, 
+            NO_PAN_VAL, NO_PAN_VAL, NO_MOVE_WIDTH_DIST, NO_MOVE_WIDTH_DIST);
+    
+    return prewarp->warpImage(image);
+  }
+
+  bool compareScore(const std::pair<float, ResultPlateScore>& firstElem, const std::pair<float, ResultPlateScore>& secondElem) {
+    return firstElem.first > secondElem.first;
+  }
+  
   AlprFullDetails ResultAggregator::getAggregateResults()
   {
     assert(all_results.size() > 0);
@@ -68,28 +110,199 @@ namespace alpr
 
     vector<vector<AlprPlateResult> > clusters = findClusters();
 
-    // Assume we have multiple results, one cluster for each unique train data (e.g., eu, eu2)
+    if (merge_strategy == MERGE_PICK_BEST)
-
-    // Now for each cluster of plates, pick the best one
-    for (unsigned int i = 0; i < clusters.size(); i++)
     {
-      float best_confidence = 0;
+      // Assume we have multiple results, one cluster for each unique train data (e.g., eu, eu2)
-      int best_index = 0;
-      for (unsigned int k = 0; k < clusters[i].size(); k++)
-      {
-        if (clusters[i][k].bestPlate.overall_confidence > best_confidence)
-        {
-          best_confidence = clusters[i][k].bestPlate.overall_confidence;
-          best_index = k;
-        }
-      }
 
-      response.results.plates.push_back(clusters[i][best_index]);
+      // Now for each cluster of plates, pick the best one
+      for (unsigned int i = 0; i < clusters.size(); i++)
+      {
+        float best_confidence = 0;
+        int best_index = 0;
+        for (unsigned int k = 0; k < clusters[i].size(); k++)
+        {
+          if (clusters[i][k].bestPlate.overall_confidence > best_confidence)
+          {
+            best_confidence = clusters[i][k].bestPlate.overall_confidence;
+            best_index = k;
+          }
+        }
+
+        response.results.plates.push_back(clusters[i][best_index]);
+      }
+    }
+    else if (merge_strategy == MERGE_COMBINE)
+    {
+      // Each cluster is the same plate, just analyzed from a slightly different 
+      // perspective.  Merge them together and score them as if they are one
+
+      const float MIN_CONFIDENCE = 75;
+      
+
+      // Factor in the position of the plate in the topN list, the confidence, and the template match status
+      // First loop is for clusters of possible plates.  If they're in separate clusters, they don't get combined, 
+      // since they are likely separate plates in the same image
+      for (unsigned int unique_plate_idx = 0; unique_plate_idx < clusters.size(); unique_plate_idx++)
+      {
+        std::map<string, ResultPlateScore> score_hash;
+        
+        // Second loop is for separate plate results for the same plate
+        for (unsigned int i = 0; i < clusters[unique_plate_idx].size(); i++)
+        {
+          // Third loop is the individual topN results for a single plate result
+          for (unsigned int j = 0; j < clusters[unique_plate_idx][i].topNPlates.size() && j < topn; j++)
+          {
+            AlprPlate plateCandidate = clusters[unique_plate_idx][i].topNPlates[j];
+            
+            if (plateCandidate.overall_confidence < MIN_CONFIDENCE)
+              continue;
+
+            float score = (plateCandidate.overall_confidence - 60) * 4;
+
+            // Add a bonus for matching the template
+            if (plateCandidate.matches_template)
+              score += 150;
+
+            // Add a bonus the higher the plate is to the #1 position
+            // and how frequently it appears there
+            float position_score_max_bonus = 65;
+            float frequency_modifier = ((float) position_score_max_bonus) / topn;
+            score += position_score_max_bonus - (j * frequency_modifier);
+            
+
+            if (score_hash.find(plateCandidate.characters) == score_hash.end())
+            {
+              ResultPlateScore newentry;
+              newentry.plate = plateCandidate;
+              newentry.score_total = 0;
+              newentry.count = 0;
+              score_hash[plateCandidate.characters] = newentry;
+            }
+
+            score_hash[plateCandidate.characters].score_total += score;
+            score_hash[plateCandidate.characters].count += 1;
+            // Use the best confidence value for a particular candidate
+            if (plateCandidate.overall_confidence > score_hash[plateCandidate.characters].plate.overall_confidence)
+              score_hash[plateCandidate.characters].plate.overall_confidence = plateCandidate.overall_confidence;
+          }
+        }
+
+        // There is a big list of results that have scores.  Sort them by top score
+        std::vector<std::pair<float, ResultPlateScore> > sorted_results;
+        std::map<string, ResultPlateScore>::iterator iter;
+        for (iter = score_hash.begin(); iter != score_hash.end(); iter++) {
+          std::pair<float,ResultPlateScore> r;
+          r.second = iter->second;
+          r.first = iter->second.score_total;
+          sorted_results.push_back(r);
+        }
+
+        std::sort(sorted_results.begin(), sorted_results.end(), compareScore);
+        
+        // output the sorted list for debugging:
+        if (config->debugGeneral)
+        {
+          cout << "Result Aggregator Scores: " << endl;
+          cout << "  " << std::setw(14) << "Plate Num"
+              << std::setw(15) << "Score"
+              << std::setw(10) << "Count"
+              << std::setw(10) << "Best conf (%)"
+              << endl;
+          
+          for (int r_idx = 0; r_idx < sorted_results.size(); r_idx++)
+          {
+            cout << "  " << std::setw(14) << sorted_results[r_idx].second.plate.characters
+                    << std::setw(15) << sorted_results[r_idx].second.score_total
+                    << std::setw(10) << sorted_results[r_idx].second.count
+                    << std::setw(10) << sorted_results[r_idx].second.plate.overall_confidence 
+                    << endl;
+
+          }
+        }
+        
+        // Figure out the best region for this cluster
+        ResultRegionScore regionResults = findBestRegion(clusters[unique_plate_idx]);
+        
+        AlprPlateResult firstResult = clusters[unique_plate_idx][0];
+        AlprPlateResult copyResult;
+        copyResult.bestPlate = sorted_results[0].second.plate;
+        copyResult.plate_index = firstResult.plate_index;
+        copyResult.region = regionResults.region;
+        copyResult.regionConfidence = regionResults.confidence;
+        copyResult.processing_time_ms = firstResult.processing_time_ms;
+        copyResult.requested_topn = firstResult.requested_topn;
+        for (int p_idx = 0; p_idx < 4; p_idx++)
+          copyResult.plate_points[p_idx] = firstResult.plate_points[p_idx];
+        
+        for (int i = 0; i < sorted_results.size(); i++)
+        {
+          if (i >= topn)
+            break;
+
+          copyResult.topNPlates.push_back(sorted_results[i].second.plate);
+        }
+        
+        response.results.plates.push_back(copyResult);
+
+      }
     }
 
     return response;
   }
   
+  ResultRegionScore ResultAggregator::findBestRegion(std::vector<AlprPlateResult> cluster) {
+
+    const float MIN_REGION_CONFIDENCE = 60;
+    
+    std::map<std::string, float> score_hash;
+    std::map<std::string, float> score_count;
+    int max_topn = 10;
+    
+    ResultRegionScore response;
+    response.confidence = 0;
+    response.region = "";
+    
+    for (unsigned int i = 0; i < cluster.size(); i++)
+    {
+      AlprPlateResult plate = cluster[i];
+      
+      if (plate.bestPlate.overall_confidence < MIN_REGION_CONFIDENCE )
+        continue;
+      
+      float score = (float) plate.regionConfidence;
+      
+      if (score_hash.count(plate.region) == 0)
+      {
+        score_hash[plate.region] = 0;
+        score_count[plate.region] = 0;
+      }
+
+      score_hash[plate.region] = score_hash[plate.region] + score;
+      score_count[plate.region] = score_count[plate.region] + 1;
+    }
+    
+    float best_score = -1;
+    std::string best_score_val = "";
+    // Now we have a hash that contains all the scores.  Iterate and find the best one and return it.
+    for(std::map<std::string, float >::iterator hash_iter=score_hash.begin(); hash_iter!=score_hash.end(); ++hash_iter) {
+      if (hash_iter->second > best_score)
+      {
+        best_score = hash_iter->second;
+        best_score_val = hash_iter->first;
+      }
+    }
+    
+    if (best_score > 0)
+    {
+      response.confidence = best_score / score_count[best_score_val];
+      response.region = best_score_val;
+    }
+    
+    return response;
+  
+  }
+
+  
   // Searches all_plates to find overlapping plates
   // Returns an array containing "clusters" (overlapping plates)
   std::vector<std::vector<AlprPlateResult> > ResultAggregator::findClusters()

src/openalpr/result_aggregator.h

@@ -22,7 +22,7 @@
 
 
 #include "alpr_impl.h"
-
+#include "prewarp.h"
 
 // Runs the analysis for multiple training sets, and aggregates the results into the best matches
 
@@ -37,10 +37,30 @@ struct PlateShapeInfo
 namespace alpr
 {
 
+  enum ResultMergeStrategy
+  {
+    MERGE_COMBINE,  // Used when running an analysis multiple times for accuracy improvement.  Merges results together
+    MERGE_PICK_BEST // Used when analyzing multiple countries.  Chooses results from one country or the other
+  };
+  
+  struct ResultPlateScore
+  {
+    AlprPlate plate;
+    float score_total;
+    int count;
+  };
+  
+  struct ResultRegionScore
+  {
+    std::string region;
+    float confidence;
+    
+  };
+  
   class ResultAggregator
   {
   public:
-    ResultAggregator();
+    ResultAggregator(ResultMergeStrategy merge_strategy, int topn, Config* config);
 
     virtual ~ResultAggregator();
 
@@ -48,11 +68,22 @@ namespace alpr
 
     AlprFullDetails getAggregateResults();
 
+    cv::Mat applyImperceptibleChange(cv::Mat image, int index);
+    
   private:
+    
+    int topn;
+    PreWarp* prewarp;
+    Config* config;
+    
     std::vector<AlprFullDetails> all_results;
 
     PlateShapeInfo getShapeInfo(AlprPlateResult plate);
 
+    ResultMergeStrategy merge_strategy;
+    
+    ResultRegionScore findBestRegion(std::vector<AlprPlateResult> cluster);
+    
     std::vector<std::vector<AlprPlateResult> > findClusters();
     int overlaps(AlprPlateResult plate, std::vector<std::vector<AlprPlateResult> > clusters);
   };