Added prewarp to ALPR analysis

src/openalpr/CMakeLists.txt

@@ -25,6 +25,7 @@ set(lpr_source_files
  edges/textlinecollection.cpp
  edges/scorekeeper.cpp
  colorfilter.cpp
+ prewarp.cpp
  transformation.cpp
  textdetection/characteranalysis.cpp
  textdetection/platemask.cpp

src/openalpr/alpr_impl.cpp

@@ -18,6 +18,7 @@
 */
 
 #include "alpr_impl.h"
+#include "prewarp.h"
 
 void plateAnalysisThread(void* arg);
 
@@ -33,6 +34,7 @@ namespace alpr
     plateDetector = ALPR_NULL_PTR;
     stateIdentifier = ALPR_NULL_PTR;
     ocr = ALPR_NULL_PTR;
+    prewarp = ALPR_NULL_PTR;
     
     // Config file or runtime dir not found.  Don't process any further.
     if (config->loaded == false)
@@ -47,6 +49,8 @@ namespace alpr
     setDetectRegion(DEFAULT_DETECT_REGION);
     this->topN = DEFAULT_TOPN;
     setDefaultRegion("");
+    
+    prewarp = new PreWarp(config);
 
   }
   AlprImpl::~AlprImpl()
@@ -61,6 +65,9 @@ namespace alpr
 
     if (ocr != ALPR_NULL_PTR)
       delete ocr;
+    
+    if (prewarp != ALPR_NULL_PTR)
+      delete prewarp;
   }
 
   bool AlprImpl::isLoaded()
@@ -96,26 +103,38 @@ namespace alpr
       return response;
     }
 
+    // Convert image to grayscale if required
+    Mat grayImg = img;
+    if (img.channels() > 2)
+      cvtColor( img, grayImg, CV_BGR2GRAY );
+    
+    // Prewarp the image and ROIs if configured]
+    std::vector<cv::Rect> warpedRegionsOfInterest = regionsOfInterest;
+    // Warp the image if prewarp is provided
+    grayImg = prewarp->warpImage(grayImg);
+    warpedRegionsOfInterest = prewarp->projectRects(regionsOfInterest, grayImg.cols, grayImg.rows, false);
+    
+    vector<PlateRegion> warpedPlateRegions;
     // Find all the candidate regions
     if (config->skipDetection == false)
     {
-      response.plateRegions = plateDetector->detect(img, regionsOfInterest);
+      warpedPlateRegions = plateDetector->detect(grayImg, warpedRegionsOfInterest);
     }
     else
     {
       // They have elected to skip plate detection.  Instead, return a list of plate regions
       // based on their regions of interest
-      for (unsigned int i = 0; i < regionsOfInterest.size(); i++)
+      for (unsigned int i = 0; i < warpedRegionsOfInterest.size(); i++)
       {
         PlateRegion pr;
-        pr.rect = cv::Rect(regionsOfInterest[i]);
-        response.plateRegions.push_back(pr);
+        pr.rect = cv::Rect(warpedRegionsOfInterest[i]);
+        warpedPlateRegions.push_back(pr);
       }
     }
 
     queue<PlateRegion> plateQueue;
-    for (unsigned int i = 0; i < response.plateRegions.size(); i++)
-      plateQueue.push(response.plateRegions[i]);
+    for (unsigned int i = 0; i < warpedPlateRegions.size(); i++)
+      plateQueue.push(warpedPlateRegions[i]);
 
     int platecount = 0;
     while(!plateQueue.empty())
@@ -123,7 +142,7 @@ namespace alpr
       PlateRegion plateRegion = plateQueue.front();
       plateQueue.pop();
 
-      PipelineData pipeline_data(img, plateRegion.rect, config);
+      PipelineData pipeline_data(img, grayImg, plateRegion.rect, config);
 
       timespec platestarttime;
       getTimeMonotonic(&platestarttime);
@@ -140,12 +159,16 @@ namespace alpr
         plateResult.regionConfidence = 0;
         plateResult.plate_index = platecount++;
 
+        // If using prewarp, remap the plate corners to the original image
+        vector<Point2f> cornerPoints = pipeline_data.plate_corners;
+        cornerPoints = prewarp->projectPoints(cornerPoints, true);
+        
         for (int pointidx = 0; pointidx < 4; pointidx++)
         {
-          plateResult.plate_points[pointidx].x = (int) pipeline_data.plate_corners[pointidx].x;
-          plateResult.plate_points[pointidx].y = (int) pipeline_data.plate_corners[pointidx].y;
+          plateResult.plate_points[pointidx].x = (int) cornerPoints[pointidx].x;
+          plateResult.plate_points[pointidx].y = (int) cornerPoints[pointidx].y;
         }
-
+        
         if (detectRegion)
         {
           stateIdentifier->recognize(&pipeline_data);
@@ -225,6 +248,10 @@ namespace alpr
 
     }
 
+    // Unwarp plate regions if necessary
+    prewarp->projectPlateRegions(warpedPlateRegions, grayImg.cols, grayImg.rows, true);
+    response.plateRegions = warpedPlateRegions;
+    
     timespec endTime;
     getTimeMonotonic(&endTime);
     response.results.total_processing_time_ms = diffclock(startTime, endTime);

src/openalpr/alpr_impl.h

@@ -32,6 +32,8 @@
 #include "detection/detector.h"
 #include "detection/detectorfactory.h"
 
+#include "prewarp.h"
+
 #include "licenseplatecandidate.h"
 #include "stateidentifier.h"
 #include "segmentation/charactersegmenter.h"
@@ -96,6 +98,7 @@ namespace alpr
       Detector* plateDetector;
       StateIdentifier* stateIdentifier;
       OCR* ocr;
+      PreWarp* prewarp;
 
       int topN;
       bool detectRegion;

src/openalpr/config.cpp

@@ -154,6 +154,8 @@ namespace alpr
 
     skipDetection = getBoolean("common", "skip_detection", false);
     
+    prewarp = getString("common", "prewarp", "");
+            
     maxPlateAngleDegrees = getInt("common", "max_plate_angle_degrees", 15);
 
     minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100);
@@ -205,6 +207,7 @@ namespace alpr
 
     debugGeneral = 	getBoolean("debug", "general",		false);
     debugTiming = 	getBoolean("debug", "timing",		false);
+    debugPrewarp = 	getBoolean("debug", "prewarp",		false);
     debugDetector = 	getBoolean("debug", "detector",		false);
     debugStateId = 	getBoolean("debug", "state_id",		false);
     debugPlateLines = 	getBoolean("debug", "plate_lines", 	false);

src/openalpr/config.h

@@ -58,6 +58,8 @@ namespace alpr
       
       bool skipDetection;
 
+      std::string prewarp;
+      
       int maxPlateAngleDegrees;
 
       float minPlateSizeWidthPx;
@@ -106,6 +108,7 @@ namespace alpr
 
       bool debugGeneral;
       bool debugTiming;
+      bool debugPrewarp;
       bool debugDetector;
       bool debugStateId;
       bool debugPlateLines;

src/openalpr/pipeline_data.cpp

@@ -8,15 +8,23 @@ namespace alpr
 
   PipelineData::PipelineData(Mat colorImage, Rect regionOfInterest, Config* config)
   {
-    this->colorImg = colorImage;
+    Mat grayImg;
     if (colorImage.channels() > 2)
     {
-      cvtColor(this->colorImg, this->grayImg, CV_BGR2GRAY);
+      cvtColor(this->colorImg, grayImg, CV_BGR2GRAY);
     }
     else
     {
-      this->grayImg = colorImage;
+      grayImg = colorImage;
     }
+    
+    PipelineData(colorImage, grayImg, regionOfInterest, config);
+  }
+  
+  PipelineData::PipelineData(Mat colorImage, Mat grayImg, Rect regionOfInterest, Config* config)
+  {
+    this->colorImg = colorImage;
+    this->grayImg = grayImg;
 
     this->regionOfInterest = regionOfInterest;
     this->config = config;

src/openalpr/pipeline_data.h

@@ -15,6 +15,7 @@ namespace alpr
 
     public:
       PipelineData(cv::Mat colorImage, cv::Rect regionOfInterest, Config* config);
+      PipelineData(cv::Mat colorImage, cv::Mat grayImage, cv::Rect regionOfInterest, Config* config);
       virtual ~PipelineData();
 
       void clearThresholds();

src/openalpr/prewarp.cpp

@@ -0,0 +1,254 @@
+/*
+ * Copyright (c) 2015 New Designs Unlimited, LLC
+ * Opensource Automated License Plate Recognition [http://www.openalpr.com]
+ *
+ * This file is part of OpenAlpr.
+ *
+ * OpenAlpr is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * version 3 as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include <opencv2/highgui/highgui.hpp>
+
+#include "prewarp.h"
+
+using namespace std;
+using namespace cv;
+
+namespace alpr
+{
+
+  PreWarp::PreWarp(Config* config) {
+    this->config = config;
+    
+    string warp_config = config->prewarp;
+    
+    // Do a cursory verification based on number of commas
+    int commacount = count(warp_config.begin(), warp_config.end(), ',');
+    
+    if (warp_config.length() < 4)
+    {
+      // No config specified.  ignore
+      if (this->config->debugPrewarp)
+        cout << "No prewarp configuration specified" << endl;
+
+      this->valid = false;
+    }
+    else if (commacount != 8)
+    {
+      if (this->config->debugPrewarp)
+        cout << "Invalid prewarp configuration" << endl;
+
+      this->valid = false;
+    }
+    else
+    {
+
+      // Parse the warp_config
+      int first_comma = warp_config.find(",");
+
+
+      string name = warp_config.substr(0, first_comma);
+      
+      if (name != "planar")
+      {
+        this->valid = false;
+      }
+      else
+      {
+        stringstream ss(warp_config.substr(first_comma + 1, warp_config.length()));
+
+        ss >> w;
+        ss.ignore();
+        ss >> h;
+        ss.ignore();
+        ss >> rotationx;
+        ss.ignore();  // Ignore comma
+        ss >> rotationy;
+        ss.ignore();  // Ignore comma
+        ss >> rotationz;
+        ss.ignore();  // Ignore comma
+        ss >> dist;
+        ss.ignore();  // Ignore comma
+        ss >> panX;
+        ss.ignore();  // Ignore comma
+        ss >> panY;
+        
+        this->valid = true;
+      }
+
+    }
+  }
+
+
+  PreWarp::~PreWarp() {
+  }
+  
+  
+  cv::Mat PreWarp::warpImage(Mat image) {
+    if (!this->valid)
+    {
+      if (this->config->debugPrewarp)
+        cout << "prewarp skipped due to missing prewarp config" << endl;
+      return image;
+    }
+    
+      
+    float width_ratio = w / ((float)image.cols);
+    float height_ratio = h / ((float)image.rows);
+
+    float rx = rotationx * width_ratio;
+    float ry = rotationy * width_ratio;
+    float px = panX / width_ratio;
+    float py = panY / height_ratio;
+
+
+    transform = findTransform(image.cols, image.rows, rx, ry, rotationz, px, py, dist);
+    
+    
+    Mat warped_image;
+  
+    warpPerspective(image, warped_image, transform, image.size(), INTER_CUBIC | WARP_INVERSE_MAP);
+
+    
+    if (this->config->debugPrewarp && this->config->debugShowImages)
+    {
+      imshow("Prewarp", warped_image);
+    }
+    return warped_image;
+  }
+
+  // Projects a "region of interest" into the new space
+  // The rect needs to be converted to points, warped, then converted back into a 
+  // bounding rectangle
+  vector<Rect> PreWarp::projectRects(vector<Rect> rects, int maxWidth, int maxHeight, bool inverse) {
+    
+    if (!this->valid)
+      return rects;
+    
+    vector<Rect> projected_rects;
+    
+    for (unsigned int i = 0; i < rects.size(); i++)
+    {
+      vector<Point2f> points;
+      points.push_back(Point(rects[i].x, rects[i].y));
+      points.push_back(Point(rects[i].x + rects[i].width, rects[i].y));
+      points.push_back(Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height));
+      points.push_back(Point(rects[i].x, rects[i].y + rects[i].height));
+      
+      vector<Point2f> projectedPoints = projectPoints(points, inverse);
+      
+      Rect projectedRect = boundingRect(projectedPoints);
+      projectedRect = expandRect(projectedRect, 0, 0, maxWidth, maxHeight);
+      projected_rects.push_back(projectedRect);
+      
+    }
+    
+    return projected_rects;
+  }
+
+  vector<Point2f> PreWarp::projectPoints(vector<Point2f> points, bool inverse) {
+    
+    if (!this->valid)
+      return points;
+    
+    vector<Point2f> output;
+    
+    if (!inverse)
+      perspectiveTransform(points, output, transform.inv());
+    else
+      perspectiveTransform(points, output, transform);
+    
+    return output;
+  }
+  
+
+  void PreWarp::projectPlateRegions(vector<PlateRegion>& plateRegions, int maxWidth, int maxHeight, bool inverse){
+    
+    if (!this->valid)
+      return;
+    
+    for (unsigned int i = 0; i < plateRegions.size(); i++)
+    {
+      vector<Rect> singleRect;
+      singleRect.push_back(plateRegions[i].rect);
+      vector<Rect> transformedRect = projectRects(singleRect, maxWidth, maxHeight, inverse);
+      plateRegions[i].rect.x = transformedRect[0].x;
+      plateRegions[i].rect.y = transformedRect[0].y;
+      plateRegions[i].rect.width = transformedRect[0].width;
+      plateRegions[i].rect.height = transformedRect[0].height;
+      
+      projectPlateRegions(plateRegions[i].children, maxWidth, maxHeight, inverse);
+    }
+  }
+  
+  cv::Mat PreWarp::findTransform(float w, float h, 
+          float rotationx, float rotationy, float rotationz, 
+          float panX, float panY, float dist) {
+
+    float alpha = rotationx;
+    float beta = rotationy;
+    float gamma = rotationz;
+    float f = 1.0;
+
+    // Projection 2D -> 3D matrix
+    Mat A1 = (Mat_<double>(4,3) <<
+        1, 0, -w/2,
+        0, 1, -h/2,
+        0, 0,    0,
+        0, 0,    1);
+    
+    // Camera Intrisecs matrix 3D -> 2D
+    Mat A2 = (Mat_<double>(3,4) <<
+        f, 0, w/2, 0,
+        0, f, h/2, 0,
+        0, 0,   1, 0);
+
+    // Rotation matrices around the X axis
+    Mat Rx = (Mat_<double>(4, 4) <<
+        1,          0,           0, 0,
+        0, cos(alpha), -sin(alpha), 0,
+        0, sin(alpha),  cos(alpha), 0,
+        0,          0,           0, 1);
+
+    // Rotation matrices around the Y axis
+    Mat Ry = (Mat_<double>(4, 4) <<
+        cos(beta), 0, sin(beta), 0,
+        0, 1, 0, 0,
+        -sin(beta), 0,  cos(beta), 0,
+        0,          0,           0, 1);
+
+    // Rotation matrices around the Z axis
+    Mat Rz = (Mat_<double>(4, 4) <<
+        cos(gamma), -sin(gamma), 0, 0,
+        sin(gamma), cos(gamma), 0, 0,
+       0, 0, 1, 0,
+        0,          0,           0, 1);
+
+    Mat R = Rx*Ry*Rz;
+
+    // Translation matrix on the Z axis 
+    Mat T = (Mat_<double>(4, 4) <<
+        1, 0, 0, panX,
+        0, 1, 0, panY,
+        0, 0, 1, dist,
+        0, 0, 0, 1);
+
+
+    return A2 * (T * (R * A1));
+
+  }
+
+  
+}
+

src/openalpr/prewarp.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2015 New Designs Unlimited, LLC
+ * Opensource Automated License Plate Recognition [http://www.openalpr.com]
+ *
+ * This file is part of OpenAlpr.
+ *
+ * OpenAlpr is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * version 3 as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifndef OPENALPR_PREWARP_H
+#define	OPENALPR_PREWARP_H
+
+#include "config.h"
+#include "utility.h"
+#include "opencv2/imgproc/imgproc.hpp"
+#include "detection/detector.h"
+
+namespace alpr
+{
+
+  class PreWarp {
+  public:
+    PreWarp(Config* config);
+    virtual ~PreWarp();
+
+    cv::Mat warpImage(cv::Mat image);
+    std::vector<cv::Point2f> projectPoints(std::vector<cv::Point2f> points, bool inverse);
+    std::vector<cv::Rect> projectRects(std::vector<cv::Rect> rects, int maxWidth, int maxHeight, bool inverse);
+    void projectPlateRegions(std::vector<PlateRegion>& plateRegions, int maxWidth, int maxHeight, bool inverse);
+
+    bool valid;
+    
+  private:
+    Config* config;
+    cv::Mat transform;
+    
+    float w, h, rotationx, rotationy, rotationz, dist, panX, panY;
+    
+    cv::Mat findTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float dist);
+  };
+
+}
+
+#endif	/* OPENALPR_PREWARP_H */
+