Renamed benchmarks directory to prevent conflicts if cmake is called from main dir

src/misc_utilities/benchmarks/benchmark.cpp

@@ -0,0 +1,315 @@
+/*
+ * Copyright (c) 2013 New Designs Unlimited, LLC
+ * Opensource Automated License Plate Recognition [http://www.openalpr.com]
+ *
+ * This file is part of OpenAlpr.
+ *
+ * OpenAlpr is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License
+ * version 3 as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "opencv2/highgui/highgui.hpp"
+#include "opencv2/imgproc/imgproc.hpp"
+
+#include <iostream>
+#include <fstream>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <numeric>      // std::accumulate
+
+#include "alpr_impl.h"
+
+#include "endtoendtest.h"
+
+#include "support/filesystem.h"
+
+using namespace std;
+using namespace cv;
+
+// Given a directory full of lp images (named [statecode]#.png) crop out the alphanumeric characters.
+// These will be used to train the OCR
+
+void outputStats(vector<double> datapoints);
+
+
+
+
+
+int main( int argc, const char** argv )
+{
+  string country;
+  string benchmarkName;
+  string inDir;
+  string outDir;
+  Mat frame;
+
+  //Check if user specify image to process
+  if(argc == 5)
+  {
+    country = argv[1];
+    benchmarkName = argv[2];
+    inDir = argv[3];
+    outDir = argv[4];
+  }
+  else
+  {
+    printf("Use:\n\t%s [country] [benchmark name] [img input dir] [results output dir]\n",argv[0]);
+    printf("\tex: %s us speed ./speed/usimages ./speed\n",argv[0]);
+    printf("\n");
+    printf("\ttest names are: speed, segocr, detection\n\n" );
+    return 0;
+  }
+
+  if (DirectoryExists(inDir.c_str()) == false)
+  {
+    printf("Input dir does not exist\n");
+    return 0;
+  }
+  if (DirectoryExists(outDir.c_str()) == false)
+  {
+    printf("Output dir does not exist\n");
+    return 0;
+  }
+
+  vector<string> files = getFilesInDir(inDir.c_str());
+  sort( files.begin(), files.end(), stringCompare );
+
+  if (benchmarkName.compare("segocr") == 0)
+  {
+    Config* config = new Config(country);
+    config->debugOff();
+
+    OCR* ocr = new OCR(config);
+
+    for (int i = 0; i< files.size(); i++)
+    {
+      if (hasEnding(files[i], ".png") || hasEnding(files[i], ".jpg"))
+      {
+        string fullpath = inDir + "/" + files[i];
+
+        frame = imread( fullpath.c_str() );
+        resize(frame, frame, Size(config->ocrImageWidthPx, config->ocrImageHeightPx));
+
+        Rect plateCoords;
+        plateCoords.x = 0;
+        plateCoords.y = 0;
+        plateCoords.width = frame.cols;
+        plateCoords.height = frame.rows;
+	
+	PipelineData pipeline_data(frame, plateCoords, config);
+
+        char statecode[3];
+        statecode[0] = files[i][0];
+        statecode[1] = files[i][1];
+        statecode[2] = '\0';
+        string statecodestr(statecode);
+
+        CharacterRegion charRegion(&pipeline_data);
+
+        if (abs(charRegion.getTopLine().angle) > 4)
+        {
+          // Rotate image:
+          Mat rotated(frame.size(), frame.type());
+          Mat rot_mat( 2, 3, CV_32FC1 );
+          Point center = Point( frame.cols/2, frame.rows/2 );
+
+          rot_mat = getRotationMatrix2D( center, charRegion.getTopLine().angle, 1.0 );
+          warpAffine( frame, rotated, rot_mat, frame.size() );
+
+          rotated.copyTo(frame);
+	  pipeline_data.crop_gray = frame;
+        }
+
+        CharacterSegmenter charSegmenter(&pipeline_data);
+        ocr->performOCR(&pipeline_data);
+        ocr->postProcessor->analyze(statecode, 25);
+
+        cout << files[i] << "," << statecode << "," << ocr->postProcessor->bestChars << endl;
+
+        imshow("Current LP", frame);
+        waitKey(5);
+      }
+    }
+
+    delete config;
+    delete ocr;
+  }
+  else if (benchmarkName.compare("detection") == 0)
+  {
+    Config config(country);
+    RegionDetector plateDetector(&config);
+
+    for (int i = 0; i< files.size(); i++)
+    {
+      if (hasEnding(files[i], ".png") || hasEnding(files[i], ".jpg"))
+      {
+        string fullpath = inDir + "/" + files[i];
+        frame = imread( fullpath.c_str() );
+
+        vector<PlateRegion> regions = plateDetector.detect(frame);
+
+        imshow("Current LP", frame);
+        waitKey(5);
+      }
+    }
+  }
+  else if (benchmarkName.compare("speed") == 0)
+  {
+    // Benchmarks speed of region detection, plate analysis, and OCR
+
+    timespec startTime;
+    timespec endTime;
+
+    Config config(country);
+    config.debugOff();
+
+    AlprImpl alpr(country);
+    alpr.config->debugOff();
+    alpr.setDetectRegion(true);
+
+    RegionDetector plateDetector(&config);
+    StateIdentifier stateIdentifier(&config);
+    OCR ocr(&config);
+
+    vector<double> endToEndTimes;
+    vector<double> regionDetectionTimes;
+    vector<double> stateIdTimes;
+    vector<double> lpAnalysisPositiveTimes;
+    vector<double> lpAnalysisNegativeTimes;
+    vector<double> ocrTimes;
+    vector<double> postProcessTimes;
+
+    for (int i = 0; i< files.size(); i++)
+    {
+      if (hasEnding(files[i], ".png") || hasEnding(files[i], ".jpg"))
+      {
+        cout << "Image: " << files[i] << endl;
+
+        string fullpath = inDir + "/" + files[i];
+        frame = imread( fullpath.c_str() );
+
+        getTime(&startTime);
+        vector<Rect> regionsOfInterest;
+        regionsOfInterest.push_back(Rect(0, 0, frame.cols, frame.rows));
+        alpr.recognize(frame, regionsOfInterest);
+        getTime(&endTime);
+        double endToEndTime = diffclock(startTime, endTime);
+        cout << " -- End to End recognition time: " << endToEndTime << "ms." << endl;
+        endToEndTimes.push_back(endToEndTime);
+
+        getTime(&startTime);
+        vector<PlateRegion> regions = plateDetector.detect(frame);
+        getTime(&endTime);
+
+        double regionDetectionTime = diffclock(startTime, endTime);
+        cout << " -- Region detection time: " << regionDetectionTime << "ms." << endl;
+        regionDetectionTimes.push_back(regionDetectionTime);
+
+        for (int z = 0; z < regions.size(); z++)
+        {
+	  
+	  PipelineData pipeline_data(frame, regions[z].rect, &config);
+	  
+          getTime(&startTime);
+	  
+          stateIdentifier.recognize(&pipeline_data);
+          getTime(&endTime);
+          double stateidTime = diffclock(startTime, endTime);
+          cout << "\tRegion " << z << ": State ID time: " << stateidTime << "ms." << endl;
+          stateIdTimes.push_back(stateidTime);
+
+          getTime(&startTime);
+          LicensePlateCandidate lp(&pipeline_data);
+          lp.recognize();
+          getTime(&endTime);
+          double analysisTime = diffclock(startTime, endTime);
+          cout << "\tRegion " << z << ": Analysis time: " << analysisTime << "ms." << endl;
+
+          if (pipeline_data.plate_area_confidence > 10)
+          {
+            lpAnalysisPositiveTimes.push_back(analysisTime);
+
+            getTime(&startTime);
+            ocr.performOCR(&pipeline_data);
+            getTime(&endTime);
+            double ocrTime = diffclock(startTime, endTime);
+            cout << "\tRegion " << z << ": OCR time: " << ocrTime << "ms." << endl;
+            ocrTimes.push_back(ocrTime);
+
+            getTime(&startTime);
+            ocr.postProcessor->analyze("", 25);
+            getTime(&endTime);
+            double postProcessTime = diffclock(startTime, endTime);
+            cout << "\tRegion " << z << ": PostProcess time: " << postProcessTime << "ms." << endl;
+            postProcessTimes.push_back(postProcessTime);
+          }
+          else
+          {
+            lpAnalysisNegativeTimes.push_back(analysisTime);
+          }
+        }
+
+        waitKey(5);
+      }
+    }
+
+    cout << endl << "---------------------" << endl;
+
+    cout << "End to End Time Statistics:" << endl;
+    outputStats(endToEndTimes);
+    cout << endl;
+
+    cout << "Region Detection Time Statistics:" << endl;
+    outputStats(regionDetectionTimes);
+    cout << endl;
+
+    cout << "State ID Time Statistics:" << endl;
+    outputStats(stateIdTimes);
+    cout << endl;
+
+    cout << "Positive Region Analysis Time Statistics:" << endl;
+    outputStats(lpAnalysisPositiveTimes);
+    cout << endl;
+
+    cout << "Negative Region Analysis Time Statistics:" << endl;
+    outputStats(lpAnalysisNegativeTimes);
+    cout << endl;
+
+    cout << "OCR Time Statistics:" << endl;
+    outputStats(ocrTimes);
+    cout << endl;
+
+    cout << "Post Processing Time Statistics:" << endl;
+    outputStats(postProcessTimes);
+    cout << endl;
+  }
+  else if (benchmarkName.compare("endtoend") == 0)
+  {
+    EndToEndTest e2eTest(inDir, outDir);
+    e2eTest.runTest(country, files);
+    
+  }
+}
+
+void outputStats(vector<double> datapoints)
+{
+  double sum = std::accumulate(datapoints.begin(), datapoints.end(), 0.0);
+  double mean = sum / datapoints.size();
+
+  std::vector<double> diff(datapoints.size());
+  std::transform(datapoints.begin(), datapoints.end(), diff.begin(),
+                 std::bind2nd(std::minus<double>(), mean));
+  double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+  double stdev = std::sqrt(sq_sum / datapoints.size());
+
+  cout << "\t" << datapoints.size() << " samples, avg: " << mean << "ms,  stdev: " << stdev << endl;
+}

src/misc_utilities/benchmarks/benchmark_utils.cpp

@@ -0,0 +1,24 @@
+
+#include "support/filesystem.h"
+#include "benchmark_utils.h"
+
+using namespace std;
+
+vector<string> filterByExtension(vector<string> fileList, string extension)
+{
+  vector<string> filteredList;
+  
+  if (extension.size() == 0)
+    return filteredList;
+  
+  if (extension[0] != '.')
+    extension = "." + extension;
+  
+  for (int i = 0; i < fileList.size(); i++)
+  {
+    if (hasEnding(fileList[i], extension))
+      filteredList.push_back(fileList[i]);
+  }
+  
+  return filteredList;
+}

src/misc_utilities/benchmarks/benchmark_utils.h

@@ -0,0 +1,9 @@
+#ifndef OPENALPR_BENCHMARKUTILS_H
+#define OPENALPR_BENCHMARKUTILS_H
+
+#include <iostream>
+
+std::vector<std::string> filterByExtension(std::vector<std::string> fileList, std::string extension);
+
+
+#endif // OPENALPR_BENCHMARKUTILS_H

src/misc_utilities/benchmarks/endtoendtest.cpp

@@ -0,0 +1,257 @@
+#include "endtoendtest.h"
+
+using namespace std;
+using namespace cv;
+
+
+EndToEndTest::EndToEndTest(string inputDir, string outputDir)
+{
+  this->inputDir = inputDir;
+  this->outputDir = outputDir;
+}
+
+
+void EndToEndTest::runTest(string country, vector<std::string> files)
+{
+
+  
+  
+  AlprImpl alpr(country);
+  alpr.config->debugOff();
+  alpr.setDetectRegion(false);
+
+  vector<EndToEndBenchmarkResult> benchmarkResults;
+
+  vector<string> textFiles = filterByExtension(files, ".txt");
+
+  for (int i = 0; i< textFiles.size(); i++)
+  {
+    cout << "Benchmarking file " << (i + 1) << " / " << textFiles.size() << " -- " << textFiles[i] << endl;
+    EndToEndBenchmarkResult benchmarkResult;
+	
+    string fulltextpath = inputDir + "/" + textFiles[i];
+    
+    ifstream inputFile(fulltextpath.c_str());
+    string line;
+    
+    getline(inputFile, line);
+    
+    istringstream ss(line);
+
+    string imgfile, plate_number;
+    int x, y, w, h;
+
+    ss >> imgfile >> x >> y >> w >> h >> plate_number;
+    
+    string fullimgpath = inputDir + "/" + imgfile;
+    
+    benchmarkResult.imageName = imgfile;
+    
+    Mat frame = imread( fullimgpath.c_str() );
+
+    Rect actualPlateRect(x, y, w, h);
+    
+    AlprFullDetails recognitionDetails = alpr.recognizeFullDetails(frame);
+    
+
+    
+    //cv::circle(frame, centerPoint, 2, Scalar(0, 0, 255), 5);
+    //drawAndWait(&frame);
+    
+    benchmarkResult.detectionFalsePositives = 0;
+    
+    for (int z = 0; z < recognitionDetails.plateRegions.size(); z++)
+    {
+      benchmarkResult.detectionFalsePositives += totalRectCount(recognitionDetails.plateRegions[z]);
+      
+      bool rectmatches = rectMatches(actualPlateRect, recognitionDetails.plateRegions[z]);
+      
+      
+      if (rectmatches)
+      {
+	// This region matches our plate_number
+	benchmarkResult.detectedPlate = true;
+	benchmarkResult.detectionFalsePositives--;
+	break;
+      }
+      
+    }
+    
+    benchmarkResult.resultsFalsePositives = recognitionDetails.results.size();
+    
+    // Determine if the top result and the top N results match the correct value
+    for (int z = 0; z < recognitionDetails.results.size(); z++)
+    {
+      //cout << "Actual: " << plate_number << endl;
+      //cout << "Candidate: " << recognitionDetails.results[z].bestPlate.characters << endl;
+      if (recognitionDetails.results[z].bestPlate.characters == plate_number)
+      {
+	benchmarkResult.topResultCorrect = true;
+	benchmarkResult.top10ResultCorrect = true;
+	benchmarkResult.resultsFalsePositives--;
+	break;
+      }
+      
+      for (int idx = 0; idx < recognitionDetails.results[z].topNPlates.size(); idx++)
+      {
+	if (recognitionDetails.results[z].topNPlates[idx].characters == plate_number)
+	{
+	  benchmarkResult.top10ResultCorrect = true;
+	  benchmarkResult.resultsFalsePositives--;
+	  break;
+	}
+      }
+      
+      if (benchmarkResult.top10ResultCorrect)
+	break;
+    }
+    
+    benchmarkResults.push_back(benchmarkResult);
+    
+  
+  }
+
+  // Print results data
+  
+  ofstream data;
+  string outputResultsFile = outputDir + "/results.txt";
+  data.open(outputResultsFile.c_str());
+  
+  data << "Image name		Detected Plate		# False Detections		Top Result Correct	Top 10 Correct		# False Results" << endl;
+  for (int i = 0; i < benchmarkResults.size(); i++)
+  {
+    EndToEndBenchmarkResult br = benchmarkResults[i];
+    data << br.imageName << "\t" << br.detectedPlate << "\t" << br.detectionFalsePositives << "\t" << br.topResultCorrect << "\t" << br.top10ResultCorrect << "\t" << br.resultsFalsePositives << endl;
+  }
+  data.close();
+  
+  // Print summary data
+  
+  
+  int totalDetections = 0;
+  int totalTopResultCorrect = 0;
+  int totalTop10Correct = 0;
+  int falseDetectionPositives = 0;
+  int falseResults = 0;
+  for (int i = 0; i < benchmarkResults.size(); i++)
+  {
+    if (benchmarkResults[i].detectedPlate) totalDetections++;
+    if (benchmarkResults[i].topResultCorrect) totalTopResultCorrect++;
+    if (benchmarkResults[i].top10ResultCorrect) totalTop10Correct++;
+    
+    falseDetectionPositives += benchmarkResults[i].detectionFalsePositives;
+    falseResults += benchmarkResults[i].resultsFalsePositives;
+  }
+  
+  // Percentage of how many are correct (higher is better)
+  float detectionScore = 100.0 * ((float) totalDetections) / ((float) benchmarkResults.size());
+  float topResultScore = 100.0 * ((float) totalTopResultCorrect) / ((float) benchmarkResults.size());
+  float top10ResultScore = 100.0 * ((float) totalTop10Correct) / ((float) benchmarkResults.size());
+  
+  // How many false positives per image (higher is worse)
+  float falseDetectionPositivesScore = ((float) falseDetectionPositives) / ((float) benchmarkResults.size());
+  float falseResultsScore = ((float) falseResults) / ((float) benchmarkResults.size());
+  
+  string outputSummaryFile = outputDir + "/summary.txt";
+  data.open(outputSummaryFile.c_str());
+  
+  data << "-------------------" << endl;
+  data << "|     SUMMARY     |" << endl;
+  data << "-------------------" << endl;
+  data << endl;
+  data << "Accuracy scores (higher is better)" << endl;
+  data << "Percent of plates DETECTED: " << detectionScore << endl;
+  data << "Percent of correct TOP10:   " << top10ResultScore << endl;
+  data << "Percent of correct MATCHES: " << topResultScore << endl;
+  data << endl;
+  data << "False Positives Score (lower is better)" << endl;
+  data << "False DETECTIONS per image: " << falseDetectionPositivesScore << endl;
+  data << "False RESULTS per image:    " << falseResultsScore << endl;
+  
+  data.close();
+  
+  
+  // Print the contents of these files now:
+  string line;
+  ifstream resultsFileIn(outputResultsFile.c_str());
+  while(getline(resultsFileIn, line))
+  {
+    cout << line << endl;
+  }
+  ifstream summaryFileIn(outputSummaryFile.c_str());
+  while(getline(summaryFileIn, line))
+  {
+    cout << line << endl;
+  }
+}
+
+
+bool EndToEndTest::rectMatches(cv::Rect actualPlate, PlateRegion candidate)
+{
+  // Determine if this region matches our plate in the image
+  // Do this simply by verifying that the center point of the plate is within the region
+  // And that the plate region is not x% larger or smaller
+
+  const float MAX_SIZE_PERCENT_LARGER = 0.65;
+
+  //int plateCenterX = actualPlate.x + (int) (((float) actualPlate.width) / 2.0);
+  //int plateCenterY = actualPlate.y + (int) (((float) actualPlate.height) / 2.0);
+  //Point centerPoint(plateCenterX, plateCenterY);
+  
+  vector<Point> requiredPoints;
+  requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.2),
+				   actualPlate.y + (int) (((float) actualPlate.height) * 0.15)
+			   ));
+  requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.8),
+				   actualPlate.y + (int) (((float) actualPlate.height) * 0.15)
+			  ));
+  requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.2),
+				  actualPlate.y + (int) (((float) actualPlate.height) * 0.85)
+			  ));
+  requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.8),
+				actualPlate.y + (int) (((float) actualPlate.height) * 0.85)
+			));
+  
+
+  float sizeDiff = 1.0 - ((float) actualPlate.area()) / ((float) candidate.rect.area());
+
+  //cout << "Candidate: " << candidate.rect.x << "," << candidate.rect.y << " " << candidate.rect.width << "-" << candidate.rect.height << endl;
+  //cout << "Actual:    " << actualPlate.x << "," << actualPlate.y << " " << actualPlate.width << "-" << actualPlate.height << endl;
+  
+  //cout << "size diff: " << sizeDiff << endl;
+  
+  bool hasAllPoints = true;
+  for (int z = 0; z < requiredPoints.size(); z++)
+  {
+    if (candidate.rect.contains(requiredPoints[z]) == false)
+      hasAllPoints = false;
+    break;
+  }
+  if ( hasAllPoints && 
+    (sizeDiff < MAX_SIZE_PERCENT_LARGER) )
+
+  {
+    return true;
+  }
+  else
+  {
+    for (int i = 0; i < candidate.children.size(); i++)
+    {
+      if (rectMatches(actualPlate, candidate.children[i]))
+	return true;
+    }
+  }
+  
+  return false;
+}
+
+int EndToEndTest::totalRectCount(PlateRegion rootCandidate)
+{
+  int childCount = 0;
+  for (int i = 0; i < rootCandidate.children.size(); i++)
+  {
+    childCount += totalRectCount(rootCandidate.children[i]);
+  }
+  
+  return childCount + 1;
+}

src/misc_utilities/benchmarks/endtoendtest.h

@@ -0,0 +1,47 @@
+#ifndef OPENALPR_ENDTOENDTEST_H
+#define OPENALPR_ENDTOENDTEST_H
+
+
+#include "opencv2/highgui/highgui.hpp"
+#include "opencv2/imgproc/imgproc.hpp"
+#include "alpr_impl.h"
+#include "benchmark_utils.h"
+
+class EndToEndTest
+{
+  public:
+    EndToEndTest(std::string inputDir, std::string outputDir);
+    void runTest(std::string country, std::vector<std::string> files);
+  
+  private:
+    
+    bool rectMatches(cv::Rect actualPlate, PlateRegion candidate);
+    int totalRectCount(PlateRegion rootCandidate);
+	
+    std::string inputDir;
+    std::string outputDir;
+  
+  
+};
+
+class EndToEndBenchmarkResult {
+  public:
+    EndToEndBenchmarkResult()
+    {
+    this->imageName = "";
+    this->detectedPlate = false;
+    this->topResultCorrect = false;
+    this->top10ResultCorrect = false;
+    this->detectionFalsePositives = 0;
+    this->resultsFalsePositives = 0;
+    }
+    
+    std::string imageName;
+    bool detectedPlate;
+    bool topResultCorrect;
+    bool top10ResultCorrect;
+    int detectionFalsePositives;
+    int resultsFalsePositives;
+};
+
+#endif	//OPENALPR_ENDTOENDTEST_H