From 716f1ca671410130f384e599bdf5dff2face63ab Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Fri, 20 Jun 2014 22:30:35 -0400 Subject: [PATCH] Updated end-to-end benchmark --- src/misc_utilities/benchmark/benchmark.cpp | 45 +-- .../benchmark/benchmark_utils.cpp | 24 ++ .../benchmark/benchmark_utils.h | 9 + src/misc_utilities/benchmark/endtoendtest.cpp | 257 ++++++++++++++++++ 4 files changed, 299 insertions(+), 36 deletions(-) create mode 100644 src/misc_utilities/benchmark/benchmark_utils.cpp create mode 100644 src/misc_utilities/benchmark/benchmark_utils.h create mode 100644 src/misc_utilities/benchmark/endtoendtest.cpp diff --git a/src/misc_utilities/benchmark/benchmark.cpp b/src/misc_utilities/benchmark/benchmark.cpp index 4c9011b..7a4f23b 100644 --- a/src/misc_utilities/benchmark/benchmark.cpp +++ b/src/misc_utilities/benchmark/benchmark.cpp @@ -28,10 +28,8 @@ #include "alpr_impl.h" -//#include "stage1.h" -//#include "stage2.h" -//#include "stateidentifier.h" -//#include "utility.h" +#include "endtoendtest.h" + #include "support/filesystem.h" using namespace std; @@ -42,6 +40,10 @@ using namespace cv; void outputStats(vector datapoints); + + + + int main( int argc, const char** argv ) { string country; @@ -284,38 +286,9 @@ int main( int argc, const char** argv ) } else if (benchmarkName.compare("endtoend") == 0) { - Alpr alpr(country); - alpr.setDetectRegion(true); - - ofstream outputdatafile; - - outputdatafile.open("results.txt"); - - for (int i = 0; i< files.size(); i++) - { - if (hasEnding(files[i], ".png")) - { - string fullpath = inDir + "/" + files[i]; - frame = imread( fullpath.c_str() ); - - vector buffer; - imencode(".bmp", frame, buffer ); - - vector results = alpr.recognize(buffer); - - outputdatafile << files[i] << ": "; - for (int z = 0; z < results.size(); z++) - { - outputdatafile << results[z].bestPlate.characters << ", "; - } - outputdatafile << endl; - - imshow("Current LP", frame); - waitKey(5); - } - } - - outputdatafile.close(); + EndToEndTest e2eTest(inDir, outDir); + e2eTest.runTest(country, files); + } } diff --git a/src/misc_utilities/benchmark/benchmark_utils.cpp b/src/misc_utilities/benchmark/benchmark_utils.cpp new file mode 100644 index 0000000..09438fa --- /dev/null +++ b/src/misc_utilities/benchmark/benchmark_utils.cpp @@ -0,0 +1,24 @@ + +#include "support/filesystem.h" +#include "benchmark_utils.h" + +using namespace std; + +vector filterByExtension(vector fileList, string extension) +{ + vector filteredList; + + if (extension.size() == 0) + return filteredList; + + if (extension[0] != '.') + extension = "." + extension; + + for (int i = 0; i < fileList.size(); i++) + { + if (hasEnding(fileList[i], extension)) + filteredList.push_back(fileList[i]); + } + + return filteredList; +} \ No newline at end of file diff --git a/src/misc_utilities/benchmark/benchmark_utils.h b/src/misc_utilities/benchmark/benchmark_utils.h new file mode 100644 index 0000000..499f712 --- /dev/null +++ b/src/misc_utilities/benchmark/benchmark_utils.h @@ -0,0 +1,9 @@ +#ifndef OPENALPR_BENCHMARKUTILS_H +#define OPENALPR_BENCHMARKUTILS_H + +#include + +std::vector filterByExtension(std::vector fileList, std::string extension); + + +#endif // OPENALPR_BENCHMARKUTILS_H diff --git a/src/misc_utilities/benchmark/endtoendtest.cpp b/src/misc_utilities/benchmark/endtoendtest.cpp new file mode 100644 index 0000000..7332d29 --- /dev/null +++ b/src/misc_utilities/benchmark/endtoendtest.cpp @@ -0,0 +1,257 @@ +#include "endtoendtest.h" + +using namespace std; +using namespace cv; + + +EndToEndTest::EndToEndTest(string inputDir, string outputDir) +{ + this->inputDir = inputDir; + this->outputDir = outputDir; +} + + +void EndToEndTest::runTest(string country, vector files) +{ + + + + AlprImpl alpr(country); + alpr.config->debugOff(); + alpr.setDetectRegion(false); + + vector benchmarkResults; + + vector textFiles = filterByExtension(files, ".txt"); + + for (int i = 0; i< textFiles.size(); i++) + { + cout << "Benchmarking file " << (i + 1) << " / " << textFiles.size() << " -- " << textFiles[i] << endl; + EndToEndBenchmarkResult benchmarkResult; + + string fulltextpath = inputDir + "/" + textFiles[i]; + + ifstream inputFile(fulltextpath.c_str()); + string line; + + getline(inputFile, line); + + istringstream ss(line); + + string imgfile, plate_number; + int x, y, w, h; + + ss >> imgfile >> x >> y >> w >> h >> plate_number; + + string fullimgpath = inputDir + "/" + imgfile; + + benchmarkResult.imageName = imgfile; + + Mat frame = imread( fullimgpath.c_str() ); + + Rect actualPlateRect(x, y, w, h); + + AlprFullDetails recognitionDetails = alpr.recognizeFullDetails(frame); + + + + //cv::circle(frame, centerPoint, 2, Scalar(0, 0, 255), 5); + //drawAndWait(&frame); + + benchmarkResult.detectionFalsePositives = 0; + + for (int z = 0; z < recognitionDetails.plateRegions.size(); z++) + { + benchmarkResult.detectionFalsePositives += totalRectCount(recognitionDetails.plateRegions[z]); + + bool rectmatches = rectMatches(actualPlateRect, recognitionDetails.plateRegions[z]); + + + if (rectmatches) + { + // This region matches our plate_number + benchmarkResult.detectedPlate = true; + benchmarkResult.detectionFalsePositives--; + break; + } + + } + + benchmarkResult.resultsFalsePositives = recognitionDetails.results.size(); + + // Determine if the top result and the top N results match the correct value + for (int z = 0; z < recognitionDetails.results.size(); z++) + { + //cout << "Actual: " << plate_number << endl; + //cout << "Candidate: " << recognitionDetails.results[z].bestPlate.characters << endl; + if (recognitionDetails.results[z].bestPlate.characters == plate_number) + { + benchmarkResult.topResultCorrect = true; + benchmarkResult.top10ResultCorrect = true; + benchmarkResult.resultsFalsePositives--; + break; + } + + for (int idx = 0; idx < recognitionDetails.results[z].topNPlates.size(); idx++) + { + if (recognitionDetails.results[z].topNPlates[idx].characters == plate_number) + { + benchmarkResult.top10ResultCorrect = true; + benchmarkResult.resultsFalsePositives--; + break; + } + } + + if (benchmarkResult.top10ResultCorrect) + break; + } + + benchmarkResults.push_back(benchmarkResult); + + + } + + // Print results data + + ofstream data; + string outputResultsFile = outputDir + "/results.txt"; + data.open(outputResultsFile.c_str()); + + data << "Image name Detected Plate # False Detections Top Result Correct Top 10 Correct # False Results" << endl; + for (int i = 0; i < benchmarkResults.size(); i++) + { + EndToEndBenchmarkResult br = benchmarkResults[i]; + data << br.imageName << "\t" << br.detectedPlate << "\t" << br.detectionFalsePositives << "\t" << br.topResultCorrect << "\t" << br.top10ResultCorrect << "\t" << br.resultsFalsePositives << endl; + } + data.close(); + + // Print summary data + + + int totalDetections = 0; + int totalTopResultCorrect = 0; + int totalTop10Correct = 0; + int falseDetectionPositives = 0; + int falseResults = 0; + for (int i = 0; i < benchmarkResults.size(); i++) + { + if (benchmarkResults[i].detectedPlate) totalDetections++; + if (benchmarkResults[i].topResultCorrect) totalTopResultCorrect++; + if (benchmarkResults[i].top10ResultCorrect) totalTop10Correct++; + + falseDetectionPositives += benchmarkResults[i].detectionFalsePositives; + falseResults += benchmarkResults[i].resultsFalsePositives; + } + + // Percentage of how many are correct (higher is better) + float detectionScore = 100.0 * ((float) totalDetections) / ((float) benchmarkResults.size()); + float topResultScore = 100.0 * ((float) totalTopResultCorrect) / ((float) benchmarkResults.size()); + float top10ResultScore = 100.0 * ((float) totalTop10Correct) / ((float) benchmarkResults.size()); + + // How many false positives per image (higher is worse) + float falseDetectionPositivesScore = ((float) falseDetectionPositives) / ((float) benchmarkResults.size()); + float falseResultsScore = ((float) falseResults) / ((float) benchmarkResults.size()); + + string outputSummaryFile = outputDir + "/summary.txt"; + data.open(outputSummaryFile.c_str()); + + data << "-------------------" << endl; + data << "| SUMMARY |" << endl; + data << "-------------------" << endl; + data << endl; + data << "Accuracy scores (higher is better)" << endl; + data << "Percent of plates DETECTED: " << detectionScore << endl; + data << "Percent of correct TOP10: " << top10ResultScore << endl; + data << "Percent of correct MATCHES: " << topResultScore << endl; + data << endl; + data << "False Positives Score (lower is better)" << endl; + data << "False DETECTIONS per image: " << falseDetectionPositivesScore << endl; + data << "False RESULTS per image: " << falseResultsScore << endl; + + data.close(); + + + // Print the contents of these files now: + string line; + ifstream resultsFileIn(outputResultsFile.c_str()); + while(getline(resultsFileIn, line)) + { + cout << line << endl; + } + ifstream summaryFileIn(outputSummaryFile.c_str()); + while(getline(summaryFileIn, line)) + { + cout << line << endl; + } +} + + +bool EndToEndTest::rectMatches(cv::Rect actualPlate, PlateRegion candidate) +{ + // Determine if this region matches our plate in the image + // Do this simply by verifying that the center point of the plate is within the region + // And that the plate region is not x% larger or smaller + + const float MAX_SIZE_PERCENT_LARGER = 0.65; + + //int plateCenterX = actualPlate.x + (int) (((float) actualPlate.width) / 2.0); + //int plateCenterY = actualPlate.y + (int) (((float) actualPlate.height) / 2.0); + //Point centerPoint(plateCenterX, plateCenterY); + + vector requiredPoints; + requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.2), + actualPlate.y + (int) (((float) actualPlate.height) * 0.15) + )); + requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.8), + actualPlate.y + (int) (((float) actualPlate.height) * 0.15) + )); + requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.2), + actualPlate.y + (int) (((float) actualPlate.height) * 0.85) + )); + requiredPoints.push_back(Point( actualPlate.x + (int) (((float) actualPlate.width) * 0.8), + actualPlate.y + (int) (((float) actualPlate.height) * 0.85) + )); + + + float sizeDiff = 1.0 - ((float) actualPlate.area()) / ((float) candidate.rect.area()); + + //cout << "Candidate: " << candidate.rect.x << "," << candidate.rect.y << " " << candidate.rect.width << "-" << candidate.rect.height << endl; + //cout << "Actual: " << actualPlate.x << "," << actualPlate.y << " " << actualPlate.width << "-" << actualPlate.height << endl; + + //cout << "size diff: " << sizeDiff << endl; + + bool hasAllPoints = true; + for (int z = 0; z < requiredPoints.size(); z++) + { + if (candidate.rect.contains(requiredPoints[z]) == false) + hasAllPoints = false; + break; + } + if ( hasAllPoints && + (sizeDiff < MAX_SIZE_PERCENT_LARGER) ) + + { + return true; + } + else + { + for (int i = 0; i < candidate.children.size(); i++) + { + if (rectMatches(actualPlate, candidate.children[i])) + return true; + } + } + + return false; +} + +int EndToEndTest::totalRectCount(PlateRegion rootCandidate) +{ + int childCount = 0; + for (int i = 0; i < rootCandidate.children.size(); i++) + { + childCount += totalRectCount(rootCandidate.children[i]); + } + + return childCount + 1; +}