From 85f52a6b8c78cc19d9177c32085c0a3e1a8c5517 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 27 Oct 2014 20:12:57 -0400 Subject: [PATCH] Wrapped OpenALPR library in "alpr" namespace. Resolves issue #60. --- src/daemon.cpp | 2 + src/main.cpp | 2 + src/misc_utilities/benchmarks/benchmark.cpp | 1 + .../benchmarks/benchmark_utils.cpp | 1 + .../benchmarks/endtoendtest.cpp | 1 + src/misc_utilities/benchmarks/endtoendtest.h | 4 +- src/misc_utilities/classifychars.cpp | 1 + src/misc_utilities/prepcharsfortraining.cpp | 1 + src/misc_utilities/sortstate.cpp | 1 + src/misc_utilities/tagplates.cpp | 1 + src/openalpr/alpr.cpp | 116 +- src/openalpr/alpr.h | 166 +- src/openalpr/alpr_impl.cpp | 849 ++++---- src/openalpr/alpr_impl.h | 94 +- src/openalpr/binarize_wolf.cpp | 336 +-- src/openalpr/binarize_wolf.h | 32 +- src/openalpr/colorfilter.cpp | 668 +++--- src/openalpr/colorfilter.h | 37 +- src/openalpr/config.cpp | 472 ++--- src/openalpr/config.h | 182 +- src/openalpr/detection/detector.cpp | 151 +- src/openalpr/detection/detector.h | 55 +- src/openalpr/detection/detectorcpu.cpp | 171 +- src/openalpr/detection/detectorcpu.h | 29 +- src/openalpr/detection/detectorfactory.cpp | 10 +- src/openalpr/detection/detectorfactory.h | 6 +- src/openalpr/edges/edgefinder.cpp | 208 +- src/openalpr/edges/edgefinder.h | 28 +- src/openalpr/edges/platecorners.cpp | 591 +++--- src/openalpr/edges/platecorners.h | 52 +- src/openalpr/edges/platelines.cpp | 380 ++-- src/openalpr/edges/platelines.h | 49 +- src/openalpr/edges/scorekeeper.cpp | 89 +- src/openalpr/edges/scorekeeper.h | 41 +- src/openalpr/edges/textlinecollection.cpp | 279 +-- src/openalpr/edges/textlinecollection.h | 62 +- src/openalpr/featurematcher.cpp | 667 +++--- src/openalpr/featurematcher.h | 61 +- src/openalpr/licenseplatecandidate.cpp | 171 +- src/openalpr/licenseplatecandidate.h | 35 +- src/openalpr/ocr.cpp | 177 +- src/openalpr/ocr.h | 25 +- src/openalpr/pipeline_data.cpp | 47 +- src/openalpr/pipeline_data.h | 80 +- src/openalpr/postprocess.cpp | 942 ++++----- src/openalpr/postprocess.h | 135 +- .../segmentation/charactersegmenter.cpp | 1863 +++++++++-------- .../segmentation/charactersegmenter.h | 79 +- src/openalpr/segmentation/segment.cpp | 54 +- src/openalpr/segmentation/segment.h | 25 +- .../segmentation/segmentationgroup.cpp | 47 +- src/openalpr/segmentation/segmentationgroup.h | 34 +- .../segmentation/verticalhistogram.cpp | 275 +-- src/openalpr/segmentation/verticalhistogram.h | 64 +- src/openalpr/stateidentifier.cpp | 111 +- src/openalpr/stateidentifier.h | 26 +- src/openalpr/support/filesystem.cpp | 185 +- src/openalpr/support/filesystem.h | 21 +- src/openalpr/support/platform.cpp | 83 +- src/openalpr/support/platform.h | 8 +- src/openalpr/support/timing.cpp | 275 +-- src/openalpr/support/timing.h | 11 +- .../textdetection/characteranalysis.cpp | 1127 +++++----- .../textdetection/characteranalysis.h | 57 +- src/openalpr/textdetection/linefinder.cpp | 409 ++-- src/openalpr/textdetection/linefinder.h | 47 +- src/openalpr/textdetection/platemask.cpp | 293 +-- src/openalpr/textdetection/platemask.h | 38 +- src/openalpr/textdetection/textcontours.cpp | 190 +- src/openalpr/textdetection/textcontours.h | 59 +- src/openalpr/textdetection/textline.cpp | 149 +- src/openalpr/textdetection/textline.h | 49 +- src/openalpr/transformation.cpp | 215 +- src/openalpr/transformation.h | 51 +- src/openalpr/utility.cpp | 745 +++---- src/openalpr/utility.h | 101 +- src/tests/test_api.cpp | 1 + src/tests/test_utility.cpp | 1 + src/video/videobuffer.cpp | 1 + 79 files changed, 7234 insertions(+), 6968 deletions(-) diff --git a/src/daemon.cpp b/src/daemon.cpp index 710166a..05b139f 100644 --- a/src/daemon.cpp +++ b/src/daemon.cpp @@ -21,6 +21,8 @@ #include #include +using namespace alpr; + // prototypes void streamRecognitionThread(void* arg); bool writeToQueue(std::string jsonResult); diff --git a/src/main.cpp b/src/main.cpp index 553e329..f6ead92 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -33,6 +33,8 @@ #include "video/videobuffer.h" #include "alpr.h" +using namespace alpr; + const std::string MAIN_WINDOW_NAME = "ALPR main window"; const bool SAVE_LAST_VIDEO_STILL = false; diff --git a/src/misc_utilities/benchmarks/benchmark.cpp b/src/misc_utilities/benchmarks/benchmark.cpp index 35b3f7f..a5407ae 100644 --- a/src/misc_utilities/benchmarks/benchmark.cpp +++ b/src/misc_utilities/benchmarks/benchmark.cpp @@ -35,6 +35,7 @@ using namespace std; using namespace cv; +using namespace alpr; // Given a directory full of lp images (named [statecode]#.png) crop out the alphanumeric characters. // These will be used to train the OCR diff --git a/src/misc_utilities/benchmarks/benchmark_utils.cpp b/src/misc_utilities/benchmarks/benchmark_utils.cpp index 09438fa..1ee3f06 100644 --- a/src/misc_utilities/benchmarks/benchmark_utils.cpp +++ b/src/misc_utilities/benchmarks/benchmark_utils.cpp @@ -3,6 +3,7 @@ #include "benchmark_utils.h" using namespace std; +using namespace alpr; vector filterByExtension(vector fileList, string extension) { diff --git a/src/misc_utilities/benchmarks/endtoendtest.cpp b/src/misc_utilities/benchmarks/endtoendtest.cpp index 65583cc..76e0086 100644 --- a/src/misc_utilities/benchmarks/endtoendtest.cpp +++ b/src/misc_utilities/benchmarks/endtoendtest.cpp @@ -2,6 +2,7 @@ using namespace std; using namespace cv; +using namespace alpr; EndToEndTest::EndToEndTest(string inputDir, string outputDir) diff --git a/src/misc_utilities/benchmarks/endtoendtest.h b/src/misc_utilities/benchmarks/endtoendtest.h index d0d2469..284df20 100644 --- a/src/misc_utilities/benchmarks/endtoendtest.h +++ b/src/misc_utilities/benchmarks/endtoendtest.h @@ -15,8 +15,8 @@ class EndToEndTest private: - bool rectMatches(cv::Rect actualPlate, PlateRegion candidate); - int totalRectCount(PlateRegion rootCandidate); + bool rectMatches(cv::Rect actualPlate, alpr::PlateRegion candidate); + int totalRectCount(alpr::PlateRegion rootCandidate); std::string inputDir; std::string outputDir; diff --git a/src/misc_utilities/classifychars.cpp b/src/misc_utilities/classifychars.cpp index a92723a..1043863 100644 --- a/src/misc_utilities/classifychars.cpp +++ b/src/misc_utilities/classifychars.cpp @@ -32,6 +32,7 @@ using namespace std; using namespace cv; +using namespace alpr; // Given a directory full of lp images (named [statecode]#.png) crop out the alphanumeric characters. // These will be used to train the OCR diff --git a/src/misc_utilities/prepcharsfortraining.cpp b/src/misc_utilities/prepcharsfortraining.cpp index e7d2359..a92e075 100644 --- a/src/misc_utilities/prepcharsfortraining.cpp +++ b/src/misc_utilities/prepcharsfortraining.cpp @@ -28,6 +28,7 @@ using namespace std; using namespace cv; +using namespace alpr; // Takes a directory full of single char images, and plops them on a big tif files // Also creates a box file so Tesseract can recognize it diff --git a/src/misc_utilities/sortstate.cpp b/src/misc_utilities/sortstate.cpp index 3dc9038..f049dfb 100644 --- a/src/misc_utilities/sortstate.cpp +++ b/src/misc_utilities/sortstate.cpp @@ -31,6 +31,7 @@ using namespace std; using namespace cv; +using namespace alpr; // Given a directory full of pre-cropped images, identify the state that each image belongs to. // This is used to sort our own positive image database as a first step before grabbing characters to use to train the OCR. diff --git a/src/misc_utilities/tagplates.cpp b/src/misc_utilities/tagplates.cpp index ba4e21e..d1beb5d 100644 --- a/src/misc_utilities/tagplates.cpp +++ b/src/misc_utilities/tagplates.cpp @@ -58,6 +58,7 @@ const int UP_ARROW_KEY= 82; using namespace std; using namespace cv; +using namespace alpr; static bool ldragging = false; static int xPos1 = 0; diff --git a/src/openalpr/alpr.cpp b/src/openalpr/alpr.cpp index d2ee8f6..8bd94f3 100644 --- a/src/openalpr/alpr.cpp +++ b/src/openalpr/alpr.cpp @@ -20,76 +20,80 @@ #include "alpr.h" #include "alpr_impl.h" -// ALPR code - -Alpr::Alpr(const std::string country, const std::string configFile, const std::string runtimeDir) -{ - impl = new AlprImpl(country, configFile, runtimeDir); -} - -Alpr::~Alpr() -{ - delete impl; -} - -AlprResults Alpr::recognize(std::string filepath) +namespace alpr { - std::ifstream ifs(filepath.c_str(), std::ios::binary|std::ios::ate); - std::ifstream::pos_type pos = ifs.tellg(); + // ALPR code - std::vector buffer(pos); + Alpr::Alpr(const std::string country, const std::string configFile, const std::string runtimeDir) + { + impl = new AlprImpl(country, configFile, runtimeDir); + } - ifs.seekg(0, std::ios::beg); - ifs.read(&buffer[0], pos); + Alpr::~Alpr() + { + delete impl; + } - return this->recognize( buffer ); -} + AlprResults Alpr::recognize(std::string filepath) + { -AlprResults Alpr::recognize(std::vector imageBytes) -{ - std::vector regionsOfInterest; - return impl->recognize(imageBytes, regionsOfInterest); -} + std::ifstream ifs(filepath.c_str(), std::ios::binary|std::ios::ate); + std::ifstream::pos_type pos = ifs.tellg(); -AlprResults Alpr::recognize(unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest) -{ - return impl->recognize(pixelData, bytesPerPixel, imgWidth, imgHeight, regionsOfInterest); -} + std::vector buffer(pos); -std::string Alpr::toJson( AlprResults results ) -{ - return AlprImpl::toJson(results); -} + ifs.seekg(0, std::ios::beg); + ifs.read(&buffer[0], pos); -AlprResults Alpr::fromJson(std::string json) { - return AlprImpl::fromJson(json); -} + return this->recognize( buffer ); + } + + AlprResults Alpr::recognize(std::vector imageBytes) + { + std::vector regionsOfInterest; + return impl->recognize(imageBytes, regionsOfInterest); + } + + AlprResults Alpr::recognize(unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest) + { + return impl->recognize(pixelData, bytesPerPixel, imgWidth, imgHeight, regionsOfInterest); + } + + std::string Alpr::toJson( AlprResults results ) + { + return AlprImpl::toJson(results); + } + + AlprResults Alpr::fromJson(std::string json) { + return AlprImpl::fromJson(json); + } -void Alpr::setDetectRegion(bool detectRegion) -{ - impl->setDetectRegion(detectRegion); -} + void Alpr::setDetectRegion(bool detectRegion) + { + impl->setDetectRegion(detectRegion); + } -void Alpr::setTopN(int topN) -{ - impl->setTopN(topN); -} + void Alpr::setTopN(int topN) + { + impl->setTopN(topN); + } -void Alpr::setDefaultRegion(std::string region) -{ - impl->setDefaultRegion(region); -} + void Alpr::setDefaultRegion(std::string region) + { + impl->setDefaultRegion(region); + } -bool Alpr::isLoaded() -{ - return impl->isLoaded(); -} + bool Alpr::isLoaded() + { + return impl->isLoaded(); + } -std::string Alpr::getVersion() -{ - return AlprImpl::getVersion(); -} + std::string Alpr::getVersion() + { + return AlprImpl::getVersion(); + } +} \ No newline at end of file diff --git a/src/openalpr/alpr.h b/src/openalpr/alpr.h index 1f774c7..45899a7 100644 --- a/src/openalpr/alpr.h +++ b/src/openalpr/alpr.h @@ -24,105 +24,109 @@ #include #include -struct AlprPlate +namespace alpr { - std::string characters; - float overall_confidence; - - bool matches_template; -}; - -struct AlprCoordinate -{ - int x; - int y; -}; - -class AlprRegionOfInterest -{ -public: - AlprRegionOfInterest(); - AlprRegionOfInterest(int x, int y, int width, int height) - { - this->x = x; - this->y = y; - this->width = width; - this->height = height; - }; - int x; - int y; - int width; - int height; -}; + struct AlprPlate + { + std::string characters; + float overall_confidence; -class AlprPlateResult -{ + bool matches_template; + }; + + struct AlprCoordinate + { + int x; + int y; + }; + + class AlprRegionOfInterest + { public: - AlprPlateResult() {}; - virtual ~AlprPlateResult() {}; + AlprRegionOfInterest(); + AlprRegionOfInterest(int x, int y, int width, int height) + { + this->x = x; + this->y = y; + this->width = width; + this->height = height; + }; - int requested_topn; + int x; + int y; + int width; + int height; + }; - AlprPlate bestPlate; - std::vector topNPlates; + class AlprPlateResult + { + public: + AlprPlateResult() {}; + virtual ~AlprPlateResult() {}; - float processing_time_ms; - AlprCoordinate plate_points[4]; + int requested_topn; - int regionConfidence; - std::string region; -}; + AlprPlate bestPlate; + std::vector topNPlates; -class AlprResults -{ - public: - AlprResults() {}; - virtual ~AlprResults() {}; + float processing_time_ms; + AlprCoordinate plate_points[4]; - long epoch_time; - int img_width; - int img_height; - float total_processing_time_ms; - - std::vector plates; - - std::vector regionsOfInterest; + int regionConfidence; + std::string region; + }; -}; + class AlprResults + { + public: + AlprResults() {}; + virtual ~AlprResults() {}; + + long epoch_time; + int img_width; + int img_height; + float total_processing_time_ms; + + std::vector plates; + + std::vector regionsOfInterest; + + }; -class AlprImpl; -class Alpr -{ + class AlprImpl; + class Alpr + { - public: - Alpr(const std::string country, const std::string configFile = "", const std::string runtimeDir = ""); - virtual ~Alpr(); + public: + Alpr(const std::string country, const std::string configFile = "", const std::string runtimeDir = ""); + virtual ~Alpr(); - void setDetectRegion(bool detectRegion); - void setTopN(int topN); - void setDefaultRegion(std::string region); + void setDetectRegion(bool detectRegion); + void setTopN(int topN); + void setDefaultRegion(std::string region); - // Recognize from an image on disk - AlprResults recognize(std::string filepath); - - // Recognize from byte data representing an encoded image (e.g., BMP, PNG, JPG, GIF etc). - AlprResults recognize(std::vector imageBytes); - - // Recognize from raw pixel data. - AlprResults recognize(unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest); + // Recognize from an image on disk + AlprResults recognize(std::string filepath); - - static std::string toJson(const AlprResults results); - static AlprResults fromJson(std::string json); + // Recognize from byte data representing an encoded image (e.g., BMP, PNG, JPG, GIF etc). + AlprResults recognize(std::vector imageBytes); - bool isLoaded(); - - static std::string getVersion(); + // Recognize from raw pixel data. + AlprResults recognize(unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest); - private: - AlprImpl* impl; -}; + static std::string toJson(const AlprResults results); + static AlprResults fromJson(std::string json); + + bool isLoaded(); + + static std::string getVersion(); + + private: + AlprImpl* impl; + }; + +} #endif // OPENALPR_APLR_H diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 02e6a56..5260d1e 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -24,471 +24,474 @@ void plateAnalysisThread(void* arg); using namespace std; using namespace cv; -AlprImpl::AlprImpl(const std::string country, const std::string configFile, const std::string runtimeDir) +namespace alpr { - config = new Config(country, configFile, runtimeDir); - - // Config file or runtime dir not found. Don't process any further. - if (config->loaded == false) + AlprImpl::AlprImpl(const std::string country, const std::string configFile, const std::string runtimeDir) { - plateDetector = ALPR_NULL_PTR; - stateIdentifier = ALPR_NULL_PTR; - ocr = ALPR_NULL_PTR; - return; + config = new Config(country, configFile, runtimeDir); + + // Config file or runtime dir not found. Don't process any further. + if (config->loaded == false) + { + plateDetector = ALPR_NULL_PTR; + stateIdentifier = ALPR_NULL_PTR; + ocr = ALPR_NULL_PTR; + return; + } + + plateDetector = createDetector(config); + stateIdentifier = new StateIdentifier(config); + ocr = new OCR(config); + setNumThreads(0); + + this->detectRegion = DEFAULT_DETECT_REGION; + this->topN = DEFAULT_TOPN; + this->defaultRegion = ""; + } - - plateDetector = createDetector(config); - stateIdentifier = new StateIdentifier(config); - ocr = new OCR(config); - setNumThreads(0); - - this->detectRegion = DEFAULT_DETECT_REGION; - this->topN = DEFAULT_TOPN; - this->defaultRegion = ""; - -} -AlprImpl::~AlprImpl() -{ - delete config; - - if (plateDetector != ALPR_NULL_PTR) - delete plateDetector; - - if (stateIdentifier != ALPR_NULL_PTR) - delete stateIdentifier; - - if (ocr != ALPR_NULL_PTR) - delete ocr; -} + AlprImpl::~AlprImpl() + { + delete config; -bool AlprImpl::isLoaded() -{ - return config->loaded; -} + if (plateDetector != ALPR_NULL_PTR) + delete plateDetector; -AlprFullDetails AlprImpl::recognizeFullDetails(cv::Mat img) -{ - std::vector regionsOfInterest; - regionsOfInterest.push_back(cv::Rect(0, 0, img.cols, img.rows)); - - return this->recognizeFullDetails(img, regionsOfInterest); -} + if (stateIdentifier != ALPR_NULL_PTR) + delete stateIdentifier; -AlprFullDetails AlprImpl::recognizeFullDetails(cv::Mat img, std::vector regionsOfInterest) -{ - timespec startTime; - getTime(&startTime); - - if (regionsOfInterest.size() == 0) + if (ocr != ALPR_NULL_PTR) + delete ocr; + } + + bool AlprImpl::isLoaded() + { + return config->loaded; + } + + AlprFullDetails AlprImpl::recognizeFullDetails(cv::Mat img) + { + std::vector regionsOfInterest; regionsOfInterest.push_back(cv::Rect(0, 0, img.cols, img.rows)); - - AlprFullDetails response; - - response.results.epoch_time = getEpochTime(); - response.results.img_width = img.cols; - response.results.img_height = img.rows; - - for (uint i = 0; i < regionsOfInterest.size(); i++) - { - response.results.regionsOfInterest.push_back(AlprRegionOfInterest(regionsOfInterest[i].x, regionsOfInterest[i].y, - regionsOfInterest[i].width, regionsOfInterest[i].height)); + + return this->recognizeFullDetails(img, regionsOfInterest); } - - if (!img.data) + + AlprFullDetails AlprImpl::recognizeFullDetails(cv::Mat img, std::vector regionsOfInterest) { - // Invalid image - if (this->config->debugGeneral) - std::cerr << "Invalid image" << std::endl; - + timespec startTime; + getTime(&startTime); + + if (regionsOfInterest.size() == 0) + regionsOfInterest.push_back(cv::Rect(0, 0, img.cols, img.rows)); + + AlprFullDetails response; + + response.results.epoch_time = getEpochTime(); + response.results.img_width = img.cols; + response.results.img_height = img.rows; + + for (uint i = 0; i < regionsOfInterest.size(); i++) + { + response.results.regionsOfInterest.push_back(AlprRegionOfInterest(regionsOfInterest[i].x, regionsOfInterest[i].y, + regionsOfInterest[i].width, regionsOfInterest[i].height)); + } + + if (!img.data) + { + // Invalid image + if (this->config->debugGeneral) + std::cerr << "Invalid image" << std::endl; + + return response; + } + + // Find all the candidate regions + response.plateRegions = plateDetector->detect(img, regionsOfInterest); + + queue plateQueue; + for (uint i = 0; i < response.plateRegions.size(); i++) + plateQueue.push(response.plateRegions[i]); + + while(!plateQueue.empty()) + { + PlateRegion plateRegion = plateQueue.front(); + plateQueue.pop(); + + PipelineData pipeline_data(img, plateRegion.rect, config); + + timespec platestarttime; + getTime(&platestarttime); + + LicensePlateCandidate lp(&pipeline_data); + + lp.recognize(); + + bool plateDetected = false; + if (pipeline_data.plate_area_confidence > 10) + { + AlprPlateResult plateResult; + plateResult.region = defaultRegion; + plateResult.regionConfidence = 0; + + for (int pointidx = 0; pointidx < 4; pointidx++) + { + plateResult.plate_points[pointidx].x = (int) pipeline_data.plate_corners[pointidx].x; + plateResult.plate_points[pointidx].y = (int) pipeline_data.plate_corners[pointidx].y; + } + + if (detectRegion) + { + stateIdentifier->recognize(&pipeline_data); + if (pipeline_data.region_confidence > 0) + { + plateResult.region = pipeline_data.region_code; + plateResult.regionConfidence = (int) pipeline_data.region_confidence; + } + } + + + + ocr->performOCR(&pipeline_data); + ocr->postProcessor.analyze(plateResult.region, topN); + const vector ppResults = ocr->postProcessor.getResults(); + + + int bestPlateIndex = 0; + + for (uint pp = 0; pp < ppResults.size(); pp++) + { + if (pp >= topN) + break; + + // Set our "best plate" match to either the first entry, or the first entry with a postprocessor template match + if (bestPlateIndex == 0 && ppResults[pp].matchesTemplate) + bestPlateIndex = pp; + + if (ppResults[pp].letters.size() >= config->postProcessMinCharacters && + ppResults[pp].letters.size() <= config->postProcessMaxCharacters) + { + AlprPlate aplate; + aplate.characters = ppResults[pp].letters; + aplate.overall_confidence = ppResults[pp].totalscore; + aplate.matches_template = ppResults[pp].matchesTemplate; + plateResult.topNPlates.push_back(aplate); + } + } + + if (plateResult.topNPlates.size() > 0) + plateResult.bestPlate = plateResult.topNPlates[bestPlateIndex]; + + timespec plateEndTime; + getTime(&plateEndTime); + plateResult.processing_time_ms = diffclock(platestarttime, plateEndTime); + + if (plateResult.topNPlates.size() > 0) + { + plateDetected = true; + response.results.plates.push_back(plateResult); + } + } + + if (!plateDetected) + { + // Not a valid plate + // Check if this plate has any children, if so, send them back up for processing + for (uint childidx = 0; childidx < plateRegion.children.size(); childidx++) + { + plateQueue.push(plateRegion.children[childidx]); + } + } + + + + + } + + timespec endTime; + getTime(&endTime); + response.results.total_processing_time_ms = diffclock(startTime, endTime); + + if (config->debugTiming) + { + cout << "Total Time to process image: " << diffclock(startTime, endTime) << "ms." << endl; + } + + if (config->debugGeneral && config->debugShowImages) + { + for (uint i = 0; i < response.plateRegions.size(); i++) + { + rectangle(img, response.plateRegions[i].rect, Scalar(0, 0, 255), 2); + } + + for (uint i = 0; i < response.results.plates.size(); i++) + { + for (int z = 0; z < 4; z++) + { + AlprCoordinate* coords = response.results.plates[i].plate_points; + Point p1(coords[z].x, coords[z].y); + Point p2(coords[(z + 1) % 4].x, coords[(z + 1) % 4].y); + line(img, p1, p2, Scalar(255,0,255), 2); + } + } + + + displayImage(config, "Main Image", img); + + // Sleep 1ms + sleep_ms(1); + + } + + + if (config->debugPauseOnFrame) + { + // Pause indefinitely until they press a key + while ((char) cv::waitKey(50) == -1) + {} + } + return response; } - // Find all the candidate regions - response.plateRegions = plateDetector->detect(img, regionsOfInterest); - - queue plateQueue; - for (uint i = 0; i < response.plateRegions.size(); i++) - plateQueue.push(response.plateRegions[i]); - - while(!plateQueue.empty()) - { - PlateRegion plateRegion = plateQueue.front(); - plateQueue.pop(); - - PipelineData pipeline_data(img, plateRegion.rect, config); - - timespec platestarttime; - getTime(&platestarttime); - - LicensePlateCandidate lp(&pipeline_data); - - lp.recognize(); - - bool plateDetected = false; - if (pipeline_data.plate_area_confidence > 10) - { - AlprPlateResult plateResult; - plateResult.region = defaultRegion; - plateResult.regionConfidence = 0; - - for (int pointidx = 0; pointidx < 4; pointidx++) - { - plateResult.plate_points[pointidx].x = (int) pipeline_data.plate_corners[pointidx].x; - plateResult.plate_points[pointidx].y = (int) pipeline_data.plate_corners[pointidx].y; - } - - if (detectRegion) - { - stateIdentifier->recognize(&pipeline_data); - if (pipeline_data.region_confidence > 0) - { - plateResult.region = pipeline_data.region_code; - plateResult.regionConfidence = (int) pipeline_data.region_confidence; - } - } + AlprResults AlprImpl::recognize( std::vector imageBytes, std::vector regionsOfInterest ) + { + cv::Mat img = cv::imdecode(cv::Mat(imageBytes), 1); - ocr->performOCR(&pipeline_data); - ocr->postProcessor.analyze(plateResult.region, topN); - const vector ppResults = ocr->postProcessor.getResults(); - - - int bestPlateIndex = 0; - - for (uint pp = 0; pp < ppResults.size(); pp++) - { - if (pp >= topN) - break; - - // Set our "best plate" match to either the first entry, or the first entry with a postprocessor template match - if (bestPlateIndex == 0 && ppResults[pp].matchesTemplate) - bestPlateIndex = pp; - - if (ppResults[pp].letters.size() >= config->postProcessMinCharacters && - ppResults[pp].letters.size() <= config->postProcessMaxCharacters) - { - AlprPlate aplate; - aplate.characters = ppResults[pp].letters; - aplate.overall_confidence = ppResults[pp].totalscore; - aplate.matches_template = ppResults[pp].matchesTemplate; - plateResult.topNPlates.push_back(aplate); - } - } - - if (plateResult.topNPlates.size() > 0) - plateResult.bestPlate = plateResult.topNPlates[bestPlateIndex]; - - timespec plateEndTime; - getTime(&plateEndTime); - plateResult.processing_time_ms = diffclock(platestarttime, plateEndTime); - - if (plateResult.topNPlates.size() > 0) - { - plateDetected = true; - response.results.plates.push_back(plateResult); - } - } - - if (!plateDetected) - { - // Not a valid plate - // Check if this plate has any children, if so, send them back up for processing - for (uint childidx = 0; childidx < plateRegion.children.size(); childidx++) - { - plateQueue.push(plateRegion.children[childidx]); - } - } - - - - + return this->recognize(img, this->convertRects(regionsOfInterest)); } - timespec endTime; - getTime(&endTime); - response.results.total_processing_time_ms = diffclock(startTime, endTime); - - if (config->debugTiming) + AlprResults AlprImpl::recognize( unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest) { - cout << "Total Time to process image: " << diffclock(startTime, endTime) << "ms." << endl; - } - - if (config->debugGeneral && config->debugShowImages) - { - for (uint i = 0; i < response.plateRegions.size(); i++) + + int arraySize = imgWidth * imgHeight * bytesPerPixel; + cv::Mat imgData = cv::Mat(arraySize, 1, CV_8U, pixelData); + cv::Mat img = imgData.reshape(bytesPerPixel, imgHeight); + + if (regionsOfInterest.size() == 0) { - rectangle(img, response.plateRegions[i].rect, Scalar(0, 0, 255), 2); - } - - for (uint i = 0; i < response.results.plates.size(); i++) - { - for (int z = 0; z < 4; z++) - { - AlprCoordinate* coords = response.results.plates[i].plate_points; - Point p1(coords[z].x, coords[z].y); - Point p2(coords[(z + 1) % 4].x, coords[(z + 1) % 4].y); - line(img, p1, p2, Scalar(255,0,255), 2); - } + AlprRegionOfInterest fullFrame(0,0, img.cols, img.rows); + + regionsOfInterest.push_back(fullFrame); } - - displayImage(config, "Main Image", img); - - // Sleep 1ms - sleep_ms(1); - + return this->recognize(img, this->convertRects(regionsOfInterest)); } - - - if (config->debugPauseOnFrame) + + AlprResults AlprImpl::recognize(cv::Mat img, std::vector regionsOfInterest) { - // Pause indefinitely until they press a key - while ((char) cv::waitKey(50) == -1) - {} + + AlprFullDetails fullDetails = recognizeFullDetails(img, regionsOfInterest); + return fullDetails.results; } - - return response; -} - -AlprResults AlprImpl::recognize( std::vector imageBytes, std::vector regionsOfInterest ) -{ - cv::Mat img = cv::imdecode(cv::Mat(imageBytes), 1); - - return this->recognize(img, this->convertRects(regionsOfInterest)); -} - -AlprResults AlprImpl::recognize( unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest) -{ - - int arraySize = imgWidth * imgHeight * bytesPerPixel; - cv::Mat imgData = cv::Mat(arraySize, 1, CV_8U, pixelData); - cv::Mat img = imgData.reshape(bytesPerPixel, imgHeight); - - if (regionsOfInterest.size() == 0) - { - AlprRegionOfInterest fullFrame(0,0, img.cols, img.rows); - - regionsOfInterest.push_back(fullFrame); - } - - return this->recognize(img, this->convertRects(regionsOfInterest)); -} - -AlprResults AlprImpl::recognize(cv::Mat img, std::vector regionsOfInterest) -{ - - AlprFullDetails fullDetails = recognizeFullDetails(img, regionsOfInterest); - return fullDetails.results; -} - - - std::vector AlprImpl::convertRects(std::vector regionsOfInterest) - { - std::vector rectRegions; - for (uint i = 0; i < regionsOfInterest.size(); i++) + std::vector AlprImpl::convertRects(std::vector regionsOfInterest) { - rectRegions.push_back(cv::Rect(regionsOfInterest[i].x, regionsOfInterest[i].y, regionsOfInterest[i].width, regionsOfInterest[i].height)); + std::vector rectRegions; + for (uint i = 0; i < regionsOfInterest.size(); i++) + { + rectRegions.push_back(cv::Rect(regionsOfInterest[i].x, regionsOfInterest[i].y, regionsOfInterest[i].width, regionsOfInterest[i].height)); + } + + return rectRegions; } - - return rectRegions; - } -string AlprImpl::toJson( const AlprResults results ) -{ - cJSON *root, *jsonResults; - root = cJSON_CreateObject(); - - - cJSON_AddNumberToObject(root,"version", 2 ); - cJSON_AddStringToObject(root,"data_type", "alpr_results" ); - - cJSON_AddNumberToObject(root,"epoch_time", results.epoch_time ); - cJSON_AddNumberToObject(root,"img_width", results.img_width ); - cJSON_AddNumberToObject(root,"img_height", results.img_height ); - cJSON_AddNumberToObject(root,"processing_time_ms", results.total_processing_time_ms ); - - // Add the regions of interest to the JSON - cJSON *rois; - cJSON_AddItemToObject(root, "regions_of_interest", rois=cJSON_CreateArray()); - for (uint i=0;ibestPlate.characters.c_str()); - cJSON_AddNumberToObject(root,"confidence", result->bestPlate.overall_confidence); - cJSON_AddNumberToObject(root,"matches_template", result->bestPlate.matches_template); - - cJSON_AddStringToObject(root,"region", result->region.c_str()); - cJSON_AddNumberToObject(root,"region_confidence", result->regionConfidence); - - cJSON_AddNumberToObject(root,"processing_time_ms", result->processing_time_ms); - cJSON_AddNumberToObject(root,"requested_topn", result->requested_topn); - - cJSON_AddItemToObject(root, "coordinates", coords=cJSON_CreateArray()); - for (int i=0;i<4;i++) - { - cJSON *coords_object; - coords_object = cJSON_CreateObject(); - cJSON_AddNumberToObject(coords_object, "x", result->plate_points[i].x); - cJSON_AddNumberToObject(coords_object, "y", result->plate_points[i].y); + cJSON_AddNumberToObject(root,"epoch_time", results.epoch_time ); + cJSON_AddNumberToObject(root,"img_width", results.img_width ); + cJSON_AddNumberToObject(root,"img_height", results.img_height ); + cJSON_AddNumberToObject(root,"processing_time_ms", results.total_processing_time_ms ); - cJSON_AddItemToArray(coords, coords_object); - } - - - cJSON_AddItemToObject(root, "candidates", candidates=cJSON_CreateArray()); - for (uint i = 0; i < result->topNPlates.size(); i++) - { - cJSON *candidate_object; - candidate_object = cJSON_CreateObject(); - cJSON_AddStringToObject(candidate_object, "plate", result->topNPlates[i].characters.c_str()); - cJSON_AddNumberToObject(candidate_object, "confidence", result->topNPlates[i].overall_confidence); - cJSON_AddNumberToObject(candidate_object, "matches_template", result->topNPlates[i].matches_template); - - cJSON_AddItemToArray(candidates, candidate_object); - } - - return root; -} - -AlprResults AlprImpl::fromJson(std::string json) { - AlprResults allResults; - - cJSON* root = cJSON_Parse(json.c_str()); - - int version = cJSON_GetObjectItem(root, "version")->valueint; - allResults.epoch_time = (long) cJSON_GetObjectItem(root, "epoch_time")->valuedouble; - allResults.img_width = cJSON_GetObjectItem(root, "img_width")->valueint; - allResults.img_height = cJSON_GetObjectItem(root, "img_height")->valueint; - allResults.total_processing_time_ms = cJSON_GetObjectItem(root, "processing_time_ms")->valueint; - - - cJSON* rois = cJSON_GetObjectItem(root,"regions_of_interest"); - int numRois = cJSON_GetArraySize(rois); - for (int c = 0; c < numRois; c++) - { - cJSON* roi = cJSON_GetArrayItem(rois, c); - int x = cJSON_GetObjectItem(roi, "x")->valueint; - int y = cJSON_GetObjectItem(roi, "y")->valueint; - int width = cJSON_GetObjectItem(roi, "width")->valueint; - int height = cJSON_GetObjectItem(roi, "height")->valueint; - - AlprRegionOfInterest alprRegion(x,y,width,height); - allResults.regionsOfInterest.push_back(alprRegion); - } - - cJSON* resultsArray = cJSON_GetObjectItem(root,"results"); - int resultsSize = cJSON_GetArraySize(resultsArray); - - for (int i = 0; i < resultsSize; i++) - { - cJSON* item = cJSON_GetArrayItem(resultsArray, i); - AlprPlateResult plate; - - //plate.bestPlate = cJSON_GetObjectItem(item, "plate")->valuestring; - plate.processing_time_ms = cJSON_GetObjectItem(item, "processing_time_ms")->valuedouble; - plate.region = cJSON_GetObjectItem(item, "region")->valuestring; - plate.regionConfidence = cJSON_GetObjectItem(item, "region_confidence")->valueint; - plate.requested_topn = cJSON_GetObjectItem(item, "requested_topn")->valueint; - - - cJSON* coordinates = cJSON_GetObjectItem(item,"coordinates"); - for (int c = 0; c < 4; c++) + // Add the regions of interest to the JSON + cJSON *rois; + cJSON_AddItemToObject(root, "regions_of_interest", rois=cJSON_CreateArray()); + for (uint i=0;ivalueint; - alprcoord.y = cJSON_GetObjectItem(coordinate, "y")->valueint; - - plate.plate_points[c] = alprcoord; + cJSON *roi_object; + roi_object = cJSON_CreateObject(); + cJSON_AddNumberToObject(roi_object, "x", results.regionsOfInterest[i].x); + cJSON_AddNumberToObject(roi_object, "y", results.regionsOfInterest[i].y); + cJSON_AddNumberToObject(roi_object, "width", results.regionsOfInterest[i].width); + cJSON_AddNumberToObject(roi_object, "height", results.regionsOfInterest[i].height); + + cJSON_AddItemToArray(rois, roi_object); } - - cJSON* candidates = cJSON_GetObjectItem(item,"candidates"); - int numCandidates = cJSON_GetArraySize(candidates); - for (int c = 0; c < numCandidates; c++) - { - cJSON* candidate = cJSON_GetArrayItem(candidates, c); - AlprPlate plateCandidate; - plateCandidate.characters = cJSON_GetObjectItem(candidate, "plate")->valuestring; - plateCandidate.overall_confidence = cJSON_GetObjectItem(candidate, "confidence")->valuedouble; - plateCandidate.matches_template = (cJSON_GetObjectItem(candidate, "matches_template")->valueint) != 0; - plate.topNPlates.push_back(plateCandidate); - - if (c == 0) + + cJSON_AddItemToObject(root, "results", jsonResults=cJSON_CreateArray()); + for (uint i = 0; i < results.plates.size(); i++) + { + cJSON *resultObj = createJsonObj( &results.plates[i] ); + cJSON_AddItemToArray(jsonResults, resultObj); + } + + // Print the JSON object to a string and return + char *out; + out=cJSON_PrintUnformatted(root); + + cJSON_Delete(root); + + string response(out); + + free(out); + return response; + } + + + + cJSON* AlprImpl::createJsonObj(const AlprPlateResult* result) + { + cJSON *root, *coords, *candidates; + + root=cJSON_CreateObject(); + + cJSON_AddStringToObject(root,"plate", result->bestPlate.characters.c_str()); + cJSON_AddNumberToObject(root,"confidence", result->bestPlate.overall_confidence); + cJSON_AddNumberToObject(root,"matches_template", result->bestPlate.matches_template); + + cJSON_AddStringToObject(root,"region", result->region.c_str()); + cJSON_AddNumberToObject(root,"region_confidence", result->regionConfidence); + + cJSON_AddNumberToObject(root,"processing_time_ms", result->processing_time_ms); + cJSON_AddNumberToObject(root,"requested_topn", result->requested_topn); + + cJSON_AddItemToObject(root, "coordinates", coords=cJSON_CreateArray()); + for (int i=0;i<4;i++) + { + cJSON *coords_object; + coords_object = cJSON_CreateObject(); + cJSON_AddNumberToObject(coords_object, "x", result->plate_points[i].x); + cJSON_AddNumberToObject(coords_object, "y", result->plate_points[i].y); + + cJSON_AddItemToArray(coords, coords_object); + } + + + cJSON_AddItemToObject(root, "candidates", candidates=cJSON_CreateArray()); + for (uint i = 0; i < result->topNPlates.size(); i++) + { + cJSON *candidate_object; + candidate_object = cJSON_CreateObject(); + cJSON_AddStringToObject(candidate_object, "plate", result->topNPlates[i].characters.c_str()); + cJSON_AddNumberToObject(candidate_object, "confidence", result->topNPlates[i].overall_confidence); + cJSON_AddNumberToObject(candidate_object, "matches_template", result->topNPlates[i].matches_template); + + cJSON_AddItemToArray(candidates, candidate_object); + } + + return root; + } + + AlprResults AlprImpl::fromJson(std::string json) { + AlprResults allResults; + + cJSON* root = cJSON_Parse(json.c_str()); + + int version = cJSON_GetObjectItem(root, "version")->valueint; + allResults.epoch_time = (long) cJSON_GetObjectItem(root, "epoch_time")->valuedouble; + allResults.img_width = cJSON_GetObjectItem(root, "img_width")->valueint; + allResults.img_height = cJSON_GetObjectItem(root, "img_height")->valueint; + allResults.total_processing_time_ms = cJSON_GetObjectItem(root, "processing_time_ms")->valueint; + + + cJSON* rois = cJSON_GetObjectItem(root,"regions_of_interest"); + int numRois = cJSON_GetArraySize(rois); + for (int c = 0; c < numRois; c++) + { + cJSON* roi = cJSON_GetArrayItem(rois, c); + int x = cJSON_GetObjectItem(roi, "x")->valueint; + int y = cJSON_GetObjectItem(roi, "y")->valueint; + int width = cJSON_GetObjectItem(roi, "width")->valueint; + int height = cJSON_GetObjectItem(roi, "height")->valueint; + + AlprRegionOfInterest alprRegion(x,y,width,height); + allResults.regionsOfInterest.push_back(alprRegion); + } + + cJSON* resultsArray = cJSON_GetObjectItem(root,"results"); + int resultsSize = cJSON_GetArraySize(resultsArray); + + for (int i = 0; i < resultsSize; i++) + { + cJSON* item = cJSON_GetArrayItem(resultsArray, i); + AlprPlateResult plate; + + //plate.bestPlate = cJSON_GetObjectItem(item, "plate")->valuestring; + plate.processing_time_ms = cJSON_GetObjectItem(item, "processing_time_ms")->valuedouble; + plate.region = cJSON_GetObjectItem(item, "region")->valuestring; + plate.regionConfidence = cJSON_GetObjectItem(item, "region_confidence")->valueint; + plate.requested_topn = cJSON_GetObjectItem(item, "requested_topn")->valueint; + + + cJSON* coordinates = cJSON_GetObjectItem(item,"coordinates"); + for (int c = 0; c < 4; c++) { - plate.bestPlate = plateCandidate; + cJSON* coordinate = cJSON_GetArrayItem(coordinates, c); + AlprCoordinate alprcoord; + alprcoord.x = cJSON_GetObjectItem(coordinate, "x")->valueint; + alprcoord.y = cJSON_GetObjectItem(coordinate, "y")->valueint; + + plate.plate_points[c] = alprcoord; } + + cJSON* candidates = cJSON_GetObjectItem(item,"candidates"); + int numCandidates = cJSON_GetArraySize(candidates); + for (int c = 0; c < numCandidates; c++) + { + cJSON* candidate = cJSON_GetArrayItem(candidates, c); + AlprPlate plateCandidate; + plateCandidate.characters = cJSON_GetObjectItem(candidate, "plate")->valuestring; + plateCandidate.overall_confidence = cJSON_GetObjectItem(candidate, "confidence")->valuedouble; + plateCandidate.matches_template = (cJSON_GetObjectItem(candidate, "matches_template")->valueint) != 0; + + plate.topNPlates.push_back(plateCandidate); + + if (c == 0) + { + plate.bestPlate = plateCandidate; + } + } + + allResults.plates.push_back(plate); } - - allResults.plates.push_back(plate); + + + cJSON_Delete(root); + + + return allResults; } - - - cJSON_Delete(root); - - - return allResults; -} -void AlprImpl::setDetectRegion(bool detectRegion) -{ - this->detectRegion = detectRegion; -} -void AlprImpl::setTopN(int topn) -{ - this->topN = topn; -} -void AlprImpl::setDefaultRegion(string region) -{ - this->defaultRegion = region; -} + void AlprImpl::setDetectRegion(bool detectRegion) + { + this->detectRegion = detectRegion; + } + void AlprImpl::setTopN(int topn) + { + this->topN = topn; + } + void AlprImpl::setDefaultRegion(string region) + { + this->defaultRegion = region; + } -std::string AlprImpl::getVersion() -{ - std::stringstream ss; - - ss << OPENALPR_MAJOR_VERSION << "." << OPENALPR_MINOR_VERSION << "." << OPENALPR_PATCH_VERSION; - return ss.str(); -} + std::string AlprImpl::getVersion() + { + std::stringstream ss; + ss << OPENALPR_MAJOR_VERSION << "." << OPENALPR_MINOR_VERSION << "." << OPENALPR_PATCH_VERSION; + return ss.str(); + } + +} \ No newline at end of file diff --git a/src/openalpr/alpr_impl.h b/src/openalpr/alpr_impl.h index b8963eb..3384627 100644 --- a/src/openalpr/alpr_impl.h +++ b/src/openalpr/alpr_impl.h @@ -52,56 +52,58 @@ #define ALPR_NULL_PTR 0 - -struct AlprFullDetails -{ - std::vector plateRegions; - AlprResults results; -}; - -class AlprImpl +namespace alpr { - public: - AlprImpl(const std::string country, const std::string configFile = "", const std::string runtimeDir = ""); - virtual ~AlprImpl(); + struct AlprFullDetails + { + std::vector plateRegions; + AlprResults results; + }; - AlprFullDetails recognizeFullDetails(cv::Mat img); - AlprFullDetails recognizeFullDetails(cv::Mat img, std::vector regionsOfInterest); - - AlprResults recognize( std::vector imageBytes, std::vector regionsOfInterest ); - AlprResults recognize( unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest ); - AlprResults recognize( cv::Mat img, std::vector regionsOfInterest ); - - void applyRegionTemplate(AlprPlateResult* result, std::string region); - - void setDetectRegion(bool detectRegion); - void setTopN(int topn); - void setDefaultRegion(std::string region); - - static std::string toJson( const AlprResults results ); - static AlprResults fromJson(std::string json); - static std::string getVersion(); - - Config* config; - - bool isLoaded(); - - private: - - Detector* plateDetector; - StateIdentifier* stateIdentifier; - OCR* ocr; - - int topN; - bool detectRegion; - std::string defaultRegion; - - std::vector convertRects(std::vector regionsOfInterest); - - static cJSON* createJsonObj(const AlprPlateResult* result); -}; + class AlprImpl + { + public: + AlprImpl(const std::string country, const std::string configFile = "", const std::string runtimeDir = ""); + virtual ~AlprImpl(); + + AlprFullDetails recognizeFullDetails(cv::Mat img); + AlprFullDetails recognizeFullDetails(cv::Mat img, std::vector regionsOfInterest); + + AlprResults recognize( std::vector imageBytes, std::vector regionsOfInterest ); + AlprResults recognize( unsigned char* pixelData, int bytesPerPixel, int imgWidth, int imgHeight, std::vector regionsOfInterest ); + AlprResults recognize( cv::Mat img, std::vector regionsOfInterest ); + + void applyRegionTemplate(AlprPlateResult* result, std::string region); + + void setDetectRegion(bool detectRegion); + void setTopN(int topn); + void setDefaultRegion(std::string region); + + static std::string toJson( const AlprResults results ); + static AlprResults fromJson(std::string json); + static std::string getVersion(); + + Config* config; + + bool isLoaded(); + + private: + + Detector* plateDetector; + StateIdentifier* stateIdentifier; + OCR* ocr; + + int topN; + bool detectRegion; + std::string defaultRegion; + + std::vector convertRects(std::vector regionsOfInterest); + + static cJSON* createJsonObj(const AlprPlateResult* result); + }; +} #endif // OPENALPR_ALPRIMPL_H \ No newline at end of file diff --git a/src/openalpr/binarize_wolf.cpp b/src/openalpr/binarize_wolf.cpp index 6143890..bb52818 100644 --- a/src/openalpr/binarize_wolf.cpp +++ b/src/openalpr/binarize_wolf.cpp @@ -36,210 +36,214 @@ using namespace std; using namespace cv; -// ************************************************************* -// glide a window across the image and -// create two maps: mean and standard deviation. -// ************************************************************* - -float calcLocalStats (Mat &im, Mat &map_m, Mat &map_s, int winx, int winy) +namespace alpr { - float m,s,max_s; - long sum, sum_sq; - uchar foo; - int wxh = winx/2; - int wyh = winy/2; - int x_firstth= wxh; - int y_lastth = im.rows-wyh-1; - int y_firstth= wyh; - float winarea = winx*winy; - max_s = 0; - for (int j = y_firstth ; j<=y_lastth; j++) + // ************************************************************* + // glide a window across the image and + // create two maps: mean and standard deviation. + // ************************************************************* + float calcLocalStats (Mat &im, Mat &map_m, Mat &map_s, int winx, int winy) { - float* mapm_rowdata = map_m.ptr(j); - float* maps_rowdata = map_s.ptr(j); - - // Calculate the initial window at the beginning of the line - sum = sum_sq = 0; - for (int wy=0 ; wy(j-wyh+wy); - for (int wx=0 ; wx max_s) - max_s = s; - mapm_rowdata[x_firstth] = m; - maps_rowdata[x_firstth] = s; + float m,s,max_s; + long sum, sum_sq; + uchar foo; + int wxh = winx/2; + int wyh = winy/2; + int x_firstth= wxh; + int y_lastth = im.rows-wyh-1; + int y_firstth= wyh; + float winarea = winx*winy; - // Shift the window, add and remove new/old values to the histogram - for (int i=1 ; i <= im.cols-winx; i++) + max_s = 0; + for (int j = y_firstth ; j<=y_lastth; j++) { - // Remove the left old column and add the right new column - for (int wy=0; wy(j); + float* maps_rowdata = map_s.ptr(j); + + // Calculate the initial window at the beginning of the line + sum = sum_sq = 0; + for (int wy=0 ; wy(j-wyh+wy); + for (int wx=0 ; wx max_s) max_s = s; - mapm_rowdata[i+wxh] = m; - maps_rowdata[i+wxh] = s; + mapm_rowdata[x_firstth] = m; + maps_rowdata[x_firstth] = s; + + // Shift the window, add and remove new/old values to the histogram + for (int i=1 ; i <= im.cols-winx; i++) + { + // Remove the left old column and add the right new column + for (int wy=0; wy max_s) + max_s = s; + mapm_rowdata[i+wxh] = m; + maps_rowdata[i+wxh] = s; + } } + + return max_s; } - return max_s; -} + /********************************************************** + * The binarization routine + **********************************************************/ -/********************************************************** - * The binarization routine - **********************************************************/ - -void NiblackSauvolaWolfJolion (Mat im, Mat output, NiblackVersion version, - int winx, int winy, float k) -{ - float dR = BINARIZEWOLF_DEFAULTDR; - - float m, s, max_s; - float th=0; - double min_I, max_I; - int wxh = winx/2; - int wyh = winy/2; - int x_firstth= wxh; - int x_lastth = im.cols-wxh-1; - int y_lastth = im.rows-wyh-1; - int y_firstth= wyh; - - // Create local statistics and store them in a float matrices - Mat map_m = Mat::zeros (im.rows, im.cols, CV_32F); - Mat map_s = Mat::zeros (im.rows, im.cols, CV_32F); - max_s = calcLocalStats (im, map_m, map_s, winx, winy); - - minMaxLoc(im, &min_I, &max_I); - - Mat thsurf (im.rows, im.cols, CV_32F); - - // Create the threshold surface, including border processing - // ---------------------------------------------------- - - for (int j = y_firstth ; j<=y_lastth; j++) + void NiblackSauvolaWolfJolion (Mat im, Mat output, NiblackVersion version, + int winx, int winy, float k) { - float* mapm_rowdata = map_m.ptr(j); - float* maps_rowdata = map_s.ptr(j); - float* thsurf_rowdata = thsurf.ptr(j); - - // NORMAL, NON-BORDER AREA IN THE MIDDLE OF THE WINDOW: - for (int i=0 ; i <= im.cols-winx; i++) + float dR = BINARIZEWOLF_DEFAULTDR; + + float m, s, max_s; + float th=0; + double min_I, max_I; + int wxh = winx/2; + int wyh = winy/2; + int x_firstth= wxh; + int x_lastth = im.cols-wxh-1; + int y_lastth = im.rows-wyh-1; + int y_firstth= wyh; + + // Create local statistics and store them in a float matrices + Mat map_m = Mat::zeros (im.rows, im.cols, CV_32F); + Mat map_s = Mat::zeros (im.rows, im.cols, CV_32F); + max_s = calcLocalStats (im, map_m, map_s, winx, winy); + + minMaxLoc(im, &min_I, &max_I); + + Mat thsurf (im.rows, im.cols, CV_32F); + + // Create the threshold surface, including border processing + // ---------------------------------------------------- + + for (int j = y_firstth ; j<=y_lastth; j++) { - m = mapm_rowdata[i+wxh]; - s = maps_rowdata[i+wxh]; + float* mapm_rowdata = map_m.ptr(j); + float* maps_rowdata = map_s.ptr(j); + float* thsurf_rowdata = thsurf.ptr(j); - // Calculate the threshold - switch (version) + // NORMAL, NON-BORDER AREA IN THE MIDDLE OF THE WINDOW: + for (int i=0 ; i <= im.cols-winx; i++) { - case NIBLACK: - th = m + k*s; - break; + m = mapm_rowdata[i+wxh]; + s = maps_rowdata[i+wxh]; - case SAUVOLA: - th = m * (1 + k*(s/dR-1)); - break; + // Calculate the threshold + switch (version) + { + case NIBLACK: + th = m + k*s; + break; - case WOLFJOLION: - th = m + k * (s/max_s-1) * (m-min_I); - break; + case SAUVOLA: + th = m * (1 + k*(s/dR-1)); + break; - default: - cerr << "Unknown threshold type in ImageThresholder::surfaceNiblackImproved()\n"; - exit (1); - } + case WOLFJOLION: + th = m + k * (s/max_s-1) * (m-min_I); + break; - thsurf_rowdata[i+wxh] = th; + default: + cerr << "Unknown threshold type in ImageThresholder::surfaceNiblackImproved()\n"; + exit (1); + } - if (i==0) - { - // LEFT BORDER - for (int i=0; i<=x_firstth; ++i) - thsurf_rowdata[i] = th; + thsurf_rowdata[i+wxh] = th; - // LEFT-UPPER CORNER + if (i==0) + { + // LEFT BORDER + for (int i=0; i<=x_firstth; ++i) + thsurf_rowdata[i] = th; + + // LEFT-UPPER CORNER + if (j==y_firstth) + for (int u=0; u(u); + for (int i=0; i<=x_firstth; ++i) + thsurf_subrowdata[i] = th; + } + + // LEFT-LOWER CORNER + if (j==y_lastth) + for (int u=y_lastth+1; u(u); + for (int i=0; i<=x_firstth; ++i) + thsurf_subrowdata[i] = th; + } + } + + // UPPER BORDER if (j==y_firstth) for (int u=0; u(u); - for (int i=0; i<=x_firstth; ++i) - thsurf_subrowdata[i] = th; - } + thsurf.fset(i+wxh,u,th); - // LEFT-LOWER CORNER + // LOWER BORDER if (j==y_lastth) for (int u=y_lastth+1; u(u); - for (int i=0; i<=x_firstth; ++i) - thsurf_subrowdata[i] = th; - } + thsurf.fset(i+wxh,u,th); } - // UPPER BORDER + // RIGHT BORDER + for (int i=x_lastth; i(u); + for (int i=x_lastth; i(u); + for (int i=x_lastth; i(u); - for (int i=x_lastth; i(u); - for (int i=x_lastth; i(y); - uchar* imdatarow = im.ptr(y); - float* thsurfdatarow = thsurf.ptr(y); - - for (int x=0; x= thsurfdatarow[x]) - outputdatarow[x]=255; - else - outputdatarow[x]=0; + uchar* outputdatarow = output.ptr(y); + uchar* imdatarow = im.ptr(y); + float* thsurfdatarow = thsurf.ptr(y); + + for (int x=0; x= thsurfdatarow[x]) + outputdatarow[x]=255; + else + outputdatarow[x]=0; + } } } -} + +} \ No newline at end of file diff --git a/src/openalpr/binarize_wolf.h b/src/openalpr/binarize_wolf.h index 27c3d1c..dd6b701 100644 --- a/src/openalpr/binarize_wolf.h +++ b/src/openalpr/binarize_wolf.h @@ -26,23 +26,27 @@ #include #include "opencv2/opencv.hpp" - -enum NiblackVersion +namespace alpr { - NIBLACK=0, - SAUVOLA, - WOLFJOLION, -}; -#define BINARIZEWOLF_VERSION "2.3 (February 26th, 2013)" -#define BINARIZEWOLF_DEFAULTDR 128 + enum NiblackVersion + { + NIBLACK=0, + SAUVOLA, + WOLFJOLION, + }; -#define uget(x,y) at(y,x) -#define uset(x,y,v) at(y,x)=v; -#define fget(x,y) at(y,x) -#define fset(x,y,v) at(y,x)=v; + #define BINARIZEWOLF_VERSION "2.3 (February 26th, 2013)" + #define BINARIZEWOLF_DEFAULTDR 128 -void NiblackSauvolaWolfJolion (cv::Mat im, cv::Mat output, NiblackVersion version, - int winx, int winy, float k); + #define uget(x,y) at(y,x) + #define uset(x,y,v) at(y,x)=v; + #define fget(x,y) at(y,x) + #define fset(x,y,v) at(y,x)=v; + + void NiblackSauvolaWolfJolion (cv::Mat im, cv::Mat output, NiblackVersion version, + int winx, int winy, float k); + +} #endif // OPENALPR_BINARIZEWOLF_H diff --git a/src/openalpr/colorfilter.cpp b/src/openalpr/colorfilter.cpp index a85cc53..9b47ce0 100644 --- a/src/openalpr/colorfilter.cpp +++ b/src/openalpr/colorfilter.cpp @@ -22,377 +22,383 @@ using namespace cv; using namespace std; -ColorFilter::ColorFilter(Mat image, Mat characterMask, Config* config) +namespace alpr { - timespec startTime; - getTime(&startTime); + - this->config = config; - - this->debug = config->debugColorFiler; - - this->grayscale = imageIsGrayscale(image); - - if (this->debug) - cout << "ColorFilter: isGrayscale = " << grayscale << endl; - - this->hsv = Mat(image.size(), image.type()); - cvtColor( image, this->hsv, CV_BGR2HSV ); - preprocessImage(); - - this->charMask = characterMask; - - this->colorMask = Mat(image.size(), CV_8U); - - findCharColors(); - - if (config->debugTiming) + ColorFilter::ColorFilter(Mat image, Mat characterMask, Config* config) { - timespec endTime; - getTime(&endTime); - cout << " -- ColorFilter Time: " << diffclock(startTime, endTime) << "ms." << endl; - } -} + timespec startTime; + getTime(&startTime); -ColorFilter::~ColorFilter() -{ -} + this->config = config; -bool ColorFilter::imageIsGrayscale(Mat image) -{ - // Check whether the original image is grayscale. If it is, we shouldn't attempt any color filter - for (int row = 0; row < image.rows; row++) - { - for (int col = 0; col < image.cols; col++) + this->debug = config->debugColorFiler; + + this->grayscale = imageIsGrayscale(image); + + if (this->debug) + cout << "ColorFilter: isGrayscale = " << grayscale << endl; + + this->hsv = Mat(image.size(), image.type()); + cvtColor( image, this->hsv, CV_BGR2HSV ); + preprocessImage(); + + this->charMask = characterMask; + + this->colorMask = Mat(image.size(), CV_8U); + + findCharColors(); + + if (config->debugTiming) { - int r = (int) image.at(row, col)[0]; - int g = (int) image.at(row, col)[1]; - int b = (int) image.at(row, col)[2]; - - if (r == g == b) - { - // So far so good - } - else - { - // Image is color. - return false; - } + timespec endTime; + getTime(&endTime); + cout << " -- ColorFilter Time: " << diffclock(startTime, endTime) << "ms." << endl; } } - return true; -} - -void ColorFilter::preprocessImage() -{ - // Equalize the brightness on the HSV channel "V" - vector channels; - split(this->hsv,channels); - Mat img_equalized = equalizeBrightness(channels[2]); - merge(channels,this->hsv); -} - -// Gets the hue/sat/val for areas that we believe are license plate characters -// Then uses that to filter the whole image and provide a mask. -void ColorFilter::findCharColors() -{ - int MINIMUM_SATURATION = 45; - - if (this->debug) - cout << "ColorFilter::findCharColors" << endl; - - //charMask.copyTo(this->colorMask); - this->colorMask = Mat::zeros(charMask.size(), CV_8U); - bitwise_not(this->colorMask, this->colorMask); - - Mat erodedCharMask(charMask.size(), CV_8U); - Mat element = getStructuringElement( 1, - Size( 2 + 1, 2+1 ), - Point( 1, 1 ) ); - erode(charMask, erodedCharMask, element); - - vector > contours; - vector hierarchy; - findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); - - vector hMeans, sMeans, vMeans; - vector hStdDevs, sStdDevs, vStdDevs; - - for (uint i = 0; i < contours.size(); i++) + ColorFilter::~ColorFilter() { - if (hierarchy[i][3] != -1) - continue; + } - Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U); + bool ColorFilter::imageIsGrayscale(Mat image) + { + // Check whether the original image is grayscale. If it is, we shouldn't attempt any color filter + for (int row = 0; row < image.rows; row++) + { + for (int col = 0; col < image.cols; col++) + { + int r = (int) image.at(row, col)[0]; + int g = (int) image.at(row, col)[1]; + int b = (int) image.at(row, col)[2]; - drawContours(singleCharMask, contours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - hierarchy - ); + if (r == g == b) + { + // So far so good + } + else + { + // Image is color. + return false; + } + } + } - // get rid of the outline by drawing a 1 pixel width black line - drawContours(singleCharMask, contours, - i, // draw this contour - cv::Scalar(0,0,0), // in - 1, - 8, - hierarchy - ); + return true; + } - //drawAndWait(&singleCharMask); + void ColorFilter::preprocessImage() + { + // Equalize the brightness on the HSV channel "V" + vector channels; + split(this->hsv,channels); + Mat img_equalized = equalizeBrightness(channels[2]); + merge(channels,this->hsv); + } - Scalar mean; - Scalar stddev; - meanStdDev(hsv, mean, stddev, singleCharMask); + // Gets the hue/sat/val for areas that we believe are license plate characters + // Then uses that to filter the whole image and provide a mask. + void ColorFilter::findCharColors() + { + int MINIMUM_SATURATION = 45; + + if (this->debug) + cout << "ColorFilter::findCharColors" << endl; + + //charMask.copyTo(this->colorMask); + this->colorMask = Mat::zeros(charMask.size(), CV_8U); + bitwise_not(this->colorMask, this->colorMask); + + Mat erodedCharMask(charMask.size(), CV_8U); + Mat element = getStructuringElement( 1, + Size( 2 + 1, 2+1 ), + Point( 1, 1 ) ); + erode(charMask, erodedCharMask, element); + + vector > contours; + vector hierarchy; + findContours(erodedCharMask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); + + vector hMeans, sMeans, vMeans; + vector hStdDevs, sStdDevs, vStdDevs; + + for (uint i = 0; i < contours.size(); i++) + { + if (hierarchy[i][3] != -1) + continue; + + Mat singleCharMask = Mat::zeros(hsv.size(), CV_8U); + + drawContours(singleCharMask, contours, + i, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + hierarchy + ); + + // get rid of the outline by drawing a 1 pixel width black line + drawContours(singleCharMask, contours, + i, // draw this contour + cv::Scalar(0,0,0), // in + 1, + 8, + hierarchy + ); + + //drawAndWait(&singleCharMask); + + Scalar mean; + Scalar stddev; + meanStdDev(hsv, mean, stddev, singleCharMask); + + if (this->debug) + { + cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <getMajorityOpinion(hMeans, .65, 30); + int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35); + int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30); + + if (sMeans[bestSatIndex] < MINIMUM_SATURATION) + return; + + bool doHueFilter = false, doSatFilter = false, doValFilter = false; + float hueMin, hueMax; + float satMin, satMax; + float valMin, valMax; + + if (this->debug) + cout << "ColorFilter Winning indices:" << endl; + if (bestHueIndex != -1) + { + doHueFilter = true; + hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]); + hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]); + + if (abs(hueMin - hueMax) < 20) + { + hueMin = hMeans[bestHueIndex] - 20; + hueMax = hMeans[bestHueIndex] + 20; + } + + if (hueMin < 0) + hueMin = 0; + if (hueMax > 180) + hueMax = 180; + + if (this->debug) + cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl; + } + if (bestSatIndex != -1) + { + doSatFilter = true; + + satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]); + satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]); + + if (abs(satMin - satMax) < 20) + { + satMin = sMeans[bestSatIndex] - 20; + satMax = sMeans[bestSatIndex] + 20; + } + + if (satMin < 0) + satMin = 0; + if (satMax > 255) + satMax = 255; + + if (this->debug) + cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl; + } + if (bestValIndex != -1) + { + doValFilter = true; + + valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]); + valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]); + + if (abs(valMin - valMax) < 20) + { + valMin = vMeans[bestValIndex] - 20; + valMax = vMeans[bestValIndex] + 20; + } + + if (valMin < 0) + valMin = 0; + if (valMax > 255) + valMax = 255; + + if (this->debug) + cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl; + } + + Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type()); + Mat imgDebug = Mat::zeros(hsv.size(), hsv.type()); + Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U); + Mat debugMask = Mat::zeros(hsv.size(), CV_8U); + bitwise_not(debugMask, debugMask); + + for (int row = 0; row < charMask.rows; row++) + { + for (int col = 0; col < charMask.cols; col++) + { + int h = (int) hsv.at(row, col)[0]; + int s = (int) hsv.at(row, col)[1]; + int v = (int) hsv.at(row, col)[2]; + + bool hPasses = true; + bool sPasses = true; + bool vPasses = true; + + int vDistance = abs(v - vMeans[bestValIndex]); + + imgDebugHueOnly.at(row, col)[0] = h; + imgDebugHueOnly.at(row, col)[1] = 255; + imgDebugHueOnly.at(row, col)[2] = 255; + + imgDebug.at(row, col)[0] = 255; + imgDebug.at(row, col)[1] = 255; + imgDebug.at(row, col)[2] = 255; + + if (doHueFilter && (h < hueMin || h > hueMax)) + { + hPasses = false; + imgDebug.at(row, col)[0] = 0; + debugMask.at(row, col) = 0; + } + if (doSatFilter && (s < satMin || s > satMax)) + { + sPasses = false; + imgDebug.at(row, col)[1] = 0; + } + if (doValFilter && (v < valMin || v > valMax)) + { + vPasses = false; + imgDebug.at(row, col)[2] = 0; + } + + //if (pixelPasses) + // colorMask.at(row, col) = 255; + //else + //imgDebug.at(row, col)[0] = hPasses & 255; + //imgDebug.at(row, col)[1] = sPasses & 255; + //imgDebug.at(row, col)[2] = vPasses & 255; + + if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) || + this->colorMask.at(row, col) = 255; + else + this->colorMask.at(row, col) = 0; + + if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses)) + { + vDistance = pow(vDistance, 0.9); + } + else + { + vDistance = pow(vDistance, 1.1); + } + if (vDistance > 255) + vDistance = 255; + imgDistanceFromCenter.at(row, col) = vDistance; + } + } + + vector debugImagesSet; if (this->debug) { - cout << "ColorFilter " << setw(3) << i << ". Mean: h: " << setw(7) << mean[0] << " s: " << setw(7) <colorMask, this->colorMask, MORPH_CLOSE, bigElement); + //dilate(this->colorMask, this->colorMask, bigElement); - int bestHueIndex = this->getMajorityOpinion(hMeans, .65, 30); - int bestSatIndex = this->getMajorityOpinion(sMeans, .65, 35); - int bestValIndex = this->getMajorityOpinion(vMeans, .65, 30); - - if (sMeans[bestSatIndex] < MINIMUM_SATURATION) - return; - - bool doHueFilter = false, doSatFilter = false, doValFilter = false; - float hueMin, hueMax; - float satMin, satMax; - float valMin, valMax; - - if (this->debug) - cout << "ColorFilter Winning indices:" << endl; - if (bestHueIndex != -1) - { - doHueFilter = true; - hueMin = hMeans[bestHueIndex] - (2 * hStdDevs[bestHueIndex]); - hueMax = hMeans[bestHueIndex] + (2 * hStdDevs[bestHueIndex]); - - if (abs(hueMin - hueMax) < 20) - { - hueMin = hMeans[bestHueIndex] - 20; - hueMax = hMeans[bestHueIndex] + 20; - } - - if (hueMin < 0) - hueMin = 0; - if (hueMax > 180) - hueMax = 180; + Mat combined(charMask.size(), charMask.type()); + bitwise_and(charMask, colorMask, combined); if (this->debug) - cout << "ColorFilter Hue: " << bestHueIndex << " : " << setw(7) << hMeans[bestHueIndex] << " -- " << hueMin << "-" << hueMax << endl; - } - if (bestSatIndex != -1) - { - doSatFilter = true; - - satMin = sMeans[bestSatIndex] - (2 * sStdDevs[bestSatIndex]); - satMax = sMeans[bestSatIndex] + (2 * sStdDevs[bestSatIndex]); - - if (abs(satMin - satMax) < 20) { - satMin = sMeans[bestSatIndex] - 20; - satMax = sMeans[bestSatIndex] + 20; - } + debugImagesSet.push_back(addLabel(colorMask, "Color Mask After")); - if (satMin < 0) - satMin = 0; - if (satMax > 255) - satMax = 255; + debugImagesSet.push_back(addLabel(combined, "Combined")); - if (this->debug) - cout << "ColorFilter Sat: " << bestSatIndex << " : " << setw(7) << sMeans[bestSatIndex] << " -- " << satMin << "-" << satMax << endl; - } - if (bestValIndex != -1) - { - doValFilter = true; + //displayImage(config, "COLOR filter Mask", colorMask); + debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug")); - valMin = vMeans[bestValIndex] - (1.5 * vStdDevs[bestValIndex]); - valMax = vMeans[bestValIndex] + (1.5 * vStdDevs[bestValIndex]); + cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR); + debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue")); - if (abs(valMin - valMax) < 20) - { - valMin = vMeans[bestValIndex] - 20; - valMax = vMeans[bestValIndex] + 20; - } + equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter); + debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance")); - if (valMin < 0) - valMin = 0; - if (valMax > 255) - valMax = 255; + debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off")); - if (this->debug) - cout << "ColorFilter Val: " << bestValIndex << " : " << setw(7) << vMeans[bestValIndex] << " -- " << valMin << "-" << valMax << endl; - } - - Mat imgDebugHueOnly = Mat::zeros(hsv.size(), hsv.type()); - Mat imgDebug = Mat::zeros(hsv.size(), hsv.type()); - Mat imgDistanceFromCenter = Mat::zeros(hsv.size(), CV_8U); - Mat debugMask = Mat::zeros(hsv.size(), CV_8U); - bitwise_not(debugMask, debugMask); - - for (int row = 0; row < charMask.rows; row++) - { - for (int col = 0; col < charMask.cols; col++) - { - int h = (int) hsv.at(row, col)[0]; - int s = (int) hsv.at(row, col)[1]; - int v = (int) hsv.at(row, col)[2]; - - bool hPasses = true; - bool sPasses = true; - bool vPasses = true; - - int vDistance = abs(v - vMeans[bestValIndex]); - - imgDebugHueOnly.at(row, col)[0] = h; - imgDebugHueOnly.at(row, col)[1] = 255; - imgDebugHueOnly.at(row, col)[2] = 255; - - imgDebug.at(row, col)[0] = 255; - imgDebug.at(row, col)[1] = 255; - imgDebug.at(row, col)[2] = 255; - - if (doHueFilter && (h < hueMin || h > hueMax)) - { - hPasses = false; - imgDebug.at(row, col)[0] = 0; - debugMask.at(row, col) = 0; - } - if (doSatFilter && (s < satMin || s > satMax)) - { - sPasses = false; - imgDebug.at(row, col)[1] = 0; - } - if (doValFilter && (v < valMin || v > valMax)) - { - vPasses = false; - imgDebug.at(row, col)[2] = 0; - } - - //if (pixelPasses) - // colorMask.at(row, col) = 255; - //else - //imgDebug.at(row, col)[0] = hPasses & 255; - //imgDebug.at(row, col)[1] = sPasses & 255; - //imgDebug.at(row, col)[2] = vPasses & 255; - - if ((hPasses) || (hPasses && sPasses))//(hPasses && vPasses) || (sPasses && vPasses) || - this->colorMask.at(row, col) = 255; - else - this->colorMask.at(row, col) = 0; - - if ((hPasses && sPasses) || (hPasses && vPasses) || (sPasses && vPasses)) - { - vDistance = pow(vDistance, 0.9); - } - else - { - vDistance = pow(vDistance, 1.1); - } - if (vDistance > 255) - vDistance = 255; - imgDistanceFromCenter.at(row, col) = vDistance; + Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3); + displayImage(config, "Color Filter Images", dashboard); } } - vector debugImagesSet; - - if (this->debug) + // Goes through an array of values, picks the winner based on the highest percentage of other values that are within the maxValDifference + // Return -1 if it fails. + int ColorFilter::getMajorityOpinion(vector values, float minPercentAgreement, float maxValDifference) { - debugImagesSet.push_back(addLabel(charMask, "Charecter mask")); - //debugImagesSet1.push_back(erodedCharMask); - Mat maskCopy(colorMask.size(), colorMask.type()); - colorMask.copyTo(maskCopy); - debugImagesSet.push_back(addLabel(maskCopy, "color Mask Before")); - } + float bestPercentAgreement = 0; + float lowestOverallDiff = 1000000000; + int bestPercentAgreementIndex = -1; - Mat bigElement = getStructuringElement( 1, - Size( 3 + 1, 3+1 ), - Point( 1, 1 ) ); - - Mat smallElement = getStructuringElement( 1, - Size( 1 + 1, 1+1 ), - Point( 1, 1 ) ); - - morphologyEx(this->colorMask, this->colorMask, MORPH_CLOSE, bigElement); - //dilate(this->colorMask, this->colorMask, bigElement); - - Mat combined(charMask.size(), charMask.type()); - bitwise_and(charMask, colorMask, combined); - - if (this->debug) - { - debugImagesSet.push_back(addLabel(colorMask, "Color Mask After")); - - debugImagesSet.push_back(addLabel(combined, "Combined")); - - //displayImage(config, "COLOR filter Mask", colorMask); - debugImagesSet.push_back(addLabel(imgDebug, "Color filter Debug")); - - cvtColor(imgDebugHueOnly, imgDebugHueOnly, CV_HSV2BGR); - debugImagesSet.push_back(addLabel(imgDebugHueOnly, "Color Filter Hue")); - - equalizeHist(imgDistanceFromCenter, imgDistanceFromCenter); - debugImagesSet.push_back(addLabel(imgDistanceFromCenter, "COLOR filter Distance")); - - debugImagesSet.push_back(addLabel(debugMask, "COLOR Hues off")); - - Mat dashboard = drawImageDashboard(debugImagesSet, imgDebugHueOnly.type(), 3); - displayImage(config, "Color Filter Images", dashboard); - } -} - -// Goes through an array of values, picks the winner based on the highest percentage of other values that are within the maxValDifference -// Return -1 if it fails. -int ColorFilter::getMajorityOpinion(vector values, float minPercentAgreement, float maxValDifference) -{ - float bestPercentAgreement = 0; - float lowestOverallDiff = 1000000000; - int bestPercentAgreementIndex = -1; - - for (uint i = 0; i < values.size(); i++) - { - int valuesInRange = 0; - float overallDiff = 0; - for (uint j = 0; j < values.size(); j++) + for (uint i = 0; i < values.size(); i++) { - float diff = abs(values[i] - values[j]); - if (diff < maxValDifference) - valuesInRange++; + int valuesInRange = 0; + float overallDiff = 0; + for (uint j = 0; j < values.size(); j++) + { + float diff = abs(values[i] - values[j]); + if (diff < maxValDifference) + valuesInRange++; - overallDiff += diff; + overallDiff += diff; + } + + float percentAgreement = ((float) valuesInRange) / ((float) values.size()); + if (overallDiff < lowestOverallDiff && percentAgreement >= bestPercentAgreement && percentAgreement >= minPercentAgreement) + { + bestPercentAgreement = percentAgreement; + bestPercentAgreementIndex = i; + lowestOverallDiff = overallDiff; + } } - float percentAgreement = ((float) valuesInRange) / ((float) values.size()); - if (overallDiff < lowestOverallDiff && percentAgreement >= bestPercentAgreement && percentAgreement >= minPercentAgreement) - { - bestPercentAgreement = percentAgreement; - bestPercentAgreementIndex = i; - lowestOverallDiff = overallDiff; - } + return bestPercentAgreementIndex; } - return bestPercentAgreementIndex; -} +} \ No newline at end of file diff --git a/src/openalpr/colorfilter.h b/src/openalpr/colorfilter.h index e5b5df0..ab00b56 100644 --- a/src/openalpr/colorfilter.h +++ b/src/openalpr/colorfilter.h @@ -27,31 +27,34 @@ #include "utility.h" #include "config.h" - -class ColorFilter +namespace alpr { - public: - ColorFilter(cv::Mat image, cv::Mat characterMask, Config* config); - virtual ~ColorFilter(); + class ColorFilter + { - cv::Mat colorMask; + public: + ColorFilter(cv::Mat image, cv::Mat characterMask, Config* config); + virtual ~ColorFilter(); - private: + cv::Mat colorMask; - Config* config; - bool debug; + private: - cv::Mat hsv; - cv::Mat charMask; + Config* config; + bool debug; - bool grayscale; + cv::Mat hsv; + cv::Mat charMask; - void preprocessImage(); - void findCharColors(); + bool grayscale; - bool imageIsGrayscale(cv::Mat image); - int getMajorityOpinion(std::vector values, float minPercentAgreement, float maxValDifference); -}; + void preprocessImage(); + void findCharColors(); + bool imageIsGrayscale(cv::Mat image); + int getMajorityOpinion(std::vector values, float minPercentAgreement, float maxValDifference); + }; + +} #endif // OPENALPR_COLORFILTER_H diff --git a/src/openalpr/config.cpp b/src/openalpr/config.cpp index 81a1fe4..e51ae10 100644 --- a/src/openalpr/config.cpp +++ b/src/openalpr/config.cpp @@ -21,264 +21,268 @@ using namespace std; -Config::Config(const std::string country, const std::string config_file, const std::string runtime_dir) +namespace alpr { - - string debug_message = ""; - - this->loaded = false; - - ini = new CSimpleIniA(); - - string configFile; - - char* envConfigFile; - envConfigFile = getenv (ENV_VARIABLE_CONFIG_FILE); - if (config_file.compare("") != 0) + + Config::Config(const std::string country, const std::string config_file, const std::string runtime_dir) { - // User has supplied a config file. Use that. - configFile = config_file; - debug_message = "Config file location provided via API"; + + string debug_message = ""; + + this->loaded = false; + + ini = new CSimpleIniA(); + + string configFile; + + char* envConfigFile; + envConfigFile = getenv (ENV_VARIABLE_CONFIG_FILE); + if (config_file.compare("") != 0) + { + // User has supplied a config file. Use that. + configFile = config_file; + debug_message = "Config file location provided via API"; + } + else if (envConfigFile != NULL) + { + // Environment variable is non-empty. Use that. + configFile = envConfigFile; + debug_message = "Config file location provided via environment variable: " + string(ENV_VARIABLE_CONFIG_FILE); + } + else if (DirectoryExists(getExeDir().c_str()) && fileExists((getExeDir() + CONFIG_FILE).c_str())) + { + configFile = getExeDir() + CONFIG_FILE; + debug_message = "Config file location provided via exe location"; + } + else + { + // Use the default + configFile = DEFAULT_CONFIG_FILE; + debug_message = "Config file location provided via default location"; + } + + //string configFile = (this->runtimeBaseDir + CONFIG_FILE); + + if (fileExists(configFile.c_str()) == false) + { + std::cerr << "--(!) Config file '" << configFile << "' does not exist!" << endl; + std::cerr << "--(!) You can specify the configuration file location via the command line " << endl; + std::cerr << "--(!) or by setting the environment variable '" << ENV_VARIABLE_CONFIG_FILE << "'" << endl; + return; + } + else if (DirectoryExists(configFile.c_str())) + { + std::cerr << "--(!) Config file '" << configFile << "' was specified as a directory, rather than a file!" << endl; + std::cerr << "--(!) Please specify the full path to the 'openalpr.conf file'" << endl; + std::cerr << "--(!) e.g., /etc/openalpr/openalpr.conf" << endl; + return; + } + + ini->LoadFile(configFile.c_str()); + + this->country = country; + + + loadValues(country); + + if (runtime_dir.compare("") != 0) + { + // User provided a runtime directory directly into the library. Use this. + this->runtimeBaseDir = runtime_dir; + } + + if ((DirectoryExists(this->runtimeBaseDir.c_str()) == false) && + (DirectoryExists((getExeDir() + RUNTIME_DIR).c_str()))) + { + // Runtime dir in the config is invalid and there is a runtime dir in the same dir as the exe. + this->runtimeBaseDir = getExeDir() + RUNTIME_DIR; + + } + + if (DirectoryExists(this->runtimeBaseDir.c_str()) == false) + { + std::cerr << "--(!) Runtime directory '" << this->runtimeBaseDir << "' does not exist!" << endl; + std::cerr << "--(!) Please update the OpenALPR config file: '" << configFile << "'" << endl; + std::cerr << "--(!) to point to the correct location of your runtime_dir" << endl; + return; + } + else if (fileExists((this->runtimeBaseDir + "/ocr/tessdata/" + this->ocrLanguage + ".traineddata").c_str()) == false) + { + std::cerr << "--(!) Runtime directory '" << this->runtimeBaseDir << "' is invalid. Missing OCR data for the country: '" << country<< "'!" << endl; + return; + } + + + if (this->debugGeneral) + { + std::cout << debug_message << endl; + } + + this->loaded = true; } - else if (envConfigFile != NULL) + Config::~Config() { - // Environment variable is non-empty. Use that. - configFile = envConfigFile; - debug_message = "Config file location provided via environment variable: " + string(ENV_VARIABLE_CONFIG_FILE); + delete ini; } - else if (DirectoryExists(getExeDir().c_str()) && fileExists((getExeDir() + CONFIG_FILE).c_str())) + + void Config::loadValues(string country) { - configFile = getExeDir() + CONFIG_FILE; - debug_message = "Config file location provided via exe location"; - } - else - { - // Use the default - configFile = DEFAULT_CONFIG_FILE; - debug_message = "Config file location provided via default location"; - } - - //string configFile = (this->runtimeBaseDir + CONFIG_FILE); - - if (fileExists(configFile.c_str()) == false) - { - std::cerr << "--(!) Config file '" << configFile << "' does not exist!" << endl; - std::cerr << "--(!) You can specify the configuration file location via the command line " << endl; - std::cerr << "--(!) or by setting the environment variable '" << ENV_VARIABLE_CONFIG_FILE << "'" << endl; - return; - } - else if (DirectoryExists(configFile.c_str())) - { - std::cerr << "--(!) Config file '" << configFile << "' was specified as a directory, rather than a file!" << endl; - std::cerr << "--(!) Please specify the full path to the 'openalpr.conf file'" << endl; - std::cerr << "--(!) e.g., /etc/openalpr/openalpr.conf" << endl; - return; - } - - ini->LoadFile(configFile.c_str()); - - this->country = country; - - - loadValues(country); - - if (runtime_dir.compare("") != 0) - { - // User provided a runtime directory directly into the library. Use this. - this->runtimeBaseDir = runtime_dir; - } - - if ((DirectoryExists(this->runtimeBaseDir.c_str()) == false) && - (DirectoryExists((getExeDir() + RUNTIME_DIR).c_str()))) - { - // Runtime dir in the config is invalid and there is a runtime dir in the same dir as the exe. - this->runtimeBaseDir = getExeDir() + RUNTIME_DIR; + + runtimeBaseDir = getString("common", "runtime_dir", "/usr/share/openalpr/runtime_data"); + + detection_iteration_increase = getFloat("common", "detection_iteration_increase", 1.1); + detectionStrictness = getInt("common", "detection_strictness", 3); + maxPlateWidthPercent = getFloat("common", "max_plate_width_percent", 100); + maxPlateHeightPercent = getFloat("common", "max_plate_height_percent", 100); + maxDetectionInputWidth = getInt("common", "max_detection_input_width", 1280); + maxDetectionInputHeight = getInt("common", "max_detection_input_height", 768); + + maxPlateAngleDegrees = getInt("common", "max_plate_angle_degrees", 15); + + minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100); + minPlateSizeHeightPx = getInt(country, "min_plate_size_height_px", 100); + + multiline = getBoolean(country, "multiline", false); + + plateWidthMM = getFloat(country, "plate_width_mm", 100); + plateHeightMM = getFloat(country, "plate_height_mm", 100); + + charHeightMM = getFloat(country, "char_height_mm", 100); + charWidthMM = getFloat(country, "char_width_mm", 100); + charWhitespaceTopMM = getFloat(country, "char_whitespace_top_mm", 100); + charWhitespaceBotMM = getFloat(country, "char_whitespace_bot_mm", 100); + + templateWidthPx = getInt(country, "template_max_width_px", 100); + templateHeightPx = getInt(country, "template_max_height_px", 100); + + float ocrImagePercent = getFloat("common", "ocr_img_size_percent", 100); + ocrImageWidthPx = round(((float) templateWidthPx) * ocrImagePercent); + ocrImageHeightPx = round(((float)templateHeightPx) * ocrImagePercent); + + + float stateIdImagePercent = getFloat("common", "state_id_img_size_percent", 100); + stateIdImageWidthPx = round(((float)templateWidthPx) * ocrImagePercent); + stateIdimageHeightPx = round(((float)templateHeightPx) * ocrImagePercent); + + + charAnalysisMinPercent = getFloat(country, "char_analysis_min_pct", 0); + charAnalysisHeightRange = getFloat(country, "char_analysis_height_range", 0); + charAnalysisHeightStepSize = getFloat(country, "char_analysis_height_step_size", 0); + charAnalysisNumSteps = getInt(country, "char_analysis_height_num_steps", 0); + + segmentationMinBoxWidthPx = getInt(country, "segmentation_min_box_width_px", 0); + segmentationMinCharHeightPercent = getFloat(country, "segmentation_min_charheight_percent", 0); + segmentationMaxCharWidthvsAverage = getFloat(country, "segmentation_max_segment_width_percent_vs_average", 0); + + plateLinesSensitivityVertical = getFloat(country, "plateline_sensitivity_vertical", 0); + plateLinesSensitivityHorizontal = getFloat(country, "plateline_sensitivity_horizontal", 0); + + ocrLanguage = getString(country, "ocr_language", "none"); + ocrMinFontSize = getInt("common", "ocr_min_font_point", 100); + + postProcessMinConfidence = getFloat("common", "postprocess_min_confidence", 100); + postProcessConfidenceSkipLevel = getFloat("common", "postprocess_confidence_skip_level", 100); + postProcessMaxSubstitutions = getInt("common", "postprocess_max_substitutions", 100); + postProcessMinCharacters = getInt("common", "postprocess_min_characters", 100); + postProcessMaxCharacters = getInt("common", "postprocess_max_characters", 100); + + debugGeneral = getBoolean("debug", "general", false); + debugTiming = getBoolean("debug", "timing", false); + debugStateId = getBoolean("debug", "state_id", false); + debugPlateLines = getBoolean("debug", "plate_lines", false); + debugPlateCorners = getBoolean("debug", "plate_corners", false); + debugCharSegmenter = getBoolean("debug", "char_segment", false); + debugCharAnalysis = getBoolean("debug", "char_analysis", false); + debugColorFiler = getBoolean("debug", "color_filter", false); + debugOcr = getBoolean("debug", "ocr", false); + debugPostProcess = getBoolean("debug", "postprocess", false); + debugShowImages = getBoolean("debug", "show_images", false); + debugPauseOnFrame = getBoolean("debug", "pause_on_frame", false); } - if (DirectoryExists(this->runtimeBaseDir.c_str()) == false) + void Config::debugOff() { - std::cerr << "--(!) Runtime directory '" << this->runtimeBaseDir << "' does not exist!" << endl; - std::cerr << "--(!) Please update the OpenALPR config file: '" << configFile << "'" << endl; - std::cerr << "--(!) to point to the correct location of your runtime_dir" << endl; - return; + debugGeneral = false; + debugTiming = false; + debugStateId = false; + debugPlateLines = false; + debugPlateCorners = false; + debugCharSegmenter = false; + debugCharAnalysis = false; + debugColorFiler = false; + debugOcr = false; + debugPostProcess = false; + debugPauseOnFrame = false; } - else if (fileExists((this->runtimeBaseDir + "/ocr/tessdata/" + this->ocrLanguage + ".traineddata").c_str()) == false) + + + string Config::getCascadeRuntimeDir() { - std::cerr << "--(!) Runtime directory '" << this->runtimeBaseDir << "' is invalid. Missing OCR data for the country: '" << country<< "'!" << endl; - return; + return this->runtimeBaseDir + CASCADE_DIR; } - - - if (this->debugGeneral) + string Config::getKeypointsRuntimeDir() { - std::cout << debug_message << endl; + return this->runtimeBaseDir + KEYPOINTS_DIR; + } + string Config::getPostProcessRuntimeDir() + { + return this->runtimeBaseDir + POSTPROCESS_DIR; + } + string Config::getTessdataPrefix() + { + return this->runtimeBaseDir + "/ocr/"; } - - this->loaded = true; -} -Config::~Config() -{ - delete ini; -} - -void Config::loadValues(string country) -{ - - runtimeBaseDir = getString("common", "runtime_dir", "/usr/share/openalpr/runtime_data"); - - detection_iteration_increase = getFloat("common", "detection_iteration_increase", 1.1); - detectionStrictness = getInt("common", "detection_strictness", 3); - maxPlateWidthPercent = getFloat("common", "max_plate_width_percent", 100); - maxPlateHeightPercent = getFloat("common", "max_plate_height_percent", 100); - maxDetectionInputWidth = getInt("common", "max_detection_input_width", 1280); - maxDetectionInputHeight = getInt("common", "max_detection_input_height", 768); - - maxPlateAngleDegrees = getInt("common", "max_plate_angle_degrees", 15); - - minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100); - minPlateSizeHeightPx = getInt(country, "min_plate_size_height_px", 100); - - multiline = getBoolean(country, "multiline", false); - - plateWidthMM = getFloat(country, "plate_width_mm", 100); - plateHeightMM = getFloat(country, "plate_height_mm", 100); - - charHeightMM = getFloat(country, "char_height_mm", 100); - charWidthMM = getFloat(country, "char_width_mm", 100); - charWhitespaceTopMM = getFloat(country, "char_whitespace_top_mm", 100); - charWhitespaceBotMM = getFloat(country, "char_whitespace_bot_mm", 100); - - templateWidthPx = getInt(country, "template_max_width_px", 100); - templateHeightPx = getInt(country, "template_max_height_px", 100); - - float ocrImagePercent = getFloat("common", "ocr_img_size_percent", 100); - ocrImageWidthPx = round(((float) templateWidthPx) * ocrImagePercent); - ocrImageHeightPx = round(((float)templateHeightPx) * ocrImagePercent); - - - float stateIdImagePercent = getFloat("common", "state_id_img_size_percent", 100); - stateIdImageWidthPx = round(((float)templateWidthPx) * ocrImagePercent); - stateIdimageHeightPx = round(((float)templateHeightPx) * ocrImagePercent); - - - charAnalysisMinPercent = getFloat(country, "char_analysis_min_pct", 0); - charAnalysisHeightRange = getFloat(country, "char_analysis_height_range", 0); - charAnalysisHeightStepSize = getFloat(country, "char_analysis_height_step_size", 0); - charAnalysisNumSteps = getInt(country, "char_analysis_height_num_steps", 0); - - segmentationMinBoxWidthPx = getInt(country, "segmentation_min_box_width_px", 0); - segmentationMinCharHeightPercent = getFloat(country, "segmentation_min_charheight_percent", 0); - segmentationMaxCharWidthvsAverage = getFloat(country, "segmentation_max_segment_width_percent_vs_average", 0); - - plateLinesSensitivityVertical = getFloat(country, "plateline_sensitivity_vertical", 0); - plateLinesSensitivityHorizontal = getFloat(country, "plateline_sensitivity_horizontal", 0); - - ocrLanguage = getString(country, "ocr_language", "none"); - ocrMinFontSize = getInt("common", "ocr_min_font_point", 100); - - postProcessMinConfidence = getFloat("common", "postprocess_min_confidence", 100); - postProcessConfidenceSkipLevel = getFloat("common", "postprocess_confidence_skip_level", 100); - postProcessMaxSubstitutions = getInt("common", "postprocess_max_substitutions", 100); - postProcessMinCharacters = getInt("common", "postprocess_min_characters", 100); - postProcessMaxCharacters = getInt("common", "postprocess_max_characters", 100); - - debugGeneral = getBoolean("debug", "general", false); - debugTiming = getBoolean("debug", "timing", false); - debugStateId = getBoolean("debug", "state_id", false); - debugPlateLines = getBoolean("debug", "plate_lines", false); - debugPlateCorners = getBoolean("debug", "plate_corners", false); - debugCharSegmenter = getBoolean("debug", "char_segment", false); - debugCharAnalysis = getBoolean("debug", "char_analysis", false); - debugColorFiler = getBoolean("debug", "color_filter", false); - debugOcr = getBoolean("debug", "ocr", false); - debugPostProcess = getBoolean("debug", "postprocess", false); - debugShowImages = getBoolean("debug", "show_images", false); - debugPauseOnFrame = getBoolean("debug", "pause_on_frame", false); - -} - -void Config::debugOff() -{ - debugGeneral = false; - debugTiming = false; - debugStateId = false; - debugPlateLines = false; - debugPlateCorners = false; - debugCharSegmenter = false; - debugCharAnalysis = false; - debugColorFiler = false; - debugOcr = false; - debugPostProcess = false; - debugPauseOnFrame = false; -} - - -string Config::getCascadeRuntimeDir() -{ - return this->runtimeBaseDir + CASCADE_DIR; -} -string Config::getKeypointsRuntimeDir() -{ - return this->runtimeBaseDir + KEYPOINTS_DIR; -} -string Config::getPostProcessRuntimeDir() -{ - return this->runtimeBaseDir + POSTPROCESS_DIR; -} -string Config::getTessdataPrefix() -{ - return this->runtimeBaseDir + "/ocr/"; -} -float Config::getFloat(string section, string key, float defaultValue) -{ - const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); - if (pszValue == NULL) + float Config::getFloat(string section, string key, float defaultValue) { - std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; - return defaultValue; + const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); + if (pszValue == NULL) + { + std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; + return defaultValue; + } + + float val = atof(pszValue); + return val; } - - float val = atof(pszValue); - return val; -} -int Config::getInt(string section, string key, int defaultValue) -{ - const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); - if (pszValue == NULL) + int Config::getInt(string section, string key, int defaultValue) { - std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; - return defaultValue; + const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); + if (pszValue == NULL) + { + std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; + return defaultValue; + } + + int val = atoi(pszValue); + return val; } - - int val = atoi(pszValue); - return val; -} -bool Config::getBoolean(string section, string key, bool defaultValue) -{ - const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); - if (pszValue == NULL) + bool Config::getBoolean(string section, string key, bool defaultValue) { - std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; - return defaultValue; + const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); + if (pszValue == NULL) + { + std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; + return defaultValue; + } + + int val = atoi(pszValue); + return val != 0; } - - int val = atoi(pszValue); - return val != 0; -} -string Config::getString(string section, string key, string defaultValue) -{ - const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); - if (pszValue == NULL) + string Config::getString(string section, string key, string defaultValue) { - std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; - return defaultValue; + const char * pszValue = ini->GetValue(section.c_str(), key.c_str(), NULL /*default*/); + if (pszValue == NULL) + { + std::cout << "Error: missing configuration entry for: " << section << "->" << key << endl; + return defaultValue; + } + + string val = string(pszValue); + return val; } - - string val = string(pszValue); - return val; -} +} \ No newline at end of file diff --git a/src/openalpr/config.h b/src/openalpr/config.h index a752441..3bdb57e 100644 --- a/src/openalpr/config.h +++ b/src/openalpr/config.h @@ -33,103 +33,105 @@ #include /* getenv */ #include - -class Config +namespace alpr { - public: - Config(const std::string country, const std::string config_file = "", const std::string runtime_dir = ""); - virtual ~Config(); + class Config + { - bool loaded; - - std::string country; - - float detection_iteration_increase; - int detectionStrictness; - float maxPlateWidthPercent; - float maxPlateHeightPercent; - int maxDetectionInputWidth; - int maxDetectionInputHeight; - - int maxPlateAngleDegrees; - - float minPlateSizeWidthPx; - float minPlateSizeHeightPx; - - bool multiline; - - float plateWidthMM; - float plateHeightMM; - - float charHeightMM; - float charWidthMM; - float charWhitespaceTopMM; - float charWhitespaceBotMM; - - int templateWidthPx; - int templateHeightPx; - - int ocrImageWidthPx; - int ocrImageHeightPx; - - int stateIdImageWidthPx; - int stateIdimageHeightPx; - - float charAnalysisMinPercent; - float charAnalysisHeightRange; - float charAnalysisHeightStepSize; - int charAnalysisNumSteps; - - float plateLinesSensitivityVertical; - float plateLinesSensitivityHorizontal; + public: + Config(const std::string country, const std::string config_file = "", const std::string runtime_dir = ""); + virtual ~Config(); - int segmentationMinBoxWidthPx; - float segmentationMinCharHeightPercent; - float segmentationMaxCharWidthvsAverage; - - std::string ocrLanguage; - int ocrMinFontSize; - - float postProcessMinConfidence; - float postProcessConfidenceSkipLevel; - uint postProcessMaxSubstitutions; - uint postProcessMinCharacters; - uint postProcessMaxCharacters; + bool loaded; - - bool debugGeneral; - bool debugTiming; - bool debugStateId; - bool debugPlateLines; - bool debugPlateCorners; - bool debugCharSegmenter; - bool debugCharAnalysis; - bool debugColorFiler; - bool debugOcr; - bool debugPostProcess; - bool debugShowImages; - bool debugPauseOnFrame; - - void debugOff(); - - std::string getKeypointsRuntimeDir(); - std::string getCascadeRuntimeDir(); - std::string getPostProcessRuntimeDir(); - std::string getTessdataPrefix(); + std::string country; -private: - CSimpleIniA* ini; + float detection_iteration_increase; + int detectionStrictness; + float maxPlateWidthPercent; + float maxPlateHeightPercent; + int maxDetectionInputWidth; + int maxDetectionInputHeight; - std::string runtimeBaseDir; - - void loadValues(std::string country); - - int getInt(std::string section, std::string key, int defaultValue); - float getFloat(std::string section, std::string key, float defaultValue); - std::string getString(std::string section, std::string key, std::string defaultValue); - bool getBoolean(std::string section, std::string key, bool defaultValue); -}; + int maxPlateAngleDegrees; + + float minPlateSizeWidthPx; + float minPlateSizeHeightPx; + + bool multiline; + + float plateWidthMM; + float plateHeightMM; + + float charHeightMM; + float charWidthMM; + float charWhitespaceTopMM; + float charWhitespaceBotMM; + + int templateWidthPx; + int templateHeightPx; + + int ocrImageWidthPx; + int ocrImageHeightPx; + + int stateIdImageWidthPx; + int stateIdimageHeightPx; + + float charAnalysisMinPercent; + float charAnalysisHeightRange; + float charAnalysisHeightStepSize; + int charAnalysisNumSteps; + + float plateLinesSensitivityVertical; + float plateLinesSensitivityHorizontal; + + int segmentationMinBoxWidthPx; + float segmentationMinCharHeightPercent; + float segmentationMaxCharWidthvsAverage; + + std::string ocrLanguage; + int ocrMinFontSize; + + float postProcessMinConfidence; + float postProcessConfidenceSkipLevel; + uint postProcessMaxSubstitutions; + uint postProcessMinCharacters; + uint postProcessMaxCharacters; + bool debugGeneral; + bool debugTiming; + bool debugStateId; + bool debugPlateLines; + bool debugPlateCorners; + bool debugCharSegmenter; + bool debugCharAnalysis; + bool debugColorFiler; + bool debugOcr; + bool debugPostProcess; + bool debugShowImages; + bool debugPauseOnFrame; + + void debugOff(); + + std::string getKeypointsRuntimeDir(); + std::string getCascadeRuntimeDir(); + std::string getPostProcessRuntimeDir(); + std::string getTessdataPrefix(); + + private: + CSimpleIniA* ini; + + std::string runtimeBaseDir; + + void loadValues(std::string country); + + int getInt(std::string section, std::string key, int defaultValue); + float getFloat(std::string section, std::string key, float defaultValue); + std::string getString(std::string section, std::string key, std::string defaultValue); + bool getBoolean(std::string section, std::string key, bool defaultValue); + }; + +} #endif // OPENALPR_CONFIG_H \ No newline at end of file diff --git a/src/openalpr/detection/detector.cpp b/src/openalpr/detection/detector.cpp index a4c4165..315f111 100644 --- a/src/openalpr/detection/detector.cpp +++ b/src/openalpr/detection/detector.cpp @@ -22,88 +22,93 @@ using namespace cv; using namespace std; -Detector::Detector(Config* config) +namespace alpr { - this->config = config; - this->scale_factor = 1.0f; -} - -Detector::~Detector() -{ - -} - -bool Detector::isLoaded() -{ - return this->loaded; -} - -vector Detector::detect(cv::Mat frame) -{ - std::vector regionsOfInterest; - regionsOfInterest.push_back(Rect(0, 0, frame.cols, frame.rows)); - return this->detect(frame, regionsOfInterest); -} - -vector Detector::detect(Mat frame, std::vector regionsOfInterest) -{ - // Must be implemented by subclass - std::vector rois; - return rois; -} - - -bool rectHasLargerArea(cv::Rect a, cv::Rect b) { return a.area() < b.area(); }; - -vector Detector::aggregateRegions(vector regions) -{ - // Combines overlapping regions into a parent->child order. - // The largest regions will be parents, and they will have children if they are within them. - // This way, when processing regions later, we can process the parents first, and only delve into the children - // If there was no plate match. Otherwise, we would process everything and that would be wasteful. - - vector orderedRegions; - vector topLevelRegions; - - // Sort the list of rect regions smallest to largest - std::sort(regions.begin(), regions.end(), rectHasLargerArea); - - // Create new PlateRegions and attach the rectangles to each - for (uint i = 0; i < regions.size(); i++) + Detector::Detector(Config* config) { - PlateRegion newRegion; - newRegion.rect = regions[i]; - orderedRegions.push_back(newRegion); + this->config = config; + this->scale_factor = 1.0f; + } - - for (uint i = 0; i < orderedRegions.size(); i++) + + Detector::~Detector() { - bool foundParent = false; - for (uint k = i + 1; k < orderedRegions.size(); k++) + + } + + bool Detector::isLoaded() + { + return this->loaded; + } + + vector Detector::detect(cv::Mat frame) + { + std::vector regionsOfInterest; + regionsOfInterest.push_back(Rect(0, 0, frame.cols, frame.rows)); + return this->detect(frame, regionsOfInterest); + } + + vector Detector::detect(Mat frame, std::vector regionsOfInterest) + { + // Must be implemented by subclass + std::vector rois; + return rois; + } + + + bool rectHasLargerArea(cv::Rect a, cv::Rect b) { return a.area() < b.area(); }; + + vector Detector::aggregateRegions(vector regions) + { + // Combines overlapping regions into a parent->child order. + // The largest regions will be parents, and they will have children if they are within them. + // This way, when processing regions later, we can process the parents first, and only delve into the children + // If there was no plate match. Otherwise, we would process everything and that would be wasteful. + + vector orderedRegions; + vector topLevelRegions; + + // Sort the list of rect regions smallest to largest + std::sort(regions.begin(), regions.end(), rectHasLargerArea); + + // Create new PlateRegions and attach the rectangles to each + for (uint i = 0; i < regions.size(); i++) { - Point center( orderedRegions[i].rect.x + (orderedRegions[i].rect.width / 2), - orderedRegions[i].rect.y + (orderedRegions[i].rect.height / 2)); - - // Check if the center of the smaller rectangle is inside the bigger rectangle. - // If so, add it to the children and continue on. - if (orderedRegions[k].rect.contains(center)) + PlateRegion newRegion; + newRegion.rect = regions[i]; + orderedRegions.push_back(newRegion); + } + + for (uint i = 0; i < orderedRegions.size(); i++) + { + bool foundParent = false; + for (uint k = i + 1; k < orderedRegions.size(); k++) { - orderedRegions[k].children.push_back(orderedRegions[i]); - foundParent = true; - break; + Point center( orderedRegions[i].rect.x + (orderedRegions[i].rect.width / 2), + orderedRegions[i].rect.y + (orderedRegions[i].rect.height / 2)); + + // Check if the center of the smaller rectangle is inside the bigger rectangle. + // If so, add it to the children and continue on. + if (orderedRegions[k].rect.contains(center)) + { + orderedRegions[k].children.push_back(orderedRegions[i]); + foundParent = true; + break; + } + } - + + if (foundParent == false) + { + // We didn't find any parents for this rectangle. Add it to the top level regions + topLevelRegions.push_back(orderedRegions[i]); + } + } - if (foundParent == false) - { - // We didn't find any parents for this rectangle. Add it to the top level regions - topLevelRegions.push_back(orderedRegions[i]); - } - + + return topLevelRegions; } - - return topLevelRegions; -} +} \ No newline at end of file diff --git a/src/openalpr/detection/detector.h b/src/openalpr/detection/detector.h index 9e721c5..d93a2e6 100644 --- a/src/openalpr/detection/detector.h +++ b/src/openalpr/detection/detector.h @@ -28,34 +28,39 @@ #include "support/timing.h" #include "constants.h" -struct PlateRegion -{ - cv::Rect rect; - std::vector children; -}; - - -class Detector +namespace alpr { - public: - Detector(Config* config); - virtual ~Detector(); - - bool isLoaded(); - std::vector detect(cv::Mat frame); - virtual std::vector detect(cv::Mat frame, std::vector regionsOfInterest); - - protected: - Config* config; - - bool loaded; - float scale_factor; - - std::vector aggregateRegions(std::vector regions); - + struct PlateRegion + { + cv::Rect rect; + std::vector children; + }; -}; + class Detector + { + + public: + Detector(Config* config); + virtual ~Detector(); + + bool isLoaded(); + std::vector detect(cv::Mat frame); + virtual std::vector detect(cv::Mat frame, std::vector regionsOfInterest); + + protected: + Config* config; + + bool loaded; + float scale_factor; + + std::vector aggregateRegions(std::vector regions); + + + + }; + +} #endif // OPENALPR_REGIONDETECTOR_H diff --git a/src/openalpr/detection/detectorcpu.cpp b/src/openalpr/detection/detectorcpu.cpp index 49820a9..92c6d58 100644 --- a/src/openalpr/detection/detectorcpu.cpp +++ b/src/openalpr/detection/detectorcpu.cpp @@ -22,97 +22,102 @@ using namespace cv; using namespace std; -DetectorCPU::DetectorCPU(Config* config) : Detector(config) { - - - - if( this->plate_cascade.load( config->getCascadeRuntimeDir() + config->country + ".xml" ) ) - { - this->loaded = true; - } - else - { - this->loaded = false; - printf("--(!)Error loading classifier\n"); - } -} - - -DetectorCPU::~DetectorCPU() { -} - -vector DetectorCPU::detect(Mat frame, std::vector regionsOfInterest) +namespace alpr { - Mat frame_gray; - cvtColor( frame, frame_gray, CV_BGR2GRAY ); - - vector detectedRegions = doCascade(frame_gray, regionsOfInterest); + DetectorCPU::DetectorCPU(Config* config) : Detector(config) { - return detectedRegions; -} -vector DetectorCPU::doCascade(Mat frame, std::vector regionsOfInterest) -{ - - if (frame.cols > config->maxDetectionInputWidth) - { - // The frame is too wide - this->scale_factor = ((float) config->maxDetectionInputWidth) / ((float) frame.cols); - - if (config->debugGeneral) - std::cout << "Input detection image is too wide. Resizing with scale: " << this->scale_factor << endl; - } - else if (frame.rows > config->maxDetectionInputHeight) - { - // The frame is too tall - this->scale_factor = ((float) config->maxDetectionInputHeight) / ((float) frame.rows); - - if (config->debugGeneral) - std::cout << "Input detection image is too tall. Resizing with scale: " << this->scale_factor << endl; - } - - int w = frame.size().width; - int h = frame.size().height; - - vector plates; - - equalizeHist( frame, frame ); - resize(frame, frame, Size(w * this->scale_factor, h * this->scale_factor)); - - //-- Detect plates - timespec startTime; - getTime(&startTime); - - float maxWidth = ((float) w) * (config->maxPlateWidthPercent / 100.0f) * this->scale_factor; - float maxHeight = ((float) h) * (config->maxPlateHeightPercent / 100.0f) * this->scale_factor; - Size minSize(config->minPlateSizeWidthPx * this->scale_factor, config->minPlateSizeHeightPx * this->scale_factor); - Size maxSize(maxWidth, maxHeight); - - plate_cascade.detectMultiScale( frame, plates, config->detection_iteration_increase, config->detectionStrictness, - 0, - //0|CV_HAAR_SCALE_IMAGE, - minSize, maxSize ); - - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "LBP Time: " << diffclock(startTime, endTime) << "ms." << endl; + if( this->plate_cascade.load( config->getCascadeRuntimeDir() + config->country + ".xml" ) ) + { + this->loaded = true; + } + else + { + this->loaded = false; + printf("--(!)Error loading classifier\n"); + } } - for( uint i = 0; i < plates.size(); i++ ) - { - plates[i].x = plates[i].x / scale_factor; - plates[i].y = plates[i].y / scale_factor; - plates[i].width = plates[i].width / scale_factor; - plates[i].height = plates[i].height / scale_factor; + + DetectorCPU::~DetectorCPU() { } - vector orderedRegions = aggregateRegions(plates); - - return orderedRegions; + vector DetectorCPU::detect(Mat frame, std::vector regionsOfInterest) + { + Mat frame_gray; + cvtColor( frame, frame_gray, CV_BGR2GRAY ); + + vector detectedRegions = doCascade(frame_gray, regionsOfInterest); + + return detectedRegions; + } + + vector DetectorCPU::doCascade(Mat frame, std::vector regionsOfInterest) + { + + + if (frame.cols > config->maxDetectionInputWidth) + { + // The frame is too wide + this->scale_factor = ((float) config->maxDetectionInputWidth) / ((float) frame.cols); + + if (config->debugGeneral) + std::cout << "Input detection image is too wide. Resizing with scale: " << this->scale_factor << endl; + } + else if (frame.rows > config->maxDetectionInputHeight) + { + // The frame is too tall + this->scale_factor = ((float) config->maxDetectionInputHeight) / ((float) frame.rows); + + if (config->debugGeneral) + std::cout << "Input detection image is too tall. Resizing with scale: " << this->scale_factor << endl; + } + + int w = frame.size().width; + int h = frame.size().height; + + vector plates; + + equalizeHist( frame, frame ); + resize(frame, frame, Size(w * this->scale_factor, h * this->scale_factor)); + + //-- Detect plates + timespec startTime; + getTime(&startTime); + + float maxWidth = ((float) w) * (config->maxPlateWidthPercent / 100.0f) * this->scale_factor; + float maxHeight = ((float) h) * (config->maxPlateHeightPercent / 100.0f) * this->scale_factor; + Size minSize(config->minPlateSizeWidthPx * this->scale_factor, config->minPlateSizeHeightPx * this->scale_factor); + Size maxSize(maxWidth, maxHeight); + + plate_cascade.detectMultiScale( frame, plates, config->detection_iteration_increase, config->detectionStrictness, + 0, + //0|CV_HAAR_SCALE_IMAGE, + minSize, maxSize ); + + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "LBP Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + for( uint i = 0; i < plates.size(); i++ ) + { + plates[i].x = plates[i].x / scale_factor; + plates[i].y = plates[i].y / scale_factor; + plates[i].width = plates[i].width / scale_factor; + plates[i].height = plates[i].height / scale_factor; + } + + vector orderedRegions = aggregateRegions(plates); + + return orderedRegions; + + } + } \ No newline at end of file diff --git a/src/openalpr/detection/detectorcpu.h b/src/openalpr/detection/detectorcpu.h index 87ccddb..e1cfecb 100644 --- a/src/openalpr/detection/detectorcpu.h +++ b/src/openalpr/detection/detectorcpu.h @@ -31,19 +31,24 @@ #include "detector.h" -class DetectorCPU : public Detector { -public: - DetectorCPU(Config* config); - virtual ~DetectorCPU(); - - std::vector detect(cv::Mat frame, std::vector regionsOfInterest); - -private: - - cv::CascadeClassifier plate_cascade; +namespace alpr +{ - std::vector doCascade(cv::Mat frame, std::vector regionsOfInterest); -}; + class DetectorCPU : public Detector { + public: + DetectorCPU(Config* config); + virtual ~DetectorCPU(); + + std::vector detect(cv::Mat frame, std::vector regionsOfInterest); + + private: + + cv::CascadeClassifier plate_cascade; + + std::vector doCascade(cv::Mat frame, std::vector regionsOfInterest); + }; + +} #endif /* OPENALPR_DETECTORCPU_H */ diff --git a/src/openalpr/detection/detectorfactory.cpp b/src/openalpr/detection/detectorfactory.cpp index f52aa89..cdc9617 100644 --- a/src/openalpr/detection/detectorfactory.cpp +++ b/src/openalpr/detection/detectorfactory.cpp @@ -1,7 +1,11 @@ #include "detectorfactory.h" -Detector* createDetector(Config* config) +namespace alpr { - return new DetectorCPU(config); -} + Detector* createDetector(Config* config) + { + return new DetectorCPU(config); + } + +} \ No newline at end of file diff --git a/src/openalpr/detection/detectorfactory.h b/src/openalpr/detection/detectorfactory.h index 5972bb3..ea5ddfc 100644 --- a/src/openalpr/detection/detectorfactory.h +++ b/src/openalpr/detection/detectorfactory.h @@ -23,7 +23,11 @@ #include "detectorcpu.h" #include "config.h" -Detector* createDetector(Config* config); +namespace alpr +{ + Detector* createDetector(Config* config); + +} #endif /* OPENALPR_DETECTORFACTORY_H */ diff --git a/src/openalpr/edges/edgefinder.cpp b/src/openalpr/edges/edgefinder.cpp index 850a280..70698bd 100644 --- a/src/openalpr/edges/edgefinder.cpp +++ b/src/openalpr/edges/edgefinder.cpp @@ -23,121 +23,125 @@ using namespace std; using namespace cv; -EdgeFinder::EdgeFinder(PipelineData* pipeline_data) { - - this->pipeline_data = pipeline_data; - - // First re-crop the area from the original picture knowing the text position - this->confidence = 0; - -} +namespace alpr +{ + EdgeFinder::EdgeFinder(PipelineData* pipeline_data) { -EdgeFinder::~EdgeFinder() { -} + this->pipeline_data = pipeline_data; -std::vector EdgeFinder::findEdgeCorners() { + // First re-crop the area from the original picture knowing the text position + this->confidence = 0; - TextLineCollection tlc(pipeline_data->textLines); - - vector corners; - - // If the character segment is especially small, just expand the existing box - // If it's a nice, long segment, then guess the correct box based on character height/position - if (tlc.longerSegment.length > tlc.charHeight * 3) - { - float charHeightToPlateWidthRatio = pipeline_data->config->plateWidthMM / pipeline_data->config->charHeightMM; - float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters - - float charHeightToPlateHeightRatio = pipeline_data->config->plateHeightMM / pipeline_data->config->charHeightMM; - float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; - - - float verticalOffset = (idealPixelHeight * 1.5 / 2); - float horizontalOffset = (idealPixelWidth * 1.25 / 2); - LineSegment topLine = tlc.centerHorizontalLine.getParallelLine(verticalOffset); - LineSegment bottomLine = tlc.centerHorizontalLine.getParallelLine(-1 * verticalOffset); - - LineSegment leftLine = tlc.centerVerticalLine.getParallelLine(-1 * horizontalOffset); - LineSegment rightLine = tlc.centerVerticalLine.getParallelLine(horizontalOffset); - - Point topLeft = topLine.intersection(leftLine); - Point topRight = topLine.intersection(rightLine); - Point botRight = bottomLine.intersection(rightLine); - Point botLeft = bottomLine.intersection(leftLine); - - corners.push_back(topLeft); - corners.push_back(topRight); - corners.push_back(botRight); - corners.push_back(botLeft); - } - else - { - - //cout << "HEYOOO!" << endl; - int expandX = (int) ((float) pipeline_data->crop_gray.cols) * 0.15f; - int expandY = (int) ((float) pipeline_data->crop_gray.rows) * 0.15f; - int w = pipeline_data->crop_gray.cols; - int h = pipeline_data->crop_gray.rows; - - corners.push_back(Point(-1 * expandX, -1 * expandY)); - corners.push_back(Point(expandX + w, -1 * expandY)); - corners.push_back(Point(expandX + w, expandY + h)); - corners.push_back(Point(-1 * expandX, expandY + h)); - -// for (int i = 0; i < 4; i++) -// { -// std::cout << "CORNER: " << corners[i].x << " - " << corners[i].y << std::endl; -// } } - // Re-crop an image (from the original image) using the new coordinates - Transformation imgTransform(pipeline_data->grayImg, pipeline_data->crop_gray, pipeline_data->regionOfInterest); - vector remappedCorners = imgTransform.transformSmallPointsToBigImage(corners); - Size cropSize = imgTransform.getCropSize(remappedCorners, - Size(pipeline_data->config->templateWidthPx, pipeline_data->config->templateHeightPx)); - - Mat transmtx = imgTransform.getTransformationMatrix(remappedCorners, cropSize); - Mat newCrop = imgTransform.crop(cropSize, transmtx); - - // Re-map the textline coordinates to the new crop - vector newLines; - for (uint i = 0; i < pipeline_data->textLines.size(); i++) - { - vector textArea = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].textArea); - vector linePolygon = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].linePolygon); - - vector textAreaRemapped; - vector linePolygonRemapped; - - textAreaRemapped = imgTransform.remapSmallPointstoCrop(textArea, transmtx); - linePolygonRemapped = imgTransform.remapSmallPointstoCrop(linePolygon, transmtx); - - newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + EdgeFinder::~EdgeFinder() { } - // Find the PlateLines for this crop - PlateLines plateLines(pipeline_data); - plateLines.processImage(newCrop, newLines, 1.05); + std::vector EdgeFinder::findEdgeCorners() { - // Get the best corners - PlateCorners cornerFinder(newCrop, &plateLines, pipeline_data, newLines); - vector smallPlateCorners = cornerFinder.findPlateCorners(); + TextLineCollection tlc(pipeline_data->textLines); - confidence = cornerFinder.confidence; + vector corners; - // Transform the best corner points back to the original image - std::vector imgArea; - imgArea.push_back(Point2f(0, 0)); - imgArea.push_back(Point2f(newCrop.cols, 0)); - imgArea.push_back(Point2f(newCrop.cols, newCrop.rows)); - imgArea.push_back(Point2f(0, newCrop.rows)); - Mat newCropTransmtx = imgTransform.getTransformationMatrix(imgArea, remappedCorners); + // If the character segment is especially small, just expand the existing box + // If it's a nice, long segment, then guess the correct box based on character height/position + if (tlc.longerSegment.length > tlc.charHeight * 3) + { + float charHeightToPlateWidthRatio = pipeline_data->config->plateWidthMM / pipeline_data->config->charHeightMM; + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters - vector cornersInOriginalImg = imgTransform.remapSmallPointstoCrop(smallPlateCorners, newCropTransmtx); + float charHeightToPlateHeightRatio = pipeline_data->config->plateHeightMM / pipeline_data->config->charHeightMM; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; - return cornersInOriginalImg; -} + float verticalOffset = (idealPixelHeight * 1.5 / 2); + float horizontalOffset = (idealPixelWidth * 1.25 / 2); + LineSegment topLine = tlc.centerHorizontalLine.getParallelLine(verticalOffset); + LineSegment bottomLine = tlc.centerHorizontalLine.getParallelLine(-1 * verticalOffset); + LineSegment leftLine = tlc.centerVerticalLine.getParallelLine(-1 * horizontalOffset); + LineSegment rightLine = tlc.centerVerticalLine.getParallelLine(horizontalOffset); + + Point topLeft = topLine.intersection(leftLine); + Point topRight = topLine.intersection(rightLine); + Point botRight = bottomLine.intersection(rightLine); + Point botLeft = bottomLine.intersection(leftLine); + + corners.push_back(topLeft); + corners.push_back(topRight); + corners.push_back(botRight); + corners.push_back(botLeft); + } + else + { + + //cout << "HEYOOO!" << endl; + int expandX = (int) ((float) pipeline_data->crop_gray.cols) * 0.15f; + int expandY = (int) ((float) pipeline_data->crop_gray.rows) * 0.15f; + int w = pipeline_data->crop_gray.cols; + int h = pipeline_data->crop_gray.rows; + + corners.push_back(Point(-1 * expandX, -1 * expandY)); + corners.push_back(Point(expandX + w, -1 * expandY)); + corners.push_back(Point(expandX + w, expandY + h)); + corners.push_back(Point(-1 * expandX, expandY + h)); + + // for (int i = 0; i < 4; i++) + // { + // std::cout << "CORNER: " << corners[i].x << " - " << corners[i].y << std::endl; + // } + } + + // Re-crop an image (from the original image) using the new coordinates + Transformation imgTransform(pipeline_data->grayImg, pipeline_data->crop_gray, pipeline_data->regionOfInterest); + vector remappedCorners = imgTransform.transformSmallPointsToBigImage(corners); + + Size cropSize = imgTransform.getCropSize(remappedCorners, + Size(pipeline_data->config->templateWidthPx, pipeline_data->config->templateHeightPx)); + + Mat transmtx = imgTransform.getTransformationMatrix(remappedCorners, cropSize); + Mat newCrop = imgTransform.crop(cropSize, transmtx); + + // Re-map the textline coordinates to the new crop + vector newLines; + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + vector textArea = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].textArea); + vector linePolygon = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].linePolygon); + + vector textAreaRemapped; + vector linePolygonRemapped; + + textAreaRemapped = imgTransform.remapSmallPointstoCrop(textArea, transmtx); + linePolygonRemapped = imgTransform.remapSmallPointstoCrop(linePolygon, transmtx); + + newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + } + + // Find the PlateLines for this crop + PlateLines plateLines(pipeline_data); + plateLines.processImage(newCrop, newLines, 1.05); + + // Get the best corners + PlateCorners cornerFinder(newCrop, &plateLines, pipeline_data, newLines); + vector smallPlateCorners = cornerFinder.findPlateCorners(); + + confidence = cornerFinder.confidence; + + // Transform the best corner points back to the original image + std::vector imgArea; + imgArea.push_back(Point2f(0, 0)); + imgArea.push_back(Point2f(newCrop.cols, 0)); + imgArea.push_back(Point2f(newCrop.cols, newCrop.rows)); + imgArea.push_back(Point2f(0, newCrop.rows)); + Mat newCropTransmtx = imgTransform.getTransformationMatrix(imgArea, remappedCorners); + + vector cornersInOriginalImg = imgTransform.remapSmallPointstoCrop(smallPlateCorners, newCropTransmtx); + + return cornersInOriginalImg; + + } + +} \ No newline at end of file diff --git a/src/openalpr/edges/edgefinder.h b/src/openalpr/edges/edgefinder.h index d0832d2..0b1f645 100644 --- a/src/openalpr/edges/edgefinder.h +++ b/src/openalpr/edges/edgefinder.h @@ -26,19 +26,23 @@ #include "platelines.h" #include "platecorners.h" -class EdgeFinder { -public: - EdgeFinder(PipelineData* pipeline_data); - virtual ~EdgeFinder(); +namespace alpr +{ - std::vector findEdgeCorners(); - - float confidence; - -private: - PipelineData* pipeline_data; - -}; + class EdgeFinder { + public: + EdgeFinder(PipelineData* pipeline_data); + virtual ~EdgeFinder(); + std::vector findEdgeCorners(); + + float confidence; + + private: + PipelineData* pipeline_data; + + }; + +} #endif /* OPENALPR_EDGEFINDER_H */ diff --git a/src/openalpr/edges/platecorners.cpp b/src/openalpr/edges/platecorners.cpp index 9e99741..e2c2d1b 100644 --- a/src/openalpr/edges/platecorners.cpp +++ b/src/openalpr/edges/platecorners.cpp @@ -22,333 +22,336 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, vector textLines) : - tlc(textLines) +namespace alpr { - this->pipelineData = pipelineData; - if (pipelineData->config->debugPlateCorners) - cout << "PlateCorners constructor" << endl; - - this->inputImage = inputImage; - this->plateLines = plateLines; - this->textLines = textLines; - - this->bestHorizontalScore = 9999999999999; - this->bestVerticalScore = 9999999999999; - - -} - -PlateCorners::~PlateCorners() -{ -} - -vector PlateCorners::findPlateCorners() -{ - if (pipelineData->config->debugPlateCorners) - cout << "PlateCorners::findPlateCorners" << endl; - - timespec startTime; - getTime(&startTime); - - int horizontalLines = this->plateLines->horizontalLines.size(); - int verticalLines = this->plateLines->verticalLines.size(); - - // layout horizontal lines - for (int h1 = NO_LINE; h1 < horizontalLines; h1++) + PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, vector textLines) : + tlc(textLines) { - for (int h2 = NO_LINE; h2 < horizontalLines; h2++) - { - if (h1 == h2 && h1 != NO_LINE) continue; + this->pipelineData = pipelineData; + + if (pipelineData->config->debugPlateCorners) + cout << "PlateCorners constructor" << endl; + + this->inputImage = inputImage; + this->plateLines = plateLines; + this->textLines = textLines; + + this->bestHorizontalScore = 9999999999999; + this->bestVerticalScore = 9999999999999; + - this->scoreHorizontals(h1, h2); - } } - // layout vertical lines - for (int v1 = NO_LINE; v1 < verticalLines; v1++) + PlateCorners::~PlateCorners() { - for (int v2 = NO_LINE; v2 < verticalLines; v2++) - { - if (v1 == v2 && v1 != NO_LINE) continue; - - this->scoreVerticals(v1, v2); - } } - if (pipelineData->config->debugPlateCorners) + vector PlateCorners::findPlateCorners() { - cout << "Drawing debug stuff..." << endl; + if (pipelineData->config->debugPlateCorners) + cout << "PlateCorners::findPlateCorners" << endl; - Mat imgCorners = Mat(inputImage.size(), inputImage.type()); - inputImage.copyTo(imgCorners); - - for (uint linenum = 0; linenum < textLines.size(); linenum++) + timespec startTime; + getTime(&startTime); + + int horizontalLines = this->plateLines->horizontalLines.size(); + int verticalLines = this->plateLines->verticalLines.size(); + + // layout horizontal lines + for (int h1 = NO_LINE; h1 < horizontalLines; h1++) { - for (int i = 0; i < 4; i++) - circle(imgCorners, textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + for (int h2 = NO_LINE; h2 < horizontalLines; h2++) + { + if (h1 == h2 && h1 != NO_LINE) continue; + + this->scoreHorizontals(h1, h2); + } } - line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); - line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); - line(imgCorners, this->bestBottom.p1, this->bestBottom.p2, Scalar(0, 0, 255), 1, CV_AA); - line(imgCorners, this->bestLeft.p1, this->bestLeft.p2, Scalar(255, 0, 0), 1, CV_AA); + // layout vertical lines + for (int v1 = NO_LINE; v1 < verticalLines; v1++) + { + for (int v2 = NO_LINE; v2 < verticalLines; v2++) + { + if (v1 == v2 && v1 != NO_LINE) continue; - displayImage(pipelineData->config, "Winning top/bottom Boundaries", imgCorners); - } - - // Check if a left/right edge has been established. - if (bestLeft.p1.x == 0 && bestLeft.p1.y == 0 && bestLeft.p2.x == 0 && bestLeft.p2.y == 0) - confidence = 0; - else if (bestTop.p1.x == 0 && bestTop.p1.y == 0 && bestTop.p2.x == 0 && bestTop.p2.y == 0) - confidence = 0; - else - confidence = 100; - - vector corners; - corners.push_back(bestTop.intersection(bestLeft)); - corners.push_back(bestTop.intersection(bestRight)); - corners.push_back(bestBottom.intersection(bestRight)); - corners.push_back(bestBottom.intersection(bestLeft)); - - if (pipelineData->config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "Plate Corners Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - return corners; -} - -void PlateCorners::scoreVerticals(int v1, int v2) -{ - ScoreKeeper scoreKeeper; - - LineSegment left; - LineSegment right; - - - float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; - float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters - - float confidenceDiff = 0; - float missingSegmentPenalty = 0; - - if (v1 == NO_LINE && v2 == NO_LINE) - { - //return; - - left = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2); - right = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2 ); - - missingSegmentPenalty = 2; - confidenceDiff += 2; - } - else if (v1 != NO_LINE && v2 != NO_LINE) - { - left = this->plateLines->verticalLines[v1].line; - right = this->plateLines->verticalLines[v2].line; - confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); - confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); - } - else if (v1 == NO_LINE && v2 != NO_LINE) - { - right = this->plateLines->verticalLines[v2].line; - left = right.getParallelLine(idealPixelWidth); - missingSegmentPenalty++; - confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); - } - else if (v1 != NO_LINE && v2 == NO_LINE) - { - left = this->plateLines->verticalLines[v1].line; - right = left.getParallelLine(-1 * idealPixelWidth); - missingSegmentPenalty++; - confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); - } - - scoreKeeper.setScore("SCORING_LINE_CONFIDENCE_WEIGHT", confidenceDiff, SCORING_LINE_CONFIDENCE_WEIGHT); - scoreKeeper.setScore("SCORING_MISSING_SEGMENT_PENALTY_VERTICAL", missingSegmentPenalty, SCORING_MISSING_SEGMENT_PENALTY_VERTICAL); - - // Make sure that the left and right lines are to the left and right of our text - // area - if (tlc.isLeftOfText(left) < 1 || tlc.isLeftOfText(right) > -1) - return; - - - ///////////////////////////////////////////////////////////////////////// - // Score angle difference from detected character box - ///////////////////////////////////////////////////////////////////////// - - float perpendicularCharAngle = tlc.charAngle - 90; - float charanglediff = abs(perpendicularCharAngle - left.angle) + abs(perpendicularCharAngle - right.angle); - - scoreKeeper.setScore("SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT", charanglediff, SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT); - - ////////////////////////////////////////////////////////////////////////// - // SCORE the shape wrt character position and height relative to position - ////////////////////////////////////////////////////////////////////////// - - Point leftMidLinePoint = left.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); - Point rightMidLinePoint = right.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); - - float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); - // normalize for image width - plateDistance = plateDistance / ((float)inputImage.cols); - - scoreKeeper.setScore("SCORING_DISTANCE_WEIGHT_VERTICAL", plateDistance, SCORING_DISTANCE_WEIGHT_VERTICAL); - - float score = scoreKeeper.getTotal(); - - if (score < this->bestVerticalScore) - { + this->scoreVerticals(v1, v2); + } + } if (pipelineData->config->debugPlateCorners) { + cout << "Drawing debug stuff..." << endl; - cout << "Vertical breakdown Score:" << endl; - - scoreKeeper.printDebugScores(); + Mat imgCorners = Mat(inputImage.size(), inputImage.type()); + inputImage.copyTo(imgCorners); + + for (uint linenum = 0; linenum < textLines.size(); linenum++) + { + for (int i = 0; i < 4; i++) + circle(imgCorners, textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + } + + line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); + line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); + line(imgCorners, this->bestBottom.p1, this->bestBottom.p2, Scalar(0, 0, 255), 1, CV_AA); + line(imgCorners, this->bestLeft.p1, this->bestLeft.p2, Scalar(255, 0, 0), 1, CV_AA); + + displayImage(pipelineData->config, "Winning top/bottom Boundaries", imgCorners); } - this->bestVerticalScore = score; - bestLeft = LineSegment(left.p1.x, left.p1.y, left.p2.x, left.p2.y); - bestRight = LineSegment(right.p1.x, right.p1.y, right.p2.x, right.p2.y); - } -} -// Score a collection of lines as a possible license plate region. -// If any segments are missing, extrapolate the missing pieces -void PlateCorners::scoreHorizontals(int h1, int h2) -{ - - ScoreKeeper scoreKeeper; + // Check if a left/right edge has been established. + if (bestLeft.p1.x == 0 && bestLeft.p1.y == 0 && bestLeft.p2.x == 0 && bestLeft.p2.y == 0) + confidence = 0; + else if (bestTop.p1.x == 0 && bestTop.p1.y == 0 && bestTop.p2.x == 0 && bestTop.p2.y == 0) + confidence = 0; + else + confidence = 100; - LineSegment top; - LineSegment bottom; + vector corners; + corners.push_back(bestTop.intersection(bestLeft)); + corners.push_back(bestTop.intersection(bestRight)); + corners.push_back(bestBottom.intersection(bestRight)); + corners.push_back(bestBottom.intersection(bestLeft)); - float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; - float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; - - float confidenceDiff = 0; - float missingSegmentPenalty = 0; - - if (h1 == NO_LINE && h2 == NO_LINE) - { -// return; - - - top = tlc.centerHorizontalLine.getParallelLine(idealPixelHeight / 2); - bottom = tlc.centerHorizontalLine.getParallelLine(-1 * idealPixelHeight / 2 ); - - missingSegmentPenalty = 2; - confidenceDiff += 2; - } - else if (h1 != NO_LINE && h2 != NO_LINE) - { - top = this->plateLines->horizontalLines[h1].line; - bottom = this->plateLines->horizontalLines[h2].line; - confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); - confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); - } - else if (h1 == NO_LINE && h2 != NO_LINE) - { - bottom = this->plateLines->horizontalLines[h2].line; - top = bottom.getParallelLine(idealPixelHeight); - missingSegmentPenalty++; - confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); - } - else if (h1 != NO_LINE && h2 == NO_LINE) - { - top = this->plateLines->horizontalLines[h1].line; - bottom = top.getParallelLine(-1 * idealPixelHeight); - missingSegmentPenalty++; - confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); - } - - scoreKeeper.setScore("SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL", missingSegmentPenalty, SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL); - //scoreKeeper.setScore("SCORING_LINE_CONFIDENCE_WEIGHT", confidenceDiff, SCORING_LINE_CONFIDENCE_WEIGHT); - - - // Make sure that the top and bottom lines are above and below - // the text area - if (tlc.isAboveText(top) < 1 || tlc.isAboveText(bottom) > -1) - return; - - // We now have 4 possible lines. Let's put them to the test and score them... - - - ////////////////////////////////////////////////////////////////////////// - // SCORE the shape wrt character position and height relative to position - ////////////////////////////////////////////////////////////////////////// - - Point topPoint = top.midpoint(); - Point botPoint = bottom.closestPointOnSegmentTo(topPoint); - float plateHeightPx = distanceBetweenPoints(topPoint, botPoint); - - // Get the height difference - - float heightRatio = tlc.charHeight / plateHeightPx; - float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); - float heightRatioDiff = abs(heightRatio - idealHeightRatio); - - scoreKeeper.setScore("SCORING_PLATEHEIGHT_WEIGHT", heightRatioDiff, SCORING_PLATEHEIGHT_WEIGHT); - - ////////////////////////////////////////////////////////////////////////// - // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle - ////////////////////////////////////////////////////////////////////////// - - Point charAreaMidPoint = tlc.centerVerticalLine.midpoint(); - Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); - Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); - - float topDistanceFromMiddle = distanceBetweenPoints(topLineSpot, charAreaMidPoint); - float bottomDistanceFromMiddle = distanceBetweenPoints(botLineSpot, charAreaMidPoint); - - float idealDistanceFromMiddle = idealPixelHeight / 2; - - float middleScore = abs(topDistanceFromMiddle - idealDistanceFromMiddle) / idealDistanceFromMiddle; - middleScore += abs(bottomDistanceFromMiddle - idealDistanceFromMiddle) / idealDistanceFromMiddle; - - scoreKeeper.setScore("SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT", middleScore, SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT); - - - ////////////////////////////////////////////////////////////// - // SCORE: the shape for angles matching the character region - ////////////////////////////////////////////////////////////// - - float charanglediff = abs(tlc.charAngle - top.angle) + abs(tlc.charAngle - bottom.angle); - - scoreKeeper.setScore("SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT", charanglediff, SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT); - - if (pipelineData->config->debugPlateCorners) - { - scoreKeeper.printDebugScores(); - Mat debugImg(this->inputImage.size(), this->inputImage.type()); - this->inputImage.copyTo(debugImg); - cvtColor(debugImg, debugImg, CV_GRAY2BGR); - line(debugImg, top.p1, top.p2, Scalar(0,0,255), 2); - line(debugImg, bottom.p1, bottom.p2, Scalar(0,0,255), 2); - //drawAndWait(&debugImg); + if (pipelineData->config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "Plate Corners Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + return corners; } - float score = scoreKeeper.getTotal(); - if (score < this->bestHorizontalScore) + void PlateCorners::scoreVerticals(int v1, int v2) { - float scorecomponent; + ScoreKeeper scoreKeeper; + + LineSegment left; + LineSegment right; + + + float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + + if (v1 == NO_LINE && v2 == NO_LINE) + { + //return; + + left = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2); + right = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2 ); + + missingSegmentPenalty = 2; + confidenceDiff += 2; + } + else if (v1 != NO_LINE && v2 != NO_LINE) + { + left = this->plateLines->verticalLines[v1].line; + right = this->plateLines->verticalLines[v2].line; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); + } + else if (v1 == NO_LINE && v2 != NO_LINE) + { + right = this->plateLines->verticalLines[v2].line; + left = right.getParallelLine(idealPixelWidth); + missingSegmentPenalty++; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); + } + else if (v1 != NO_LINE && v2 == NO_LINE) + { + left = this->plateLines->verticalLines[v1].line; + right = left.getParallelLine(-1 * idealPixelWidth); + missingSegmentPenalty++; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); + } + + scoreKeeper.setScore("SCORING_LINE_CONFIDENCE_WEIGHT", confidenceDiff, SCORING_LINE_CONFIDENCE_WEIGHT); + scoreKeeper.setScore("SCORING_MISSING_SEGMENT_PENALTY_VERTICAL", missingSegmentPenalty, SCORING_MISSING_SEGMENT_PENALTY_VERTICAL); + + // Make sure that the left and right lines are to the left and right of our text + // area + if (tlc.isLeftOfText(left) < 1 || tlc.isLeftOfText(right) > -1) + return; + + + ///////////////////////////////////////////////////////////////////////// + // Score angle difference from detected character box + ///////////////////////////////////////////////////////////////////////// + + float perpendicularCharAngle = tlc.charAngle - 90; + float charanglediff = abs(perpendicularCharAngle - left.angle) + abs(perpendicularCharAngle - right.angle); + + scoreKeeper.setScore("SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT", charanglediff, SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT); + + ////////////////////////////////////////////////////////////////////////// + // SCORE the shape wrt character position and height relative to position + ////////////////////////////////////////////////////////////////////////// + + Point leftMidLinePoint = left.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); + Point rightMidLinePoint = right.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); + + float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); + // normalize for image width + plateDistance = plateDistance / ((float)inputImage.cols); + + scoreKeeper.setScore("SCORING_DISTANCE_WEIGHT_VERTICAL", plateDistance, SCORING_DISTANCE_WEIGHT_VERTICAL); + + float score = scoreKeeper.getTotal(); + + if (score < this->bestVerticalScore) + { + + if (pipelineData->config->debugPlateCorners) + { + + cout << "Vertical breakdown Score:" << endl; + + scoreKeeper.printDebugScores(); + } + + this->bestVerticalScore = score; + bestLeft = LineSegment(left.p1.x, left.p1.y, left.p2.x, left.p2.y); + bestRight = LineSegment(right.p1.x, right.p1.y, right.p2.x, right.p2.y); + } + } + // Score a collection of lines as a possible license plate region. + // If any segments are missing, extrapolate the missing pieces + void PlateCorners::scoreHorizontals(int h1, int h2) + { + + ScoreKeeper scoreKeeper; + + LineSegment top; + LineSegment bottom; + + float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; + + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + + if (h1 == NO_LINE && h2 == NO_LINE) + { + // return; + + + top = tlc.centerHorizontalLine.getParallelLine(idealPixelHeight / 2); + bottom = tlc.centerHorizontalLine.getParallelLine(-1 * idealPixelHeight / 2 ); + + missingSegmentPenalty = 2; + confidenceDiff += 2; + } + else if (h1 != NO_LINE && h2 != NO_LINE) + { + top = this->plateLines->horizontalLines[h1].line; + bottom = this->plateLines->horizontalLines[h2].line; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); + } + else if (h1 == NO_LINE && h2 != NO_LINE) + { + bottom = this->plateLines->horizontalLines[h2].line; + top = bottom.getParallelLine(idealPixelHeight); + missingSegmentPenalty++; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); + } + else if (h1 != NO_LINE && h2 == NO_LINE) + { + top = this->plateLines->horizontalLines[h1].line; + bottom = top.getParallelLine(-1 * idealPixelHeight); + missingSegmentPenalty++; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); + } + + scoreKeeper.setScore("SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL", missingSegmentPenalty, SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL); + //scoreKeeper.setScore("SCORING_LINE_CONFIDENCE_WEIGHT", confidenceDiff, SCORING_LINE_CONFIDENCE_WEIGHT); + + + // Make sure that the top and bottom lines are above and below + // the text area + if (tlc.isAboveText(top) < 1 || tlc.isAboveText(bottom) > -1) + return; + + // We now have 4 possible lines. Let's put them to the test and score them... + + + ////////////////////////////////////////////////////////////////////////// + // SCORE the shape wrt character position and height relative to position + ////////////////////////////////////////////////////////////////////////// + + Point topPoint = top.midpoint(); + Point botPoint = bottom.closestPointOnSegmentTo(topPoint); + float plateHeightPx = distanceBetweenPoints(topPoint, botPoint); + + // Get the height difference + + float heightRatio = tlc.charHeight / plateHeightPx; + float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); + float heightRatioDiff = abs(heightRatio - idealHeightRatio); + + scoreKeeper.setScore("SCORING_PLATEHEIGHT_WEIGHT", heightRatioDiff, SCORING_PLATEHEIGHT_WEIGHT); + + ////////////////////////////////////////////////////////////////////////// + // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle + ////////////////////////////////////////////////////////////////////////// + + Point charAreaMidPoint = tlc.centerVerticalLine.midpoint(); + Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); + Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); + + float topDistanceFromMiddle = distanceBetweenPoints(topLineSpot, charAreaMidPoint); + float bottomDistanceFromMiddle = distanceBetweenPoints(botLineSpot, charAreaMidPoint); + + float idealDistanceFromMiddle = idealPixelHeight / 2; + + float middleScore = abs(topDistanceFromMiddle - idealDistanceFromMiddle) / idealDistanceFromMiddle; + middleScore += abs(bottomDistanceFromMiddle - idealDistanceFromMiddle) / idealDistanceFromMiddle; + + scoreKeeper.setScore("SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT", middleScore, SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT); + + + ////////////////////////////////////////////////////////////// + // SCORE: the shape for angles matching the character region + ////////////////////////////////////////////////////////////// + + float charanglediff = abs(tlc.charAngle - top.angle) + abs(tlc.charAngle - bottom.angle); + + scoreKeeper.setScore("SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT", charanglediff, SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT); if (pipelineData->config->debugPlateCorners) { - cout << "Horizontal breakdown Score:" << endl; scoreKeeper.printDebugScores(); + Mat debugImg(this->inputImage.size(), this->inputImage.type()); + this->inputImage.copyTo(debugImg); + cvtColor(debugImg, debugImg, CV_GRAY2BGR); + line(debugImg, top.p1, top.p2, Scalar(0,0,255), 2); + line(debugImg, bottom.p1, bottom.p2, Scalar(0,0,255), 2); + //drawAndWait(&debugImg); + + } + + float score = scoreKeeper.getTotal(); + if (score < this->bestHorizontalScore) + { + float scorecomponent; + + if (pipelineData->config->debugPlateCorners) + { + cout << "Horizontal breakdown Score:" << endl; + scoreKeeper.printDebugScores(); + } + this->bestHorizontalScore = score; + bestTop = LineSegment(top.p1.x, top.p1.y, top.p2.x, top.p2.y); + bestBottom = LineSegment(bottom.p1.x, bottom.p1.y, bottom.p2.x, bottom.p2.y); } - this->bestHorizontalScore = score; - bestTop = LineSegment(top.p1.x, top.p1.y, top.p2.x, top.p2.y); - bestBottom = LineSegment(bottom.p1.x, bottom.p1.y, bottom.p2.x, bottom.p2.y); } + } - - diff --git a/src/openalpr/edges/platecorners.h b/src/openalpr/edges/platecorners.h index ff73012..0538855 100644 --- a/src/openalpr/edges/platecorners.h +++ b/src/openalpr/edges/platecorners.h @@ -40,40 +40,42 @@ #define SCORING_LINE_CONFIDENCE_WEIGHT 18.0 - - -class PlateCorners +namespace alpr { - public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, std::vector textLines) ; - - virtual ~PlateCorners(); + class PlateCorners + { - std::vector findPlateCorners(); + public: + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, std::vector textLines) ; - float confidence; + virtual ~PlateCorners(); - private: + std::vector findPlateCorners(); - PipelineData* pipelineData; - cv::Mat inputImage; + float confidence; - std::vector textLines; - TextLineCollection tlc; - - float bestHorizontalScore; - float bestVerticalScore; - LineSegment bestTop; - LineSegment bestBottom; - LineSegment bestLeft; - LineSegment bestRight; + private: - PlateLines* plateLines; + PipelineData* pipelineData; + cv::Mat inputImage; - void scoreHorizontals( int h1, int h2 ); - void scoreVerticals( int v1, int v2 ); + std::vector textLines; + TextLineCollection tlc; -}; + float bestHorizontalScore; + float bestVerticalScore; + LineSegment bestTop; + LineSegment bestBottom; + LineSegment bestLeft; + LineSegment bestRight; + PlateLines* plateLines; + + void scoreHorizontals( int h1, int h2 ); + void scoreVerticals( int v1, int v2 ); + + }; + +} #endif // OPENALPR_PLATELINES_H diff --git a/src/openalpr/edges/platelines.cpp b/src/openalpr/edges/platelines.cpp index 7c37b75..3515c40 100644 --- a/src/openalpr/edges/platelines.cpp +++ b/src/openalpr/edges/platelines.cpp @@ -23,230 +23,234 @@ using namespace cv; using namespace std; const float MIN_CONFIDENCE = 0.3; - - -PlateLines::PlateLines(PipelineData* pipelineData) + +namespace alpr { - this->pipelineData = pipelineData; - - this->debug = pipelineData->config->debugPlateLines; - if (debug) - cout << "PlateLines constructor" << endl; -} - -PlateLines::~PlateLines() -{ -} - -void PlateLines::processImage(Mat inputImage, vector textLines, float sensitivity) -{ - if (this->debug) - cout << "PlateLines findLines" << endl; - - timespec startTime; - getTime(&startTime); - - - // Ignore input images that are pure white or pure black - Scalar avgPixelIntensity = mean(inputImage); - if (avgPixelIntensity[0] == 255) - return; - else if (avgPixelIntensity[0] == 0) - return; - - // Do a bilateral filter to clean the noise but keep edges sharp - Mat smoothed(inputImage.size(), inputImage.type()); - adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45); - - - int morph_elem = 2; - int morph_size = 2; - Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - - - Mat edges(inputImage.size(), inputImage.type()); - Canny(smoothed, edges, 66, 133); - - // Create a mask that is dilated based on the detected characters - - - Mat mask = Mat::zeros(inputImage.size(), CV_8U); - - for (uint i = 0; i < textLines.size(); i++) + PlateLines::PlateLines(PipelineData* pipelineData) { - vector > polygons; - polygons.push_back(textLines[i].textArea); - fillPoly(mask, polygons, Scalar(255,255,255)); + this->pipelineData = pipelineData; + + this->debug = pipelineData->config->debugPlateLines; + + if (debug) + cout << "PlateLines constructor" << endl; } - - - dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); - bitwise_not(mask, mask); - - // AND canny edges with the character mask - bitwise_and(edges, mask, edges); - - - vector hlines = this->getLines(edges, sensitivity, false); - vector vlines = this->getLines(edges, sensitivity, true); - for (uint i = 0; i < hlines.size(); i++) - this->horizontalLines.push_back(hlines[i]); - for (uint i = 0; i < vlines.size(); i++) - this->verticalLines.push_back(vlines[i]); - - // if debug is enabled, draw the image - if (this->debug) + PlateLines::~PlateLines() { - Mat debugImgHoriz(edges.size(), edges.type()); - Mat debugImgVert(edges.size(), edges.type()); - edges.copyTo(debugImgHoriz); - edges.copyTo(debugImgVert); - cvtColor(debugImgHoriz,debugImgHoriz,CV_GRAY2BGR); - cvtColor(debugImgVert,debugImgVert,CV_GRAY2BGR); + } - for( size_t i = 0; i < this->horizontalLines.size(); i++ ) + void PlateLines::processImage(Mat inputImage, vector textLines, float sensitivity) + { + if (this->debug) + cout << "PlateLines findLines" << endl; + + timespec startTime; + getTime(&startTime); + + + // Ignore input images that are pure white or pure black + Scalar avgPixelIntensity = mean(inputImage); + if (avgPixelIntensity[0] == 255) + return; + else if (avgPixelIntensity[0] == 0) + return; + + // Do a bilateral filter to clean the noise but keep edges sharp + Mat smoothed(inputImage.size(), inputImage.type()); + adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45); + + + int morph_elem = 2; + int morph_size = 2; + Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + + + Mat edges(inputImage.size(), inputImage.type()); + Canny(smoothed, edges, 66, 133); + + // Create a mask that is dilated based on the detected characters + + + Mat mask = Mat::zeros(inputImage.size(), CV_8U); + + for (uint i = 0; i < textLines.size(); i++) { - line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); + vector > polygons; + polygons.push_back(textLines[i].textArea); + fillPoly(mask, polygons, Scalar(255,255,255)); } - for( size_t i = 0; i < this->verticalLines.size(); i++ ) + + + dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); + bitwise_not(mask, mask); + + // AND canny edges with the character mask + bitwise_and(edges, mask, edges); + + + vector hlines = this->getLines(edges, sensitivity, false); + vector vlines = this->getLines(edges, sensitivity, true); + for (uint i = 0; i < hlines.size(); i++) + this->horizontalLines.push_back(hlines[i]); + for (uint i = 0; i < vlines.size(); i++) + this->verticalLines.push_back(vlines[i]); + + // if debug is enabled, draw the image + if (this->debug) { - line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); + Mat debugImgHoriz(edges.size(), edges.type()); + Mat debugImgVert(edges.size(), edges.type()); + edges.copyTo(debugImgHoriz); + edges.copyTo(debugImgVert); + cvtColor(debugImgHoriz,debugImgHoriz,CV_GRAY2BGR); + cvtColor(debugImgVert,debugImgVert,CV_GRAY2BGR); + + for( size_t i = 0; i < this->horizontalLines.size(); i++ ) + { + line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); + } + + for( size_t i = 0; i < this->verticalLines.size(); i++ ) + { + line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); + } + + vector images; + images.push_back(debugImgHoriz); + images.push_back(debugImgVert); + + Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1); + displayImage(pipelineData->config, "Hough Lines", dashboard); } - vector images; - images.push_back(debugImgHoriz); - images.push_back(debugImgVert); + if (pipelineData->config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "Plate Lines Time: " << diffclock(startTime, endTime) << "ms." << endl; + } - Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1); - displayImage(pipelineData->config, "Hough Lines", dashboard); } - if (pipelineData->config->debugTiming) + + + + vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) { - timespec endTime; - getTime(&endTime); - cout << "Plate Lines Time: " << diffclock(startTime, endTime) << "ms." << endl; - } + if (this->debug) + cout << "PlateLines::getLines" << endl; -} + static int HORIZONTAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityHorizontal; + static int VERTICAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityVertical; + vector allLines; + vector filteredLines; - - -vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) -{ - if (this->debug) - cout << "PlateLines::getLines" << endl; - - static int HORIZONTAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityHorizontal; - static int VERTICAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityVertical; - - vector allLines; - vector filteredLines; - - int sensitivity; - if (vertical) - sensitivity = VERTICAL_SENSITIVITY * (1.0 / sensitivityMultiplier); - else - sensitivity = HORIZONTAL_SENSITIVITY * (1.0 / sensitivityMultiplier); - - HoughLines( edges, allLines, 1, CV_PI/180, sensitivity, 0, 0 ); - - for( size_t i = 0; i < allLines.size(); i++ ) - { - float rho = allLines[i][0], theta = allLines[i][1]; - Point pt1, pt2; - double a = cos(theta), b = sin(theta); - double x0 = a*rho, y0 = b*rho; - - double angle = theta * (180 / CV_PI); - pt1.x = cvRound(x0 + 1000*(-b)); - pt1.y = cvRound(y0 + 1000*(a)); - pt2.x = cvRound(x0 - 1000*(-b)); - pt2.y = cvRound(y0 - 1000*(a)); - + int sensitivity; if (vertical) - { - if (angle < 20 || angle > 340 || (angle > 160 && angle < 210)) - { - // good vertical - - LineSegment line; - if (pt1.y <= pt2.y) - line = LineSegment(pt2.x, pt2.y, pt1.x, pt1.y); - else - line = LineSegment(pt1.x, pt1.y, pt2.x, pt2.y); - - // Get rid of the -1000, 1000 stuff. Terminate at the edges of the image - // Helps with debugging/rounding issues later - LineSegment top(0, 0, edges.cols, 0); - LineSegment bottom(0, edges.rows, edges.cols, edges.rows); - Point p1 = line.intersection(bottom); - Point p2 = line.intersection(top); - - PlateLine plateLine; - plateLine.line = LineSegment(p1.x, p1.y, p2.x, p2.y); - plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; - filteredLines.push_back(plateLine); - } - } + sensitivity = VERTICAL_SENSITIVITY * (1.0 / sensitivityMultiplier); else + sensitivity = HORIZONTAL_SENSITIVITY * (1.0 / sensitivityMultiplier); + + HoughLines( edges, allLines, 1, CV_PI/180, sensitivity, 0, 0 ); + + for( size_t i = 0; i < allLines.size(); i++ ) { - if ( (angle > 70 && angle < 110) || (angle > 250 && angle < 290)) + float rho = allLines[i][0], theta = allLines[i][1]; + Point pt1, pt2; + double a = cos(theta), b = sin(theta); + double x0 = a*rho, y0 = b*rho; + + double angle = theta * (180 / CV_PI); + pt1.x = cvRound(x0 + 1000*(-b)); + pt1.y = cvRound(y0 + 1000*(a)); + pt2.x = cvRound(x0 - 1000*(-b)); + pt2.y = cvRound(y0 - 1000*(a)); + + if (vertical) { - // good horizontal + if (angle < 20 || angle > 340 || (angle > 160 && angle < 210)) + { + // good vertical - LineSegment line; - if (pt1.x <= pt2.x) - line = LineSegment(pt1.x, pt1.y, pt2.x, pt2.y); - else - line =LineSegment(pt2.x, pt2.y, pt1.x, pt1.y); + LineSegment line; + if (pt1.y <= pt2.y) + line = LineSegment(pt2.x, pt2.y, pt1.x, pt1.y); + else + line = LineSegment(pt1.x, pt1.y, pt2.x, pt2.y); - // Get rid of the -1000, 1000 stuff. Terminate at the edges of the image - // Helps with debugging/ rounding issues later - int newY1 = line.getPointAt(0); - int newY2 = line.getPointAt(edges.cols); + // Get rid of the -1000, 1000 stuff. Terminate at the edges of the image + // Helps with debugging/rounding issues later + LineSegment top(0, 0, edges.cols, 0); + LineSegment bottom(0, edges.rows, edges.cols, edges.rows); + Point p1 = line.intersection(bottom); + Point p2 = line.intersection(top); - PlateLine plateLine; - plateLine.line = LineSegment(0, newY1, edges.cols, newY2); - plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; - filteredLines.push_back(plateLine); + PlateLine plateLine; + plateLine.line = LineSegment(p1.x, p1.y, p2.x, p2.y); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); + } + } + else + { + if ( (angle > 70 && angle < 110) || (angle > 250 && angle < 290)) + { + // good horizontal + + LineSegment line; + if (pt1.x <= pt2.x) + line = LineSegment(pt1.x, pt1.y, pt2.x, pt2.y); + else + line =LineSegment(pt2.x, pt2.y, pt1.x, pt1.y); + + // Get rid of the -1000, 1000 stuff. Terminate at the edges of the image + // Helps with debugging/ rounding issues later + int newY1 = line.getPointAt(0); + int newY2 = line.getPointAt(edges.cols); + + PlateLine plateLine; + plateLine.line = LineSegment(0, newY1, edges.cols, newY2); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); + } } } + + return filteredLines; } - return filteredLines; -} - -Mat PlateLines::customGrayscaleConversion(Mat src) -{ - Mat img_hsv; - cvtColor(src,img_hsv,CV_BGR2HSV); - - Mat grayscale = Mat(img_hsv.size(), CV_8U ); - Mat hue(img_hsv.size(), CV_8U ); - - for (int row = 0; row < img_hsv.rows; row++) + Mat PlateLines::customGrayscaleConversion(Mat src) { - for (int col = 0; col < img_hsv.cols; col++) + Mat img_hsv; + cvtColor(src,img_hsv,CV_BGR2HSV); + + Mat grayscale = Mat(img_hsv.size(), CV_8U ); + Mat hue(img_hsv.size(), CV_8U ); + + for (int row = 0; row < img_hsv.rows; row++) { - int h = (int) img_hsv.at(row, col)[0]; - //int s = (int) img_hsv.at(row, col)[1]; - int v = (int) img_hsv.at(row, col)[2]; + for (int col = 0; col < img_hsv.cols; col++) + { + int h = (int) img_hsv.at(row, col)[0]; + //int s = (int) img_hsv.at(row, col)[1]; + int v = (int) img_hsv.at(row, col)[2]; - int pixval = pow(v, 1.05); + int pixval = pow(v, 1.05); - if (pixval > 255) - pixval = 255; - grayscale.at(row, col) = pixval; + if (pixval > 255) + pixval = 255; + grayscale.at(row, col) = pixval; - hue.at(row, col) = h * (255.0 / 180.0); + hue.at(row, col) = h * (255.0 / 180.0); + } } + + //displayImage(config, "Hue", hue); + return grayscale; } - //displayImage(config, "Hue", hue); - return grayscale; -} +} \ No newline at end of file diff --git a/src/openalpr/edges/platelines.h b/src/openalpr/edges/platelines.h index 4dde7ed..41e36a6 100644 --- a/src/openalpr/edges/platelines.h +++ b/src/openalpr/edges/platelines.h @@ -27,34 +27,39 @@ #include "config.h" #include "pipeline_data.h" -struct PlateLine -{ - LineSegment line; - float confidence; -}; - -class PlateLines +namespace alpr { - public: - PlateLines(PipelineData* pipelineData); - virtual ~PlateLines(); + struct PlateLine + { + LineSegment line; + float confidence; + }; - void processImage(cv::Mat img, std::vector textLines, float sensitivity=1.0); + class PlateLines + { - std::vector horizontalLines; - std::vector verticalLines; + public: + PlateLines(PipelineData* pipelineData); + virtual ~PlateLines(); - std::vector winningCorners; + void processImage(cv::Mat img, std::vector textLines, float sensitivity=1.0); - private: - - PipelineData* pipelineData; - bool debug; + std::vector horizontalLines; + std::vector verticalLines; - cv::Mat customGrayscaleConversion(cv::Mat src); - void findLines(cv::Mat inputImage); - std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); -}; + std::vector winningCorners; + + private: + + PipelineData* pipelineData; + bool debug; + + cv::Mat customGrayscaleConversion(cv::Mat src); + void findLines(cv::Mat inputImage); + std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); + }; + +} #endif // OPENALPR_PLATELINES_H diff --git a/src/openalpr/edges/scorekeeper.cpp b/src/openalpr/edges/scorekeeper.cpp index a7fcfe3..35aa3d9 100644 --- a/src/openalpr/edges/scorekeeper.cpp +++ b/src/openalpr/edges/scorekeeper.cpp @@ -21,55 +21,60 @@ #include "scorekeeper.h" -ScoreKeeper::ScoreKeeper() { -} +namespace alpr +{ + + ScoreKeeper::ScoreKeeper() { + } -ScoreKeeper::~ScoreKeeper() { -} + ScoreKeeper::~ScoreKeeper() { + } -void ScoreKeeper::setScore(std::string weight_id, float score, float weight) { + void ScoreKeeper::setScore(std::string weight_id, float score, float weight) { - // Assume that we never set this value twice - weight_ids.push_back(weight_id); - scores.push_back(score); - weights.push_back(weight); -} + // Assume that we never set this value twice + weight_ids.push_back(weight_id); + scores.push_back(score); + weights.push_back(weight); + } -float ScoreKeeper::getTotal() { + float ScoreKeeper::getTotal() { - float score = 0; - - for (unsigned int i = 0; i < weights.size(); i++) - { - score += scores[i] * weights[i]; + float score = 0; + + for (unsigned int i = 0; i < weights.size(); i++) + { + score += scores[i] * weights[i]; + } + + return score; + } + + void ScoreKeeper::printDebugScores() { + + int longest_weight_id = 0; + for (unsigned int i = 0; i < weight_ids.size(); i++) + { + if (weight_ids[i].length() > longest_weight_id) + longest_weight_id = weight_ids[i].length(); + } + + float total = getTotal(); + + for (unsigned int i = 0; i < weight_ids.size(); i++) + { + float percent_of_total = (scores[i] * weights[i]) / total * 100; + + std::cout << " - " << std::setw(longest_weight_id + 1) << std::left << weight_ids[i] << + " Weighted Score: " << std::setw(10) << std::left << (scores[i] * weights[i]) << + " Orig Score: " << std::setw(10) << std::left << scores[i] << + " (" << percent_of_total << "% of total)" << std::endl; + } + + + std::cout << "Total: " << total << std::endl; } - return score; -} - -void ScoreKeeper::printDebugScores() { - - int longest_weight_id = 0; - for (unsigned int i = 0; i < weight_ids.size(); i++) - { - if (weight_ids[i].length() > longest_weight_id) - longest_weight_id = weight_ids[i].length(); - } - - float total = getTotal(); - - for (unsigned int i = 0; i < weight_ids.size(); i++) - { - float percent_of_total = (scores[i] * weights[i]) / total * 100; - - std::cout << " - " << std::setw(longest_weight_id + 1) << std::left << weight_ids[i] << - " Weighted Score: " << std::setw(10) << std::left << (scores[i] * weights[i]) << - " Orig Score: " << std::setw(10) << std::left << scores[i] << - " (" << percent_of_total << "% of total)" << std::endl; - } - - - std::cout << "Total: " << total << std::endl; } \ No newline at end of file diff --git a/src/openalpr/edges/scorekeeper.h b/src/openalpr/edges/scorekeeper.h index 9bc773f..3ee3929 100644 --- a/src/openalpr/edges/scorekeeper.h +++ b/src/openalpr/edges/scorekeeper.h @@ -24,25 +24,30 @@ #include #include -class ScoreKeeper { -public: - ScoreKeeper(); - virtual ~ScoreKeeper(); - - void setScore(std::string weight_id, float score, float weight); - - float getTotal(); - - void printDebugScores(); - -private: +namespace alpr +{ - std::vector weight_ids; - std::vector weights; - - std::vector scores; - -}; + class ScoreKeeper { + public: + ScoreKeeper(); + virtual ~ScoreKeeper(); + + void setScore(std::string weight_id, float score, float weight); + + float getTotal(); + + void printDebugScores(); + + private: + + std::vector weight_ids; + std::vector weights; + + std::vector scores; + + }; + +} #endif /* OPENALPR_SCOREKEEPER_H */ diff --git a/src/openalpr/edges/textlinecollection.cpp b/src/openalpr/edges/textlinecollection.cpp index a0bc7fb..980ee74 100644 --- a/src/openalpr/edges/textlinecollection.cpp +++ b/src/openalpr/edges/textlinecollection.cpp @@ -10,151 +10,156 @@ using namespace cv; using namespace std; -TextLineCollection::TextLineCollection(std::vector textLines) { - - - charHeight = 0; - charAngle = 0; - for (uint i = 0; i < textLines.size(); i++) - { - charHeight += textLines[i].lineHeight; - charAngle += textLines[i].angle; - - } - charHeight = charHeight / textLines.size(); - charAngle = charAngle / textLines.size(); - - this->topCharArea = textLines[0].charBoxTop; - this->bottomCharArea = textLines[0].charBoxBottom; - for (uint i = 1; i < textLines.size(); i++) - { - - if (this->topCharArea.isPointBelowLine(textLines[i].charBoxTop.midpoint()) == false) - this->topCharArea = textLines[i].charBoxTop; - - if (this->bottomCharArea.isPointBelowLine(textLines[i].charBoxBottom.midpoint())) - this->bottomCharArea = textLines[i].charBoxBottom; +namespace alpr +{ + + TextLineCollection::TextLineCollection(std::vector textLines) { + + + charHeight = 0; + charAngle = 0; + for (uint i = 0; i < textLines.size(); i++) + { + charHeight += textLines[i].lineHeight; + charAngle += textLines[i].angle; + + } + charHeight = charHeight / textLines.size(); + charAngle = charAngle / textLines.size(); + + this->topCharArea = textLines[0].charBoxTop; + this->bottomCharArea = textLines[0].charBoxBottom; + for (uint i = 1; i < textLines.size(); i++) + { + + if (this->topCharArea.isPointBelowLine(textLines[i].charBoxTop.midpoint()) == false) + this->topCharArea = textLines[i].charBoxTop; + + if (this->bottomCharArea.isPointBelowLine(textLines[i].charBoxBottom.midpoint())) + this->bottomCharArea = textLines[i].charBoxBottom; + + } + + longerSegment = this->bottomCharArea; + shorterSegment = this->topCharArea; + if (this->topCharArea.length > this->bottomCharArea.length) + { + longerSegment = this->topCharArea; + shorterSegment = this->bottomCharArea; + } + + findCenterHorizontal(); + findCenterVertical(); + // Center Vertical Line + } - - longerSegment = this->bottomCharArea; - shorterSegment = this->topCharArea; - if (this->topCharArea.length > this->bottomCharArea.length) - { - longerSegment = this->topCharArea; - shorterSegment = this->bottomCharArea; + + cv::Mat TextLineCollection::getDebugImage(cv::Size imageSize) { + + Mat debugImage = Mat::zeros(imageSize, CV_8U); + line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); + line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); + + return debugImage; + } - - findCenterHorizontal(); - findCenterVertical(); - // Center Vertical Line - - -} - -cv::Mat TextLineCollection::getDebugImage(cv::Size imageSize) { - - Mat debugImage = Mat::zeros(imageSize, CV_8U); - line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); - line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); - - return debugImage; - -} -// Returns 1 for above, 0 for within, and -1 for below -int TextLineCollection::isAboveText(LineSegment line) { - // Test four points (left and right corner of top and bottom line) - - Point topLeft = line.closestPointOnSegmentTo(topCharArea.p1); - Point topRight = line.closestPointOnSegmentTo(topCharArea.p2); - - bool lineIsBelowTop = topCharArea.isPointBelowLine(topLeft) || topCharArea.isPointBelowLine(topRight); - - if (!lineIsBelowTop) - return 1; - - Point bottomLeft = line.closestPointOnSegmentTo(bottomCharArea.p1); - Point bottomRight = line.closestPointOnSegmentTo(bottomCharArea.p2); - - bool lineIsBelowBottom = bottomCharArea.isPointBelowLine(bottomLeft) && - bottomCharArea.isPointBelowLine(bottomRight); - - if (lineIsBelowBottom) - return -1; - - return 0; - -} + // Returns 1 for above, 0 for within, and -1 for below + int TextLineCollection::isAboveText(LineSegment line) { + // Test four points (left and right corner of top and bottom line) -// Returns 1 for left, 0 for within, and -1 for to the right -int TextLineCollection::isLeftOfText(LineSegment line) { + Point topLeft = line.closestPointOnSegmentTo(topCharArea.p1); + Point topRight = line.closestPointOnSegmentTo(topCharArea.p2); - LineSegment leftSide = LineSegment(bottomCharArea.p1, topCharArea.p1); - - Point topLeft = line.closestPointOnSegmentTo(leftSide.p2); - Point bottomLeft = line.closestPointOnSegmentTo(leftSide.p1); - - bool lineIsAboveLeft = (!leftSide.isPointBelowLine(topLeft)) && (!leftSide.isPointBelowLine(bottomLeft)); - - if (lineIsAboveLeft) - return 1; - - LineSegment rightSide = LineSegment(bottomCharArea.p2, topCharArea.p2); - - Point topRight = line.closestPointOnSegmentTo(rightSide.p2); - Point bottomRight = line.closestPointOnSegmentTo(rightSide.p1); - - - bool lineIsBelowRight = rightSide.isPointBelowLine(topRight) && rightSide.isPointBelowLine(bottomRight); - - if (lineIsBelowRight) - return -1; - - return 0; -} + bool lineIsBelowTop = topCharArea.isPointBelowLine(topLeft) || topCharArea.isPointBelowLine(topRight); -void TextLineCollection::findCenterHorizontal() { - // To find the center horizontal line: - // Find the longer of the lines (if multiline) - // Get the nearest point on the bottom-most line for the - // left and right - + if (!lineIsBelowTop) + return 1; - - Point leftP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p1); - Point leftP2 = longerSegment.p1; - LineSegment left = LineSegment(leftP1, leftP2); - - Point leftMidpoint = left.midpoint(); - - - - Point rightP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p2); - Point rightP2 = longerSegment.p2; - LineSegment right = LineSegment(rightP1, rightP2); - - Point rightMidpoint = right.midpoint(); - - this->centerHorizontalLine = LineSegment(leftMidpoint, rightMidpoint); - -} + Point bottomLeft = line.closestPointOnSegmentTo(bottomCharArea.p1); + Point bottomRight = line.closestPointOnSegmentTo(bottomCharArea.p2); -void TextLineCollection::findCenterVertical() { - // To find the center vertical line: - // Choose the longest line (if multiline) - // Get the midpoint - // Draw a line up/down using the closest point on the bottom line - + bool lineIsBelowBottom = bottomCharArea.isPointBelowLine(bottomLeft) && + bottomCharArea.isPointBelowLine(bottomRight); - Point p1 = longerSegment.midpoint(); - - Point p2 = shorterSegment.closestPointOnSegmentTo(p1); - - // Draw bottom to top - if (p1.y < p2.y) - this->centerVerticalLine = LineSegment(p1, p2); - else - this->centerVerticalLine = LineSegment(p2, p1); -} + if (lineIsBelowBottom) + return -1; + + return 0; + + } + + // Returns 1 for left, 0 for within, and -1 for to the right + int TextLineCollection::isLeftOfText(LineSegment line) { + + LineSegment leftSide = LineSegment(bottomCharArea.p1, topCharArea.p1); + + Point topLeft = line.closestPointOnSegmentTo(leftSide.p2); + Point bottomLeft = line.closestPointOnSegmentTo(leftSide.p1); + + bool lineIsAboveLeft = (!leftSide.isPointBelowLine(topLeft)) && (!leftSide.isPointBelowLine(bottomLeft)); + + if (lineIsAboveLeft) + return 1; + + LineSegment rightSide = LineSegment(bottomCharArea.p2, topCharArea.p2); + + Point topRight = line.closestPointOnSegmentTo(rightSide.p2); + Point bottomRight = line.closestPointOnSegmentTo(rightSide.p1); + + + bool lineIsBelowRight = rightSide.isPointBelowLine(topRight) && rightSide.isPointBelowLine(bottomRight); + + if (lineIsBelowRight) + return -1; + + return 0; + } + + void TextLineCollection::findCenterHorizontal() { + // To find the center horizontal line: + // Find the longer of the lines (if multiline) + // Get the nearest point on the bottom-most line for the + // left and right + + + + Point leftP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p1); + Point leftP2 = longerSegment.p1; + LineSegment left = LineSegment(leftP1, leftP2); + + Point leftMidpoint = left.midpoint(); + + + + Point rightP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p2); + Point rightP2 = longerSegment.p2; + LineSegment right = LineSegment(rightP1, rightP2); + + Point rightMidpoint = right.midpoint(); + + this->centerHorizontalLine = LineSegment(leftMidpoint, rightMidpoint); + + } + + void TextLineCollection::findCenterVertical() { + // To find the center vertical line: + // Choose the longest line (if multiline) + // Get the midpoint + // Draw a line up/down using the closest point on the bottom line + + + Point p1 = longerSegment.midpoint(); + + Point p2 = shorterSegment.closestPointOnSegmentTo(p1); + + // Draw bottom to top + if (p1.y < p2.y) + this->centerVerticalLine = LineSegment(p1, p2); + else + this->centerVerticalLine = LineSegment(p2, p1); + } + +} \ No newline at end of file diff --git a/src/openalpr/edges/textlinecollection.h b/src/openalpr/edges/textlinecollection.h index cea83e0..a7c59b4 100644 --- a/src/openalpr/edges/textlinecollection.h +++ b/src/openalpr/edges/textlinecollection.h @@ -13,37 +13,41 @@ #include "opencv2/imgproc/imgproc.hpp" #include "textdetection/textline.h" - -class TextLineCollection +namespace alpr { -public: - TextLineCollection(std::vector textLines); - - int isLeftOfText(LineSegment line); - int isAboveText(LineSegment line); - - LineSegment centerHorizontalLine; - LineSegment centerVerticalLine; - LineSegment longerSegment; - LineSegment shorterSegment; - - float charHeight; - float charAngle; - - cv::Mat getDebugImage(cv::Size imageSize); - -private: - - LineSegment topCharArea; - LineSegment bottomCharArea; - - - cv::Mat textMask; - - void findCenterHorizontal(); - void findCenterVertical(); -}; + class TextLineCollection + { + public: + TextLineCollection(std::vector textLines); + + int isLeftOfText(LineSegment line); + int isAboveText(LineSegment line); + + LineSegment centerHorizontalLine; + LineSegment centerVerticalLine; + + LineSegment longerSegment; + LineSegment shorterSegment; + + float charHeight; + float charAngle; + + cv::Mat getDebugImage(cv::Size imageSize); + + private: + + LineSegment topCharArea; + LineSegment bottomCharArea; + + + cv::Mat textMask; + + void findCenterHorizontal(); + void findCenterVertical(); + }; + +} #endif /* OPENALPR_TEXTLINECOLLECTION_H */ diff --git a/src/openalpr/featurematcher.cpp b/src/openalpr/featurematcher.cpp index feeb8e6..b15e4fb 100644 --- a/src/openalpr/featurematcher.cpp +++ b/src/openalpr/featurematcher.cpp @@ -22,369 +22,374 @@ using namespace cv; using namespace std; -//const int DEFAULT_QUERY_FEATURES = 305; -//const int DEFAULT_TRAINING_FEATURES = 305; -const float MAX_DISTANCE_TO_MATCH = 100.0f; - -FeatureMatcher::FeatureMatcher(Config* config) +namespace alpr { - this->config = config; - //this->descriptorMatcher = DescriptorMatcher::create( "BruteForce-HammingLUT" ); - this->descriptorMatcher = new BFMatcher(NORM_HAMMING, false); + //const int DEFAULT_QUERY_FEATURES = 305; + //const int DEFAULT_TRAINING_FEATURES = 305; + const float MAX_DISTANCE_TO_MATCH = 100.0f; - //this->descriptorMatcher = DescriptorMatcher::create( "FlannBased" ); - - this->detector = new FastFeatureDetector(10, true); - this->extractor = new BRISK(10, 1, 0.9); -} - -FeatureMatcher::~FeatureMatcher() -{ - for (uint i = 0; i < trainingImgKeypoints.size(); i++) - trainingImgKeypoints[i].clear(); - trainingImgKeypoints.clear(); - - descriptorMatcher.release(); - detector.release(); - extractor.release(); -} - -bool FeatureMatcher::isLoaded() -{ - if( detector.empty() || extractor.empty() || descriptorMatcher.empty() ) + FeatureMatcher::FeatureMatcher(Config* config) { - return false; + this->config = config; + + //this->descriptorMatcher = DescriptorMatcher::create( "BruteForce-HammingLUT" ); + this->descriptorMatcher = new BFMatcher(NORM_HAMMING, false); + + //this->descriptorMatcher = DescriptorMatcher::create( "FlannBased" ); + + this->detector = new FastFeatureDetector(10, true); + this->extractor = new BRISK(10, 1, 0.9); } - return true; -} - -int FeatureMatcher::numTrainingElements() -{ - return billMapping.size(); -} - -void FeatureMatcher::surfStyleMatching( const Mat& queryDescriptors, vector queryKeypoints, - vector& matches12 ) -{ - vector > matchesKnn; - - this->descriptorMatcher->radiusMatch(queryDescriptors, matchesKnn, MAX_DISTANCE_TO_MATCH); - - vector tempMatches; - _surfStyleMatching(queryDescriptors, matchesKnn, tempMatches); - - crisscrossFiltering(queryKeypoints, tempMatches, matches12); -} - -void FeatureMatcher::_surfStyleMatching(const Mat& queryDescriptors, vector > matchesKnn, vector& matches12) -{ - //objectMatches.clear(); - //objectMatches.resize(objectIds.size()); - //cout << "starting matcher" << matchesKnn.size() << endl; - for (int descInd = 0; descInd < queryDescriptors.rows; descInd++) + FeatureMatcher::~FeatureMatcher() { - //const std::vector & matches = matchesKnn[descInd]; - //cout << "two: " << descInd << ":" << matches.size() << endl; + for (uint i = 0; i < trainingImgKeypoints.size(); i++) + trainingImgKeypoints[i].clear(); + trainingImgKeypoints.clear(); - // Check to make sure we have 2 matches. I think this is always the case, but it doesn't hurt to be sure - if (matchesKnn[descInd].size() > 1) - { - // Next throw out matches with a crappy score - // Ignore... already handled by the radiusMatch - //if (matchesKnn[descInd][0].distance < MAX_DISTANCE_TO_MATCH) - //{ - float ratioThreshold = 0.75; - - // Check if both matches came from the same image. If they both came from the same image, score them slightly less harshly - if (matchesKnn[descInd][0].imgIdx == matchesKnn[descInd][1].imgIdx) - { - ratioThreshold = 0.85; - } - - if ((matchesKnn[descInd][0].distance / matchesKnn[descInd][1].distance) < ratioThreshold) - { - bool already_exists = false; - // Quickly run through the matches we've already added and make sure it's not a duplicate... - for (uint q = 0; q < matches12.size(); q++) - { - if (matchesKnn[descInd][0].queryIdx == matches12[q].queryIdx) - { - already_exists = true; - break; - } - else if ((matchesKnn[descInd][0].trainIdx == matches12[q].trainIdx) && - (matchesKnn[descInd][0].imgIdx == matches12[q].imgIdx)) - { - already_exists = true; - break; - } - } - - // Good match. - if (already_exists == false) - matches12.push_back(matchesKnn[descInd][0]); - } - - //} - } - else if (matchesKnn[descInd].size() == 1) - { - // Only match? Does this ever happen? - matches12.push_back(matchesKnn[descInd][0]); - } - // In the ratio test, we will compare the quality of a match with the next match that is not from the same object: - // we can accept several matches with similar scores as long as they are for the same object. Those should not be - // part of the model anyway as they are not discriminative enough - - //for (unsigned int first_index = 0; first_index < matches.size(); ++first_index) - //{ - //matches12.push_back(match); - //} + descriptorMatcher.release(); + detector.release(); + extractor.release(); } -} -// Compares the matches keypoints for parallel lines. Removes matches that are criss-crossing too much -// We assume that license plates won't be upside-down or backwards. So expect lines to be closely parallel -void FeatureMatcher::crisscrossFiltering(const vector queryKeypoints, const vector inputMatches, vector &outputMatches) -{ - Rect crissCrossAreaVertical(0, 0, config->stateIdImageWidthPx, config->stateIdimageHeightPx * 2); - Rect crissCrossAreaHorizontal(0, 0, config->stateIdImageWidthPx * 2, config->stateIdimageHeightPx); - - for (uint i = 0; i < billMapping.size(); i++) + bool FeatureMatcher::isLoaded() { - vector matchesForOnePlate; - for (uint j = 0; j < inputMatches.size(); j++) + if( detector.empty() || extractor.empty() || descriptorMatcher.empty() ) { - if (inputMatches[j].imgIdx == (int) i) - matchesForOnePlate.push_back(inputMatches[j]); + return false; } - // For each plate, compare the lines for the keypoints (training image and query image) - // go through each line between keypoints and filter out matches that are criss-crossing - vector vlines; - vector hlines; - vector matchIdx; - - for (uint j = 0; j < matchesForOnePlate.size(); j++) - { - KeyPoint tkp = trainingImgKeypoints[i][matchesForOnePlate[j].trainIdx]; - KeyPoint qkp = queryKeypoints[matchesForOnePlate[j].queryIdx]; - - vlines.push_back(LineSegment(tkp.pt.x, tkp.pt.y + config->stateIdimageHeightPx, qkp.pt.x, qkp.pt.y)); - hlines.push_back(LineSegment(tkp.pt.x, tkp.pt.y, qkp.pt.x + config->stateIdImageWidthPx, qkp.pt.y)); - matchIdx.push_back(j); - } - - // Iterate through each line (n^2) removing the one with the most criss-crosses until there are none left. - int mostIntersections = 1; - while (mostIntersections > 0 && vlines.size() > 0) - { - int mostIntersectionsIndex = -1; - mostIntersections = 0; - - for (uint j = 0; j < vlines.size(); j++) - { - int intrCount = 0; - for (uint q = 0; q < vlines.size(); q++) - { - Point vintr = vlines[j].intersection(vlines[q]); - Point hintr = hlines[j].intersection(hlines[q]); - float vangleDiff = abs(vlines[j].angle - vlines[q].angle); - float hangleDiff = abs(hlines[j].angle - hlines[q].angle); - if (vintr.inside(crissCrossAreaVertical) && vangleDiff > 10) - { - intrCount++; - } - else if (hintr.inside(crissCrossAreaHorizontal) && hangleDiff > 10) - { - intrCount++; - } - } - - if (intrCount > mostIntersections) - { - mostIntersections = intrCount; - mostIntersectionsIndex = j; - } - } - - if (mostIntersectionsIndex >= 0) - { - if (this->config->debugStateId) - cout << "Filtered intersection! " << billMapping[i] << endl; - vlines.erase(vlines.begin() + mostIntersectionsIndex); - hlines.erase(hlines.begin() + mostIntersectionsIndex); - matchIdx.erase(matchIdx.begin() + mostIntersectionsIndex); - } - } - - // Push the non-crisscrosses back on the list - for (uint j = 0; j < matchIdx.size(); j++) - { - outputMatches.push_back(matchesForOnePlate[matchIdx[j]]); - } - } -} - -// Returns true if successful, false otherwise -bool FeatureMatcher::loadRecognitionSet(string country) -{ - std::ostringstream out; - out << config->getKeypointsRuntimeDir() << "/" << country << "/"; - string country_dir = out.str(); - - if (DirectoryExists(country_dir.c_str())) - { - vector trainImages; - vector plateFiles = getFilesInDir(country_dir.c_str()); - - for (uint i = 0; i < plateFiles.size(); i++) - { - if (hasEnding(plateFiles[i], ".jpg") == false) - continue; - - string fullpath = country_dir + plateFiles[i]; - Mat img = imread( fullpath ); - - // convert to gray and resize to the size of the templates - cvtColor(img, img, CV_BGR2GRAY); - resize(img, img, getSizeMaintainingAspect(img, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); - - if( img.empty() ) - { - cout << "Can not read images" << endl; - return -1; - } - - Mat descriptors; - - vector keypoints; - detector->detect( img, keypoints ); - extractor->compute(img, keypoints, descriptors); - - if (descriptors.cols > 0) - { - billMapping.push_back(plateFiles[i].substr(0, 2)); - trainImages.push_back(descriptors); - trainingImgKeypoints.push_back(keypoints); - } - } - - this->descriptorMatcher->add(trainImages); - this->descriptorMatcher->train(); - return true; } - return false; -} - -RecognitionResult FeatureMatcher::recognize( const Mat& queryImg, bool drawOnImage, Mat* outputImage, - bool debug_on, vector debug_matches_array - ) -{ - RecognitionResult result; - - result.haswinner = false; - result.confidence = 0; - - Mat queryDescriptors; - vector queryKeypoints; - - detector->detect( queryImg, queryKeypoints ); - extractor->compute(queryImg, queryKeypoints, queryDescriptors); - - if (queryKeypoints.size() <= 5) + int FeatureMatcher::numTrainingElements() { - // Cut it loose if there's less than 5 keypoints... nothing would ever match anyway and it could crash the matcher. - if (drawOnImage) + return billMapping.size(); + } + + void FeatureMatcher::surfStyleMatching( const Mat& queryDescriptors, vector queryKeypoints, + vector& matches12 ) + { + vector > matchesKnn; + + this->descriptorMatcher->radiusMatch(queryDescriptors, matchesKnn, MAX_DISTANCE_TO_MATCH); + + vector tempMatches; + _surfStyleMatching(queryDescriptors, matchesKnn, tempMatches); + + crisscrossFiltering(queryKeypoints, tempMatches, matches12); + } + + void FeatureMatcher::_surfStyleMatching(const Mat& queryDescriptors, vector > matchesKnn, vector& matches12) + { + //objectMatches.clear(); + //objectMatches.resize(objectIds.size()); + //cout << "starting matcher" << matchesKnn.size() << endl; + for (int descInd = 0; descInd < queryDescriptors.rows; descInd++) { - drawKeypoints( queryImg, queryKeypoints, *outputImage, CV_RGB(0, 255, 0), DrawMatchesFlags::DEFAULT ); - } - return result; - } + //const std::vector & matches = matchesKnn[descInd]; + //cout << "two: " << descInd << ":" << matches.size() << endl; - vector filteredMatches; - - surfStyleMatching( queryDescriptors, queryKeypoints, filteredMatches ); - - // Create and initialize the counts to 0 - std::vector bill_match_counts( billMapping.size() ); - - for (uint i = 0; i < billMapping.size(); i++) - { - bill_match_counts[i] = 0; - } - - for (uint i = 0; i < filteredMatches.size(); i++) - { - bill_match_counts[filteredMatches[i].imgIdx]++; - //if (filteredMatches[i].imgIdx - } - - float max_count = 0; // represented as a percent (0 to 100) - int secondmost_count = 0; - int maxcount_index = -1; - for (uint i = 0; i < billMapping.size(); i++) - { - if (bill_match_counts[i] > max_count && bill_match_counts[i] >= 4) - { - secondmost_count = max_count; - if (secondmost_count <= 2) // A value of 1 or 2 is effectively 0 - secondmost_count = 0; - - max_count = bill_match_counts[i]; - maxcount_index = i; - } - } - - float score = ((max_count - secondmost_count - 3) / 10) * 100; - if (score < 0) - score = 0; - else if (score > 100) - score = 100; - - if (score > 0) - { - result.haswinner = true; - result.winner = billMapping[maxcount_index]; - result.confidence = score; - - if (drawOnImage) - { - vector positiveMatches; - for (uint i = 0; i < filteredMatches.size(); i++) + // Check to make sure we have 2 matches. I think this is always the case, but it doesn't hurt to be sure + if (matchesKnn[descInd].size() > 1) { - if (filteredMatches[i].imgIdx == maxcount_index) + // Next throw out matches with a crappy score + // Ignore... already handled by the radiusMatch + //if (matchesKnn[descInd][0].distance < MAX_DISTANCE_TO_MATCH) + //{ + float ratioThreshold = 0.75; + + // Check if both matches came from the same image. If they both came from the same image, score them slightly less harshly + if (matchesKnn[descInd][0].imgIdx == matchesKnn[descInd][1].imgIdx) { - positiveMatches.push_back( queryKeypoints[filteredMatches[i].queryIdx] ); + ratioThreshold = 0.85; + } + + if ((matchesKnn[descInd][0].distance / matchesKnn[descInd][1].distance) < ratioThreshold) + { + bool already_exists = false; + // Quickly run through the matches we've already added and make sure it's not a duplicate... + for (uint q = 0; q < matches12.size(); q++) + { + if (matchesKnn[descInd][0].queryIdx == matches12[q].queryIdx) + { + already_exists = true; + break; + } + else if ((matchesKnn[descInd][0].trainIdx == matches12[q].trainIdx) && + (matchesKnn[descInd][0].imgIdx == matches12[q].imgIdx)) + { + already_exists = true; + break; + } + } + + // Good match. + if (already_exists == false) + matches12.push_back(matchesKnn[descInd][0]); + } + + //} + } + else if (matchesKnn[descInd].size() == 1) + { + // Only match? Does this ever happen? + matches12.push_back(matchesKnn[descInd][0]); + } + // In the ratio test, we will compare the quality of a match with the next match that is not from the same object: + // we can accept several matches with similar scores as long as they are for the same object. Those should not be + // part of the model anyway as they are not discriminative enough + + //for (unsigned int first_index = 0; first_index < matches.size(); ++first_index) + //{ + //matches12.push_back(match); + //} + } + } + + // Compares the matches keypoints for parallel lines. Removes matches that are criss-crossing too much + // We assume that license plates won't be upside-down or backwards. So expect lines to be closely parallel + void FeatureMatcher::crisscrossFiltering(const vector queryKeypoints, const vector inputMatches, vector &outputMatches) + { + Rect crissCrossAreaVertical(0, 0, config->stateIdImageWidthPx, config->stateIdimageHeightPx * 2); + Rect crissCrossAreaHorizontal(0, 0, config->stateIdImageWidthPx * 2, config->stateIdimageHeightPx); + + for (uint i = 0; i < billMapping.size(); i++) + { + vector matchesForOnePlate; + for (uint j = 0; j < inputMatches.size(); j++) + { + if (inputMatches[j].imgIdx == (int) i) + matchesForOnePlate.push_back(inputMatches[j]); + } + + // For each plate, compare the lines for the keypoints (training image and query image) + // go through each line between keypoints and filter out matches that are criss-crossing + vector vlines; + vector hlines; + vector matchIdx; + + for (uint j = 0; j < matchesForOnePlate.size(); j++) + { + KeyPoint tkp = trainingImgKeypoints[i][matchesForOnePlate[j].trainIdx]; + KeyPoint qkp = queryKeypoints[matchesForOnePlate[j].queryIdx]; + + vlines.push_back(LineSegment(tkp.pt.x, tkp.pt.y + config->stateIdimageHeightPx, qkp.pt.x, qkp.pt.y)); + hlines.push_back(LineSegment(tkp.pt.x, tkp.pt.y, qkp.pt.x + config->stateIdImageWidthPx, qkp.pt.y)); + matchIdx.push_back(j); + } + + // Iterate through each line (n^2) removing the one with the most criss-crosses until there are none left. + int mostIntersections = 1; + while (mostIntersections > 0 && vlines.size() > 0) + { + int mostIntersectionsIndex = -1; + mostIntersections = 0; + + for (uint j = 0; j < vlines.size(); j++) + { + int intrCount = 0; + for (uint q = 0; q < vlines.size(); q++) + { + Point vintr = vlines[j].intersection(vlines[q]); + Point hintr = hlines[j].intersection(hlines[q]); + float vangleDiff = abs(vlines[j].angle - vlines[q].angle); + float hangleDiff = abs(hlines[j].angle - hlines[q].angle); + if (vintr.inside(crissCrossAreaVertical) && vangleDiff > 10) + { + intrCount++; + } + else if (hintr.inside(crissCrossAreaHorizontal) && hangleDiff > 10) + { + intrCount++; + } + } + + if (intrCount > mostIntersections) + { + mostIntersections = intrCount; + mostIntersectionsIndex = j; + } + } + + if (mostIntersectionsIndex >= 0) + { + if (this->config->debugStateId) + cout << "Filtered intersection! " << billMapping[i] << endl; + vlines.erase(vlines.begin() + mostIntersectionsIndex); + hlines.erase(hlines.begin() + mostIntersectionsIndex); + matchIdx.erase(matchIdx.begin() + mostIntersectionsIndex); } } - Mat tmpImg; - drawKeypoints( queryImg, queryKeypoints, tmpImg, CV_RGB(185, 0, 0), DrawMatchesFlags::DEFAULT ); - drawKeypoints( tmpImg, positiveMatches, *outputImage, CV_RGB(0, 255, 0), DrawMatchesFlags::DEFAULT ); - - if (result.haswinner) + // Push the non-crisscrosses back on the list + for (uint j = 0; j < matchIdx.size(); j++) { - std::ostringstream out; - out << result.winner << " (" << result.confidence << "%)"; - - // we detected a bill, let the people know! - //putText(*outputImage, out.str(), Point(15, 27), FONT_HERSHEY_DUPLEX, 1.1, CV_RGB(0, 0, 0), 2); + outputMatches.push_back(matchesForOnePlate[matchIdx[j]]); } } } - if (this->config->debugStateId) + // Returns true if successful, false otherwise + bool FeatureMatcher::loadRecognitionSet(string country) { - for (uint i = 0; i < billMapping.size(); i++) + std::ostringstream out; + out << config->getKeypointsRuntimeDir() << "/" << country << "/"; + string country_dir = out.str(); + + if (DirectoryExists(country_dir.c_str())) { - cout << billMapping[i] << " : " << bill_match_counts[i] << endl; + vector trainImages; + vector plateFiles = getFilesInDir(country_dir.c_str()); + + for (uint i = 0; i < plateFiles.size(); i++) + { + if (hasEnding(plateFiles[i], ".jpg") == false) + continue; + + string fullpath = country_dir + plateFiles[i]; + Mat img = imread( fullpath ); + + // convert to gray and resize to the size of the templates + cvtColor(img, img, CV_BGR2GRAY); + resize(img, img, getSizeMaintainingAspect(img, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); + + if( img.empty() ) + { + cout << "Can not read images" << endl; + return -1; + } + + Mat descriptors; + + vector keypoints; + detector->detect( img, keypoints ); + extractor->compute(img, keypoints, descriptors); + + if (descriptors.cols > 0) + { + billMapping.push_back(plateFiles[i].substr(0, 2)); + trainImages.push_back(descriptors); + trainingImgKeypoints.push_back(keypoints); + } + } + + this->descriptorMatcher->add(trainImages); + this->descriptorMatcher->train(); + + return true; } + + return false; } - return result; -} + RecognitionResult FeatureMatcher::recognize( const Mat& queryImg, bool drawOnImage, Mat* outputImage, + bool debug_on, vector debug_matches_array + ) + { + RecognitionResult result; + + result.haswinner = false; + result.confidence = 0; + + Mat queryDescriptors; + vector queryKeypoints; + + detector->detect( queryImg, queryKeypoints ); + extractor->compute(queryImg, queryKeypoints, queryDescriptors); + + if (queryKeypoints.size() <= 5) + { + // Cut it loose if there's less than 5 keypoints... nothing would ever match anyway and it could crash the matcher. + if (drawOnImage) + { + drawKeypoints( queryImg, queryKeypoints, *outputImage, CV_RGB(0, 255, 0), DrawMatchesFlags::DEFAULT ); + } + return result; + } + + vector filteredMatches; + + surfStyleMatching( queryDescriptors, queryKeypoints, filteredMatches ); + + // Create and initialize the counts to 0 + std::vector bill_match_counts( billMapping.size() ); + + for (uint i = 0; i < billMapping.size(); i++) + { + bill_match_counts[i] = 0; + } + + for (uint i = 0; i < filteredMatches.size(); i++) + { + bill_match_counts[filteredMatches[i].imgIdx]++; + //if (filteredMatches[i].imgIdx + } + + float max_count = 0; // represented as a percent (0 to 100) + int secondmost_count = 0; + int maxcount_index = -1; + for (uint i = 0; i < billMapping.size(); i++) + { + if (bill_match_counts[i] > max_count && bill_match_counts[i] >= 4) + { + secondmost_count = max_count; + if (secondmost_count <= 2) // A value of 1 or 2 is effectively 0 + secondmost_count = 0; + + max_count = bill_match_counts[i]; + maxcount_index = i; + } + } + + float score = ((max_count - secondmost_count - 3) / 10) * 100; + if (score < 0) + score = 0; + else if (score > 100) + score = 100; + + if (score > 0) + { + result.haswinner = true; + result.winner = billMapping[maxcount_index]; + result.confidence = score; + + if (drawOnImage) + { + vector positiveMatches; + for (uint i = 0; i < filteredMatches.size(); i++) + { + if (filteredMatches[i].imgIdx == maxcount_index) + { + positiveMatches.push_back( queryKeypoints[filteredMatches[i].queryIdx] ); + } + } + + Mat tmpImg; + drawKeypoints( queryImg, queryKeypoints, tmpImg, CV_RGB(185, 0, 0), DrawMatchesFlags::DEFAULT ); + drawKeypoints( tmpImg, positiveMatches, *outputImage, CV_RGB(0, 255, 0), DrawMatchesFlags::DEFAULT ); + + if (result.haswinner) + { + std::ostringstream out; + out << result.winner << " (" << result.confidence << "%)"; + + // we detected a bill, let the people know! + //putText(*outputImage, out.str(), Point(15, 27), FONT_HERSHEY_DUPLEX, 1.1, CV_RGB(0, 0, 0), 2); + } + } + } + + if (this->config->debugStateId) + { + for (uint i = 0; i < billMapping.size(); i++) + { + cout << billMapping[i] << " : " << bill_match_counts[i] << endl; + } + } + + return result; + } + +} \ No newline at end of file diff --git a/src/openalpr/featurematcher.h b/src/openalpr/featurematcher.h index c44cf23..2088790 100644 --- a/src/openalpr/featurematcher.h +++ b/src/openalpr/featurematcher.h @@ -30,48 +30,51 @@ #include "utility.h" #include "config.h" - -struct RecognitionResult -{ - bool haswinner; - std::string winner; - int confidence; -} ; - -class FeatureMatcher +namespace alpr { - public: - FeatureMatcher(Config* config); - virtual ~FeatureMatcher(); + struct RecognitionResult + { + bool haswinner; + std::string winner; + int confidence; + } ; - RecognitionResult recognize( const cv::Mat& queryImg, bool drawOnImage, cv::Mat* outputImage, - bool debug_on, std::vector debug_matches_array ); + class FeatureMatcher + { - bool loadRecognitionSet(std::string country); + public: + FeatureMatcher(Config* config); + virtual ~FeatureMatcher(); - bool isLoaded(); + RecognitionResult recognize( const cv::Mat& queryImg, bool drawOnImage, cv::Mat* outputImage, + bool debug_on, std::vector debug_matches_array ); - int numTrainingElements(); + bool loadRecognitionSet(std::string country); - private: - Config* config; + bool isLoaded(); - cv::Ptr descriptorMatcher; - cv::Ptr detector; - cv::Ptr extractor; + int numTrainingElements(); - std::vector > trainingImgKeypoints; + private: + Config* config; - void _surfStyleMatching(const cv::Mat& queryDescriptors, std::vector > matchesKnn, std::vector& matches12); + cv::Ptr descriptorMatcher; + cv::Ptr detector; + cv::Ptr extractor; - void crisscrossFiltering(const std::vector queryKeypoints, const std::vector inputMatches, std::vector &outputMatches); + std::vector > trainingImgKeypoints; - std::vector billMapping; + void _surfStyleMatching(const cv::Mat& queryDescriptors, std::vector > matchesKnn, std::vector& matches12); - void surfStyleMatching( const cv::Mat& queryDescriptors, std::vector queryKeypoints, - std::vector& matches12 ); + void crisscrossFiltering(const std::vector queryKeypoints, const std::vector inputMatches, std::vector &outputMatches); -}; + std::vector billMapping; + void surfStyleMatching( const cv::Mat& queryDescriptors, std::vector queryKeypoints, + std::vector& matches12 ); + + }; + +} #endif // OPENALPR_FEATUREMATCHER_H diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index c00c4dc..5db9504 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -26,101 +26,104 @@ using namespace std; using namespace cv; -LicensePlateCandidate::LicensePlateCandidate(PipelineData* pipeline_data) +namespace alpr { - this->pipeline_data = pipeline_data; - this->config = pipeline_data->config; -} - -LicensePlateCandidate::~LicensePlateCandidate() -{ - delete charSegmenter; -} - -// Must delete this pointer in parent class -void LicensePlateCandidate::recognize() -{ - charSegmenter = NULL; - - pipeline_data->plate_area_confidence = 0; - pipeline_data->isMultiline = config->multiline; - - - Rect expandedRegion = this->pipeline_data->regionOfInterest; - - pipeline_data->crop_gray = Mat(this->pipeline_data->grayImg, expandedRegion); - resize(pipeline_data->crop_gray, pipeline_data->crop_gray, Size(config->templateWidthPx, config->templateHeightPx)); - - - CharacterAnalysis textAnalysis(pipeline_data); - - if (textAnalysis.confidence > 10) + LicensePlateCandidate::LicensePlateCandidate(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; - EdgeFinder edgeFinder(pipeline_data); - - pipeline_data->plate_corners = edgeFinder.findEdgeCorners(); - - if (edgeFinder.confidence > 0) + } + + LicensePlateCandidate::~LicensePlateCandidate() + { + delete charSegmenter; + } + + // Must delete this pointer in parent class + void LicensePlateCandidate::recognize() + { + charSegmenter = NULL; + + pipeline_data->plate_area_confidence = 0; + pipeline_data->isMultiline = config->multiline; + + + Rect expandedRegion = this->pipeline_data->regionOfInterest; + + pipeline_data->crop_gray = Mat(this->pipeline_data->grayImg, expandedRegion); + resize(pipeline_data->crop_gray, pipeline_data->crop_gray, Size(config->templateWidthPx, config->templateHeightPx)); + + + CharacterAnalysis textAnalysis(pipeline_data); + + if (textAnalysis.confidence > 10) { - timespec startTime; - getTime(&startTime); + EdgeFinder edgeFinder(pipeline_data); + pipeline_data->plate_corners = edgeFinder.findEdgeCorners(); - Mat originalCrop = pipeline_data->crop_gray; - - Transformation imgTransform(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion); - - Size cropSize = imgTransform.getCropSize(pipeline_data->plate_corners, - Size(pipeline_data->config->ocrImageWidthPx, pipeline_data->config->ocrImageHeightPx)); - Mat transmtx = imgTransform.getTransformationMatrix(pipeline_data->plate_corners, cropSize); - pipeline_data->crop_gray = imgTransform.crop(cropSize, transmtx); - - - if (this->config->debugGeneral) - displayImage(config, "quadrilateral", pipeline_data->crop_gray); - - - - // Apply a perspective transformation to the TextLine objects - // to match the newly deskewed license plate crop - vector newLines; - for (uint i = 0; i < pipeline_data->textLines.size(); i++) - { - vector textArea = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].textArea); - vector linePolygon = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].linePolygon); - - vector textAreaRemapped; - vector linePolygonRemapped; - - textAreaRemapped = imgTransform.remapSmallPointstoCrop(textArea, transmtx); - linePolygonRemapped = imgTransform.remapSmallPointstoCrop(linePolygon, transmtx); - - newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); - } - - pipeline_data->textLines.clear(); - for (uint i = 0; i < newLines.size(); i++) - pipeline_data->textLines.push_back(newLines[i]); - - - - if (config->debugTiming) + if (edgeFinder.confidence > 0) { - timespec endTime; - getTime(&endTime); - cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; + + timespec startTime; + getTime(&startTime); + + + Mat originalCrop = pipeline_data->crop_gray; + + Transformation imgTransform(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion); + + Size cropSize = imgTransform.getCropSize(pipeline_data->plate_corners, + Size(pipeline_data->config->ocrImageWidthPx, pipeline_data->config->ocrImageHeightPx)); + Mat transmtx = imgTransform.getTransformationMatrix(pipeline_data->plate_corners, cropSize); + pipeline_data->crop_gray = imgTransform.crop(cropSize, transmtx); + + + if (this->config->debugGeneral) + displayImage(config, "quadrilateral", pipeline_data->crop_gray); + + + + // Apply a perspective transformation to the TextLine objects + // to match the newly deskewed license plate crop + vector newLines; + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + vector textArea = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].textArea); + vector linePolygon = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].linePolygon); + + vector textAreaRemapped; + vector linePolygonRemapped; + + textAreaRemapped = imgTransform.remapSmallPointstoCrop(textArea, transmtx); + linePolygonRemapped = imgTransform.remapSmallPointstoCrop(linePolygon, transmtx); + + newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + } + + pipeline_data->textLines.clear(); + for (uint i = 0; i < newLines.size(); i++) + pipeline_data->textLines.push_back(newLines[i]); + + + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + charSegmenter = new CharacterSegmenter(pipeline_data); + + + pipeline_data->plate_area_confidence = 100; } - - charSegmenter = new CharacterSegmenter(pipeline_data); - - pipeline_data->plate_area_confidence = 100; } - } + } - - diff --git a/src/openalpr/licenseplatecandidate.h b/src/openalpr/licenseplatecandidate.h index 3093006..2591a6c 100644 --- a/src/openalpr/licenseplatecandidate.h +++ b/src/openalpr/licenseplatecandidate.h @@ -37,31 +37,32 @@ #include "config.h" #include "pipeline_data.h" -//vector getCharacterRegions(Mat frame, vector regionsOfInterest); -//vector getCharSegmentsBetweenLines(Mat img, vector > contours, LineSegment top, LineSegment bottom); - -class LicensePlateCandidate +namespace alpr { - public: - LicensePlateCandidate(PipelineData* pipeline_data); - virtual ~LicensePlateCandidate(); + class LicensePlateCandidate + { + + public: + LicensePlateCandidate(PipelineData* pipeline_data); + virtual ~LicensePlateCandidate(); - void recognize(); + void recognize(); - private: - PipelineData* pipeline_data; - Config* config; + private: + PipelineData* pipeline_data; + Config* config; - CharacterSegmenter* charSegmenter; + CharacterSegmenter* charSegmenter; - cv::Mat filterByCharacterHue(std::vector > charRegionContours); - std::vector findPlateCorners(cv::Mat inputImage, PlateLines plateLines, CharacterAnalysis textAnalysis); // top-left, top-right, bottom-right, bottom-left + cv::Mat filterByCharacterHue(std::vector > charRegionContours); + std::vector findPlateCorners(cv::Mat inputImage, PlateLines plateLines, CharacterAnalysis textAnalysis); // top-left, top-right, bottom-right, bottom-left - cv::Size getCropSize(std::vector areaCorners); - -}; + cv::Size getCropSize(std::vector areaCorners); + }; + +} #endif // OPENALPR_LICENSEPLATECANDIDATE_H diff --git a/src/openalpr/ocr.cpp b/src/openalpr/ocr.cpp index 26ade9e..31b9199 100644 --- a/src/openalpr/ocr.cpp +++ b/src/openalpr/ocr.cpp @@ -23,112 +23,117 @@ using namespace std; using namespace cv; using namespace tesseract; -OCR::OCR(Config* config) -: postProcessor(config) +namespace alpr { - const string EXPECTED_TESSERACT_VERSION = "3.03"; - - this->config = config; - - if (startsWith(tesseract.Version(), EXPECTED_TESSERACT_VERSION) == false) + OCR::OCR(Config* config) + : postProcessor(config) { - std::cerr << "Warning: You are running an unsupported version of Tesseract." << endl; - std::cerr << "Expecting version " << EXPECTED_TESSERACT_VERSION << ", your version is: " << tesseract.Version() << endl; - } - - // Tesseract requires the prefix directory to be set as an env variable - tesseract.Init(config->getTessdataPrefix().c_str(), config->ocrLanguage.c_str() ); - tesseract.SetVariable("save_blob_choices", "T"); - tesseract.SetPageSegMode(PSM_SINGLE_CHAR); -} + const string EXPECTED_TESSERACT_VERSION = "3.03"; -OCR::~OCR() -{ - tesseract.Clear(); -} + this->config = config; -void OCR::performOCR(PipelineData* pipeline_data) -{ - timespec startTime; - getTime(&startTime); - postProcessor.clear(); - - // Don't waste time on OCR processing if it is impossible to get sufficient characters - if (pipeline_data->charRegions.size() < config->postProcessMinCharacters) - return; - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - // Make it black text on white background - bitwise_not(pipeline_data->thresholds[i], pipeline_data->thresholds[i]); - tesseract.SetImage((uchar*) pipeline_data->thresholds[i].data, - pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height, - pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1()); - - for (uint j = 0; j < pipeline_data->charRegions.size(); j++) + if (startsWith(tesseract.Version(), EXPECTED_TESSERACT_VERSION) == false) { - Rect expandedRegion = expandRect( pipeline_data->charRegions[j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ; + std::cerr << "Warning: You are running an unsupported version of Tesseract." << endl; + std::cerr << "Expecting version " << EXPECTED_TESSERACT_VERSION << ", your version is: " << tesseract.Version() << endl; + } - tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height); - tesseract.Recognize(NULL); + // Tesseract requires the prefix directory to be set as an env variable + tesseract.Init(config->getTessdataPrefix().c_str(), config->ocrLanguage.c_str() ); + tesseract.SetVariable("save_blob_choices", "T"); + tesseract.SetPageSegMode(PSM_SINGLE_CHAR); + } - tesseract::ResultIterator* ri = tesseract.GetIterator(); - tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL; - do + OCR::~OCR() + { + tesseract.Clear(); + } + + void OCR::performOCR(PipelineData* pipeline_data) + { + timespec startTime; + getTime(&startTime); + + postProcessor.clear(); + + // Don't waste time on OCR processing if it is impossible to get sufficient characters + if (pipeline_data->charRegions.size() < config->postProcessMinCharacters) + return; + + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + // Make it black text on white background + bitwise_not(pipeline_data->thresholds[i], pipeline_data->thresholds[i]); + tesseract.SetImage((uchar*) pipeline_data->thresholds[i].data, + pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height, + pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1()); + + for (uint j = 0; j < pipeline_data->charRegions.size(); j++) { - const char* symbol = ri->GetUTF8Text(level); - float conf = ri->Confidence(level); + Rect expandedRegion = expandRect( pipeline_data->charRegions[j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ; - bool dontcare; - int fontindex = 0; - int pointsize = 0; - const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex); + tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height); + tesseract.Recognize(NULL); - if(symbol != 0 && pointsize >= config->ocrMinFontSize) + tesseract::ResultIterator* ri = tesseract.GetIterator(); + tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL; + do { - postProcessor.addLetter(string(symbol), j, conf); + const char* symbol = ri->GetUTF8Text(level); + float conf = ri->Confidence(level); + + bool dontcare; + int fontindex = 0; + int pointsize = 0; + const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex); + + if(symbol != 0 && pointsize >= config->ocrMinFontSize) + { + postProcessor.addLetter(string(symbol), j, conf); + + if (this->config->debugOcr) + printf("charpos%d: threshold %d: symbol %s, conf: %f font: %s (index %d) size %dpx", j, i, symbol, conf, fontName, fontindex, pointsize); + + bool indent = false; + tesseract::ChoiceIterator ci(*ri); + do + { + const char* choice = ci.GetUTF8Text(); + + postProcessor.addLetter(string(choice), j, ci.Confidence()); + + //letterScores.addScore(*choice, j, ci.Confidence() - MIN_CONFIDENCE); + if (this->config->debugOcr) + { + if (indent) printf("\t\t "); + printf("\t- "); + printf("%s conf: %f\n", choice, ci.Confidence()); + } + + indent = true; + } + while(ci.Next()); + } if (this->config->debugOcr) - printf("charpos%d: threshold %d: symbol %s, conf: %f font: %s (index %d) size %dpx", j, i, symbol, conf, fontName, fontindex, pointsize); + printf("---------------------------------------------\n"); - bool indent = false; - tesseract::ChoiceIterator ci(*ri); - do - { - const char* choice = ci.GetUTF8Text(); - - postProcessor.addLetter(string(choice), j, ci.Confidence()); - - //letterScores.addScore(*choice, j, ci.Confidence() - MIN_CONFIDENCE); - if (this->config->debugOcr) - { - if (indent) printf("\t\t "); - printf("\t- "); - printf("%s conf: %f\n", choice, ci.Confidence()); - } - - indent = true; - } - while(ci.Next()); + delete[] symbol; } + while((ri->Next(level))); - if (this->config->debugOcr) - printf("---------------------------------------------\n"); - - delete[] symbol; + delete ri; } - while((ri->Next(level))); + } - delete ri; + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "OCR Time: " << diffclock(startTime, endTime) << "ms." << endl; } } - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "OCR Time: " << diffclock(startTime, endTime) << "ms." << endl; - } -} +} \ No newline at end of file diff --git a/src/openalpr/ocr.h b/src/openalpr/ocr.h index 714f7bc..808cd0a 100644 --- a/src/openalpr/ocr.h +++ b/src/openalpr/ocr.h @@ -34,22 +34,27 @@ #include "tesseract/baseapi.h" -class OCR +namespace alpr { - public: - OCR(Config* config); - virtual ~OCR(); + class OCR + { - void performOCR(PipelineData* pipeline_data); + public: + OCR(Config* config); + virtual ~OCR(); - PostProcess postProcessor; + void performOCR(PipelineData* pipeline_data); - private: - Config* config; + PostProcess postProcessor; - tesseract::TessBaseAPI tesseract; + private: + Config* config; -}; + tesseract::TessBaseAPI tesseract; + + }; + +} #endif // OPENALPR_OCR_H diff --git a/src/openalpr/pipeline_data.cpp b/src/openalpr/pipeline_data.cpp index cc3cfe5..13b55b4 100644 --- a/src/openalpr/pipeline_data.cpp +++ b/src/openalpr/pipeline_data.cpp @@ -3,29 +3,34 @@ using namespace cv; using namespace std; -PipelineData::PipelineData(Mat colorImage, Rect regionOfInterest, Config* config) +namespace alpr { - this->colorImg = colorImage; - cvtColor(this->colorImg, this->grayImg, CV_BGR2GRAY); - this->regionOfInterest = regionOfInterest; - this->config = config; - - this->region_confidence = 0; - - plate_inverted = false; -} - -PipelineData::~PipelineData() -{ - clearThresholds(); -} - -void PipelineData::clearThresholds() -{ - for (uint i = 0; i < thresholds.size(); i++) + PipelineData::PipelineData(Mat colorImage, Rect regionOfInterest, Config* config) { - thresholds[i].release(); + this->colorImg = colorImage; + cvtColor(this->colorImg, this->grayImg, CV_BGR2GRAY); + + this->regionOfInterest = regionOfInterest; + this->config = config; + + this->region_confidence = 0; + + plate_inverted = false; } - thresholds.clear(); + + PipelineData::~PipelineData() + { + clearThresholds(); + } + + void PipelineData::clearThresholds() + { + for (uint i = 0; i < thresholds.size(); i++) + { + thresholds[i].release(); + } + thresholds.clear(); + } + } diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 6c71e8a..2c09453 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -7,52 +7,56 @@ #include "config.h" #include "textdetection/textline.h" -class PipelineData +namespace alpr { - public: - PipelineData(cv::Mat colorImage, cv::Rect regionOfInterest, Config* config); - virtual ~PipelineData(); + class PipelineData + { - void clearThresholds(); - - // Inputs - Config* config; - - cv::Mat colorImg; - cv::Mat grayImg; - cv::Rect regionOfInterest; - - bool isMultiline; - - cv::Mat crop_gray; - - bool hasPlateBorder; - cv::Mat plateBorderMask; - std::vector textLines; - - std::vector thresholds; - - std::vector plate_corners; + public: + PipelineData(cv::Mat colorImage, cv::Rect regionOfInterest, Config* config); + virtual ~PipelineData(); - - // Outputs - bool plate_inverted; - - std::string region_code; - float region_confidence; - + void clearThresholds(); - float plate_area_confidence; - - std::vector charRegions; - + // Inputs + Config* config; + + cv::Mat colorImg; + cv::Mat grayImg; + cv::Rect regionOfInterest; + + bool isMultiline; + + cv::Mat crop_gray; + + bool hasPlateBorder; + cv::Mat plateBorderMask; + std::vector textLines; + + std::vector thresholds; + + std::vector plate_corners; + + + // Outputs + bool plate_inverted; + + std::string region_code; + float region_confidence; + + + float plate_area_confidence; + + std::vector charRegions; - // OCR - -}; + // OCR + + }; + +} #endif // OPENALPR_PIPELINEDATA_H \ No newline at end of file diff --git a/src/openalpr/postprocess.cpp b/src/openalpr/postprocess.cpp index 1f5dc72..fa7c2b7 100644 --- a/src/openalpr/postprocess.cpp +++ b/src/openalpr/postprocess.cpp @@ -21,500 +21,506 @@ using namespace std; -PostProcess::PostProcess(Config* config) +namespace alpr { - this->config = config; + - stringstream filename; - filename << config->getPostProcessRuntimeDir() << "/" << config->country << ".patterns"; - - std::ifstream infile(filename.str().c_str()); - - string region, pattern; - while (infile >> region >> pattern) + PostProcess::PostProcess(Config* config) { - RegexRule* rule = new RegexRule(region, pattern); - //cout << "REGION: " << region << " PATTERN: " << pattern << endl; + this->config = config; - if (rules.find(region) == rules.end()) + stringstream filename; + filename << config->getPostProcessRuntimeDir() << "/" << config->country << ".patterns"; + + std::ifstream infile(filename.str().c_str()); + + string region, pattern; + while (infile >> region >> pattern) { - vector newRule; - newRule.push_back(rule); - rules[region] = newRule; + RegexRule* rule = new RegexRule(region, pattern); + //cout << "REGION: " << region << " PATTERN: " << pattern << endl; + + if (rules.find(region) == rules.end()) + { + vector newRule; + newRule.push_back(rule); + rules[region] = newRule; + } + else + { + vector oldRule = rules[region]; + oldRule.push_back(rule); + rules[region] = oldRule; + } + } + + //vector test = rules["base"]; + //for (int i = 0; i < test.size(); i++) + // cout << "Rule: " << test[i].regex << endl; + } + + PostProcess::~PostProcess() + { + // TODO: Delete all entries in rules vector + map >::iterator iter; + + for (iter = rules.begin(); iter != rules.end(); ++iter) + { + for (int i = 0; i < iter->second.size(); i++) + { + delete iter->second[i]; + } + } + } + + void PostProcess::addLetter(string letter, int charposition, float score) + { + if (score < config->postProcessMinConfidence) + return; + + insertLetter(letter, charposition, score); + + if (score < config->postProcessConfidenceSkipLevel) + { + float adjustedScore = abs(config->postProcessConfidenceSkipLevel - score) + config->postProcessMinConfidence; + insertLetter(SKIP_CHAR, charposition, adjustedScore ); + } + + //if (letter == '0') + //{ + // insertLetter('O', charposition, score - 0.5); + //} + } + + void PostProcess::insertLetter(string letter, int charposition, float score) + { + score = score - config->postProcessMinConfidence; + + int existingIndex = -1; + if (letters.size() < charposition + 1) + { + for (int i = letters.size(); i < charposition + 1; i++) + { + vector tmp; + letters.push_back(tmp); + } + } + + for (int i = 0; i < letters[charposition].size(); i++) + { + if (letters[charposition][i].letter == letter && + letters[charposition][i].charposition == charposition) + { + existingIndex = i; + break; + } + } + + if (existingIndex == -1) + { + Letter newLetter; + newLetter.charposition = charposition; + newLetter.letter = letter; + newLetter.occurences = 1; + newLetter.totalscore = score; + letters[charposition].push_back(newLetter); } else { - vector oldRule = rules[region]; - oldRule.push_back(rule); - rules[region] = oldRule; + letters[charposition][existingIndex].occurences = letters[charposition][existingIndex].occurences + 1; + letters[charposition][existingIndex].totalscore = letters[charposition][existingIndex].totalscore + score; } } - //vector test = rules["base"]; - //for (int i = 0; i < test.size(); i++) - // cout << "Rule: " << test[i].regex << endl; -} - -PostProcess::~PostProcess() -{ - // TODO: Delete all entries in rules vector - map >::iterator iter; - - for (iter = rules.begin(); iter != rules.end(); ++iter) + void PostProcess::clear() { - for (int i = 0; i < iter->second.size(); i++) - { - delete iter->second[i]; - } - } -} - -void PostProcess::addLetter(string letter, int charposition, float score) -{ - if (score < config->postProcessMinConfidence) - return; - - insertLetter(letter, charposition, score); - - if (score < config->postProcessConfidenceSkipLevel) - { - float adjustedScore = abs(config->postProcessConfidenceSkipLevel - score) + config->postProcessMinConfidence; - insertLetter(SKIP_CHAR, charposition, adjustedScore ); - } - - //if (letter == '0') - //{ - // insertLetter('O', charposition, score - 0.5); - //} -} - -void PostProcess::insertLetter(string letter, int charposition, float score) -{ - score = score - config->postProcessMinConfidence; - - int existingIndex = -1; - if (letters.size() < charposition + 1) - { - for (int i = letters.size(); i < charposition + 1; i++) - { - vector tmp; - letters.push_back(tmp); - } - } - - for (int i = 0; i < letters[charposition].size(); i++) - { - if (letters[charposition][i].letter == letter && - letters[charposition][i].charposition == charposition) - { - existingIndex = i; - break; - } - } - - if (existingIndex == -1) - { - Letter newLetter; - newLetter.charposition = charposition; - newLetter.letter = letter; - newLetter.occurences = 1; - newLetter.totalscore = score; - letters[charposition].push_back(newLetter); - } - else - { - letters[charposition][existingIndex].occurences = letters[charposition][existingIndex].occurences + 1; - letters[charposition][existingIndex].totalscore = letters[charposition][existingIndex].totalscore + score; - } -} - -void PostProcess::clear() -{ - for (int i = 0; i < letters.size(); i++) - { - letters[i].clear(); - } - letters.resize(0); - - unknownCharPositions.clear(); - unknownCharPositions.resize(0); - allPossibilities.clear(); - //allPossibilities.resize(0); - - bestChars = ""; - matchesTemplate = false; -} - -void PostProcess::analyze(string templateregion, int topn) -{ - timespec startTime; - getTime(&startTime); - - // Get a list of missing positions - for (int i = letters.size() -1; i >= 0; i--) - { - if (letters[i].size() == 0) - { - unknownCharPositions.push_back(i); - } - } - - if (letters.size() == 0) - return; - - // Sort the letters as they are - for (int i = 0; i < letters.size(); i++) - { - if (letters[i].size() > 0) - sort(letters[i].begin(), letters[i].end(), letterCompare); - } - - if (this->config->debugPostProcess) - { - // Print all letters for (int i = 0; i < letters.size(); i++) { - for (int j = 0; j < letters[i].size(); j++) - cout << "PostProcess Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl; + letters[i].clear(); } + letters.resize(0); + + unknownCharPositions.clear(); + unknownCharPositions.resize(0); + allPossibilities.clear(); + //allPossibilities.resize(0); + + bestChars = ""; + matchesTemplate = false; } - // Prune the letters based on the topN value. - // If our topN value is 3, for example, we can get rid of a lot of low scoring letters - // because it would be impossible for them to be a part of our topN results. - vector maxDepth = getMaxDepth(topn); - - for (int i = 0; i < letters.size(); i++) + void PostProcess::analyze(string templateregion, int topn) { - for (int k = letters[i].size() - 1; k > maxDepth[i]; k--) + timespec startTime; + getTime(&startTime); + + // Get a list of missing positions + for (int i = letters.size() -1; i >= 0; i--) { - letters[i].erase(letters[i].begin() + k); - } - } - - //getTopN(); - vector tmp; - findAllPermutations(tmp, 0, config->postProcessMaxSubstitutions); - - timespec sortStartTime; - getTime(&sortStartTime); - - int numelements = topn; - if (allPossibilities.size() < topn) - numelements = allPossibilities.size() - 1; - - partial_sort( allPossibilities.begin(), allPossibilities.begin() + numelements, allPossibilities.end(), wordCompare ); - - if (config->debugTiming) - { - timespec sortEndTime; - getTime(&sortEndTime); - cout << " -- PostProcess Sort Time: " << diffclock(sortStartTime, sortEndTime) << "ms." << endl; - } - - matchesTemplate = false; - - if (templateregion != "") - { - vector regionRules = rules[templateregion]; - - for (int i = 0; i < allPossibilities.size(); i++) - { - for (int j = 0; j < regionRules.size(); j++) + if (letters[i].size() == 0) { - allPossibilities[i].matchesTemplate = regionRules[j]->match(allPossibilities[i].letters); - if (allPossibilities[i].matchesTemplate) + unknownCharPositions.push_back(i); + } + } + + if (letters.size() == 0) + return; + + // Sort the letters as they are + for (int i = 0; i < letters.size(); i++) + { + if (letters[i].size() > 0) + sort(letters[i].begin(), letters[i].end(), letterCompare); + } + + if (this->config->debugPostProcess) + { + // Print all letters + for (int i = 0; i < letters.size(); i++) + { + for (int j = 0; j < letters[i].size(); j++) + cout << "PostProcess Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl; + } + } + + // Prune the letters based on the topN value. + // If our topN value is 3, for example, we can get rid of a lot of low scoring letters + // because it would be impossible for them to be a part of our topN results. + vector maxDepth = getMaxDepth(topn); + + for (int i = 0; i < letters.size(); i++) + { + for (int k = letters[i].size() - 1; k > maxDepth[i]; k--) + { + letters[i].erase(letters[i].begin() + k); + } + } + + //getTopN(); + vector tmp; + findAllPermutations(tmp, 0, config->postProcessMaxSubstitutions); + + timespec sortStartTime; + getTime(&sortStartTime); + + int numelements = topn; + if (allPossibilities.size() < topn) + numelements = allPossibilities.size() - 1; + + partial_sort( allPossibilities.begin(), allPossibilities.begin() + numelements, allPossibilities.end(), wordCompare ); + + if (config->debugTiming) + { + timespec sortEndTime; + getTime(&sortEndTime); + cout << " -- PostProcess Sort Time: " << diffclock(sortStartTime, sortEndTime) << "ms." << endl; + } + + matchesTemplate = false; + + if (templateregion != "") + { + vector regionRules = rules[templateregion]; + + for (int i = 0; i < allPossibilities.size(); i++) + { + for (int j = 0; j < regionRules.size(); j++) { - allPossibilities[i].letters = regionRules[j]->filterSkips(allPossibilities[i].letters); - //bestChars = regionRules[j]->filterSkips(allPossibilities[i].letters); - matchesTemplate = true; + allPossibilities[i].matchesTemplate = regionRules[j]->match(allPossibilities[i].letters); + if (allPossibilities[i].matchesTemplate) + { + allPossibilities[i].letters = regionRules[j]->filterSkips(allPossibilities[i].letters); + //bestChars = regionRules[j]->filterSkips(allPossibilities[i].letters); + matchesTemplate = true; + break; + } + } + + if (i >= topn - 1) + break; + //if (matchesTemplate || i >= TOP_N - 1) + //break; + } + } + + if (matchesTemplate) + { + for (int z = 0; z < allPossibilities.size(); z++) + { + if (allPossibilities[z].matchesTemplate) + { + bestChars = allPossibilities[z].letters; + break; + } + } + } + else + { + bestChars = allPossibilities[0].letters; + } + + // Now adjust the confidence scores to a percentage value + if (allPossibilities.size() > 0) + { + float maxPercentScore = calculateMaxConfidenceScore(); + float highestRelativeScore = (float) allPossibilities[0].totalscore; + + for (int i = 0; i < allPossibilities.size(); i++) + { + allPossibilities[i].totalscore = maxPercentScore * (allPossibilities[i].totalscore / highestRelativeScore); + } + } + + if (this->config->debugPostProcess) + { + // Print top words + for (int i = 0; i < allPossibilities.size(); i++) + { + cout << "Top " << topn << " Possibilities: " << allPossibilities[i].letters << " :\t" << allPossibilities[i].totalscore; + if (allPossibilities[i].letters == bestChars) + cout << " <--- "; + cout << endl; + + if (i >= topn - 1) + break; + } + cout << allPossibilities.size() << " total permutations" << endl; + } + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "PostProcess Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + if (this->config->debugPostProcess) + cout << "PostProcess Analysis Complete: " << bestChars << " -- MATCH: " << matchesTemplate << endl; + } + + float PostProcess::calculateMaxConfidenceScore() + { + // Take the best score for each char position and average it. + + float totalScore = 0; + int numScores = 0; + // Get a list of missing positions + for (int i = 0; i < letters.size(); i++) + { + if (letters[i].size() > 0) + { + totalScore += (letters[i][0].totalscore / letters[i][0].occurences) + config->postProcessMinConfidence; + numScores++; + } + } + + if (numScores == 0) + return 0; + + return totalScore / ((float) numScores); + } + + // Finds the minimum number of letters to include in the recursive sorting algorithm. + // For example, if I have letters + // A-200 B-100 C-100 + // X-99 Y-95 Z-90 + // Q-55 R-80 + // And my topN value was 3, this would return: + // 0, 1, 1 + // Which represents: + // A-200 B-100 C-100 + // Y-95 Z-90 + vector PostProcess::getMaxDepth(int topn) + { + vector depth; + for (int i = 0; i < letters.size(); i++) + depth.push_back(0); + + int nextLeastDropCharPos = getNextLeastDrop(depth); + while (nextLeastDropCharPos != -1) + { + if (getPermutationCount(depth) >= topn) + break; + + depth[nextLeastDropCharPos] = depth[nextLeastDropCharPos] + 1; + + nextLeastDropCharPos = getNextLeastDrop(depth); + } + + return depth; + } + + int PostProcess::getPermutationCount(vector depth) + { + int permutationCount = 1; + for (int i = 0; i < depth.size(); i++) + { + permutationCount *= (depth[i] + 1); + } + + return permutationCount; + } + + int PostProcess::getNextLeastDrop(vector depth) + { + int nextLeastDropCharPos = -1; + float leastNextDrop = 99999999999; + + for (int i = 0; i < letters.size(); i++) + { + if (depth[i] + 1 >= letters[i].size()) + continue; + + float drop = letters[i][depth[i]].totalscore - letters[i][depth[i]+1].totalscore; + + if (drop < leastNextDrop) + { + nextLeastDropCharPos = i; + leastNextDrop = drop; + } + } + + return nextLeastDropCharPos; + } + + const vector PostProcess::getResults() + { + return this->allPossibilities; + } + + void PostProcess::findAllPermutations(vector prevletters, int charPos, int substitutionsLeft) + { + if (substitutionsLeft < 0) + return; + + // Add my letter to the chain and recurse + for (int i = 0; i < letters[charPos].size(); i++) + { + if (charPos == letters.size() - 1) + { + // Last letter, add the word + PPResult possibility; + possibility.letters = ""; + possibility.totalscore = 0; + possibility.matchesTemplate = false; + for (int z = 0; z < prevletters.size(); z++) + { + if (prevletters[z].letter != SKIP_CHAR) + possibility.letters = possibility.letters + prevletters[z].letter; + possibility.totalscore = possibility.totalscore + prevletters[z].totalscore; + } + + if (letters[charPos][i].letter != SKIP_CHAR) + possibility.letters = possibility.letters + letters[charPos][i].letter; + possibility.totalscore = possibility.totalscore +letters[charPos][i].totalscore; + + allPossibilities.push_back(possibility); + } + else + { + prevletters.push_back(letters[charPos][i]); + + float scorePercentDiff = abs( letters[charPos][0].totalscore - letters[charPos][i].totalscore ) / letters[charPos][0].totalscore; + if (i != 0 && letters[charPos][i].letter != SKIP_CHAR && scorePercentDiff > 0.10f ) + findAllPermutations(prevletters, charPos + 1, substitutionsLeft - 1); + else + findAllPermutations(prevletters, charPos + 1, substitutionsLeft); + + prevletters.pop_back(); + } + } + + if (letters[charPos].size() == 0) + { + // No letters for this char position... + // Just pass it along + findAllPermutations(prevletters, charPos + 1, substitutionsLeft); + } + } + + bool wordCompare( const PPResult &left, const PPResult &right ) + { + if (left.totalscore < right.totalscore) + return false; + return true; + } + + bool letterCompare( const Letter &left, const Letter &right ) + { + if (left.totalscore < right.totalscore) + return false; + return true; + } + + RegexRule::RegexRule(string region, string pattern) + { + this->original = pattern; + this->region = region; + + numchars = 0; + for (int i = 0; i < pattern.size(); i++) + { + if (pattern.at(i) == '[') + { + while (pattern.at(i) != ']' ) + { + this->regex = this->regex + pattern.at(i); + i++; + } + this->regex = this->regex + ']'; + } + else if (pattern.at(i) == '?') + { + this->regex = this->regex + '.'; + this->skipPositions.push_back(numchars); + } + else if (pattern.at(i) == '@') + { + this->regex = this->regex + "\\a"; + } + else if (pattern.at(i) == '#') + { + this->regex = this->regex + "\\d"; + } + + numchars++; + } + + trexp.Compile(this->regex.c_str()); + + //cout << "AA " << this->region << ": " << original << " regex: " << regex << endl; + //for (int z = 0; z < this->skipPositions.size(); z++) + // cout << "AA Skip position: " << skipPositions[z] << endl; + } + + bool RegexRule::match(string text) + { + if (text.length() != numchars) + return false; + + return trexp.Match(text.c_str()); + } + + string RegexRule::filterSkips(string text) + { + string response = ""; + for (int i = 0; i < text.size(); i++) + { + bool skip = false; + for (int j = 0; j < skipPositions.size(); j++) + { + if (skipPositions[j] == i) + { + skip = true; break; } } - if (i >= topn - 1) - break; - //if (matchesTemplate || i >= TOP_N - 1) - //break; - } - } - - if (matchesTemplate) - { - for (int z = 0; z < allPossibilities.size(); z++) - { - if (allPossibilities[z].matchesTemplate) - { - bestChars = allPossibilities[z].letters; - break; - } - } - } - else - { - bestChars = allPossibilities[0].letters; - } - - // Now adjust the confidence scores to a percentage value - if (allPossibilities.size() > 0) - { - float maxPercentScore = calculateMaxConfidenceScore(); - float highestRelativeScore = (float) allPossibilities[0].totalscore; - - for (int i = 0; i < allPossibilities.size(); i++) - { - allPossibilities[i].totalscore = maxPercentScore * (allPossibilities[i].totalscore / highestRelativeScore); - } - } - - if (this->config->debugPostProcess) - { - // Print top words - for (int i = 0; i < allPossibilities.size(); i++) - { - cout << "Top " << topn << " Possibilities: " << allPossibilities[i].letters << " :\t" << allPossibilities[i].totalscore; - if (allPossibilities[i].letters == bestChars) - cout << " <--- "; - cout << endl; - - if (i >= topn - 1) - break; - } - cout << allPossibilities.size() << " total permutations" << endl; - } - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "PostProcess Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - if (this->config->debugPostProcess) - cout << "PostProcess Analysis Complete: " << bestChars << " -- MATCH: " << matchesTemplate << endl; -} - -float PostProcess::calculateMaxConfidenceScore() -{ - // Take the best score for each char position and average it. - - float totalScore = 0; - int numScores = 0; - // Get a list of missing positions - for (int i = 0; i < letters.size(); i++) - { - if (letters[i].size() > 0) - { - totalScore += (letters[i][0].totalscore / letters[i][0].occurences) + config->postProcessMinConfidence; - numScores++; - } - } - - if (numScores == 0) - return 0; - - return totalScore / ((float) numScores); -} - -// Finds the minimum number of letters to include in the recursive sorting algorithm. -// For example, if I have letters -// A-200 B-100 C-100 -// X-99 Y-95 Z-90 -// Q-55 R-80 -// And my topN value was 3, this would return: -// 0, 1, 1 -// Which represents: -// A-200 B-100 C-100 -// Y-95 Z-90 -vector PostProcess::getMaxDepth(int topn) -{ - vector depth; - for (int i = 0; i < letters.size(); i++) - depth.push_back(0); - - int nextLeastDropCharPos = getNextLeastDrop(depth); - while (nextLeastDropCharPos != -1) - { - if (getPermutationCount(depth) >= topn) - break; - - depth[nextLeastDropCharPos] = depth[nextLeastDropCharPos] + 1; - - nextLeastDropCharPos = getNextLeastDrop(depth); - } - - return depth; -} - -int PostProcess::getPermutationCount(vector depth) -{ - int permutationCount = 1; - for (int i = 0; i < depth.size(); i++) - { - permutationCount *= (depth[i] + 1); - } - - return permutationCount; -} - -int PostProcess::getNextLeastDrop(vector depth) -{ - int nextLeastDropCharPos = -1; - float leastNextDrop = 99999999999; - - for (int i = 0; i < letters.size(); i++) - { - if (depth[i] + 1 >= letters[i].size()) - continue; - - float drop = letters[i][depth[i]].totalscore - letters[i][depth[i]+1].totalscore; - - if (drop < leastNextDrop) - { - nextLeastDropCharPos = i; - leastNextDrop = drop; - } - } - - return nextLeastDropCharPos; -} - -const vector PostProcess::getResults() -{ - return this->allPossibilities; -} - -void PostProcess::findAllPermutations(vector prevletters, int charPos, int substitutionsLeft) -{ - if (substitutionsLeft < 0) - return; - - // Add my letter to the chain and recurse - for (int i = 0; i < letters[charPos].size(); i++) - { - if (charPos == letters.size() - 1) - { - // Last letter, add the word - PPResult possibility; - possibility.letters = ""; - possibility.totalscore = 0; - possibility.matchesTemplate = false; - for (int z = 0; z < prevletters.size(); z++) - { - if (prevletters[z].letter != SKIP_CHAR) - possibility.letters = possibility.letters + prevletters[z].letter; - possibility.totalscore = possibility.totalscore + prevletters[z].totalscore; - } - - if (letters[charPos][i].letter != SKIP_CHAR) - possibility.letters = possibility.letters + letters[charPos][i].letter; - possibility.totalscore = possibility.totalscore +letters[charPos][i].totalscore; - - allPossibilities.push_back(possibility); - } - else - { - prevletters.push_back(letters[charPos][i]); - - float scorePercentDiff = abs( letters[charPos][0].totalscore - letters[charPos][i].totalscore ) / letters[charPos][0].totalscore; - if (i != 0 && letters[charPos][i].letter != SKIP_CHAR && scorePercentDiff > 0.10f ) - findAllPermutations(prevletters, charPos + 1, substitutionsLeft - 1); - else - findAllPermutations(prevletters, charPos + 1, substitutionsLeft); - - prevletters.pop_back(); - } - } - - if (letters[charPos].size() == 0) - { - // No letters for this char position... - // Just pass it along - findAllPermutations(prevletters, charPos + 1, substitutionsLeft); - } -} - -bool wordCompare( const PPResult &left, const PPResult &right ) -{ - if (left.totalscore < right.totalscore) - return false; - return true; -} - -bool letterCompare( const Letter &left, const Letter &right ) -{ - if (left.totalscore < right.totalscore) - return false; - return true; -} - -RegexRule::RegexRule(string region, string pattern) -{ - this->original = pattern; - this->region = region; - - numchars = 0; - for (int i = 0; i < pattern.size(); i++) - { - if (pattern.at(i) == '[') - { - while (pattern.at(i) != ']' ) - { - this->regex = this->regex + pattern.at(i); - i++; - } - this->regex = this->regex + ']'; - } - else if (pattern.at(i) == '?') - { - this->regex = this->regex + '.'; - this->skipPositions.push_back(numchars); - } - else if (pattern.at(i) == '@') - { - this->regex = this->regex + "\\a"; - } - else if (pattern.at(i) == '#') - { - this->regex = this->regex + "\\d"; + if (skip == false) + response = response + text[i]; } - numchars++; + return response; } - trexp.Compile(this->regex.c_str()); - - //cout << "AA " << this->region << ": " << original << " regex: " << regex << endl; - //for (int z = 0; z < this->skipPositions.size(); z++) - // cout << "AA Skip position: " << skipPositions[z] << endl; -} - -bool RegexRule::match(string text) -{ - if (text.length() != numchars) - return false; - - return trexp.Match(text.c_str()); -} - -string RegexRule::filterSkips(string text) -{ - string response = ""; - for (int i = 0; i < text.size(); i++) - { - bool skip = false; - for (int j = 0; j < skipPositions.size(); j++) - { - if (skipPositions[j] == i) - { - skip = true; - break; - } - } - - if (skip == false) - response = response + text[i]; - } - - return response; -} +} \ No newline at end of file diff --git a/src/openalpr/postprocess.h b/src/openalpr/postprocess.h index bc55ed1..8bc98e3 100644 --- a/src/openalpr/postprocess.h +++ b/src/openalpr/postprocess.h @@ -32,96 +32,81 @@ #define SKIP_CHAR "~" -struct Letter +namespace alpr { - std::string letter; - int charposition; - float totalscore; - int occurences; -}; -struct PPResult -{ - std::string letters; - float totalscore; - bool matchesTemplate; -}; + struct Letter + { + std::string letter; + int charposition; + float totalscore; + int occurences; + }; -bool wordCompare( const PPResult &left, const PPResult &right ); -bool letterCompare( const Letter &left, const Letter &right ); - -class RegexRule -{ - public: - RegexRule(std::string region, std::string pattern); - - bool match(std::string text); - std::string filterSkips(std::string text); - - private: - int numchars; - TRexpp trexp; - std::string original; - std::string regex; - std::string region; - std::vector skipPositions; -}; - -class PostProcess -{ - public: - PostProcess(Config* config); - ~PostProcess(); - - void addLetter(std::string letter, int charposition, float score); - - void clear(); - void analyze(std::string templateregion, int topn); - - std::string bestChars; + struct PPResult + { + std::string letters; + float totalscore; bool matchesTemplate; + }; - const std::vector getResults(); + bool wordCompare( const PPResult &left, const PPResult &right ); + bool letterCompare( const Letter &left, const Letter &right ); - private: - Config* config; - //void getTopN(); - void findAllPermutations(std::vector prevletters, int charPos, int substitutionsLeft); + class RegexRule + { + public: + RegexRule(std::string region, std::string pattern); - void insertLetter(std::string letter, int charPosition, float score); + bool match(std::string text); + std::string filterSkips(std::string text); - std::map > rules; + private: + int numchars; + TRexpp trexp; + std::string original; + std::string regex; + std::string region; + std::vector skipPositions; + }; - float calculateMaxConfidenceScore(); + class PostProcess + { + public: + PostProcess(Config* config); + ~PostProcess(); - std::vector > letters; - std::vector unknownCharPositions; + void addLetter(std::string letter, int charposition, float score); - std::vector allPossibilities; + void clear(); + void analyze(std::string templateregion, int topn); - // Functions used to prune the list of letters (based on topn) to improve performance - std::vector getMaxDepth(int topn); - int getPermutationCount(std::vector depth); - int getNextLeastDrop(std::vector depth); -}; + std::string bestChars; + bool matchesTemplate; -/* -class LetterScores -{ - public: - LetterScores(int numCharPositions); + const std::vector getResults(); - void addScore(char letter, int charposition, float score); + private: + Config* config; + //void getTopN(); + void findAllPermutations(std::vector prevletters, int charPos, int substitutionsLeft); - vector getBestScore(); - float getConfidence(); + void insertLetter(std::string letter, int charPosition, float score); - private: - int numCharPositions; + std::map > rules; - vector letters; - vector charpositions; - vector scores; -}; -*/ + float calculateMaxConfidenceScore(); + + std::vector > letters; + std::vector unknownCharPositions; + + std::vector allPossibilities; + + // Functions used to prune the list of letters (based on topn) to improve performance + std::vector getMaxDepth(int topn); + int getPermutationCount(std::vector depth); + int getNextLeastDrop(std::vector depth); + }; + +} #endif // OPENALPR_POSTPROCESS_H diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 065a5ad..6ed8667 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -24,1058 +24,1061 @@ using namespace cv; using namespace std; -CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) +namespace alpr { - this->pipeline_data = pipeline_data; - this->config = pipeline_data->config; - this->confidence = 0; - - if (this->config->debugCharSegmenter) - cout << "Starting CharacterSegmenter" << endl; - - //CharacterRegion charRegion(img, debug); - - timespec startTime; - getTime(&startTime); - - if (pipeline_data->plate_inverted) - bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - pipeline_data->clearThresholds(); - pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - - // TODO: Perhaps a bilateral filter would be better here. - medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); - - if (this->config->debugCharSegmenter) - cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; - - if (pipeline_data->plate_inverted) - bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - - - if (this->config->debugCharSegmenter) + CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) { - displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); - } + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; -// if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) -// { -// Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); -// charAnalysis->bestThreshold.copyTo(img_contours); -// cvtColor(img_contours, img_contours, CV_GRAY2RGB); -// -// vector > allowedContours; -// for (uint i = 0; i < charAnalysis->bestContours.size(); i++) -// { -// if (charAnalysis->bestContours.goodIndices[i]) -// allowedContours.push_back(charAnalysis->bestContours.contours[i]); -// } -// -// drawContours(img_contours, charAnalysis->bestContours.contours, -// -1, // draw all contours -// cv::Scalar(255,0,0), // in blue -// 1); // with a thickness of 1 -// -// drawContours(img_contours, allowedContours, -// -1, // draw all contours -// cv::Scalar(0,255,0), // in green -// 1); // with a thickness of 1 -// -// -// line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); -// line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); -// -// -// Mat bordered = addLabel(img_contours, "Best Contours"); -// imgDbgGeneral.push_back(bordered); -// } + this->confidence = 0; + if (this->config->debugCharSegmenter) + cout << "Starting CharacterSegmenter" << endl; - - for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) - { - this->top = pipeline_data->textLines[lineidx].topLine; - this->bottom = pipeline_data->textLines[lineidx].bottomLine; - - float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight; - float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM; - float avgCharWidth = avgCharHeight / height_to_width_ratio; - - removeSmallContours(pipeline_data->thresholds, avgCharHeight, pipeline_data->textLines[lineidx]); - - // Do the histogram analysis to figure out char regions + //CharacterRegion charRegion(img, debug); timespec startTime; getTime(&startTime); - vector allHistograms; + if (pipeline_data->plate_inverted) + bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - vector lineBoxes; - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); + // TODO: Perhaps a bilateral filter would be better here. + medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); - fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255)); + if (this->config->debugCharSegmenter) + cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; - VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); + if (pipeline_data->plate_inverted) + bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - if (this->config->debugCharSegmenter) - { - Mat histoCopy(vertHistogram.histoImg.size(), vertHistogram.histoImg.type()); - //vertHistogram.copyTo(histoCopy); - cvtColor(vertHistogram.histoImg, histoCopy, CV_GRAY2RGB); - - string label = "threshold: " + toString(i); - allHistograms.push_back(addLabel(histoCopy, label)); - } - -// - float score = 0; - vector charBoxes = getHistogramBoxes(vertHistogram, avgCharWidth, avgCharHeight, &score); - - if (this->config->debugCharSegmenter) - { - for (uint cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++) - { - rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0)); - } - - Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 1); - displayImage(config, "Char seg histograms", histDashboard); - } - - for (uint z = 0; z < charBoxes.size(); z++) - lineBoxes.push_back(charBoxes[z]); - //drawAndWait(&histogramMask); - } - - float medianCharWidth = avgCharWidth; - vector widthValues; - // Compute largest char width - for (uint i = 0; i < lineBoxes.size(); i++) - { - widthValues.push_back(lineBoxes[i].width); - } - - medianCharWidth = median(widthValues.data(), widthValues.size()); - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], lineBoxes, medianCharWidth); if (this->config->debugCharSegmenter) { - // Setup the dashboard images to show the cleaning filters + displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); + } + + // if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) + // { + // Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); + // charAnalysis->bestThreshold.copyTo(img_contours); + // cvtColor(img_contours, img_contours, CV_GRAY2RGB); + // + // vector > allowedContours; + // for (uint i = 0; i < charAnalysis->bestContours.size(); i++) + // { + // if (charAnalysis->bestContours.goodIndices[i]) + // allowedContours.push_back(charAnalysis->bestContours.contours[i]); + // } + // + // drawContours(img_contours, charAnalysis->bestContours.contours, + // -1, // draw all contours + // cv::Scalar(255,0,0), // in blue + // 1); // with a thickness of 1 + // + // drawContours(img_contours, allowedContours, + // -1, // draw all contours + // cv::Scalar(0,255,0), // in green + // 1); // with a thickness of 1 + // + // + // line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); + // line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); + // + // + // Mat bordered = addLabel(img_contours, "Best Contours"); + // imgDbgGeneral.push_back(bordered); + // } + + + + for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) + { + this->top = pipeline_data->textLines[lineidx].topLine; + this->bottom = pipeline_data->textLines[lineidx].bottomLine; + + float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight; + float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM; + float avgCharWidth = avgCharHeight / height_to_width_ratio; + + removeSmallContours(pipeline_data->thresholds, avgCharHeight, pipeline_data->textLines[lineidx]); + + // Do the histogram analysis to figure out char regions + + timespec startTime; + getTime(&startTime); + + vector allHistograms; + + vector lineBoxes; for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - Mat cleanImg = Mat::zeros(pipeline_data->thresholds[i].size(), pipeline_data->thresholds[i].type()); - Mat boxMask = getCharBoxMask(pipeline_data->thresholds[i], candidateBoxes); - pipeline_data->thresholds[i].copyTo(cleanImg); - bitwise_and(cleanImg, boxMask, cleanImg); - cvtColor(cleanImg, cleanImg, CV_GRAY2BGR); + Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); - for (uint c = 0; c < candidateBoxes.size(); c++) - rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1); - imgDbgCleanStages.push_back(cleanImg); + fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255)); + + VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); + + if (this->config->debugCharSegmenter) + { + Mat histoCopy(vertHistogram.histoImg.size(), vertHistogram.histoImg.type()); + //vertHistogram.copyTo(histoCopy); + cvtColor(vertHistogram.histoImg, histoCopy, CV_GRAY2RGB); + + string label = "threshold: " + toString(i); + allHistograms.push_back(addLabel(histoCopy, label)); + } + + // + float score = 0; + vector charBoxes = getHistogramBoxes(vertHistogram, avgCharWidth, avgCharHeight, &score); + + if (this->config->debugCharSegmenter) + { + for (uint cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++) + { + rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0)); + } + + Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 1); + displayImage(config, "Char seg histograms", histDashboard); + } + + for (uint z = 0; z < charBoxes.size(); z++) + lineBoxes.push_back(charBoxes[z]); + //drawAndWait(&histogramMask); + } + + float medianCharWidth = avgCharWidth; + vector widthValues; + // Compute largest char width + for (uint i = 0; i < lineBoxes.size(); i++) + { + widthValues.push_back(lineBoxes[i].width); + } + + medianCharWidth = median(widthValues.data(), widthValues.size()); + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], lineBoxes, medianCharWidth); + + if (this->config->debugCharSegmenter) + { + // Setup the dashboard images to show the cleaning filters + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + Mat cleanImg = Mat::zeros(pipeline_data->thresholds[i].size(), pipeline_data->thresholds[i].type()); + Mat boxMask = getCharBoxMask(pipeline_data->thresholds[i], candidateBoxes); + pipeline_data->thresholds[i].copyTo(cleanImg); + bitwise_and(cleanImg, boxMask, cleanImg); + cvtColor(cleanImg, cleanImg, CV_GRAY2BGR); + + for (uint c = 0; c < candidateBoxes.size(); c++) + rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1); + imgDbgCleanStages.push_back(cleanImg); + } + } + + getTime(&startTime); + + filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); + candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); + candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth); + cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes); + + candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); + + for (uint cbox = 0; cbox < candidateBoxes.size(); cbox++) + pipeline_data->charRegions.push_back(candidateBoxes[cbox]); + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << " -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + if (this->config->debugCharSegmenter) + { + Mat imgDash = drawImageDashboard(pipeline_data->thresholds, CV_8U, 3); + displayImage(config, "Segmentation after cleaning", imgDash); + + Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2); + displayImage(config, "Segmentation General", generalDash); + + Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3); + displayImage(config, "Segmentation Clean Filters", cleanImgDash); } } - getTime(&startTime); - - filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth); - cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes); - - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - - for (uint cbox = 0; cbox < candidateBoxes.size(); cbox++) - pipeline_data->charRegions.push_back(candidateBoxes[cbox]); + cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); if (config->debugTiming) { timespec endTime; getTime(&endTime); - cout << " -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - if (this->config->debugCharSegmenter) - { - Mat imgDash = drawImageDashboard(pipeline_data->thresholds, CV_8U, 3); - displayImage(config, "Segmentation after cleaning", imgDash); - - Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2); - displayImage(config, "Segmentation General", generalDash); - - Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3); - displayImage(config, "Segmentation Clean Filters", cleanImgDash); + cout << "Character Segmenter Time: " << diffclock(startTime, endTime) << "ms." << endl; } } - - cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); - if (config->debugTiming) + CharacterSegmenter::~CharacterSegmenter() { - timespec endTime; - getTime(&endTime); - cout << "Character Segmenter Time: " << diffclock(startTime, endTime) << "ms." << endl; + } -} -CharacterSegmenter::~CharacterSegmenter() -{ - -} - -// Given a histogram and the horizontal line boundaries, respond with an array of boxes where the characters are -// Scores the histogram quality as well based on num chars, char volume, and even separation -vector CharacterSegmenter::getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score) -{ - float MIN_HISTOGRAM_HEIGHT = avgCharHeight * config->segmentationMinCharHeightPercent; - - float MAX_SEGMENT_WIDTH = avgCharWidth * config->segmentationMaxCharWidthvsAverage; - - //float MIN_BOX_AREA = (avgCharWidth * avgCharHeight) * 0.25; - - int pxLeniency = 2; - - vector charBoxes; - vector allBoxes = get1DHits(histogram.histoImg, pxLeniency); - - for (uint i = 0; i < allBoxes.size(); i++) + // Given a histogram and the horizontal line boundaries, respond with an array of boxes where the characters are + // Scores the histogram quality as well based on num chars, char volume, and even separation + vector CharacterSegmenter::getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score) { - if (allBoxes[i].width >= config->segmentationMinBoxWidthPx && allBoxes[i].width <= MAX_SEGMENT_WIDTH && - allBoxes[i].height > MIN_HISTOGRAM_HEIGHT ) + float MIN_HISTOGRAM_HEIGHT = avgCharHeight * config->segmentationMinCharHeightPercent; + + float MAX_SEGMENT_WIDTH = avgCharWidth * config->segmentationMaxCharWidthvsAverage; + + //float MIN_BOX_AREA = (avgCharWidth * avgCharHeight) * 0.25; + + int pxLeniency = 2; + + vector charBoxes; + vector allBoxes = get1DHits(histogram.histoImg, pxLeniency); + + for (uint i = 0; i < allBoxes.size(); i++) { - charBoxes.push_back(allBoxes[i]); - } - else if (allBoxes[i].width > avgCharWidth * 2 && allBoxes[i].width < MAX_SEGMENT_WIDTH * 2 && allBoxes[i].height > MIN_HISTOGRAM_HEIGHT) - { - // rectangle(histogram.histoImg, allBoxes[i], Scalar(255, 0, 0) ); - // drawAndWait(&histogram.histoImg); - // Try to split up doubles into two good char regions, check for a break between 40% and 60% - int leftEdge = allBoxes[i].x + (int) (((float) allBoxes[i].width) * 0.4f); - int rightEdge = allBoxes[i].x + (int) (((float) allBoxes[i].width) * 0.6f); - - int minX = histogram.getLocalMinimum(leftEdge, rightEdge); - int maxXChar1 = histogram.getLocalMaximum(allBoxes[i].x, minX); - int maxXChar2 = histogram.getLocalMaximum(minX, allBoxes[i].x + allBoxes[i].width); - int minHeight = histogram.getHeightAt(minX); - - int maxHeightChar1 = histogram.getHeightAt(maxXChar1); - int maxHeightChar2 = histogram.getHeightAt(maxXChar2); - - if (maxHeightChar1 > MIN_HISTOGRAM_HEIGHT && minHeight < (0.25 * ((float) maxHeightChar1))) + if (allBoxes[i].width >= config->segmentationMinBoxWidthPx && allBoxes[i].width <= MAX_SEGMENT_WIDTH && + allBoxes[i].height > MIN_HISTOGRAM_HEIGHT ) { - // Add a box for Char1 - Point botRight = Point(minX - 1, allBoxes[i].y + allBoxes[i].height); - charBoxes.push_back(Rect(allBoxes[i].tl(), botRight) ); + charBoxes.push_back(allBoxes[i]); } - if (maxHeightChar2 > MIN_HISTOGRAM_HEIGHT && minHeight < (0.25 * ((float) maxHeightChar2))) - { - // Add a box for Char2 - Point topLeft = Point(minX + 1, allBoxes[i].y); - charBoxes.push_back(Rect(topLeft, allBoxes[i].br()) ); - } - } - } - - return charBoxes; -} - -vector CharacterSegmenter::getBestCharBoxes(Mat img, vector charBoxes, float avgCharWidth) -{ - float MAX_SEGMENT_WIDTH = avgCharWidth * 1.65; - - // This histogram is based on how many char boxes (from ALL of the many thresholded images) are covering each column - // Makes a sort of histogram from all the previous char boxes. Figures out the best fit from that. - - Mat histoImg = Mat::zeros(Size(img.cols, img.rows), CV_8U); - - int columnCount; - - for (int col = 0; col < img.cols; col++) - { - columnCount = 0; - - for (uint i = 0; i < charBoxes.size(); i++) - { - if (col >= charBoxes[i].x && col < (charBoxes[i].x + charBoxes[i].width)) - columnCount++; - } - - // Fill the line of the histogram - for (; columnCount > 0; columnCount--) - histoImg.at(histoImg.rows - columnCount, col) = 255; - } - - VerticalHistogram histogram(histoImg, Mat::ones(histoImg.size(), CV_8U)); - - // Go through each row in the histoImg and score it. Try to find the single line that gives me the most right-sized character regions (based on avgCharWidth) - - int bestRowIndex = 0; - float bestRowScore = 0; - vector bestBoxes; - - for (int row = 0; row < histoImg.rows; row++) - { - vector validBoxes; - vector allBoxes = get1DHits(histoImg, row); - - if (this->config->debugCharSegmenter) - cout << "All Boxes size " << allBoxes.size() << endl; - - if (allBoxes.size() == 0) - break; - - float rowScore = 0; - - for (uint boxidx = 0; boxidx < allBoxes.size(); boxidx++) - { - int w = allBoxes[boxidx].width; - if (w >= config->segmentationMinBoxWidthPx && w <= MAX_SEGMENT_WIDTH) - { - float widthDiffPixels = abs(w - avgCharWidth); - float widthDiffPercent = widthDiffPixels / avgCharWidth; - rowScore += 10 * (1 - widthDiffPercent); - - if (widthDiffPercent < 0.25) // Bonus points when it's close to the average character width - rowScore += 8; - - validBoxes.push_back(allBoxes[boxidx]); - } - else if (w > avgCharWidth * 2 && w <= MAX_SEGMENT_WIDTH * 2 ) + else if (allBoxes[i].width > avgCharWidth * 2 && allBoxes[i].width < MAX_SEGMENT_WIDTH * 2 && allBoxes[i].height > MIN_HISTOGRAM_HEIGHT) { + // rectangle(histogram.histoImg, allBoxes[i], Scalar(255, 0, 0) ); + // drawAndWait(&histogram.histoImg); // Try to split up doubles into two good char regions, check for a break between 40% and 60% - int leftEdge = allBoxes[boxidx].x + (int) (((float) allBoxes[boxidx].width) * 0.4f); - int rightEdge = allBoxes[boxidx].x + (int) (((float) allBoxes[boxidx].width) * 0.6f); + int leftEdge = allBoxes[i].x + (int) (((float) allBoxes[i].width) * 0.4f); + int rightEdge = allBoxes[i].x + (int) (((float) allBoxes[i].width) * 0.6f); int minX = histogram.getLocalMinimum(leftEdge, rightEdge); - int maxXChar1 = histogram.getLocalMaximum(allBoxes[boxidx].x, minX); - int maxXChar2 = histogram.getLocalMaximum(minX, allBoxes[boxidx].x + allBoxes[boxidx].width); + int maxXChar1 = histogram.getLocalMaximum(allBoxes[i].x, minX); + int maxXChar2 = histogram.getLocalMaximum(minX, allBoxes[i].x + allBoxes[i].width); int minHeight = histogram.getHeightAt(minX); int maxHeightChar1 = histogram.getHeightAt(maxXChar1); int maxHeightChar2 = histogram.getHeightAt(maxXChar2); - if ( minHeight < (0.25 * ((float) maxHeightChar1))) + if (maxHeightChar1 > MIN_HISTOGRAM_HEIGHT && minHeight < (0.25 * ((float) maxHeightChar1))) { // Add a box for Char1 - Point botRight = Point(minX - 1, allBoxes[boxidx].y + allBoxes[boxidx].height); - validBoxes.push_back(Rect(allBoxes[boxidx].tl(), botRight) ); + Point botRight = Point(minX - 1, allBoxes[i].y + allBoxes[i].height); + charBoxes.push_back(Rect(allBoxes[i].tl(), botRight) ); } - if ( minHeight < (0.25 * ((float) maxHeightChar2))) + if (maxHeightChar2 > MIN_HISTOGRAM_HEIGHT && minHeight < (0.25 * ((float) maxHeightChar2))) { // Add a box for Char2 - Point topLeft = Point(minX + 1, allBoxes[boxidx].y); - validBoxes.push_back(Rect(topLeft, allBoxes[boxidx].br()) ); + Point topLeft = Point(minX + 1, allBoxes[i].y); + charBoxes.push_back(Rect(topLeft, allBoxes[i].br()) ); } } } - if (rowScore > bestRowScore) - { - bestRowScore = rowScore; - bestRowIndex = row; - bestBoxes = validBoxes; - } + return charBoxes; } - if (this->config->debugCharSegmenter) + vector CharacterSegmenter::getBestCharBoxes(Mat img, vector charBoxes, float avgCharWidth) { - cvtColor(histoImg, histoImg, CV_GRAY2BGR); - line(histoImg, Point(0, histoImg.rows - 1 - bestRowIndex), Point(histoImg.cols, histoImg.rows - 1 - bestRowIndex), Scalar(0, 255, 0)); + float MAX_SEGMENT_WIDTH = avgCharWidth * 1.65; - Mat imgBestBoxes(img.size(), img.type()); - img.copyTo(imgBestBoxes); - cvtColor(imgBestBoxes, imgBestBoxes, CV_GRAY2BGR); - for (uint i = 0; i < bestBoxes.size(); i++) - rectangle(imgBestBoxes, bestBoxes[i], Scalar(0, 255, 0)); + // This histogram is based on how many char boxes (from ALL of the many thresholded images) are covering each column + // Makes a sort of histogram from all the previous char boxes. Figures out the best fit from that. - this->imgDbgGeneral.push_back(addLabel(histoImg, "All Histograms")); - this->imgDbgGeneral.push_back(addLabel(imgBestBoxes, "Best Boxes")); - } + Mat histoImg = Mat::zeros(Size(img.cols, img.rows), CV_8U); - return bestBoxes; -} + int columnCount; -vector CharacterSegmenter::get1DHits(Mat img, int yOffset) -{ - vector hits; - - bool onSegment = false; - int curSegmentLength = 0; - for (int col = 0; col < img.cols; col++) - { - bool isOn = img.at(img.rows - 1 - yOffset, col); - if (isOn) + for (int col = 0; col < img.cols; col++) { - // We're on a segment. Increment the length - onSegment = true; - curSegmentLength++; - } + columnCount = 0; - if (onSegment && (isOn == false || (col == img.cols - 1))) - { - // A segment just ended or we're at the very end of the row and we're on a segment - Point topLeft = Point(col - curSegmentLength, top.getPointAt(col - curSegmentLength) - 1); - Point botRight = Point(col, bottom.getPointAt(col) + 1); - hits.push_back(Rect(topLeft, botRight)); - - onSegment = false; - curSegmentLength = 0; - } - } - - return hits; -} - -void CharacterSegmenter::removeSmallContours(vector thresholds, float avgCharHeight, TextLine textLine) -{ - //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks - const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; - - Mat textLineMask = Mat::zeros(thresholds[0].size(), CV_8U); - fillConvexPoly(textLineMask, textLine.linePolygon.data(), textLine.linePolygon.size(), Scalar(255,255,255)); - - for (uint i = 0; i < thresholds.size(); i++) - { - vector > contours; - vector hierarchy; - Mat thresholdsCopy = Mat::zeros(thresholds[i].size(), thresholds[i].type()); - - thresholds[i].copyTo(thresholdsCopy, textLineMask); - findContours(thresholdsCopy, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); - - for (uint c = 0; c < contours.size(); c++) - { - if (contours[c].size() == 0) - continue; - - Rect mr = boundingRect(contours[c]); - if (mr.height < MIN_CONTOUR_HEIGHT) + for (uint i = 0; i < charBoxes.size(); i++) { - // Erase it - drawContours(thresholds[i], contours, c, Scalar(0, 0, 0), -1); - continue; + if (col >= charBoxes[i].x && col < (charBoxes[i].x + charBoxes[i].width)) + columnCount++; } + + // Fill the line of the histogram + for (; columnCount > 0; columnCount--) + histoImg.at(histoImg.rows - columnCount, col) = 255; } - } -} -vector CharacterSegmenter::combineCloseBoxes( vector charBoxes, float biggestCharWidth) -{ - vector newCharBoxes; + VerticalHistogram histogram(histoImg, Mat::ones(histoImg.size(), CV_8U)); - for (uint i = 0; i < charBoxes.size(); i++) - { - if (i == charBoxes.size() - 1) + // Go through each row in the histoImg and score it. Try to find the single line that gives me the most right-sized character regions (based on avgCharWidth) + + int bestRowIndex = 0; + float bestRowScore = 0; + vector bestBoxes; + + for (int row = 0; row < histoImg.rows; row++) { - newCharBoxes.push_back(charBoxes[i]); - break; - } - float bigWidth = (charBoxes[i + 1].x + charBoxes[i + 1].width - charBoxes[i].x); + vector validBoxes; + vector allBoxes = get1DHits(histoImg, row); - float w1Diff = abs(charBoxes[i].width - biggestCharWidth); - float w2Diff = abs(charBoxes[i + 1].width - biggestCharWidth); - float bigDiff = abs(bigWidth - biggestCharWidth); - bigDiff *= 1.3; // Make it a little harder to merge boxes. - - if (bigDiff < w1Diff && bigDiff < w2Diff) - { - Rect bigRect(charBoxes[i].x, charBoxes[i].y, bigWidth, charBoxes[i].height); - newCharBoxes.push_back(bigRect); if (this->config->debugCharSegmenter) + cout << "All Boxes size " << allBoxes.size() << endl; + + if (allBoxes.size() == 0) + break; + + float rowScore = 0; + + for (uint boxidx = 0; boxidx < allBoxes.size(); boxidx++) { - for (uint z = 0; z < pipeline_data->thresholds.size(); z++) + int w = allBoxes[boxidx].width; + if (w >= config->segmentationMinBoxWidthPx && w <= MAX_SEGMENT_WIDTH) { - Point center(bigRect.x + bigRect.width / 2, bigRect.y + bigRect.height / 2); - RotatedRect rrect(center, Size2f(bigRect.width, bigRect.height + (bigRect.height / 2)), 0); - ellipse(imgDbgCleanStages[z], rrect, Scalar(0,255,0), 1); + float widthDiffPixels = abs(w - avgCharWidth); + float widthDiffPercent = widthDiffPixels / avgCharWidth; + rowScore += 10 * (1 - widthDiffPercent); + + if (widthDiffPercent < 0.25) // Bonus points when it's close to the average character width + rowScore += 8; + + validBoxes.push_back(allBoxes[boxidx]); } - cout << "Merging 2 boxes -- " << i << " and " << i + 1 << endl; - } - - i++; - } - else - { - newCharBoxes.push_back(charBoxes[i]); - } - } - - return newCharBoxes; -} - -void CharacterSegmenter::cleanCharRegions(vector thresholds, vector charRegions) -{ - const float MIN_SPECKLE_HEIGHT_PERCENT = 0.13; - const float MIN_SPECKLE_WIDTH_PX = 3; - const float MIN_CONTOUR_AREA_PERCENT = 0.1; - const float MIN_CONTOUR_HEIGHT_PERCENT = config->segmentationMinCharHeightPercent; - - Mat mask = getCharBoxMask(thresholds[0], charRegions); - - for (uint i = 0; i < thresholds.size(); i++) - { - bitwise_and(thresholds[i], mask, thresholds[i]); - vector > contours; - - Mat tempImg(thresholds[i].size(), thresholds[i].type()); - thresholds[i].copyTo(tempImg); - - //Mat element = getStructuringElement( 1, -// Size( 2 + 1, 2+1 ), -// Point( 1, 1 ) ); - //dilate(thresholds[i], tempImg, element); - //morphologyEx(thresholds[i], tempImg, MORPH_CLOSE, element); - //drawAndWait(&tempImg); - - findContours(tempImg, contours, RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - - for (uint j = 0; j < charRegions.size(); j++) - { - const float MIN_SPECKLE_HEIGHT = ((float)charRegions[j].height) * MIN_SPECKLE_HEIGHT_PERCENT; - const float MIN_CONTOUR_AREA = ((float)charRegions[j].area()) * MIN_CONTOUR_AREA_PERCENT; - - int tallestContourHeight = 0; - float totalArea = 0; - for (uint c = 0; c < contours.size(); c++) - { - if (contours[c].size() == 0) - continue; - if (charRegions[j].contains(contours[c][0]) == false) - continue; - - Rect r = boundingRect(contours[c]); - - if (r.height <= MIN_SPECKLE_HEIGHT || r.width <= MIN_SPECKLE_WIDTH_PX) + else if (w > avgCharWidth * 2 && w <= MAX_SEGMENT_WIDTH * 2 ) { - // Erase this speckle - drawContours(thresholds[i], contours, c, Scalar(0,0,0), CV_FILLED); + // Try to split up doubles into two good char regions, check for a break between 40% and 60% + int leftEdge = allBoxes[boxidx].x + (int) (((float) allBoxes[boxidx].width) * 0.4f); + int rightEdge = allBoxes[boxidx].x + (int) (((float) allBoxes[boxidx].width) * 0.6f); - if (this->config->debugCharSegmenter) + int minX = histogram.getLocalMinimum(leftEdge, rightEdge); + int maxXChar1 = histogram.getLocalMaximum(allBoxes[boxidx].x, minX); + int maxXChar2 = histogram.getLocalMaximum(minX, allBoxes[boxidx].x + allBoxes[boxidx].width); + int minHeight = histogram.getHeightAt(minX); + + int maxHeightChar1 = histogram.getHeightAt(maxXChar1); + int maxHeightChar2 = histogram.getHeightAt(maxXChar2); + + if ( minHeight < (0.25 * ((float) maxHeightChar1))) { - drawContours(imgDbgCleanStages[i], contours, c, COLOR_DEBUG_SPECKLES, CV_FILLED); + // Add a box for Char1 + Point botRight = Point(minX - 1, allBoxes[boxidx].y + allBoxes[boxidx].height); + validBoxes.push_back(Rect(allBoxes[boxidx].tl(), botRight) ); + } + if ( minHeight < (0.25 * ((float) maxHeightChar2))) + { + // Add a box for Char2 + Point topLeft = Point(minX + 1, allBoxes[boxidx].y); + validBoxes.push_back(Rect(topLeft, allBoxes[boxidx].br()) ); } } - else - { - if (r.height > tallestContourHeight) - tallestContourHeight = r.height; - - totalArea += contourArea(contours[c]); - } - //else if (r.height > tallestContourHeight) - //{ - // tallestContourIndex = c; - // tallestContourHeight = h; - //} } - if (totalArea < MIN_CONTOUR_AREA) + if (rowScore > bestRowScore) { - // Character is not voluminous enough. Erase it. - if (this->config->debugCharSegmenter) - { - cout << "Character CLEAN: (area) removing box " << j << " in threshold " << i << " -- Area " << totalArea << " < " << MIN_CONTOUR_AREA << endl; - - Rect boxTop(charRegions[j].x, charRegions[j].y - 10, charRegions[j].width, 10); - rectangle(imgDbgCleanStages[i], boxTop, COLOR_DEBUG_MIN_AREA, -1); - } - - rectangle(thresholds[i], charRegions[j], Scalar(0, 0, 0), -1); - } - else if (tallestContourHeight < ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT)) - { - // This character is too short. Black the whole thing out - if (this->config->debugCharSegmenter) - { - cout << "Character CLEAN: (height) removing box " << j << " in threshold " << i << " -- Height " << tallestContourHeight << " < " << ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT) << endl; - - Rect boxBottom(charRegions[j].x, charRegions[j].y + charRegions[j].height, charRegions[j].width, 10); - rectangle(imgDbgCleanStages[i], boxBottom, COLOR_DEBUG_MIN_HEIGHT, -1); - } - rectangle(thresholds[i], charRegions[j], Scalar(0, 0, 0), -1); - } - } - - Mat closureElement = getStructuringElement( 1, - Size( 2 + 1, 2+1 ), - Point( 1, 1 ) ); - - //morphologyEx(thresholds[i], thresholds[i], MORPH_OPEN, element); - - //dilate(thresholds[i], thresholds[i], element); - //erode(thresholds[i], thresholds[i], element); - - morphologyEx(thresholds[i], thresholds[i], MORPH_CLOSE, closureElement); - - // Lastly, draw a clipping line between each character boxes - for (uint j = 0; j < charRegions.size(); j++) - { - line(thresholds[i], Point(charRegions[j].x - 1, charRegions[j].y), Point(charRegions[j].x - 1, charRegions[j].y + charRegions[j].height), Scalar(0, 0, 0)); - line(thresholds[i], Point(charRegions[j].x + charRegions[j].width + 1, charRegions[j].y), Point(charRegions[j].x + charRegions[j].width + 1, charRegions[j].y + charRegions[j].height), Scalar(0, 0, 0)); - } - } -} - -void CharacterSegmenter::cleanBasedOnColor(vector thresholds, Mat colorMask, vector charRegions) -{ - // If I knock out x% of the contour area from this thing (after applying the color filter) - // Consider it a bad news bear. REmove the whole area. - const float MIN_PERCENT_CHUNK_REMOVED = 0.6; - - for (uint i = 0; i < thresholds.size(); i++) - { - for (uint j = 0; j < charRegions.size(); j++) - { - Mat boxChar = Mat::zeros(thresholds[i].size(), CV_8U); - rectangle(boxChar, charRegions[j], Scalar(255,255,255), CV_FILLED); - - bitwise_and(thresholds[i], boxChar, boxChar); - - float meanBefore = mean(boxChar, boxChar)[0]; - - Mat thresholdCopy(thresholds[i].size(), CV_8U); - bitwise_and(colorMask, boxChar, thresholdCopy); - - float meanAfter = mean(thresholdCopy, boxChar)[0]; - - if (meanAfter < meanBefore * (1-MIN_PERCENT_CHUNK_REMOVED)) - { - rectangle(thresholds[i], charRegions[j], Scalar(0,0,0), CV_FILLED); - - if (this->config->debugCharSegmenter) - { - //vector tmpDebug; - //Mat thresholdCopy2 = Mat::zeros(thresholds[i].size(), CV_8U); - //rectangle(thresholdCopy2, charRegions[j], Scalar(255,255,255), CV_FILLED); - //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask")); - //bitwise_and(thresholds[i], thresholdCopy2, thresholdCopy2); - //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask + Thresh")); - //bitwise_and(colorMask, thresholdCopy2, thresholdCopy2); - //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask + Thresh + Color")); -// -// Mat tmpytmp = addLabel(thresholdCopy2, "box Mask + Thresh + Color"); -// Mat tmpx = drawImageDashboard(tmpDebug, tmpytmp.type(), 1); - //drawAndWait( &tmpx ); - - cout << "Segmentation Filter Clean by Color: Removed Threshold " << i << " charregion " << j << endl; - cout << "Segmentation Filter Clean by Color: before=" << meanBefore << " after=" << meanAfter << endl; - - Point topLeft(charRegions[j].x, charRegions[j].y); - circle(imgDbgCleanStages[i], topLeft, 5, COLOR_DEBUG_COLORFILTER, CV_FILLED); - } - } - } - } -} - -void CharacterSegmenter::cleanMostlyFullBoxes(vector thresholds, const vector charRegions) -{ - float MAX_FILLED = 0.95 * 255; - - for (uint i = 0; i < charRegions.size(); i++) - { - Mat mask = Mat::zeros(thresholds[0].size(), CV_8U); - rectangle(mask, charRegions[i], Scalar(255,255,255), -1); - - for (uint j = 0; j < thresholds.size(); j++) - { - if (mean(thresholds[j], mask)[0] > MAX_FILLED) - { - // Empty the rect - rectangle(thresholds[j], charRegions[i], Scalar(0,0,0), -1); - - if (this->config->debugCharSegmenter) - { - Rect inset(charRegions[i].x + 2, charRegions[i].y - 2, charRegions[i].width - 4, charRegions[i].height - 4); - rectangle(imgDbgCleanStages[j], inset, COLOR_DEBUG_FULLBOX, 1); - } - } - } - } -} - -vector CharacterSegmenter::filterMostlyEmptyBoxes(vector thresholds, const vector charRegions) -{ - // Of the n thresholded images, if box 3 (for example) is empty in half (for example) of the thresholded images, - // clear all data for every box #3. - - //const float MIN_AREA_PERCENT = 0.1; - const float MIN_CONTOUR_HEIGHT_PERCENT = config->segmentationMinCharHeightPercent; - - Mat mask = getCharBoxMask(thresholds[0], charRegions); - - vector boxScores(charRegions.size()); - - for (uint i = 0; i < charRegions.size(); i++) - boxScores[i] = 0; - - for (uint i = 0; i < thresholds.size(); i++) - { - for (uint j = 0; j < charRegions.size(); j++) - { - //float minArea = charRegions[j].area() * MIN_AREA_PERCENT; - - Mat tempImg = Mat::zeros(thresholds[i].size(), thresholds[i].type()); - rectangle(tempImg, charRegions[j], Scalar(255,255,255), CV_FILLED); - bitwise_and(thresholds[i], tempImg, tempImg); - - vector > contours; - findContours(tempImg, contours, RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - - vector allPointsInBox; - for (uint c = 0; c < contours.size(); c++) - { - if (contours[c].size() == 0) - continue; - - for (uint z = 0; z < contours[c].size(); z++) - allPointsInBox.push_back(contours[c][z]); - } - - float height = 0; - if (allPointsInBox.size() > 0) - { - height = boundingRect(allPointsInBox).height; - } - - if (height >= ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT)) - { - boxScores[j] = boxScores[j] + 1; - } - else if (this->config->debugCharSegmenter) - { - drawX(imgDbgCleanStages[i], charRegions[j], COLOR_DEBUG_EMPTYFILTER, 3); - } - } - } - - vector newCharRegions; - - int maxBoxScore = 0; - for (uint i = 0; i < charRegions.size(); i++) - { - if (boxScores[i] > maxBoxScore) - maxBoxScore = boxScores[i]; - } - - // Need a good char sample in at least 50% of the boxes for it to be valid. - int MIN_FULL_BOXES = maxBoxScore * 0.49; - - // Now check each score. If it's below the minimum, remove the charRegion - for (uint i = 0; i < charRegions.size(); i++) - { - if (boxScores[i] > MIN_FULL_BOXES) - newCharRegions.push_back(charRegions[i]); - else - { - // Erase the box from the Mat... mainly for debug purposes - if (this->config->debugCharSegmenter) - { - cout << "Mostly Empty Filter: box index: " << i; - cout << " this box had a score of : " << boxScores[i];; - cout << " MIN_FULL_BOXES: " << MIN_FULL_BOXES << endl;; - - for (uint z = 0; z < thresholds.size(); z++) - { - rectangle(thresholds[z], charRegions[i], Scalar(0,0,0), -1); - - drawX(imgDbgCleanStages[z], charRegions[i], COLOR_DEBUG_EMPTYFILTER, 1); - } + bestRowScore = rowScore; + bestRowIndex = row; + bestBoxes = validBoxes; } } if (this->config->debugCharSegmenter) - cout << " Box Score: " << boxScores[i] << endl; + { + cvtColor(histoImg, histoImg, CV_GRAY2BGR); + line(histoImg, Point(0, histoImg.rows - 1 - bestRowIndex), Point(histoImg.cols, histoImg.rows - 1 - bestRowIndex), Scalar(0, 255, 0)); + + Mat imgBestBoxes(img.size(), img.type()); + img.copyTo(imgBestBoxes); + cvtColor(imgBestBoxes, imgBestBoxes, CV_GRAY2BGR); + for (uint i = 0; i < bestBoxes.size(); i++) + rectangle(imgBestBoxes, bestBoxes[i], Scalar(0, 255, 0)); + + this->imgDbgGeneral.push_back(addLabel(histoImg, "All Histograms")); + this->imgDbgGeneral.push_back(addLabel(imgBestBoxes, "Best Boxes")); + } + + return bestBoxes; } - return newCharRegions; -} - -void CharacterSegmenter::filterEdgeBoxes(vector thresholds, const vector charRegions, float avgCharWidth, float avgCharHeight) -{ - const float MIN_ANGLE_FOR_ROTATION = 0.4; - int MIN_CONNECTED_EDGE_PIXELS = (avgCharHeight * 1.5); - - // Sometimes the rectangle won't be very tall, making it impossible to detect an edge - // Adjust for this here. - int alternate = thresholds[0].rows * 0.92; - if (alternate < MIN_CONNECTED_EDGE_PIXELS && alternate > avgCharHeight) - MIN_CONNECTED_EDGE_PIXELS = alternate; - - // - // Pay special attention to the edge boxes. If it's a skinny box, and the vertical height extends above our bounds... remove it. - //while (charBoxes.size() > 0 && charBoxes[charBoxes.size() - 1].width < MIN_SEGMENT_WIDTH_EDGES) - // charBoxes.erase(charBoxes.begin() + charBoxes.size() - 1); - // Now filter the "edge" boxes. We don't want to include skinny boxes on the edges, since these could be plate boundaries - //while (charBoxes.size() > 0 && charBoxes[0].width < MIN_SEGMENT_WIDTH_EDGES) - // charBoxes.erase(charBoxes.begin() + 0); - - // TECHNIQUE #1 - // Check for long vertical lines. Once the line is too long, mask the whole region - - if (charRegions.size() <= 1) - return; - - // Check both sides to see where the edges are - // The first starts at the right edge of the leftmost char region and works its way left - // The second starts at the left edge of the rightmost char region and works its way right. - // We start by rotating the threshold image to the correct angle - // then check each column 1 by 1. - - vector leftEdges; - vector rightEdges; - - for (uint i = 0; i < thresholds.size(); i++) + vector CharacterSegmenter::get1DHits(Mat img, int yOffset) { - Mat rotated; + vector hits; - if (top.angle > MIN_ANGLE_FOR_ROTATION) + bool onSegment = false; + int curSegmentLength = 0; + for (int col = 0; col < img.cols; col++) { - // Rotate image: - rotated = Mat(thresholds[i].size(), thresholds[i].type()); - Mat rot_mat( 2, 3, CV_32FC1 ); - Point center = Point( thresholds[i].cols/2, thresholds[i].rows/2 ); - - rot_mat = getRotationMatrix2D( center, top.angle, 1.0 ); - warpAffine( thresholds[i], rotated, rot_mat, thresholds[i].size() ); - } - else - { - rotated = thresholds[i]; - } - - int leftEdgeX = 0; - int rightEdgeX = rotated.cols; - // Do the left side - int col = charRegions[0].x + charRegions[0].width; - while (col >= 0) - { - int rowLength = getLongestBlobLengthBetweenLines(rotated, col); - - if (rowLength > MIN_CONNECTED_EDGE_PIXELS) + bool isOn = img.at(img.rows - 1 - yOffset, col); + if (isOn) { - leftEdgeX = col; - break; + // We're on a segment. Increment the length + onSegment = true; + curSegmentLength++; } - col--; + if (onSegment && (isOn == false || (col == img.cols - 1))) + { + // A segment just ended or we're at the very end of the row and we're on a segment + Point topLeft = Point(col - curSegmentLength, top.getPointAt(col - curSegmentLength) - 1); + Point botRight = Point(col, bottom.getPointAt(col) + 1); + hits.push_back(Rect(topLeft, botRight)); + + onSegment = false; + curSegmentLength = 0; + } } - col = charRegions[charRegions.size() - 1].x; - while (col < rotated.cols) - { - int rowLength = getLongestBlobLengthBetweenLines(rotated, col); + return hits; + } - if (rowLength > MIN_CONNECTED_EDGE_PIXELS) + void CharacterSegmenter::removeSmallContours(vector thresholds, float avgCharHeight, TextLine textLine) + { + //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks + const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; + + Mat textLineMask = Mat::zeros(thresholds[0].size(), CV_8U); + fillConvexPoly(textLineMask, textLine.linePolygon.data(), textLine.linePolygon.size(), Scalar(255,255,255)); + + for (uint i = 0; i < thresholds.size(); i++) + { + vector > contours; + vector hierarchy; + Mat thresholdsCopy = Mat::zeros(thresholds[i].size(), thresholds[i].type()); + + thresholds[i].copyTo(thresholdsCopy, textLineMask); + findContours(thresholdsCopy, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); + + for (uint c = 0; c < contours.size(); c++) { - rightEdgeX = col; + if (contours[c].size() == 0) + continue; + + Rect mr = boundingRect(contours[c]); + if (mr.height < MIN_CONTOUR_HEIGHT) + { + // Erase it + drawContours(thresholds[i], contours, c, Scalar(0, 0, 0), -1); + continue; + } + } + } + } + + vector CharacterSegmenter::combineCloseBoxes( vector charBoxes, float biggestCharWidth) + { + vector newCharBoxes; + + for (uint i = 0; i < charBoxes.size(); i++) + { + if (i == charBoxes.size() - 1) + { + newCharBoxes.push_back(charBoxes[i]); break; } - col++; + float bigWidth = (charBoxes[i + 1].x + charBoxes[i + 1].width - charBoxes[i].x); + + float w1Diff = abs(charBoxes[i].width - biggestCharWidth); + float w2Diff = abs(charBoxes[i + 1].width - biggestCharWidth); + float bigDiff = abs(bigWidth - biggestCharWidth); + bigDiff *= 1.3; // Make it a little harder to merge boxes. + + if (bigDiff < w1Diff && bigDiff < w2Diff) + { + Rect bigRect(charBoxes[i].x, charBoxes[i].y, bigWidth, charBoxes[i].height); + newCharBoxes.push_back(bigRect); + if (this->config->debugCharSegmenter) + { + for (uint z = 0; z < pipeline_data->thresholds.size(); z++) + { + Point center(bigRect.x + bigRect.width / 2, bigRect.y + bigRect.height / 2); + RotatedRect rrect(center, Size2f(bigRect.width, bigRect.height + (bigRect.height / 2)), 0); + ellipse(imgDbgCleanStages[z], rrect, Scalar(0,255,0), 1); + } + cout << "Merging 2 boxes -- " << i << " and " << i + 1 << endl; + } + + i++; + } + else + { + newCharBoxes.push_back(charBoxes[i]); + } } - if (leftEdgeX != 0) - leftEdges.push_back(leftEdgeX); - if (rightEdgeX != thresholds[i].cols) - rightEdges.push_back(rightEdgeX); + return newCharBoxes; } - int leftEdge = 0; - int rightEdge = thresholds[0].cols; - - // Assign the edge values to the SECOND closest value - if (leftEdges.size() > 1) + void CharacterSegmenter::cleanCharRegions(vector thresholds, vector charRegions) { - sort (leftEdges.begin(), leftEdges.begin()+leftEdges.size()); - leftEdge = leftEdges[leftEdges.size() - 2] + 1; - } - if (rightEdges.size() > 1) - { - sort (rightEdges.begin(), rightEdges.begin()+rightEdges.size()); - rightEdge = rightEdges[1] - 1; - } + const float MIN_SPECKLE_HEIGHT_PERCENT = 0.13; + const float MIN_SPECKLE_WIDTH_PX = 3; + const float MIN_CONTOUR_AREA_PERCENT = 0.1; + const float MIN_CONTOUR_HEIGHT_PERCENT = config->segmentationMinCharHeightPercent; - if (leftEdge != 0 || rightEdge != thresholds[0].cols) - { - Mat mask = Mat::zeros(thresholds[0].size(), CV_8U); - rectangle(mask, Point(leftEdge, 0), Point(rightEdge, thresholds[0].rows), Scalar(255,255,255), -1); - - if (top.angle > MIN_ANGLE_FOR_ROTATION) - { - // Rotate mask: - Mat rot_mat( 2, 3, CV_32FC1 ); - Point center = Point( mask.cols/2, mask.rows/2 ); - - rot_mat = getRotationMatrix2D( center, top.angle * -1, 1.0 ); - warpAffine( mask, mask, rot_mat, mask.size() ); - } - - // If our edge mask covers more than x% of the char region, mask the whole thing... - const float MAX_COVERAGE_PERCENT = 0.6; - int leftCoveragePx = leftEdge - charRegions[0].x; - float leftCoveragePercent = ((float) leftCoveragePx) / ((float) charRegions[0].width); - float rightCoveragePx = (charRegions[charRegions.size() -1].x + charRegions[charRegions.size() -1].width) - rightEdge; - float rightCoveragePercent = ((float) rightCoveragePx) / ((float) charRegions[charRegions.size() -1].width); - if ((leftCoveragePercent > MAX_COVERAGE_PERCENT) || - (charRegions[0].width - leftCoveragePx < config->segmentationMinBoxWidthPx)) - { - rectangle(mask, charRegions[0], Scalar(0,0,0), -1); // Mask the whole region - if (this->config->debugCharSegmenter) - cout << "Edge Filter: Entire left region is erased" << endl; - } - if ((rightCoveragePercent > MAX_COVERAGE_PERCENT) || - (charRegions[charRegions.size() -1].width - rightCoveragePx < config->segmentationMinBoxWidthPx)) - { - rectangle(mask, charRegions[charRegions.size() -1], Scalar(0,0,0), -1); - if (this->config->debugCharSegmenter) - cout << "Edge Filter: Entire right region is erased" << endl; - } + Mat mask = getCharBoxMask(thresholds[0], charRegions); for (uint i = 0; i < thresholds.size(); i++) { bitwise_and(thresholds[i], mask, thresholds[i]); - } + vector > contours; - if (this->config->debugCharSegmenter) - { - cout << "Edge Filter: left=" << leftEdge << " right=" << rightEdge << endl; - Mat bordered = addLabel(mask, "Edge Filter #1"); - imgDbgGeneral.push_back(bordered); + Mat tempImg(thresholds[i].size(), thresholds[i].type()); + thresholds[i].copyTo(tempImg); - Mat invertedMask(mask.size(), mask.type()); - bitwise_not(mask, invertedMask); - for (uint z = 0; z < imgDbgCleanStages.size(); z++) - fillMask(imgDbgCleanStages[z], invertedMask, Scalar(0,0,255)); - } - } + //Mat element = getStructuringElement( 1, + // Size( 2 + 1, 2+1 ), + // Point( 1, 1 ) ); + //dilate(thresholds[i], tempImg, element); + //morphologyEx(thresholds[i], tempImg, MORPH_CLOSE, element); + //drawAndWait(&tempImg); -} + findContours(tempImg, contours, RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); -int CharacterSegmenter::getLongestBlobLengthBetweenLines(Mat img, int col) -{ - int longestBlobLength = 0; - - bool onSegment = false; - bool wasbetweenLines = false; - float curSegmentLength = 0; - for (int row = 0; row < img.rows; row++) - { - bool isbetweenLines = false; - - bool isOn = img.at(row, col); - // check two rows at a time. - if (!isOn && col < img.cols) - isOn = img.at(row, col); - - if (isOn) - { - // We're on a segment. Increment the length - isbetweenLines = top.isPointBelowLine(Point(col, row)) && !bottom.isPointBelowLine(Point(col, row)); - float incrementBy = 1; - - // Add a little extra to the score if this is outside of the lines - if (!isbetweenLines) - incrementBy = 1.1; - - onSegment = true; - curSegmentLength += incrementBy; - } - if (isOn && isbetweenLines) - { - wasbetweenLines = true; - } - - if (onSegment && (isOn == false || (row == img.rows - 1))) - { - if (wasbetweenLines && curSegmentLength > longestBlobLength) - longestBlobLength = curSegmentLength; - - onSegment = false; - isbetweenLines = false; - curSegmentLength = 0; - } - } - - return longestBlobLength; -} - -// Checks to see if a skinny, tall line (extending above or below the char Height) is inside the given box. -// Returns the contour index if true. -1 otherwise -int CharacterSegmenter::isSkinnyLineInsideBox(Mat threshold, Rect box, vector > contours, vector hierarchy, float avgCharWidth, float avgCharHeight) -{ - float MIN_EDGE_CONTOUR_HEIGHT = avgCharHeight * 1.25; - - // Sometimes the threshold is smaller than the MIN_EDGE_CONTOUR_HEIGHT. - // In that case, adjust to be smaller - int alternate = threshold.rows * 0.92; - if (alternate < MIN_EDGE_CONTOUR_HEIGHT && alternate > avgCharHeight) - MIN_EDGE_CONTOUR_HEIGHT = alternate; - - Rect slightlySmallerBox(box.x, box.y, box.width, box.height); - Mat boxMask = Mat::zeros(threshold.size(), CV_8U); - rectangle(boxMask, slightlySmallerBox, Scalar(255, 255, 255), -1); - - for (uint i = 0; i < contours.size(); i++) - { - // Only bother with the big boxes - if (boundingRect(contours[i]).height < MIN_EDGE_CONTOUR_HEIGHT) - continue; - - Mat tempImg = Mat::zeros(threshold.size(), CV_8U); - drawContours(tempImg, contours, i, Scalar(255,255,255), -1, 8, hierarchy, 1); - bitwise_and(tempImg, boxMask, tempImg); - - vector > subContours; - findContours(tempImg, subContours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - int tallestContourIdx = -1; - int tallestContourHeight = 0; - int tallestContourWidth = 0; - float tallestContourArea = 0; - for (uint s = 0; s < subContours.size(); s++) - { - Rect r = boundingRect(subContours[s]); - if (r.height > tallestContourHeight) + for (uint j = 0; j < charRegions.size(); j++) { - tallestContourIdx = s; - tallestContourHeight = r.height; - tallestContourWidth = r.width; - tallestContourArea = contourArea(subContours[s]); + const float MIN_SPECKLE_HEIGHT = ((float)charRegions[j].height) * MIN_SPECKLE_HEIGHT_PERCENT; + const float MIN_CONTOUR_AREA = ((float)charRegions[j].area()) * MIN_CONTOUR_AREA_PERCENT; + + int tallestContourHeight = 0; + float totalArea = 0; + for (uint c = 0; c < contours.size(); c++) + { + if (contours[c].size() == 0) + continue; + if (charRegions[j].contains(contours[c][0]) == false) + continue; + + Rect r = boundingRect(contours[c]); + + if (r.height <= MIN_SPECKLE_HEIGHT || r.width <= MIN_SPECKLE_WIDTH_PX) + { + // Erase this speckle + drawContours(thresholds[i], contours, c, Scalar(0,0,0), CV_FILLED); + + if (this->config->debugCharSegmenter) + { + drawContours(imgDbgCleanStages[i], contours, c, COLOR_DEBUG_SPECKLES, CV_FILLED); + } + } + else + { + if (r.height > tallestContourHeight) + tallestContourHeight = r.height; + + totalArea += contourArea(contours[c]); + } + //else if (r.height > tallestContourHeight) + //{ + // tallestContourIndex = c; + // tallestContourHeight = h; + //} + } + + if (totalArea < MIN_CONTOUR_AREA) + { + // Character is not voluminous enough. Erase it. + if (this->config->debugCharSegmenter) + { + cout << "Character CLEAN: (area) removing box " << j << " in threshold " << i << " -- Area " << totalArea << " < " << MIN_CONTOUR_AREA << endl; + + Rect boxTop(charRegions[j].x, charRegions[j].y - 10, charRegions[j].width, 10); + rectangle(imgDbgCleanStages[i], boxTop, COLOR_DEBUG_MIN_AREA, -1); + } + + rectangle(thresholds[i], charRegions[j], Scalar(0, 0, 0), -1); + } + else if (tallestContourHeight < ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT)) + { + // This character is too short. Black the whole thing out + if (this->config->debugCharSegmenter) + { + cout << "Character CLEAN: (height) removing box " << j << " in threshold " << i << " -- Height " << tallestContourHeight << " < " << ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT) << endl; + + Rect boxBottom(charRegions[j].x, charRegions[j].y + charRegions[j].height, charRegions[j].width, 10); + rectangle(imgDbgCleanStages[i], boxBottom, COLOR_DEBUG_MIN_HEIGHT, -1); + } + rectangle(thresholds[i], charRegions[j], Scalar(0, 0, 0), -1); + } } - } - if (tallestContourIdx != -1) - { - //cout << "Edge Filter: " << tallestContourHeight << " -- " << avgCharHeight << endl; - if (tallestContourHeight >= avgCharHeight * 0.9 && - ((tallestContourWidth < config->segmentationMinBoxWidthPx) || (tallestContourArea < avgCharWidth * avgCharHeight * 0.1))) + Mat closureElement = getStructuringElement( 1, + Size( 2 + 1, 2+1 ), + Point( 1, 1 ) ); + + //morphologyEx(thresholds[i], thresholds[i], MORPH_OPEN, element); + + //dilate(thresholds[i], thresholds[i], element); + //erode(thresholds[i], thresholds[i], element); + + morphologyEx(thresholds[i], thresholds[i], MORPH_CLOSE, closureElement); + + // Lastly, draw a clipping line between each character boxes + for (uint j = 0; j < charRegions.size(); j++) { - cout << "Edge Filter: Avg contour width: " << avgCharWidth << " This guy is: " << tallestContourWidth << endl; - cout << "Edge Filter: tallestContourArea: " << tallestContourArea << " Minimum: " << avgCharWidth * avgCharHeight * 0.1 << endl; - return i; + line(thresholds[i], Point(charRegions[j].x - 1, charRegions[j].y), Point(charRegions[j].x - 1, charRegions[j].y + charRegions[j].height), Scalar(0, 0, 0)); + line(thresholds[i], Point(charRegions[j].x + charRegions[j].width + 1, charRegions[j].y), Point(charRegions[j].x + charRegions[j].width + 1, charRegions[j].y + charRegions[j].height), Scalar(0, 0, 0)); } } } - return -1; + void CharacterSegmenter::cleanBasedOnColor(vector thresholds, Mat colorMask, vector charRegions) + { + // If I knock out x% of the contour area from this thing (after applying the color filter) + // Consider it a bad news bear. REmove the whole area. + const float MIN_PERCENT_CHUNK_REMOVED = 0.6; + + for (uint i = 0; i < thresholds.size(); i++) + { + for (uint j = 0; j < charRegions.size(); j++) + { + Mat boxChar = Mat::zeros(thresholds[i].size(), CV_8U); + rectangle(boxChar, charRegions[j], Scalar(255,255,255), CV_FILLED); + + bitwise_and(thresholds[i], boxChar, boxChar); + + float meanBefore = mean(boxChar, boxChar)[0]; + + Mat thresholdCopy(thresholds[i].size(), CV_8U); + bitwise_and(colorMask, boxChar, thresholdCopy); + + float meanAfter = mean(thresholdCopy, boxChar)[0]; + + if (meanAfter < meanBefore * (1-MIN_PERCENT_CHUNK_REMOVED)) + { + rectangle(thresholds[i], charRegions[j], Scalar(0,0,0), CV_FILLED); + + if (this->config->debugCharSegmenter) + { + //vector tmpDebug; + //Mat thresholdCopy2 = Mat::zeros(thresholds[i].size(), CV_8U); + //rectangle(thresholdCopy2, charRegions[j], Scalar(255,255,255), CV_FILLED); + //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask")); + //bitwise_and(thresholds[i], thresholdCopy2, thresholdCopy2); + //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask + Thresh")); + //bitwise_and(colorMask, thresholdCopy2, thresholdCopy2); + //tmpDebug.push_back(addLabel(thresholdCopy2, "box Mask + Thresh + Color")); + // + // Mat tmpytmp = addLabel(thresholdCopy2, "box Mask + Thresh + Color"); + // Mat tmpx = drawImageDashboard(tmpDebug, tmpytmp.type(), 1); + //drawAndWait( &tmpx ); + + cout << "Segmentation Filter Clean by Color: Removed Threshold " << i << " charregion " << j << endl; + cout << "Segmentation Filter Clean by Color: before=" << meanBefore << " after=" << meanAfter << endl; + + Point topLeft(charRegions[j].x, charRegions[j].y); + circle(imgDbgCleanStages[i], topLeft, 5, COLOR_DEBUG_COLORFILTER, CV_FILLED); + } + } + } + } + } + + void CharacterSegmenter::cleanMostlyFullBoxes(vector thresholds, const vector charRegions) + { + float MAX_FILLED = 0.95 * 255; + + for (uint i = 0; i < charRegions.size(); i++) + { + Mat mask = Mat::zeros(thresholds[0].size(), CV_8U); + rectangle(mask, charRegions[i], Scalar(255,255,255), -1); + + for (uint j = 0; j < thresholds.size(); j++) + { + if (mean(thresholds[j], mask)[0] > MAX_FILLED) + { + // Empty the rect + rectangle(thresholds[j], charRegions[i], Scalar(0,0,0), -1); + + if (this->config->debugCharSegmenter) + { + Rect inset(charRegions[i].x + 2, charRegions[i].y - 2, charRegions[i].width - 4, charRegions[i].height - 4); + rectangle(imgDbgCleanStages[j], inset, COLOR_DEBUG_FULLBOX, 1); + } + } + } + } + } + + vector CharacterSegmenter::filterMostlyEmptyBoxes(vector thresholds, const vector charRegions) + { + // Of the n thresholded images, if box 3 (for example) is empty in half (for example) of the thresholded images, + // clear all data for every box #3. + + //const float MIN_AREA_PERCENT = 0.1; + const float MIN_CONTOUR_HEIGHT_PERCENT = config->segmentationMinCharHeightPercent; + + Mat mask = getCharBoxMask(thresholds[0], charRegions); + + vector boxScores(charRegions.size()); + + for (uint i = 0; i < charRegions.size(); i++) + boxScores[i] = 0; + + for (uint i = 0; i < thresholds.size(); i++) + { + for (uint j = 0; j < charRegions.size(); j++) + { + //float minArea = charRegions[j].area() * MIN_AREA_PERCENT; + + Mat tempImg = Mat::zeros(thresholds[i].size(), thresholds[i].type()); + rectangle(tempImg, charRegions[j], Scalar(255,255,255), CV_FILLED); + bitwise_and(thresholds[i], tempImg, tempImg); + + vector > contours; + findContours(tempImg, contours, RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + + vector allPointsInBox; + for (uint c = 0; c < contours.size(); c++) + { + if (contours[c].size() == 0) + continue; + + for (uint z = 0; z < contours[c].size(); z++) + allPointsInBox.push_back(contours[c][z]); + } + + float height = 0; + if (allPointsInBox.size() > 0) + { + height = boundingRect(allPointsInBox).height; + } + + if (height >= ((float) charRegions[j].height * MIN_CONTOUR_HEIGHT_PERCENT)) + { + boxScores[j] = boxScores[j] + 1; + } + else if (this->config->debugCharSegmenter) + { + drawX(imgDbgCleanStages[i], charRegions[j], COLOR_DEBUG_EMPTYFILTER, 3); + } + } + } + + vector newCharRegions; + + int maxBoxScore = 0; + for (uint i = 0; i < charRegions.size(); i++) + { + if (boxScores[i] > maxBoxScore) + maxBoxScore = boxScores[i]; + } + + // Need a good char sample in at least 50% of the boxes for it to be valid. + int MIN_FULL_BOXES = maxBoxScore * 0.49; + + // Now check each score. If it's below the minimum, remove the charRegion + for (uint i = 0; i < charRegions.size(); i++) + { + if (boxScores[i] > MIN_FULL_BOXES) + newCharRegions.push_back(charRegions[i]); + else + { + // Erase the box from the Mat... mainly for debug purposes + if (this->config->debugCharSegmenter) + { + cout << "Mostly Empty Filter: box index: " << i; + cout << " this box had a score of : " << boxScores[i];; + cout << " MIN_FULL_BOXES: " << MIN_FULL_BOXES << endl;; + + for (uint z = 0; z < thresholds.size(); z++) + { + rectangle(thresholds[z], charRegions[i], Scalar(0,0,0), -1); + + drawX(imgDbgCleanStages[z], charRegions[i], COLOR_DEBUG_EMPTYFILTER, 1); + } + } + } + + if (this->config->debugCharSegmenter) + cout << " Box Score: " << boxScores[i] << endl; + } + + return newCharRegions; + } + + void CharacterSegmenter::filterEdgeBoxes(vector thresholds, const vector charRegions, float avgCharWidth, float avgCharHeight) + { + const float MIN_ANGLE_FOR_ROTATION = 0.4; + int MIN_CONNECTED_EDGE_PIXELS = (avgCharHeight * 1.5); + + // Sometimes the rectangle won't be very tall, making it impossible to detect an edge + // Adjust for this here. + int alternate = thresholds[0].rows * 0.92; + if (alternate < MIN_CONNECTED_EDGE_PIXELS && alternate > avgCharHeight) + MIN_CONNECTED_EDGE_PIXELS = alternate; + + // + // Pay special attention to the edge boxes. If it's a skinny box, and the vertical height extends above our bounds... remove it. + //while (charBoxes.size() > 0 && charBoxes[charBoxes.size() - 1].width < MIN_SEGMENT_WIDTH_EDGES) + // charBoxes.erase(charBoxes.begin() + charBoxes.size() - 1); + // Now filter the "edge" boxes. We don't want to include skinny boxes on the edges, since these could be plate boundaries + //while (charBoxes.size() > 0 && charBoxes[0].width < MIN_SEGMENT_WIDTH_EDGES) + // charBoxes.erase(charBoxes.begin() + 0); + + // TECHNIQUE #1 + // Check for long vertical lines. Once the line is too long, mask the whole region + + if (charRegions.size() <= 1) + return; + + // Check both sides to see where the edges are + // The first starts at the right edge of the leftmost char region and works its way left + // The second starts at the left edge of the rightmost char region and works its way right. + // We start by rotating the threshold image to the correct angle + // then check each column 1 by 1. + + vector leftEdges; + vector rightEdges; + + for (uint i = 0; i < thresholds.size(); i++) + { + Mat rotated; + + if (top.angle > MIN_ANGLE_FOR_ROTATION) + { + // Rotate image: + rotated = Mat(thresholds[i].size(), thresholds[i].type()); + Mat rot_mat( 2, 3, CV_32FC1 ); + Point center = Point( thresholds[i].cols/2, thresholds[i].rows/2 ); + + rot_mat = getRotationMatrix2D( center, top.angle, 1.0 ); + warpAffine( thresholds[i], rotated, rot_mat, thresholds[i].size() ); + } + else + { + rotated = thresholds[i]; + } + + int leftEdgeX = 0; + int rightEdgeX = rotated.cols; + // Do the left side + int col = charRegions[0].x + charRegions[0].width; + while (col >= 0) + { + int rowLength = getLongestBlobLengthBetweenLines(rotated, col); + + if (rowLength > MIN_CONNECTED_EDGE_PIXELS) + { + leftEdgeX = col; + break; + } + + col--; + } + + col = charRegions[charRegions.size() - 1].x; + while (col < rotated.cols) + { + int rowLength = getLongestBlobLengthBetweenLines(rotated, col); + + if (rowLength > MIN_CONNECTED_EDGE_PIXELS) + { + rightEdgeX = col; + break; + } + col++; + } + + if (leftEdgeX != 0) + leftEdges.push_back(leftEdgeX); + if (rightEdgeX != thresholds[i].cols) + rightEdges.push_back(rightEdgeX); + } + + int leftEdge = 0; + int rightEdge = thresholds[0].cols; + + // Assign the edge values to the SECOND closest value + if (leftEdges.size() > 1) + { + sort (leftEdges.begin(), leftEdges.begin()+leftEdges.size()); + leftEdge = leftEdges[leftEdges.size() - 2] + 1; + } + if (rightEdges.size() > 1) + { + sort (rightEdges.begin(), rightEdges.begin()+rightEdges.size()); + rightEdge = rightEdges[1] - 1; + } + + if (leftEdge != 0 || rightEdge != thresholds[0].cols) + { + Mat mask = Mat::zeros(thresholds[0].size(), CV_8U); + rectangle(mask, Point(leftEdge, 0), Point(rightEdge, thresholds[0].rows), Scalar(255,255,255), -1); + + if (top.angle > MIN_ANGLE_FOR_ROTATION) + { + // Rotate mask: + Mat rot_mat( 2, 3, CV_32FC1 ); + Point center = Point( mask.cols/2, mask.rows/2 ); + + rot_mat = getRotationMatrix2D( center, top.angle * -1, 1.0 ); + warpAffine( mask, mask, rot_mat, mask.size() ); + } + + // If our edge mask covers more than x% of the char region, mask the whole thing... + const float MAX_COVERAGE_PERCENT = 0.6; + int leftCoveragePx = leftEdge - charRegions[0].x; + float leftCoveragePercent = ((float) leftCoveragePx) / ((float) charRegions[0].width); + float rightCoveragePx = (charRegions[charRegions.size() -1].x + charRegions[charRegions.size() -1].width) - rightEdge; + float rightCoveragePercent = ((float) rightCoveragePx) / ((float) charRegions[charRegions.size() -1].width); + if ((leftCoveragePercent > MAX_COVERAGE_PERCENT) || + (charRegions[0].width - leftCoveragePx < config->segmentationMinBoxWidthPx)) + { + rectangle(mask, charRegions[0], Scalar(0,0,0), -1); // Mask the whole region + if (this->config->debugCharSegmenter) + cout << "Edge Filter: Entire left region is erased" << endl; + } + if ((rightCoveragePercent > MAX_COVERAGE_PERCENT) || + (charRegions[charRegions.size() -1].width - rightCoveragePx < config->segmentationMinBoxWidthPx)) + { + rectangle(mask, charRegions[charRegions.size() -1], Scalar(0,0,0), -1); + if (this->config->debugCharSegmenter) + cout << "Edge Filter: Entire right region is erased" << endl; + } + + for (uint i = 0; i < thresholds.size(); i++) + { + bitwise_and(thresholds[i], mask, thresholds[i]); + } + + if (this->config->debugCharSegmenter) + { + cout << "Edge Filter: left=" << leftEdge << " right=" << rightEdge << endl; + Mat bordered = addLabel(mask, "Edge Filter #1"); + imgDbgGeneral.push_back(bordered); + + Mat invertedMask(mask.size(), mask.type()); + bitwise_not(mask, invertedMask); + for (uint z = 0; z < imgDbgCleanStages.size(); z++) + fillMask(imgDbgCleanStages[z], invertedMask, Scalar(0,0,255)); + } + } + + } + + int CharacterSegmenter::getLongestBlobLengthBetweenLines(Mat img, int col) + { + int longestBlobLength = 0; + + bool onSegment = false; + bool wasbetweenLines = false; + float curSegmentLength = 0; + for (int row = 0; row < img.rows; row++) + { + bool isbetweenLines = false; + + bool isOn = img.at(row, col); + // check two rows at a time. + if (!isOn && col < img.cols) + isOn = img.at(row, col); + + if (isOn) + { + // We're on a segment. Increment the length + isbetweenLines = top.isPointBelowLine(Point(col, row)) && !bottom.isPointBelowLine(Point(col, row)); + float incrementBy = 1; + + // Add a little extra to the score if this is outside of the lines + if (!isbetweenLines) + incrementBy = 1.1; + + onSegment = true; + curSegmentLength += incrementBy; + } + if (isOn && isbetweenLines) + { + wasbetweenLines = true; + } + + if (onSegment && (isOn == false || (row == img.rows - 1))) + { + if (wasbetweenLines && curSegmentLength > longestBlobLength) + longestBlobLength = curSegmentLength; + + onSegment = false; + isbetweenLines = false; + curSegmentLength = 0; + } + } + + return longestBlobLength; + } + + // Checks to see if a skinny, tall line (extending above or below the char Height) is inside the given box. + // Returns the contour index if true. -1 otherwise + int CharacterSegmenter::isSkinnyLineInsideBox(Mat threshold, Rect box, vector > contours, vector hierarchy, float avgCharWidth, float avgCharHeight) + { + float MIN_EDGE_CONTOUR_HEIGHT = avgCharHeight * 1.25; + + // Sometimes the threshold is smaller than the MIN_EDGE_CONTOUR_HEIGHT. + // In that case, adjust to be smaller + int alternate = threshold.rows * 0.92; + if (alternate < MIN_EDGE_CONTOUR_HEIGHT && alternate > avgCharHeight) + MIN_EDGE_CONTOUR_HEIGHT = alternate; + + Rect slightlySmallerBox(box.x, box.y, box.width, box.height); + Mat boxMask = Mat::zeros(threshold.size(), CV_8U); + rectangle(boxMask, slightlySmallerBox, Scalar(255, 255, 255), -1); + + for (uint i = 0; i < contours.size(); i++) + { + // Only bother with the big boxes + if (boundingRect(contours[i]).height < MIN_EDGE_CONTOUR_HEIGHT) + continue; + + Mat tempImg = Mat::zeros(threshold.size(), CV_8U); + drawContours(tempImg, contours, i, Scalar(255,255,255), -1, 8, hierarchy, 1); + bitwise_and(tempImg, boxMask, tempImg); + + vector > subContours; + findContours(tempImg, subContours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + int tallestContourIdx = -1; + int tallestContourHeight = 0; + int tallestContourWidth = 0; + float tallestContourArea = 0; + for (uint s = 0; s < subContours.size(); s++) + { + Rect r = boundingRect(subContours[s]); + if (r.height > tallestContourHeight) + { + tallestContourIdx = s; + tallestContourHeight = r.height; + tallestContourWidth = r.width; + tallestContourArea = contourArea(subContours[s]); + } + } + + if (tallestContourIdx != -1) + { + //cout << "Edge Filter: " << tallestContourHeight << " -- " << avgCharHeight << endl; + if (tallestContourHeight >= avgCharHeight * 0.9 && + ((tallestContourWidth < config->segmentationMinBoxWidthPx) || (tallestContourArea < avgCharWidth * avgCharHeight * 0.1))) + { + cout << "Edge Filter: Avg contour width: " << avgCharWidth << " This guy is: " << tallestContourWidth << endl; + cout << "Edge Filter: tallestContourArea: " << tallestContourArea << " Minimum: " << avgCharWidth * avgCharHeight * 0.1 << endl; + return i; + } + } + } + + return -1; + } + + Mat CharacterSegmenter::getCharBoxMask(Mat img_threshold, vector charBoxes) + { + Mat mask = Mat::zeros(img_threshold.size(), CV_8U); + for (uint i = 0; i < charBoxes.size(); i++) + rectangle(mask, charBoxes[i], Scalar(255, 255, 255), -1); + + return mask; + } + } - -Mat CharacterSegmenter::getCharBoxMask(Mat img_threshold, vector charBoxes) -{ - Mat mask = Mat::zeros(img_threshold.size(), CV_8U); - for (uint i = 0; i < charBoxes.size(); i++) - rectangle(mask, charBoxes[i], Scalar(255, 255, 255), -1); - - return mask; -} - - diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 54b9c22..831c643 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -29,58 +29,61 @@ #include "textdetection/textcontours.h" #include "pipeline_data.h" - -//const float MIN_BOX_WIDTH_PX = 4; // 4 pixels - -const cv::Scalar COLOR_DEBUG_EDGE(0,0,255); // Red -const cv::Scalar COLOR_DEBUG_SPECKLES(0,0,255); // Red -const cv::Scalar COLOR_DEBUG_MIN_HEIGHT(255,0,0); // Blue -const cv::Scalar COLOR_DEBUG_MIN_AREA(255,0,0); // Blue -const cv::Scalar COLOR_DEBUG_FULLBOX(255,255,0); // Blue-green -const cv::Scalar COLOR_DEBUG_COLORFILTER(255,0,255); // Magenta -const cv::Scalar COLOR_DEBUG_EMPTYFILTER(0,255,255); // Yellow - -class CharacterSegmenter +namespace alpr { - public: - CharacterSegmenter(PipelineData* pipeline_data); - virtual ~CharacterSegmenter(); - int confidence; + const cv::Scalar COLOR_DEBUG_EDGE(0,0,255); // Red + const cv::Scalar COLOR_DEBUG_SPECKLES(0,0,255); // Red + const cv::Scalar COLOR_DEBUG_MIN_HEIGHT(255,0,0); // Blue + const cv::Scalar COLOR_DEBUG_MIN_AREA(255,0,0); // Blue + const cv::Scalar COLOR_DEBUG_FULLBOX(255,255,0); // Blue-green + const cv::Scalar COLOR_DEBUG_COLORFILTER(255,0,255); // Magenta + const cv::Scalar COLOR_DEBUG_EMPTYFILTER(0,255,255); // Yellow + + class CharacterSegmenter + { + + public: + CharacterSegmenter(PipelineData* pipeline_data); + virtual ~CharacterSegmenter(); + + int confidence; - private: - Config* config; - PipelineData* pipeline_data; - + private: + Config* config; + PipelineData* pipeline_data; - LineSegment top; - LineSegment bottom; - std::vector imgDbgGeneral; - std::vector imgDbgCleanStages; + LineSegment top; + LineSegment bottom; - cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); + std::vector imgDbgGeneral; + std::vector imgDbgCleanStages; - void removeSmallContours(std::vector thresholds, float avgCharHeight, TextLine textLine); + cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); - std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); - std::vector getBestCharBoxes(cv::Mat img, std::vector charBoxes, float avgCharWidth); - std::vector combineCloseBoxes( std::vector charBoxes, float avgCharWidth); + void removeSmallContours(std::vector thresholds, float avgCharHeight, TextLine textLine); - std::vector get1DHits(cv::Mat img, int yOffset); + std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); + std::vector getBestCharBoxes(cv::Mat img, std::vector charBoxes, float avgCharWidth); + std::vector combineCloseBoxes( std::vector charBoxes, float avgCharWidth); - void cleanCharRegions(std::vector thresholds, std::vector charRegions); - void cleanBasedOnColor(std::vector thresholds, cv::Mat colorMask, std::vector charRegions); - void cleanMostlyFullBoxes(std::vector thresholds, const std::vector charRegions); - std::vector filterMostlyEmptyBoxes(std::vector thresholds, const std::vector charRegions); - void filterEdgeBoxes(std::vector thresholds, const std::vector charRegions, float avgCharWidth, float avgCharHeight); + std::vector get1DHits(cv::Mat img, int yOffset); - int getLongestBlobLengthBetweenLines(cv::Mat img, int col); + void cleanCharRegions(std::vector thresholds, std::vector charRegions); + void cleanBasedOnColor(std::vector thresholds, cv::Mat colorMask, std::vector charRegions); + void cleanMostlyFullBoxes(std::vector thresholds, const std::vector charRegions); + std::vector filterMostlyEmptyBoxes(std::vector thresholds, const std::vector charRegions); + void filterEdgeBoxes(std::vector thresholds, const std::vector charRegions, float avgCharWidth, float avgCharHeight); - int isSkinnyLineInsideBox(cv::Mat threshold, cv::Rect box, std::vector > contours, std::vector hierarchy, float avgCharWidth, float avgCharHeight); + int getLongestBlobLengthBetweenLines(cv::Mat img, int col); -}; + int isSkinnyLineInsideBox(cv::Mat threshold, cv::Rect box, std::vector > contours, std::vector hierarchy, float avgCharWidth, float avgCharHeight); + + }; + +} #endif // OPENALPR_CHARACTERSEGMENTER_H diff --git a/src/openalpr/segmentation/segment.cpp b/src/openalpr/segmentation/segment.cpp index 2c5bb95..8c088d1 100644 --- a/src/openalpr/segmentation/segment.cpp +++ b/src/openalpr/segmentation/segment.cpp @@ -19,32 +19,36 @@ #include "segment.h" -Segment::Segment(cv::Rect newSegment) -{ - this->segment = newSegment; -} - -Segment::~Segment() +namespace alpr { -} + Segment::Segment(cv::Rect newSegment) + { + this->segment = newSegment; + } -bool Segment::matches(cv::Rect newSegment) -{ - // Compare the two segments with a given leniency - const float WIDTH_LENIENCY_MIN = 0.25; - const float WIDTH_LENIENCY_MAX = 0.20; - - float left_min = segment.x - (((float)segment.width) * WIDTH_LENIENCY_MIN); - float left_max = segment.x + (((float)segment.width) * WIDTH_LENIENCY_MAX); - float right_min = (segment.x + segment.width) - (((float)segment.width) * WIDTH_LENIENCY_MIN); - float right_max = (segment.x + segment.width) + (((float)segment.width) * WIDTH_LENIENCY_MAX); - - int newSegRight = newSegment.x + newSegment.width; - if (newSegment.x >= left_min && newSegment.x <= left_max && - newSegRight >= right_min && newSegRight <= right_max) - return true; - - return false; -} + Segment::~Segment() + { + } + + bool Segment::matches(cv::Rect newSegment) + { + // Compare the two segments with a given leniency + const float WIDTH_LENIENCY_MIN = 0.25; + const float WIDTH_LENIENCY_MAX = 0.20; + + float left_min = segment.x - (((float)segment.width) * WIDTH_LENIENCY_MIN); + float left_max = segment.x + (((float)segment.width) * WIDTH_LENIENCY_MAX); + float right_min = (segment.x + segment.width) - (((float)segment.width) * WIDTH_LENIENCY_MIN); + float right_max = (segment.x + segment.width) + (((float)segment.width) * WIDTH_LENIENCY_MAX); + + int newSegRight = newSegment.x + newSegment.width; + if (newSegment.x >= left_min && newSegment.x <= left_max && + newSegRight >= right_min && newSegRight <= right_max) + return true; + + return false; + } + +} \ No newline at end of file diff --git a/src/openalpr/segmentation/segment.h b/src/openalpr/segmentation/segment.h index c07f9a5..8fa939c 100644 --- a/src/openalpr/segmentation/segment.h +++ b/src/openalpr/segmentation/segment.h @@ -25,17 +25,22 @@ #include "opencv2/imgproc/imgproc.hpp" -class Segment -{ +namespace alpr + { - public: - Segment(cv::Rect newSegment); - virtual ~Segment(); + class Segment + { - cv::Rect segment; - - bool matches(cv::Rect newSegment); - -}; + public: + Segment(cv::Rect newSegment); + virtual ~Segment(); + + cv::Rect segment; + + bool matches(cv::Rect newSegment); + + }; + +} #endif // OPENALPR_SEGMENTATIONGROUP_H diff --git a/src/openalpr/segmentation/segmentationgroup.cpp b/src/openalpr/segmentation/segmentationgroup.cpp index 117f094..4a541ca 100644 --- a/src/openalpr/segmentation/segmentationgroup.cpp +++ b/src/openalpr/segmentation/segmentationgroup.cpp @@ -19,31 +19,36 @@ #include "segmentationgroup.h" -SegmentationGroup::SegmentationGroup() -{ - -} - -SegmentationGroup::~SegmentationGroup() +namespace alpr { -} - -void SegmentationGroup::add(int segmentID) -{ - this->segmentIDs.push_back(segmentID); -} - -bool SegmentationGroup::equals(SegmentationGroup otherGroup) -{ - if (segmentIDs.size() != otherGroup.segmentIDs.size()) - return false; - - for (int i = 0; i < segmentIDs.size(); i++) + SegmentationGroup::SegmentationGroup() { - if (otherGroup.segmentIDs[i] != segmentIDs[i]) + + } + + SegmentationGroup::~SegmentationGroup() + { + + } + + void SegmentationGroup::add(int segmentID) + { + this->segmentIDs.push_back(segmentID); + } + + bool SegmentationGroup::equals(SegmentationGroup otherGroup) + { + if (segmentIDs.size() != otherGroup.segmentIDs.size()) return false; + + for (int i = 0; i < segmentIDs.size(); i++) + { + if (otherGroup.segmentIDs[i] != segmentIDs[i]) + return false; + } + + return true; } - return true; } \ No newline at end of file diff --git a/src/openalpr/segmentation/segmentationgroup.h b/src/openalpr/segmentation/segmentationgroup.h index cd12e73..40d5762 100644 --- a/src/openalpr/segmentation/segmentationgroup.h +++ b/src/openalpr/segmentation/segmentationgroup.h @@ -27,24 +27,28 @@ #include "segment.h" - -class SegmentationGroup +namespace alpr { - public: - SegmentationGroup(); - virtual ~SegmentationGroup(); + class SegmentationGroup + { - void add(int segmentID); - - std::vector segmentIDs; - - bool equals(SegmentationGroup otherGroup); - + public: + SegmentationGroup(); + virtual ~SegmentationGroup(); - private: - float strength; // Debuggin purposes -- how many threshold segmentations match this one perfectly - -}; + void add(int segmentID); + + std::vector segmentIDs; + + bool equals(SegmentationGroup otherGroup); + + + private: + float strength; // Debuggin purposes -- how many threshold segmentations match this one perfectly + + }; + +} #endif // OPENALPR_SEGMENTATIONGROUP_H diff --git a/src/openalpr/segmentation/verticalhistogram.cpp b/src/openalpr/segmentation/verticalhistogram.cpp index 2ba9733..b480acb 100644 --- a/src/openalpr/segmentation/verticalhistogram.cpp +++ b/src/openalpr/segmentation/verticalhistogram.cpp @@ -22,160 +22,165 @@ using namespace cv; using namespace std; -VerticalHistogram::VerticalHistogram(Mat inputImage, Mat mask) +namespace alpr { - analyzeImage(inputImage, mask); -} -VerticalHistogram::~VerticalHistogram() -{ - histoImg.release(); - colHeights.clear(); -} - -void VerticalHistogram::analyzeImage(Mat inputImage, Mat mask) -{ - highestPeak = 0; - lowestValley = inputImage.rows; - - histoImg = Mat::zeros(inputImage.size(), CV_8U); - - int columnCount; - - for (int col = 0; col < inputImage.cols; col++) + VerticalHistogram::VerticalHistogram(Mat inputImage, Mat mask) { - columnCount = 0; - - for (int row = 0; row < inputImage.rows; row++) - { - if (inputImage.at(row, col) > 0 && mask.at(row, col) > 0) - columnCount++; - } - - this->colHeights.push_back(columnCount); - - if (columnCount < lowestValley) - lowestValley = columnCount; - if (columnCount > highestPeak) - highestPeak = columnCount; - - for (; columnCount > 0; columnCount--) - histoImg.at(inputImage.rows - columnCount, col) = 255; + analyzeImage(inputImage, mask); } -} -int VerticalHistogram::getLocalMinimum(int leftX, int rightX) -{ - int minimum = histoImg.rows + 1; - int lowestX = leftX; - - for (int i = leftX; i <= rightX; i++) + VerticalHistogram::~VerticalHistogram() { - if (colHeights[i] < minimum) + histoImg.release(); + colHeights.clear(); + } + + void VerticalHistogram::analyzeImage(Mat inputImage, Mat mask) + { + highestPeak = 0; + lowestValley = inputImage.rows; + + histoImg = Mat::zeros(inputImage.size(), CV_8U); + + int columnCount; + + for (int col = 0; col < inputImage.cols; col++) { - lowestX = i; - minimum = colHeights[i]; + columnCount = 0; + + for (int row = 0; row < inputImage.rows; row++) + { + if (inputImage.at(row, col) > 0 && mask.at(row, col) > 0) + columnCount++; + } + + this->colHeights.push_back(columnCount); + + if (columnCount < lowestValley) + lowestValley = columnCount; + if (columnCount > highestPeak) + highestPeak = columnCount; + + for (; columnCount > 0; columnCount--) + histoImg.at(inputImage.rows - columnCount, col) = 255; } } - return lowestX; -} - -int VerticalHistogram::getLocalMaximum(int leftX, int rightX) -{ - int maximum = -1; - int highestX = leftX; - - for (int i = leftX; i <= rightX; i++) + int VerticalHistogram::getLocalMinimum(int leftX, int rightX) { - if (colHeights[i] > maximum) + int minimum = histoImg.rows + 1; + int lowestX = leftX; + + for (int i = leftX; i <= rightX; i++) { - highestX = i; - maximum = colHeights[i]; + if (colHeights[i] < minimum) + { + lowestX = i; + minimum = colHeights[i]; + } + } + + return lowestX; + } + + int VerticalHistogram::getLocalMaximum(int leftX, int rightX) + { + int maximum = -1; + int highestX = leftX; + + for (int i = leftX; i <= rightX; i++) + { + if (colHeights[i] > maximum) + { + highestX = i; + maximum = colHeights[i]; + } + } + + return highestX; + } + + int VerticalHistogram::getHeightAt(int x) + { + return colHeights[x]; + } + + void VerticalHistogram::findValleys() + { + //int MINIMUM_PEAK_HEIGHT = (int) (((float) highestPeak) * 0.75); + + int totalWidth = colHeights.size(); + + int midpoint = ((highestPeak - lowestValley) / 2) + lowestValley; + + HistogramDirection prevDirection = FALLING; + + int relativePeakHeight = 0; + //int valleyStart = 0; + + for (int i = 0; i < totalWidth; i++) + { + bool aboveMidpoint = (colHeights[i] >= midpoint); + + if (aboveMidpoint) + { + if (colHeights[i] > relativePeakHeight) + relativePeakHeight = colHeights[i]; + + prevDirection = FLAT; + } + else + { + relativePeakHeight = 0; + + HistogramDirection direction = getHistogramDirection(i); + + if ((prevDirection == FALLING || prevDirection == FLAT) && direction == RISING) + { + } + else if ((prevDirection == FALLING || prevDirection == FLAT) && direction == RISING) + { + } + } } } - return highestX; -} - -int VerticalHistogram::getHeightAt(int x) -{ - return colHeights[x]; -} - -void VerticalHistogram::findValleys() -{ - //int MINIMUM_PEAK_HEIGHT = (int) (((float) highestPeak) * 0.75); - - int totalWidth = colHeights.size(); - - int midpoint = ((highestPeak - lowestValley) / 2) + lowestValley; - - HistogramDirection prevDirection = FALLING; - - int relativePeakHeight = 0; - //int valleyStart = 0; - - for (int i = 0; i < totalWidth; i++) + HistogramDirection VerticalHistogram::getHistogramDirection(uint index) { - bool aboveMidpoint = (colHeights[i] >= midpoint); + int EXTRA_WIDTH_TO_AVERAGE = 2; - if (aboveMidpoint) + float trailingAverage = 0; + float forwardAverage = 0; + + int trailStartIndex = index - EXTRA_WIDTH_TO_AVERAGE; + if (trailStartIndex < 0) + trailStartIndex = 0; + uint forwardEndIndex = index + EXTRA_WIDTH_TO_AVERAGE; + if (forwardEndIndex >= colHeights.size()) + forwardEndIndex = colHeights.size() - 1; + + for (int i = index; i >= trailStartIndex; i--) { - if (colHeights[i] > relativePeakHeight) - relativePeakHeight = colHeights[i]; - - prevDirection = FLAT; + trailingAverage += colHeights[i]; } + trailingAverage = trailingAverage / ((float) (1 + index - trailStartIndex)); + + for (uint i = index; i <= forwardEndIndex; i++) + { + forwardAverage += colHeights[i]; + } + forwardAverage = forwardAverage / ((float) (1 + forwardEndIndex - index)); + + float diff = forwardAverage - trailingAverage; + float minDiff = ((float) (highestPeak - lowestValley)) * 0.10; + + if (diff > minDiff) + return RISING; + else if (diff < minDiff) + return FALLING; else - { - relativePeakHeight = 0; - - HistogramDirection direction = getHistogramDirection(i); - - if ((prevDirection == FALLING || prevDirection == FLAT) && direction == RISING) - { - } - else if ((prevDirection == FALLING || prevDirection == FLAT) && direction == RISING) - { - } - } + return FLAT; } -} -HistogramDirection VerticalHistogram::getHistogramDirection(uint index) -{ - int EXTRA_WIDTH_TO_AVERAGE = 2; - - float trailingAverage = 0; - float forwardAverage = 0; - - int trailStartIndex = index - EXTRA_WIDTH_TO_AVERAGE; - if (trailStartIndex < 0) - trailStartIndex = 0; - uint forwardEndIndex = index + EXTRA_WIDTH_TO_AVERAGE; - if (forwardEndIndex >= colHeights.size()) - forwardEndIndex = colHeights.size() - 1; - - for (int i = index; i >= trailStartIndex; i--) - { - trailingAverage += colHeights[i]; - } - trailingAverage = trailingAverage / ((float) (1 + index - trailStartIndex)); - - for (uint i = index; i <= forwardEndIndex; i++) - { - forwardAverage += colHeights[i]; - } - forwardAverage = forwardAverage / ((float) (1 + forwardEndIndex - index)); - - float diff = forwardAverage - trailingAverage; - float minDiff = ((float) (highestPeak - lowestValley)) * 0.10; - - if (diff > minDiff) - return RISING; - else if (diff < minDiff) - return FALLING; - else - return FLAT; -} +} \ No newline at end of file diff --git a/src/openalpr/segmentation/verticalhistogram.h b/src/openalpr/segmentation/verticalhistogram.h index 92e554e..8eea36e 100644 --- a/src/openalpr/segmentation/verticalhistogram.h +++ b/src/openalpr/segmentation/verticalhistogram.h @@ -22,43 +22,47 @@ #include "opencv2/imgproc/imgproc.hpp" - -struct Valley -{ - int startIndex; - int endIndex; - int width; - int pixelsWithin; -}; - -enum HistogramDirection { RISING, FALLING, FLAT }; - -class VerticalHistogram +namespace alpr { - public: - VerticalHistogram(cv::Mat inputImage, cv::Mat mask); - virtual ~VerticalHistogram(); + struct Valley + { + int startIndex; + int endIndex; + int width; + int pixelsWithin; + }; - cv::Mat histoImg; + enum HistogramDirection { RISING, FALLING, FLAT }; - // Returns the lowest X position between two points. - int getLocalMinimum(int leftX, int rightX); - // Returns the highest X position between two points. - int getLocalMaximum(int leftX, int rightX); + class VerticalHistogram + { - int getHeightAt(int x); + public: + VerticalHistogram(cv::Mat inputImage, cv::Mat mask); + virtual ~VerticalHistogram(); - private: - std::vector colHeights; - int highestPeak; - int lowestValley; - std::vector valleys; + cv::Mat histoImg; - void analyzeImage(cv::Mat inputImage, cv::Mat mask); - void findValleys(); + // Returns the lowest X position between two points. + int getLocalMinimum(int leftX, int rightX); + // Returns the highest X position between two points. + int getLocalMaximum(int leftX, int rightX); - HistogramDirection getHistogramDirection(uint index); -}; + int getHeightAt(int x); + private: + std::vector colHeights; + int highestPeak; + int lowestValley; + std::vector valleys; + + void analyzeImage(cv::Mat inputImage, cv::Mat mask); + void findValleys(); + + HistogramDirection getHistogramDirection(uint index); + }; + +} + #endif // OPENALPR_VERTICALHISTOGRAM_H diff --git a/src/openalpr/stateidentifier.cpp b/src/openalpr/stateidentifier.cpp index f9aee36..d7977ef 100644 --- a/src/openalpr/stateidentifier.cpp +++ b/src/openalpr/stateidentifier.cpp @@ -23,67 +23,72 @@ using namespace cv; using namespace std; -StateIdentifier::StateIdentifier(Config* config) +namespace alpr { - this->config = config; - featureMatcher = new FeatureMatcher(config); - - if (featureMatcher->isLoaded() == false) + StateIdentifier::StateIdentifier(Config* config) { - cout << "Can not create detector or descriptor extractor or descriptor matcher of given types" << endl; - return; + this->config = config; + + featureMatcher = new FeatureMatcher(config); + + if (featureMatcher->isLoaded() == false) + { + cout << "Can not create detector or descriptor extractor or descriptor matcher of given types" << endl; + return; + } + + featureMatcher->loadRecognitionSet(config->country); } - featureMatcher->loadRecognitionSet(config->country); -} - -StateIdentifier::~StateIdentifier() -{ - delete featureMatcher; -} - - -// Attempts to recognize the plate. Returns a confidence level. Updates the region code and confidence -// If region is found, returns true. -bool StateIdentifier::recognize(PipelineData* pipeline_data) -{ - timespec startTime; - getTime(&startTime); - - Mat plateImg = Mat(pipeline_data->grayImg, pipeline_data->regionOfInterest); - - resize(plateImg, plateImg, getSizeMaintainingAspect(plateImg, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); - - - Mat debugImg(plateImg.size(), plateImg.type()); - plateImg.copyTo(debugImg); - vector matchesArray(featureMatcher->numTrainingElements()); - - RecognitionResult result = featureMatcher->recognize(plateImg, true, &debugImg, true, matchesArray ); - - if (this->config->debugStateId) + StateIdentifier::~StateIdentifier() { - displayImage(config, "State Identifier1", plateImg); - displayImage(config, "State Identifier", debugImg); - cout << result.haswinner << " : " << result.confidence << " : " << result.winner << endl; + delete featureMatcher; } - if (config->debugTiming) + + // Attempts to recognize the plate. Returns a confidence level. Updates the region code and confidence + // If region is found, returns true. + bool StateIdentifier::recognize(PipelineData* pipeline_data) { - timespec endTime; - getTime(&endTime); - cout << "State Identification Time: " << diffclock(startTime, endTime) << "ms." << endl; + timespec startTime; + getTime(&startTime); + + Mat plateImg = Mat(pipeline_data->grayImg, pipeline_data->regionOfInterest); + + resize(plateImg, plateImg, getSizeMaintainingAspect(plateImg, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); + + + Mat debugImg(plateImg.size(), plateImg.type()); + plateImg.copyTo(debugImg); + vector matchesArray(featureMatcher->numTrainingElements()); + + RecognitionResult result = featureMatcher->recognize(plateImg, true, &debugImg, true, matchesArray ); + + if (this->config->debugStateId) + { + displayImage(config, "State Identifier1", plateImg); + displayImage(config, "State Identifier", debugImg); + cout << result.haswinner << " : " << result.confidence << " : " << result.winner << endl; + } + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "State Identification Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + if (result.haswinner == false) + return 0; + + pipeline_data->region_code = result.winner; + pipeline_data->region_confidence = result.confidence; + + if (result.confidence >= 10) + return true; + + return false; } - if (result.haswinner == false) - return 0; - - pipeline_data->region_code = result.winner; - pipeline_data->region_confidence = result.confidence; - - if (result.confidence >= 10) - return true; - - return false; -} +} \ No newline at end of file diff --git a/src/openalpr/stateidentifier.h b/src/openalpr/stateidentifier.h index a42474e..cb7185d 100644 --- a/src/openalpr/stateidentifier.h +++ b/src/openalpr/stateidentifier.h @@ -27,24 +27,28 @@ #include "config.h" #include "pipeline_data.h" -class StateIdentifier +namespace alpr { - public: - StateIdentifier(Config* config); - virtual ~StateIdentifier(); + class StateIdentifier + { - bool recognize(PipelineData* pipeline_data); + public: + StateIdentifier(Config* config); + virtual ~StateIdentifier(); - //int confidence; + bool recognize(PipelineData* pipeline_data); - protected: - Config* config; + //int confidence; - private: + protected: + Config* config; - FeatureMatcher* featureMatcher; + private: -}; + FeatureMatcher* featureMatcher; + }; + +} #endif // OPENALPR_STATEIDENTIFIER_H diff --git a/src/openalpr/support/filesystem.cpp b/src/openalpr/support/filesystem.cpp index 84c2f2f..8431220 100644 --- a/src/openalpr/support/filesystem.cpp +++ b/src/openalpr/support/filesystem.cpp @@ -1,109 +1,114 @@ #include "filesystem.h" -bool startsWith(std::string const &fullString, std::string const &prefix) +namespace alpr { - if(fullString.substr(0, prefix.size()).compare(prefix) == 0) { - return true; - } - - return false; -} -bool hasEnding (std::string const &fullString, std::string const &ending) -{ - if (fullString.length() >= ending.length()) - { - return (0 == fullString.compare (fullString.length() - ending.length(), ending.length(), ending)); - } - else + bool startsWith(std::string const &fullString, std::string const &prefix) { + if(fullString.substr(0, prefix.size()).compare(prefix) == 0) { + return true; + } + return false; } -} -bool hasEndingInsensitive(const std::string& fullString, const std::string& ending) -{ - if (fullString.length() < ending.length()) - return false; - - int startidx = fullString.length() - ending.length(); - - for (unsigned int i = startidx; i < fullString.length(); ++i) - if (tolower(fullString[i]) != tolower(ending[i - startidx])) - return false; - return true; -} - -bool DirectoryExists( const char* pzPath ) -{ - if ( pzPath == NULL) return false; - - DIR *pDir; - bool bExists = false; - - pDir = opendir (pzPath); - - if (pDir != NULL) + bool hasEnding (std::string const &fullString, std::string const &ending) { - bExists = true; - (void) closedir (pDir); - } - - return bExists; -} - -bool fileExists( const char* pzPath ) -{ - if (pzPath == NULL) return false; - - bool fExists = false; - std::ifstream f(pzPath); - fExists = f.is_open(); - f.close(); - return fExists; -} - -std::vector getFilesInDir(const char* dirPath) -{ - DIR *dir; - - std::vector files; - - struct dirent *ent; - if ((dir = opendir (dirPath)) != NULL) - { - /* print all the files and directories within directory */ - while ((ent = readdir (dir)) != NULL) + if (fullString.length() >= ending.length()) { - if (strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0) - files.push_back(ent->d_name); + return (0 == fullString.compare (fullString.length() - ending.length(), ending.length(), ending)); + } + else + { + return false; } - closedir (dir); } - else + + bool hasEndingInsensitive(const std::string& fullString, const std::string& ending) { - /* could not open directory */ - perror (""); + if (fullString.length() < ending.length()) + return false; + + int startidx = fullString.length() - ending.length(); + + for (unsigned int i = startidx; i < fullString.length(); ++i) + if (tolower(fullString[i]) != tolower(ending[i - startidx])) + return false; + return true; + } + + bool DirectoryExists( const char* pzPath ) + { + if ( pzPath == NULL) return false; + + DIR *pDir; + bool bExists = false; + + pDir = opendir (pzPath); + + if (pDir != NULL) + { + bExists = true; + (void) closedir (pDir); + } + + return bExists; + } + + bool fileExists( const char* pzPath ) + { + if (pzPath == NULL) return false; + + bool fExists = false; + std::ifstream f(pzPath); + fExists = f.is_open(); + f.close(); + return fExists; + } + + std::vector getFilesInDir(const char* dirPath) + { + DIR *dir; + + std::vector files; + + struct dirent *ent; + if ((dir = opendir (dirPath)) != NULL) + { + /* print all the files and directories within directory */ + while ((ent = readdir (dir)) != NULL) + { + if (strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0) + files.push_back(ent->d_name); + } + closedir (dir); + } + else + { + /* could not open directory */ + perror (""); + return files; + } + return files; } - return files; -} - -bool stringCompare( const std::string &left, const std::string &right ) -{ - for( std::string::const_iterator lit = left.begin(), rit = right.begin(); lit != left.end() && rit != right.end(); ++lit, ++rit ) - if( tolower( *lit ) < tolower( *rit ) ) + bool stringCompare( const std::string &left, const std::string &right ) + { + for( std::string::const_iterator lit = left.begin(), rit = right.begin(); lit != left.end() && rit != right.end(); ++lit, ++rit ) + if( tolower( *lit ) < tolower( *rit ) ) + return true; + else if( tolower( *lit ) > tolower( *rit ) ) + return false; + if( left.size() < right.size() ) return true; - else if( tolower( *lit ) > tolower( *rit ) ) - return false; - if( left.size() < right.size() ) - return true; - return false; -} + return false; + } -std::string filenameWithoutExtension(std::string filename) -{ - int lastindex = filename.find_last_of("."); - return filename.substr(0, lastindex); + std::string filenameWithoutExtension(std::string filename) + { + int lastindex = filename.find_last_of("."); + return filename.substr(0, lastindex); + } + } \ No newline at end of file diff --git a/src/openalpr/support/filesystem.h b/src/openalpr/support/filesystem.h index 8f358ee..5b4e214 100644 --- a/src/openalpr/support/filesystem.h +++ b/src/openalpr/support/filesystem.h @@ -17,16 +17,21 @@ #include #include -bool startsWith(std::string const &fullString, std::string const &prefix); -bool hasEnding (std::string const &fullString, std::string const &ending); -bool hasEndingInsensitive(const std::string& fullString, const std::string& ending); +namespace alpr +{ -std::string filenameWithoutExtension(std::string filename); + bool startsWith(std::string const &fullString, std::string const &prefix); + bool hasEnding (std::string const &fullString, std::string const &ending); + bool hasEndingInsensitive(const std::string& fullString, const std::string& ending); -bool DirectoryExists( const char* pzPath ); -bool fileExists( const char* pzPath ); -std::vector getFilesInDir(const char* dirPath); + std::string filenameWithoutExtension(std::string filename); -bool stringCompare( const std::string &left, const std::string &right ); + bool DirectoryExists( const char* pzPath ); + bool fileExists( const char* pzPath ); + std::vector getFilesInDir(const char* dirPath); + + bool stringCompare( const std::string &left, const std::string &right ); + +} #endif // FILESYSTEM_H diff --git a/src/openalpr/support/platform.cpp b/src/openalpr/support/platform.cpp index 72a26c5..185e257 100644 --- a/src/openalpr/support/platform.cpp +++ b/src/openalpr/support/platform.cpp @@ -1,45 +1,50 @@ #include "platform.h" -void sleep_ms(int sleepMs) +namespace alpr { - #ifdef WINDOWS - Sleep(sleepMs); - #else - usleep(sleepMs * 1000); // usleep takes sleep time in us (1 millionth of a second) - #endif -} -std::string getExeDir() -{ - #ifdef WINDOWS - TCHAR szEXEPath[2048]; - std::stringstream ss; - GetModuleFileName(NULL, szEXEPath, 2048); - ss << szEXEPath; + void sleep_ms(int sleepMs) + { + #ifdef WINDOWS + Sleep(sleepMs); + #else + usleep(sleepMs * 1000); // usleep takes sleep time in us (1 millionth of a second) + #endif + } - std::string exeFile = ss.str(); - std::string directory; - const size_t last_slash_idx = exeFile.rfind('\\'); - if (std::string::npos != last_slash_idx) - { - directory = exeFile.substr(0, last_slash_idx); - } - return directory; - #else - char buffer[2048]; - - readlink("/proc/self/exe", buffer, 2048); - - std::stringstream ss; - ss << buffer; - std::string exeFile = ss.str(); - std::string directory; - - const size_t last_slash_idx = exeFile.rfind('/'); - if (std::string::npos != last_slash_idx) - { - directory = exeFile.substr(0, last_slash_idx); - } - return directory; - #endif + std::string getExeDir() + { + #ifdef WINDOWS + TCHAR szEXEPath[2048]; + std::stringstream ss; + GetModuleFileName(NULL, szEXEPath, 2048); + ss << szEXEPath; + + std::string exeFile = ss.str(); + std::string directory; + const size_t last_slash_idx = exeFile.rfind('\\'); + if (std::string::npos != last_slash_idx) + { + directory = exeFile.substr(0, last_slash_idx); + } + return directory; + #else + char buffer[2048]; + + readlink("/proc/self/exe", buffer, 2048); + + std::stringstream ss; + ss << buffer; + std::string exeFile = ss.str(); + std::string directory; + + const size_t last_slash_idx = exeFile.rfind('/'); + if (std::string::npos != last_slash_idx) + { + directory = exeFile.substr(0, last_slash_idx); + } + return directory; + #endif + } + } \ No newline at end of file diff --git a/src/openalpr/support/platform.h b/src/openalpr/support/platform.h index 4912f0c..07b15a4 100644 --- a/src/openalpr/support/platform.h +++ b/src/openalpr/support/platform.h @@ -10,11 +10,13 @@ #include #endif +namespace alpr +{ + void sleep_ms(int sleepMs); + std::string getExeDir(); -void sleep_ms(int sleepMs); - -std::string getExeDir(); +} #endif //OPENALPR_PLATFORM_H diff --git a/src/openalpr/support/timing.cpp b/src/openalpr/support/timing.cpp index 6b16997..f585076 100644 --- a/src/openalpr/support/timing.cpp +++ b/src/openalpr/support/timing.cpp @@ -1,159 +1,164 @@ #include "timing.h" -timespec diff(timespec start, timespec end); - -#ifdef WINDOWS - -// Windows timing code -LARGE_INTEGER getFILETIMEoffset() +namespace alpr { - SYSTEMTIME s; - FILETIME f; - LARGE_INTEGER t; - s.wYear = 1970; - s.wMonth = 1; - s.wDay = 1; - s.wHour = 0; - s.wMinute = 0; - s.wSecond = 0; - s.wMilliseconds = 0; - SystemTimeToFileTime(&s, &f); - t.QuadPart = f.dwHighDateTime; - t.QuadPart <<= 32; - t.QuadPart |= f.dwLowDateTime; - return (t); -} + timespec diff(timespec start, timespec end); -int clock_gettime(int X, timespec *tv) -{ - LARGE_INTEGER t; - FILETIME f; - double microseconds; - static LARGE_INTEGER offset; - static double frequencyToMicroseconds; - static int initialized = 0; - static BOOL usePerformanceCounter = 0; + #ifdef WINDOWS - if (!initialized) + // Windows timing code + LARGE_INTEGER getFILETIMEoffset() { - LARGE_INTEGER performanceFrequency; - initialized = 1; - usePerformanceCounter = QueryPerformanceFrequency(&performanceFrequency); - if (usePerformanceCounter) - { - QueryPerformanceCounter(&offset); - frequencyToMicroseconds = (double)performanceFrequency.QuadPart / 1000000.; - } - else - { - offset = getFILETIMEoffset(); - frequencyToMicroseconds = 10.; - } - } - if (usePerformanceCounter) QueryPerformanceCounter(&t); - else - { - GetSystemTimeAsFileTime(&f); + SYSTEMTIME s; + FILETIME f; + LARGE_INTEGER t; + + s.wYear = 1970; + s.wMonth = 1; + s.wDay = 1; + s.wHour = 0; + s.wMinute = 0; + s.wSecond = 0; + s.wMilliseconds = 0; + SystemTimeToFileTime(&s, &f); t.QuadPart = f.dwHighDateTime; t.QuadPart <<= 32; t.QuadPart |= f.dwLowDateTime; + return (t); } - t.QuadPart -= offset.QuadPart; - microseconds = (double)t.QuadPart / frequencyToMicroseconds; - t.QuadPart = microseconds; - tv->tv_sec = t.QuadPart / 1000000; - tv->tv_usec = t.QuadPart % 1000000; - return (0); -} - -void getTime(timespec* time) -{ - clock_gettime(0, time); -} - -double diffclock(timespec time1,timespec time2) -{ - timespec delta = diff(time1,time2); - double milliseconds = (delta.tv_sec * 1000) + (((double) delta.tv_usec) / 1000.0); - - return milliseconds; -} - -timespec diff(timespec start, timespec end) -{ - timespec temp; - if ((end.tv_usec-start.tv_usec)<0) + int clock_gettime(int X, timespec *tv) { - temp.tv_sec = end.tv_sec-start.tv_sec-1; - temp.tv_usec = 1000000+end.tv_usec-start.tv_usec; + LARGE_INTEGER t; + FILETIME f; + double microseconds; + static LARGE_INTEGER offset; + static double frequencyToMicroseconds; + static int initialized = 0; + static BOOL usePerformanceCounter = 0; + + if (!initialized) + { + LARGE_INTEGER performanceFrequency; + initialized = 1; + usePerformanceCounter = QueryPerformanceFrequency(&performanceFrequency); + if (usePerformanceCounter) + { + QueryPerformanceCounter(&offset); + frequencyToMicroseconds = (double)performanceFrequency.QuadPart / 1000000.; + } + else + { + offset = getFILETIMEoffset(); + frequencyToMicroseconds = 10.; + } + } + if (usePerformanceCounter) QueryPerformanceCounter(&t); + else + { + GetSystemTimeAsFileTime(&f); + t.QuadPart = f.dwHighDateTime; + t.QuadPart <<= 32; + t.QuadPart |= f.dwLowDateTime; + } + + t.QuadPart -= offset.QuadPart; + microseconds = (double)t.QuadPart / frequencyToMicroseconds; + t.QuadPart = microseconds; + tv->tv_sec = t.QuadPart / 1000000; + tv->tv_usec = t.QuadPart % 1000000; + return (0); } - else + + void getTime(timespec* time) { - temp.tv_sec = end.tv_sec-start.tv_sec; - temp.tv_usec = end.tv_usec-start.tv_usec; + clock_gettime(0, time); } - return temp; -} - -long getEpochTime() -{ - return std::time(0) * 1000; -} - -#else - -void getTime(timespec* time) -{ -#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time - clock_serv_t cclock; - mach_timespec_t mts; - host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); - clock_get_time(cclock, &mts); - mach_port_deallocate(mach_task_self(), cclock); - time->tv_sec = mts.tv_sec; - time->tv_nsec = mts.tv_nsec; -#else - clock_gettime(CLOCK_MONOTONIC, time); -#endif -} - -double diffclock(timespec time1,timespec time2) -{ - timespec delta = diff(time1,time2); - double milliseconds = (delta.tv_sec * 1000) + (((double) delta.tv_nsec) / 1000000.0); - - return milliseconds; -} - -timespec diff(timespec start, timespec end) -{ - timespec temp; - if ((end.tv_nsec-start.tv_nsec)<0) + double diffclock(timespec time1,timespec time2) { - temp.tv_sec = end.tv_sec-start.tv_sec-1; - temp.tv_nsec = 1000000000+end.tv_nsec-start.tv_nsec; + timespec delta = diff(time1,time2); + double milliseconds = (delta.tv_sec * 1000) + (((double) delta.tv_usec) / 1000.0); + + return milliseconds; } - else + + timespec diff(timespec start, timespec end) { - temp.tv_sec = end.tv_sec-start.tv_sec; - temp.tv_nsec = end.tv_nsec-start.tv_nsec; + timespec temp; + if ((end.tv_usec-start.tv_usec)<0) + { + temp.tv_sec = end.tv_sec-start.tv_sec-1; + temp.tv_usec = 1000000+end.tv_usec-start.tv_usec; + } + else + { + temp.tv_sec = end.tv_sec-start.tv_sec; + temp.tv_usec = end.tv_usec-start.tv_usec; + } + return temp; } - return temp; + + + long getEpochTime() + { + return std::time(0) * 1000; + } + + #else + + void getTime(timespec* time) + { + #ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time + clock_serv_t cclock; + mach_timespec_t mts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + time->tv_sec = mts.tv_sec; + time->tv_nsec = mts.tv_nsec; + #else + clock_gettime(CLOCK_MONOTONIC, time); + #endif + } + + double diffclock(timespec time1,timespec time2) + { + timespec delta = diff(time1,time2); + double milliseconds = (delta.tv_sec * 1000) + (((double) delta.tv_nsec) / 1000000.0); + + return milliseconds; + } + + timespec diff(timespec start, timespec end) + { + timespec temp; + if ((end.tv_nsec-start.tv_nsec)<0) + { + temp.tv_sec = end.tv_sec-start.tv_sec-1; + temp.tv_nsec = 1000000000+end.tv_nsec-start.tv_nsec; + } + else + { + temp.tv_sec = end.tv_sec-start.tv_sec; + temp.tv_nsec = end.tv_nsec-start.tv_nsec; + } + return temp; + } + + + long getEpochTime() + { + struct timeval tp; + gettimeofday(&tp, NULL); + long ms = tp.tv_sec * 1000 + tp.tv_usec / 1000; + + return ms; + } + + #endif + } -long getEpochTime() -{ - struct timeval tp; - gettimeofday(&tp, NULL); - long ms = tp.tv_sec * 1000 + tp.tv_usec / 1000; - - return ms; -} - -#endif - - diff --git a/src/openalpr/support/timing.h b/src/openalpr/support/timing.h index 2de04d2..a89bc64 100644 --- a/src/openalpr/support/timing.h +++ b/src/openalpr/support/timing.h @@ -23,9 +23,14 @@ #define timespec timeval #endif -void getTime(timespec* time); -double diffclock(timespec time1,timespec time2); +namespace alpr +{ -long getEpochTime(); + void getTime(timespec* time); + double diffclock(timespec time1,timespec time2); + + long getEpochTime(); + +} #endif diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 1b9f25b..af60ec8 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -25,629 +25,634 @@ using namespace cv; using namespace std; -bool sort_text_line(TextLine i, TextLine j) { return (i.topLine.p1.y < j.topLine.p1.y); } - -CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) -{ - this->pipeline_data = pipeline_data; - this->config = pipeline_data->config; - - this->confidence = 0; - - if (this->config->debugCharAnalysis) - cout << "Starting CharacterAnalysis identification" << endl; - - this->analyze(); -} - -CharacterAnalysis::~CharacterAnalysis() +namespace alpr { -} + bool sort_text_line(TextLine i, TextLine j) { return (i.topLine.p1.y < j.topLine.p1.y); } -void CharacterAnalysis::analyze() -{ - pipeline_data->clearThresholds(); - pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - - - - timespec startTime; - getTime(&startTime); - - pipeline_data->textLines.clear(); - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) { - TextContours tc(pipeline_data->thresholds[i]); + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; - allTextContours.push_back(tc); + this->confidence = 0; + + if (this->config->debugCharAnalysis) + cout << "Starting CharacterAnalysis identification" << endl; + + this->analyze(); } - if (config->debugTiming) + CharacterAnalysis::~CharacterAnalysis() { - timespec endTime; - getTime(&endTime); - cout << " -- Character Analysis Find Contours Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - //Mat img_equalized = equalizeBrightness(img_gray); - getTime(&startTime); - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - this->filter(pipeline_data->thresholds[i], allTextContours[i]); - - if (config->debugCharAnalysis) - cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl; } - if (config->debugTiming) + void CharacterAnalysis::analyze() { - timespec endTime; - getTime(&endTime); - cout << " -- Character Analysis Filter Time: " << diffclock(startTime, endTime) << "ms." << endl; - } + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - PlateMask plateMask(pipeline_data); - plateMask.findOuterBoxMask(allTextContours); - - pipeline_data->hasPlateBorder = plateMask.hasPlateMask; - pipeline_data->plateBorderMask = plateMask.getMask(); - if (plateMask.hasPlateMask) - { - // Filter out bad contours now that we have an outer box mask... + + timespec startTime; + getTime(&startTime); + + pipeline_data->textLines.clear(); + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - filterByOuterMask(allTextContours[i]); - } - } + TextContours tc(pipeline_data->thresholds[i]); - int bestFitScore = -1; - int bestFitIndex = -1; - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - - int segmentCount = allTextContours[i].getGoodIndicesCount(); - - if (segmentCount > bestFitScore) - { - bestFitScore = segmentCount; - bestFitIndex = i; - bestThreshold = pipeline_data->thresholds[i]; - bestContours = allTextContours[i]; - } - } - - if (this->config->debugCharAnalysis) - cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl; - - if (bestFitScore <= 1) - return; - - //getColorMask(img, allContours, allHierarchy, charSegments); - - if (this->config->debugCharAnalysis) - { - Mat img_contours = bestContours.drawDebugImage(bestThreshold); - - displayImage(config, "Matching Contours", img_contours); - } - - LineFinder lf(pipeline_data); - vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); - - vector tempTextLines; - for (uint i = 0; i < linePolygons.size(); i++) - { - vector linePolygon = linePolygons[i]; - - LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); - LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); - - vector textArea = getCharArea(topLine, bottomLine); - - TextLine textLine(textArea, linePolygon); - - tempTextLines.push_back(textLine); - } - - filterBetweenLines(bestThreshold, bestContours, tempTextLines); - - // Sort the lines from top to bottom. - std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line); - - // Now that we've filtered a few more contours, re-do the text area. - for (uint i = 0; i < tempTextLines.size(); i++) - { - vector updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); - vector linePolygon = tempTextLines[i].linePolygon; - if (updatedTextArea.size() > 0 && linePolygon.size() > 0) - { - pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); - } - - } - - pipeline_data->plate_inverted = isPlateInverted(); - - - if (pipeline_data->textLines.size() > 0) - { - int confidenceDrainers = 0; - int charSegmentCount = this->bestContours.getGoodIndicesCount(); - if (charSegmentCount == 1) - confidenceDrainers += 91; - else if (charSegmentCount < 5) - confidenceDrainers += (5 - charSegmentCount) * 10; - - // Use the angle for the first line -- assume they'll always be parallel for multi-line plates - int absangle = abs(pipeline_data->textLines[0].topLine.angle); - if (absangle > config->maxPlateAngleDegrees) - confidenceDrainers += 91; - else if (absangle > 1) - confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ; - - // If a multiline plate has only one line, disqualify - if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2) - confidenceDrainers += 95; - - if (confidenceDrainers >= 100) - this->confidence=1; - else - this->confidence = 100 - confidenceDrainers; - } - - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "Character Analysis Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - // Draw debug dashboard - if (this->pipeline_data->config->debugCharAnalysis && pipeline_data->textLines.size() > 0) - { - vector tempDash; - for (uint z = 0; z < pipeline_data->thresholds.size(); z++) - { - Mat tmp(pipeline_data->thresholds[z].size(), pipeline_data->thresholds[z].type()); - pipeline_data->thresholds[z].copyTo(tmp); - cvtColor(tmp, tmp, CV_GRAY2BGR); - - tempDash.push_back(tmp); + allTextContours.push_back(tc); } - Mat bestVal(this->bestThreshold.size(), this->bestThreshold.type()); - this->bestThreshold.copyTo(bestVal); - cvtColor(bestVal, bestVal, CV_GRAY2BGR); - - for (uint z = 0; z < this->bestContours.size(); z++) + if (config->debugTiming) { - Scalar dcolor(255,0,0); - if (this->bestContours.goodIndices[z]) - dcolor = Scalar(0,255,0); - drawContours(bestVal, this->bestContours.contours, z, dcolor, 1); + timespec endTime; + getTime(&endTime); + cout << " -- Character Analysis Find Contours Time: " << diffclock(startTime, endTime) << "ms." << endl; } - tempDash.push_back(bestVal); - displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3)); - } -} + //Mat img_equalized = equalizeBrightness(img_gray); + getTime(&startTime); - - -Mat CharacterAnalysis::getCharacterMask() -{ - Mat charMask = Mat::zeros(bestThreshold.size(), CV_8U); - - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestContours.goodIndices[i] == false) - continue; - - drawContours(charMask, bestContours.contours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - bestContours.hierarchy, - 1 - ); - - } - - return charMask; -} - - -void CharacterAnalysis::filter(Mat img, TextContours& textContours) -{ - static int STARTING_MIN_HEIGHT = round (((float) img.rows) * config->charAnalysisMinPercent); - static int STARTING_MAX_HEIGHT = round (((float) img.rows) * (config->charAnalysisMinPercent + config->charAnalysisHeightRange)); - static int HEIGHT_STEP = round (((float) img.rows) * config->charAnalysisHeightStepSize); - static int NUM_STEPS = config->charAnalysisNumSteps; - - int bestFitScore = -1; - - vector bestIndices; - - for (int i = 0; i < NUM_STEPS; i++) - { - - //vector goodIndices(contours.size()); - for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; - - this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - - int goodIndices = textContours.getGoodIndicesCount(); - if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... - continue; - - this->filterContourHoles(textContours); - - goodIndices = textContours.getGoodIndicesCount(); - if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... - continue; - - - int segmentCount = textContours.getGoodIndicesCount(); - - if (segmentCount > bestFitScore) + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - bestFitScore = segmentCount; - bestIndices = textContours.getIndicesCopy(); - } - } + this->filter(pipeline_data->thresholds[i], allTextContours[i]); - textContours.setIndices(bestIndices); -} - -// Goes through the contours for the plate and picks out possible char segments based on min/max height -void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx) -{ - float idealAspect=config->charWidthMM / config->charHeightMM; - float aspecttolerance=0.25; - - - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i] == false) - continue; - - textContours.goodIndices[i] = false; // Set it to not included unless it proves valid - - //Create bounding rect of object - Rect mr= boundingRect(textContours.contours[i]); - - float minWidth = mr.height * 0.2; - //Crop image - - //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl; - if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) - { - float charAspect= (float)mr.width/(float)mr.height; - - //cout << " -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl; - if (abs(charAspect - idealAspect) < aspecttolerance) - textContours.goodIndices[i] = true; - } - } - -} - -void CharacterAnalysis::filterContourHoles(TextContours& textContours) -{ - - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i] == false) - continue; - - textContours.goodIndices[i] = false; // Set it to not included unless it proves valid - - int parentIndex = textContours.hierarchy[i][3]; - - if (parentIndex >= 0 && textContours.goodIndices[parentIndex]) - { - // this contour is a child of an already identified contour. REMOVE it - if (this->config->debugCharAnalysis) - { - cout << "filterContourHoles: contour index: " << i << endl; - } - } - else - { - textContours.goodIndices[i] = true; - } - } - -} - -// Goes through the contours for the plate and picks out possible char segments based on min/max height -// returns a vector of indices corresponding to valid contours -void CharacterAnalysis::filterByParentContour( TextContours& textContours) -{ - - vector parentIDs; - vector votes; - - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i] == false) - continue; - - textContours.goodIndices[i] = false; // Set it to not included unless it proves - - int voteIndex = -1; - int parentID = textContours.hierarchy[i][3]; - // check if parentID is already in the lsit - for (uint j = 0; j < parentIDs.size(); j++) - { - if (parentIDs[j] == parentID) - { - voteIndex = j; - break; - } - } - if (voteIndex == -1) - { - parentIDs.push_back(parentID); - votes.push_back(1); - } - else - { - votes[voteIndex] = votes[voteIndex] + 1; - } - } - - // Tally up the votes, pick the winner - int totalVotes = 0; - int winningParentId = 0; - int highestVotes = 0; - for (uint i = 0; i < parentIDs.size(); i++) - { - if (votes[i] > highestVotes) - { - winningParentId = parentIDs[i]; - highestVotes = votes[i]; - } - totalVotes += votes[i]; - } - - // Now filter out all the contours with a different parent ID (assuming the totalVotes > 2) - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i] == false) - continue; - - if (totalVotes <= 2) - { - textContours.goodIndices[i] = true; - } - else if (textContours.hierarchy[i][3] == winningParentId) - { - textContours.goodIndices[i] = true; - } - } - -} - -void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector textLines ) -{ - static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; - static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; - - if (textLines.size() == 0) - return; - - vector validPoints; - - - // Create a white mask for the area inside the polygon - Mat outerMask = Mat::zeros(img.size(), CV_8U); - - for (uint i = 0; i < textLines.size(); i++) - fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255)); - - // For each contour, determine if enough of it is between the lines to qualify - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i] == false) - continue; - - float percentInsideMask = getContourAreaPercentInsideMask(outerMask, - textContours.contours, - textContours.hierarchy, - (int) i); - - - - if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) - { - // Not enough area is inside the lines. if (config->debugCharAnalysis) - cout << "Rejecting due to insufficient area" << endl; - textContours.goodIndices[i] = false; - - continue; + cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl; } - - - // now check to make sure that the top and bottom of the contour are near enough to the lines - - // First get the high and low point for the contour - // Remember that origin is top-left, so the top Y values are actually closer to 0. - Rect brect = boundingRect(textContours.contours[i]); - int xmiddle = brect.x + (brect.width / 2); - Point topMiddle = Point(xmiddle, brect.y); - Point botMiddle = Point(xmiddle, brect.y+brect.height); - - // Get the absolute distance from the top and bottom lines - - for (uint i = 0; i < textLines.size(); i++) + + if (config->debugTiming) { - Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(topMiddle); - Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(botMiddle); + timespec endTime; + getTime(&endTime); + cout << " -- Character Analysis Filter Time: " << diffclock(startTime, endTime) << "ms." << endl; + } - float absTopDistance = distanceBetweenPoints(closestTopPoint, topMiddle); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, botMiddle); + PlateMask plateMask(pipeline_data); + plateMask.findOuterBoxMask(allTextContours); - float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + pipeline_data->hasPlateBorder = plateMask.hasPlateMask; + pipeline_data->plateBorderMask = plateMask.getMask(); - if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + if (plateMask.hasPlateMask) + { + // Filter out bad contours now that we have an outer box mask... + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - // It's ok, leave it as-is. + filterByOuterMask(allTextContours[i]); + } + } + + int bestFitScore = -1; + int bestFitIndex = -1; + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + + int segmentCount = allTextContours[i].getGoodIndicesCount(); + + if (segmentCount > bestFitScore) + { + bestFitScore = segmentCount; + bestFitIndex = i; + bestThreshold = pipeline_data->thresholds[i]; + bestContours = allTextContours[i]; + } + } + + if (this->config->debugCharAnalysis) + cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl; + + if (bestFitScore <= 1) + return; + + //getColorMask(img, allContours, allHierarchy, charSegments); + + if (this->config->debugCharAnalysis) + { + Mat img_contours = bestContours.drawDebugImage(bestThreshold); + + displayImage(config, "Matching Contours", img_contours); + } + + LineFinder lf(pipeline_data); + vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); + + vector tempTextLines; + for (uint i = 0; i < linePolygons.size(); i++) + { + vector linePolygon = linePolygons[i]; + + LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + + vector textArea = getCharArea(topLine, bottomLine); + + TextLine textLine(textArea, linePolygon); + + tempTextLines.push_back(textLine); + } + + filterBetweenLines(bestThreshold, bestContours, tempTextLines); + + // Sort the lines from top to bottom. + std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line); + + // Now that we've filtered a few more contours, re-do the text area. + for (uint i = 0; i < tempTextLines.size(); i++) + { + vector updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); + vector linePolygon = tempTextLines[i].linePolygon; + if (updatedTextArea.size() > 0 && linePolygon.size() > 0) + { + pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); + } + + } + + pipeline_data->plate_inverted = isPlateInverted(); + + + if (pipeline_data->textLines.size() > 0) + { + int confidenceDrainers = 0; + int charSegmentCount = this->bestContours.getGoodIndicesCount(); + if (charSegmentCount == 1) + confidenceDrainers += 91; + else if (charSegmentCount < 5) + confidenceDrainers += (5 - charSegmentCount) * 10; + + // Use the angle for the first line -- assume they'll always be parallel for multi-line plates + int absangle = abs(pipeline_data->textLines[0].topLine.angle); + if (absangle > config->maxPlateAngleDegrees) + confidenceDrainers += 91; + else if (absangle > 1) + confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ; + + // If a multiline plate has only one line, disqualify + if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2) + confidenceDrainers += 95; + + if (confidenceDrainers >= 100) + this->confidence=1; + else + this->confidence = 100 - confidenceDrainers; + } + + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "Character Analysis Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + // Draw debug dashboard + if (this->pipeline_data->config->debugCharAnalysis && pipeline_data->textLines.size() > 0) + { + vector tempDash; + for (uint z = 0; z < pipeline_data->thresholds.size(); z++) + { + Mat tmp(pipeline_data->thresholds[z].size(), pipeline_data->thresholds[z].type()); + pipeline_data->thresholds[z].copyTo(tmp); + cvtColor(tmp, tmp, CV_GRAY2BGR); + + tempDash.push_back(tmp); + } + + Mat bestVal(this->bestThreshold.size(), this->bestThreshold.type()); + this->bestThreshold.copyTo(bestVal); + cvtColor(bestVal, bestVal, CV_GRAY2BGR); + + for (uint z = 0; z < this->bestContours.size(); z++) + { + Scalar dcolor(255,0,0); + if (this->bestContours.goodIndices[z]) + dcolor = Scalar(0,255,0); + drawContours(bestVal, this->bestContours.contours, z, dcolor, 1); + } + tempDash.push_back(bestVal); + displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3)); + } + } + + + + + Mat CharacterAnalysis::getCharacterMask() + { + Mat charMask = Mat::zeros(bestThreshold.size(), CV_8U); + + for (uint i = 0; i < bestContours.size(); i++) + { + if (bestContours.goodIndices[i] == false) + continue; + + drawContours(charMask, bestContours.contours, + i, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + bestContours.hierarchy, + 1 + ); + + } + + return charMask; + } + + + void CharacterAnalysis::filter(Mat img, TextContours& textContours) + { + static int STARTING_MIN_HEIGHT = round (((float) img.rows) * config->charAnalysisMinPercent); + static int STARTING_MAX_HEIGHT = round (((float) img.rows) * (config->charAnalysisMinPercent + config->charAnalysisHeightRange)); + static int HEIGHT_STEP = round (((float) img.rows) * config->charAnalysisHeightStepSize); + static int NUM_STEPS = config->charAnalysisNumSteps; + + int bestFitScore = -1; + + vector bestIndices; + + for (int i = 0; i < NUM_STEPS; i++) + { + + //vector goodIndices(contours.size()); + for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; + + this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); + + int goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... + continue; + + this->filterContourHoles(textContours); + + goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... + continue; + + + int segmentCount = textContours.getGoodIndicesCount(); + + if (segmentCount > bestFitScore) + { + bestFitScore = segmentCount; + bestIndices = textContours.getIndicesCopy(); + } + } + + textContours.setIndices(bestIndices); + } + + // Goes through the contours for the plate and picks out possible char segments based on min/max height + void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx) + { + float idealAspect=config->charWidthMM / config->charHeightMM; + float aspecttolerance=0.25; + + + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + + //Create bounding rect of object + Rect mr= boundingRect(textContours.contours[i]); + + float minWidth = mr.height * 0.2; + //Crop image + + //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl; + if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) + { + float charAspect= (float)mr.width/(float)mr.height; + + //cout << " -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl; + if (abs(charAspect - idealAspect) < aspecttolerance) + textContours.goodIndices[i] = true; + } + } + + } + + void CharacterAnalysis::filterContourHoles(TextContours& textContours) + { + + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + + int parentIndex = textContours.hierarchy[i][3]; + + if (parentIndex >= 0 && textContours.goodIndices[parentIndex]) + { + // this contour is a child of an already identified contour. REMOVE it + if (this->config->debugCharAnalysis) + { + cout << "filterContourHoles: contour index: " << i << endl; + } } else { - - textContours.goodIndices[i] = false; - if (config->debugCharAnalysis) - cout << "Rejecting due to top/bottom points that are out of range" << endl; + textContours.goodIndices[i] = true; } } - + } -} - -void CharacterAnalysis::filterByOuterMask(TextContours& textContours) -{ - float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; - float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; - - if (this->pipeline_data->hasPlateBorder == false) - return; - - - cv::Mat plateMask = pipeline_data->plateBorderMask; - - Mat tempMaskedContour = Mat::zeros(plateMask.size(), CV_8U); - Mat tempFullContour = Mat::zeros(plateMask.size(), CV_8U); - - int charsInsideMask = 0; - int totalChars = 0; - - vector originalindices; - for (uint i = 0; i < textContours.size(); i++) - originalindices.push_back(textContours.goodIndices[i]); - - for (uint i=0; i < textContours.size(); i++) + // Goes through the contours for the plate and picks out possible char segments based on min/max height + // returns a vector of indices corresponding to valid contours + void CharacterAnalysis::filterByParentContour( TextContours& textContours) { - if (textContours.goodIndices[i] == false) - continue; - totalChars++; + vector parentIDs; + vector votes; - drawContours(tempFullContour, textContours.contours, i, Scalar(255,255,255), CV_FILLED, 8, textContours.hierarchy); - bitwise_and(tempFullContour, plateMask, tempMaskedContour); - - float beforeMaskWhiteness = mean(tempFullContour)[0]; - float afterMaskWhiteness = mean(tempMaskedContour)[0]; - - if (afterMaskWhiteness / beforeMaskWhiteness > MINIMUM_PERCENT_LEFT_AFTER_MASK) + for (uint i = 0; i < textContours.size(); i++) { - charsInsideMask++; - textContours.goodIndices[i] = true; + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves + + int voteIndex = -1; + int parentID = textContours.hierarchy[i][3]; + // check if parentID is already in the lsit + for (uint j = 0; j < parentIDs.size(); j++) + { + if (parentIDs[j] == parentID) + { + voteIndex = j; + break; + } + } + if (voteIndex == -1) + { + parentIDs.push_back(parentID); + votes.push_back(1); + } + else + { + votes[voteIndex] = votes[voteIndex] + 1; + } } + + // Tally up the votes, pick the winner + int totalVotes = 0; + int winningParentId = 0; + int highestVotes = 0; + for (uint i = 0; i < parentIDs.size(); i++) + { + if (votes[i] > highestVotes) + { + winningParentId = parentIDs[i]; + highestVotes = votes[i]; + } + totalVotes += votes[i]; + } + + // Now filter out all the contours with a different parent ID (assuming the totalVotes > 2) + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + if (totalVotes <= 2) + { + textContours.goodIndices[i] = true; + } + else if (textContours.hierarchy[i][3] == winningParentId) + { + textContours.goodIndices[i] = true; + } + } + } - if (totalChars == 0) + void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector textLines ) { - textContours.goodIndices = originalindices; - return; + static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; + static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; + + if (textLines.size() == 0) + return; + + vector validPoints; + + + // Create a white mask for the area inside the polygon + Mat outerMask = Mat::zeros(img.size(), CV_8U); + + for (uint i = 0; i < textLines.size(); i++) + fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255)); + + // For each contour, determine if enough of it is between the lines to qualify + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + float percentInsideMask = getContourAreaPercentInsideMask(outerMask, + textContours.contours, + textContours.hierarchy, + (int) i); + + + + if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) + { + // Not enough area is inside the lines. + if (config->debugCharAnalysis) + cout << "Rejecting due to insufficient area" << endl; + textContours.goodIndices[i] = false; + + continue; + } + + + // now check to make sure that the top and bottom of the contour are near enough to the lines + + // First get the high and low point for the contour + // Remember that origin is top-left, so the top Y values are actually closer to 0. + Rect brect = boundingRect(textContours.contours[i]); + int xmiddle = brect.x + (brect.width / 2); + Point topMiddle = Point(xmiddle, brect.y); + Point botMiddle = Point(xmiddle, brect.y+brect.height); + + // Get the absolute distance from the top and bottom lines + + for (uint i = 0; i < textLines.size(); i++) + { + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(topMiddle); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(botMiddle); + + float absTopDistance = distanceBetweenPoints(closestTopPoint, topMiddle); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, botMiddle); + + float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + + if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + { + // It's ok, leave it as-is. + } + else + { + + textContours.goodIndices[i] = false; + if (config->debugCharAnalysis) + cout << "Rejecting due to top/bottom points that are out of range" << endl; + } + } + + } + } - // Check to make sure that this is a valid box. If the box is too small (e.g., 1 char is inside, and 3 are outside) - // then don't use this to filter. - float percentCharsInsideMask = ((float) charsInsideMask) / ((float) totalChars); - if (percentCharsInsideMask < MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK) + void CharacterAnalysis::filterByOuterMask(TextContours& textContours) { - textContours.goodIndices = originalindices; - return; + float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; + float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; + + if (this->pipeline_data->hasPlateBorder == false) + return; + + + cv::Mat plateMask = pipeline_data->plateBorderMask; + + Mat tempMaskedContour = Mat::zeros(plateMask.size(), CV_8U); + Mat tempFullContour = Mat::zeros(plateMask.size(), CV_8U); + + int charsInsideMask = 0; + int totalChars = 0; + + vector originalindices; + for (uint i = 0; i < textContours.size(); i++) + originalindices.push_back(textContours.goodIndices[i]); + + for (uint i=0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + totalChars++; + + drawContours(tempFullContour, textContours.contours, i, Scalar(255,255,255), CV_FILLED, 8, textContours.hierarchy); + bitwise_and(tempFullContour, plateMask, tempMaskedContour); + + float beforeMaskWhiteness = mean(tempFullContour)[0]; + float afterMaskWhiteness = mean(tempMaskedContour)[0]; + + if (afterMaskWhiteness / beforeMaskWhiteness > MINIMUM_PERCENT_LEFT_AFTER_MASK) + { + charsInsideMask++; + textContours.goodIndices[i] = true; + } + } + + if (totalChars == 0) + { + textContours.goodIndices = originalindices; + return; + } + + // Check to make sure that this is a valid box. If the box is too small (e.g., 1 char is inside, and 3 are outside) + // then don't use this to filter. + float percentCharsInsideMask = ((float) charsInsideMask) / ((float) totalChars); + if (percentCharsInsideMask < MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK) + { + textContours.goodIndices = originalindices; + return; + } + } -} + bool CharacterAnalysis::isPlateInverted() + { + Mat charMask = getCharacterMask(); -bool CharacterAnalysis::isPlateInverted() -{ - Mat charMask = getCharacterMask(); - - Scalar meanVal = mean(bestThreshold, charMask)[0]; + Scalar meanVal = mean(bestThreshold, charMask)[0]; - if (this->config->debugCharAnalysis) - cout << "CharacterAnalysis, plate inverted: MEAN: " << meanVal << " : " << bestThreshold.type() << endl; + if (this->config->debugCharAnalysis) + cout << "CharacterAnalysis, plate inverted: MEAN: " << meanVal << " : " << bestThreshold.type() << endl; - if (meanVal[0] < 100) // Half would be 122.5. Give it a little extra oomf before saying it needs inversion. Most states aren't inverted. - return true; + if (meanVal[0] < 100) // Half would be 122.5. Give it a little extra oomf before saying it needs inversion. Most states aren't inverted. + return true; - return false; -} - -bool CharacterAnalysis::verifySize(Mat r, float minHeightPx, float maxHeightPx) -{ - //Char sizes 45x90 - float aspect=config->charWidthMM / config->charHeightMM; - float charAspect= (float)r.cols/(float)r.rows; - float error=0.35; - //float minHeight=TEMPLATE_PLATE_HEIGHT * .35; - //float maxHeight=TEMPLATE_PLATE_HEIGHT * .65; - //We have a different aspect ratio for number 1, and it can be ~0.2 - float minAspect=0.2; - float maxAspect=aspect+aspect*error; - //area of pixels - float area=countNonZero(r); - //bb area - float bbArea=r.cols*r.rows; - //% of pixel in area - float percPixels=area/bbArea; - - //if(DEBUG) - //cout << "Aspect: "<< aspect << " ["<< minAspect << "," << maxAspect << "] " << "Area "<< percPixels <<" Char aspect " << charAspect << " Height char "<< r.rows << "\n"; - if(percPixels < 0.8 && charAspect > minAspect && charAspect < maxAspect && r.rows >= minHeightPx && r.rows < maxHeightPx) - return true; - else return false; -} + } -vector CharacterAnalysis::getCharArea(LineSegment topLine, LineSegment bottomLine) -{ - const int MAX = 100000; - const int MIN= -1; - - int leftX = MAX; - int rightX = MIN; - - for (uint i = 0; i < bestContours.size(); i++) + bool CharacterAnalysis::verifySize(Mat r, float minHeightPx, float maxHeightPx) { - if (bestContours.goodIndices[i] == false) - continue; + //Char sizes 45x90 + float aspect=config->charWidthMM / config->charHeightMM; + float charAspect= (float)r.cols/(float)r.rows; + float error=0.35; + //float minHeight=TEMPLATE_PLATE_HEIGHT * .35; + //float maxHeight=TEMPLATE_PLATE_HEIGHT * .65; + //We have a different aspect ratio for number 1, and it can be ~0.2 + float minAspect=0.2; + float maxAspect=aspect+aspect*error; + //area of pixels + float area=countNonZero(r); + //bb area + float bbArea=r.cols*r.rows; + //% of pixel in area + float percPixels=area/bbArea; - for (uint z = 0; z < bestContours.contours[i].size(); z++) + //if(DEBUG) + //cout << "Aspect: "<< aspect << " ["<< minAspect << "," << maxAspect << "] " << "Area "<< percPixels <<" Char aspect " << charAspect << " Height char "<< r.rows << "\n"; + if(percPixels < 0.8 && charAspect > minAspect && charAspect < maxAspect && r.rows >= minHeightPx && r.rows < maxHeightPx) + return true; + else + return false; + } + + vector CharacterAnalysis::getCharArea(LineSegment topLine, LineSegment bottomLine) + { + const int MAX = 100000; + const int MIN= -1; + + int leftX = MAX; + int rightX = MIN; + + for (uint i = 0; i < bestContours.size(); i++) { - if (bestContours.contours[i][z].x < leftX) - leftX = bestContours.contours[i][z].x; - if (bestContours.contours[i][z].x > rightX) - rightX = bestContours.contours[i][z].x; + if (bestContours.goodIndices[i] == false) + continue; + + for (uint z = 0; z < bestContours.contours[i].size(); z++) + { + if (bestContours.contours[i][z].x < leftX) + leftX = bestContours.contours[i][z].x; + if (bestContours.contours[i][z].x > rightX) + rightX = bestContours.contours[i][z].x; + } } + + vector charArea; + if (leftX != MAX && rightX != MIN) + { + Point tl(leftX, topLine.getPointAt(leftX)); + Point tr(rightX, topLine.getPointAt(rightX)); + Point br(rightX, bottomLine.getPointAt(rightX)); + Point bl(leftX, bottomLine.getPointAt(leftX)); + charArea.push_back(tl); + charArea.push_back(tr); + charArea.push_back(br); + charArea.push_back(bl); + } + + return charArea; } - vector charArea; - if (leftX != MAX && rightX != MIN) - { - Point tl(leftX, topLine.getPointAt(leftX)); - Point tr(rightX, topLine.getPointAt(rightX)); - Point br(rightX, bottomLine.getPointAt(rightX)); - Point bl(leftX, bottomLine.getPointAt(leftX)); - charArea.push_back(tl); - charArea.push_back(tr); - charArea.push_back(br); - charArea.push_back(bl); - } - - return charArea; -} +} \ No newline at end of file diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 830ea32..2479172 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -29,46 +29,51 @@ #include "platemask.h" #include "linefinder.h" -class CharacterAnalysis +namespace alpr { - public: - CharacterAnalysis(PipelineData* pipeline_data); - virtual ~CharacterAnalysis(); + class CharacterAnalysis + { - int confidence; + public: + CharacterAnalysis(PipelineData* pipeline_data); + virtual ~CharacterAnalysis(); - cv::Mat bestThreshold; - - TextContours bestContours; + int confidence; + + cv::Mat bestThreshold; + + TextContours bestContours; - std::vector allTextContours; + std::vector allTextContours; - void analyze(); + void analyze(); - cv::Mat getCharacterMask(); + cv::Mat getCharacterMask(); - private: - PipelineData* pipeline_data; - Config* config; + private: + PipelineData* pipeline_data; + Config* config; - cv::Mat findOuterBoxMask( ); + cv::Mat findOuterBoxMask( ); - bool isPlateInverted(); - void filter(cv::Mat img, TextContours& textContours); + bool isPlateInverted(); + void filter(cv::Mat img, TextContours& textContours); - void filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx); - void filterByParentContour( TextContours& textContours ); - void filterContourHoles(TextContours& textContours); - void filterByOuterMask(TextContours& textContours); + void filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx); + void filterByParentContour( TextContours& textContours ); + void filterContourHoles(TextContours& textContours); + void filterByOuterMask(TextContours& textContours); - std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); - void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); + std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); + void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); - bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); + bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - -}; + + }; + +} #endif // OPENALPR_CHARACTERANALYSIS_H diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index 3ef2b7f..6a5ee3e 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -26,228 +26,233 @@ using namespace std; using namespace cv; -LineFinder::LineFinder(PipelineData* pipeline_data) { - this->pipeline_data = pipeline_data; -} - -LineFinder::~LineFinder() { -} - -vector > LineFinder::findLines(Mat image, const TextContours contours) +namespace alpr { - const float MIN_AREA_TO_IGNORE = 0.65; - - vector > linesFound; - - cvtColor(image, image, CV_GRAY2BGR); - - vector charPoints; - - for (uint i = 0; i < contours.contours.size(); i++) - { - if (contours.goodIndices[i] == false) - continue; - - charPoints.push_back( CharPointInfo(contours.contours[i], i) ); + + LineFinder::LineFinder(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; } - - vector bestLine = getBestLine(contours, charPoints); - - if (bestLine.size() > 0) - linesFound.push_back(bestLine); - - if (pipeline_data->isMultiline) + + LineFinder::~LineFinder() { + } + + vector > LineFinder::findLines(Mat image, const TextContours contours) { - // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. - // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it. - - vector remainingPoints; + const float MIN_AREA_TO_IGNORE = 0.65; + + vector > linesFound; + + cvtColor(image, image, CV_GRAY2BGR); + + vector charPoints; + + for (uint i = 0; i < contours.contours.size(); i++) + { + if (contours.goodIndices[i] == false) + continue; + + charPoints.push_back( CharPointInfo(contours.contours[i], i) ); + } + + vector bestLine = getBestLine(contours, charPoints); + + if (bestLine.size() > 0) + linesFound.push_back(bestLine); + + if (pipeline_data->isMultiline) + { + // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. + // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it. + + vector remainingPoints; + for (uint i = 0; i < charPoints.size(); i++) + { + Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U); + fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255)); + + float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); + + if (percentInside < MIN_AREA_TO_IGNORE) + { + remainingPoints.push_back(charPoints[i]); + } + } + + vector nextBestLine = getBestLine(contours, remainingPoints); + + if (nextBestLine.size() > 0) + linesFound.push_back(nextBestLine); + } + + + return linesFound; + } + + + // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width + vector LineFinder::getBestLine(const TextContours contours, vector charPoints) + { + vector bestStripe; + + // Find the best fit line segment that is parallel with the most char segments + if (charPoints.size() <= 1) + { + // Maybe do something about this later, for now let's just ignore + return bestStripe; + } + + + vector charheights; for (uint i = 0; i < charPoints.size(); i++) + charheights.push_back(charPoints[i].boundingBox.height); + float medianCharHeight = median(charheights.data(), charheights.size()); + + + + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < charPoints.size() - 1; i++) { - Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U); - fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255)); - - float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); - - if (percentInside < MIN_AREA_TO_IGNORE) + for (uint k = i+1; k < charPoints.size(); k++) { - remainingPoints.push_back(charPoints[i]); + + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) + { + leftCPIndex = i; + rightCPIndex = k; + } + else + { + leftCPIndex = k; + rightCPIndex = i; + } + + + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); + + + // Only allow lines that have a sane angle + // if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + // abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + // { + // topLines.push_back(top); + // bottomLines.push_back(bottom); + // } + + LineSegment parallelBot = top.getParallelLine(medianCharHeight * -1); + LineSegment parallelTop = bottom.getParallelLine(medianCharHeight); + + // Only allow lines that have a sane angle + if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(parallelBot.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(top); + bottomLines.push_back(parallelBot); + } + + // Only allow lines that have a sane angle + if (abs(parallelTop.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(parallelTop); + bottomLines.push_back(bottom); + } } } - - vector nextBestLine = getBestLine(contours, remainingPoints); - - if (nextBestLine.size() > 0) - linesFound.push_back(nextBestLine); - } - - - return linesFound; -} + + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < charPoints.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); + + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (bestScore < 0) + return bestStripe; + + if (pipeline_data->config->debugCharAnalysis) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + displayImage(pipeline_data->config, "Winning lines", tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); -// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector LineFinder::getBestLine(const TextContours contours, vector charPoints) -{ - vector bestStripe; - - // Find the best fit line segment that is parallel with the most char segments - if (charPoints.size() <= 1) - { - // Maybe do something about this later, for now let's just ignore return bestStripe; } - - - vector charheights; - for (uint i = 0; i < charPoints.size(); i++) - charheights.push_back(charPoints[i].boundingBox.height); - float medianCharHeight = median(charheights.data(), charheights.size()); + + CharPointInfo::CharPointInfo(vector contour, int index) { + this->contourIndex = index; - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charPoints.size() - 1; i++) - { - for (uint k = i+1; k < charPoints.size(); k++) - { - - int leftCPIndex, rightCPIndex; - if (charPoints[i].top.x < charPoints[k].top.x) - { - leftCPIndex = i; - rightCPIndex = k; - } - else - { - leftCPIndex = k; - rightCPIndex = i; - } + this->boundingBox = cv::boundingRect( Mat(contour) ); - LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); - LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); + int x = boundingBox.x + (boundingBox.width / 2); + int y = boundingBox.y; + this->top = Point(x, y); - // Only allow lines that have a sane angle -// if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && -// abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) -// { -// topLines.push_back(top); -// bottomLines.push_back(bottom); -// } + x = boundingBox.x + (boundingBox.width / 2); + y = boundingBox.y + boundingBox.height; - LineSegment parallelBot = top.getParallelLine(medianCharHeight * -1); - LineSegment parallelTop = bottom.getParallelLine(medianCharHeight); + this->bottom = Point(x,y); - // Only allow lines that have a sane angle - if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && - abs(parallelBot.angle) <= pipeline_data->config->maxPlateAngleDegrees) - { - topLines.push_back(top); - bottomLines.push_back(parallelBot); - } - - // Only allow lines that have a sane angle - if (abs(parallelTop.angle) <= pipeline_data->config->maxPlateAngleDegrees && - abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) - { - topLines.push_back(parallelTop); - bottomLines.push_back(bottom); - } - } } - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charPoints.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); - float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); - - float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; - float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; - float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; - float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (bestScore < 0) - return bestStripe; - - if (pipeline_data->config->debugCharAnalysis) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - - Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); - cvtColor(tempImg, tempImg, CV_GRAY2BGR); - - cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - displayImage(pipeline_data->config, "Winning lines", tempImg); - } - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); - - - return bestStripe; -} - -CharPointInfo::CharPointInfo(vector contour, int index) { - - - this->contourIndex = index; - - this->boundingBox = cv::boundingRect( Mat(contour) ); - - - int x = boundingBox.x + (boundingBox.width / 2); - int y = boundingBox.y; - - this->top = Point(x, y); - - x = boundingBox.x + (boundingBox.width / 2); - y = boundingBox.y + boundingBox.height; - - this->bottom = Point(x,y); - -} +} \ No newline at end of file diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h index 7b7398f..2d546d1 100644 --- a/src/openalpr/textdetection/linefinder.h +++ b/src/openalpr/textdetection/linefinder.h @@ -27,29 +27,34 @@ #include "textline.h" #include "pipeline_data.h" -class CharPointInfo +namespace alpr { -public: - CharPointInfo(std::vector contour, int index); - - cv::Rect boundingBox; - cv::Point top; - cv::Point bottom; - int contourIndex; - -}; -class LineFinder { -public: - LineFinder(PipelineData* pipeline_data); - virtual ~LineFinder(); - - std::vector > findLines(cv::Mat image, const TextContours contours); -private: - PipelineData* pipeline_data; - - std::vector getBestLine(const TextContours contours, std::vector charPoints); -}; + class CharPointInfo + { + public: + CharPointInfo(std::vector contour, int index); + + cv::Rect boundingBox; + cv::Point top; + cv::Point bottom; + int contourIndex; + + }; + + class LineFinder { + public: + LineFinder(PipelineData* pipeline_data); + virtual ~LineFinder(); + + std::vector > findLines(cv::Mat image, const TextContours contours); + private: + PipelineData* pipeline_data; + + std::vector getBestLine(const TextContours contours, std::vector charPoints); + }; + +} #endif /* OPENALPR_LINEFINDER_H */ diff --git a/src/openalpr/textdetection/platemask.cpp b/src/openalpr/textdetection/platemask.cpp index 04dabbd..4eee470 100644 --- a/src/openalpr/textdetection/platemask.cpp +++ b/src/openalpr/textdetection/platemask.cpp @@ -22,178 +22,183 @@ using namespace std; using namespace cv; -PlateMask::PlateMask(PipelineData* pipeline_data) { - this->pipeline_data = pipeline_data; - this->hasPlateMask = false; - -} - - -PlateMask::~PlateMask() { -} - -cv::Mat PlateMask::getMask() { - return this->plateMask; -} - -void PlateMask::findOuterBoxMask( vector contours ) +namespace alpr { - double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. - int winningIndex = -1; - int winningParentId = -1; - int bestCharCount = 0; - double lowestArea = 99999999999999; + PlateMask::PlateMask(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; + this->hasPlateMask = false; - if (pipeline_data->config->debugCharAnalysis) - cout << "CharacterAnalysis::findOuterBoxMask" << endl; - - for (uint imgIndex = 0; imgIndex < contours.size(); imgIndex++) - { - //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); - - int charsRecognized = 0; - int parentId = -1; - bool hasParent = false; - for (uint i = 0; i < contours[imgIndex].goodIndices.size(); i++) - { - if (contours[imgIndex].goodIndices[i]) charsRecognized++; - if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1) - { - parentId = contours[imgIndex].hierarchy[i][3]; - hasParent = true; - } - } - - if (charsRecognized == 0) - continue; - - if (hasParent) - { - double boxArea = contourArea(contours[imgIndex].contours[parentId]); - if (boxArea < min_parent_area) - continue; - - if ((charsRecognized > bestCharCount) || - (charsRecognized == bestCharCount && boxArea < lowestArea)) - //(boxArea < lowestArea) - { - bestCharCount = charsRecognized; - winningIndex = imgIndex; - winningParentId = parentId; - lowestArea = boxArea; - } - } } - if (pipeline_data->config->debugCharAnalysis) - cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; - if (winningIndex != -1 && bestCharCount >= 3) + PlateMask::~PlateMask() { + } + + cv::Mat PlateMask::getMask() { + return this->plateMask; + } + + void PlateMask::findOuterBoxMask( vector contours ) { - int longestChildIndex = -1; - double longestChildLength = 0; - // Find the child with the longest permiter/arc length ( just for kicks) - for (uint i = 0; i < contours[winningIndex].size(); i++) + double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. + + int winningIndex = -1; + int winningParentId = -1; + int bestCharCount = 0; + double lowestArea = 99999999999999; + + if (pipeline_data->config->debugCharAnalysis) + cout << "CharacterAnalysis::findOuterBoxMask" << endl; + + for (uint imgIndex = 0; imgIndex < contours.size(); imgIndex++) { - for (uint j = 0; j < contours[winningIndex].size(); j++) + //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); + + int charsRecognized = 0; + int parentId = -1; + bool hasParent = false; + for (uint i = 0; i < contours[imgIndex].goodIndices.size(); i++) { - if (contours[winningIndex].hierarchy[j][3] == winningParentId) + if (contours[imgIndex].goodIndices[i]) charsRecognized++; + if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1) { - double arclength = arcLength(contours[winningIndex].contours[j], false); - if (arclength > longestChildLength) - { - longestChildIndex = j; - longestChildLength = arclength; - } + parentId = contours[imgIndex].hierarchy[i][3]; + hasParent = true; + } + } + + if (charsRecognized == 0) + continue; + + if (hasParent) + { + double boxArea = contourArea(contours[imgIndex].contours[parentId]); + if (boxArea < min_parent_area) + continue; + + if ((charsRecognized > bestCharCount) || + (charsRecognized == bestCharCount && boxArea < lowestArea)) + //(boxArea < lowestArea) + { + bestCharCount = charsRecognized; + winningIndex = imgIndex; + winningParentId = parentId; + lowestArea = boxArea; } } } - Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + if (pipeline_data->config->debugCharAnalysis) + cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; - // get rid of the outline by drawing a 1 pixel width black line - drawContours(mask, contours[winningIndex].contours, - winningParentId, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - contours[winningIndex].hierarchy, - 0 - ); - - // Morph Open the mask to get rid of any little connectors to non-plate portions - int morph_elem = 2; - int morph_size = 3; - Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - - //morphologyEx( mask, mask, MORPH_CLOSE, element ); - morphologyEx( mask, mask, MORPH_OPEN, element ); - - //morph_size = 1; - //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - //dilate(mask, mask, element); - - // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. - // We'll want to do the contour again and find the larges one so that we remove the clipped portion. - - vector > contoursSecondRound; - - findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - int biggestContourIndex = -1; - double largestArea = 0; - for (uint c = 0; c < contoursSecondRound.size(); c++) + if (winningIndex != -1 && bestCharCount >= 3) { - double area = contourArea(contoursSecondRound[c]); - if (area > largestArea) + int longestChildIndex = -1; + double longestChildLength = 0; + // Find the child with the longest permiter/arc length ( just for kicks) + for (uint i = 0; i < contours[winningIndex].size(); i++) { - biggestContourIndex = c; - largestArea = area; + for (uint j = 0; j < contours[winningIndex].size(); j++) + { + if (contours[winningIndex].hierarchy[j][3] == winningParentId) + { + double arclength = arcLength(contours[winningIndex].contours[j], false); + if (arclength > longestChildLength) + { + longestChildIndex = j; + longestChildLength = arclength; + } + } + } } - } - if (biggestContourIndex != -1) - { - mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - vector smoothedMaskPoints; - approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); - - vector > tempvec; - tempvec.push_back(smoothedMaskPoints); - //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); - drawContours(mask, tempvec, - 0, // draw this contour + // get rid of the outline by drawing a 1 pixel width black line + drawContours(mask, contours[winningIndex].contours, + winningParentId, // draw this contour cv::Scalar(255,255,255), // in CV_FILLED, 8, contours[winningIndex].hierarchy, 0 ); + + // Morph Open the mask to get rid of any little connectors to non-plate portions + int morph_elem = 2; + int morph_size = 3; + Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + + //morphologyEx( mask, mask, MORPH_CLOSE, element ); + morphologyEx( mask, mask, MORPH_OPEN, element ); + + //morph_size = 1; + //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + //dilate(mask, mask, element); + + // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. + // We'll want to do the contour again and find the larges one so that we remove the clipped portion. + + vector > contoursSecondRound; + + findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + int biggestContourIndex = -1; + double largestArea = 0; + for (uint c = 0; c < contoursSecondRound.size(); c++) + { + double area = contourArea(contoursSecondRound[c]); + if (area > largestArea) + { + biggestContourIndex = c; + largestArea = area; + } + } + + if (biggestContourIndex != -1) + { + mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + vector smoothedMaskPoints; + approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); + + vector > tempvec; + tempvec.push_back(smoothedMaskPoints); + //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); + drawContours(mask, tempvec, + 0, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + contours[winningIndex].hierarchy, + 0 + ); + } + + if (pipeline_data->config->debugCharAnalysis) + { + vector debugImgs; + Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); + + debugImgs.push_back(mask); + debugImgs.push_back(pipeline_data->thresholds[winningIndex]); + debugImgs.push_back(debugImgMasked); + + Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); + displayImage(pipeline_data->config, "Winning outer box", dashboard); + } + + hasPlateMask = true; + this->plateMask = mask; } - if (pipeline_data->config->debugCharAnalysis) - { - vector debugImgs; - Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + hasPlateMask = false; + Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); + bitwise_not(fullMask, fullMask); - pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); - - debugImgs.push_back(mask); - debugImgs.push_back(pipeline_data->thresholds[winningIndex]); - debugImgs.push_back(debugImgMasked); - - Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); - displayImage(pipeline_data->config, "Winning outer box", dashboard); - } - - hasPlateMask = true; - this->plateMask = mask; + this->plateMask = fullMask; } - hasPlateMask = false; - Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); - bitwise_not(fullMask, fullMask); - - this->plateMask = fullMask; -} +} \ No newline at end of file diff --git a/src/openalpr/textdetection/platemask.h b/src/openalpr/textdetection/platemask.h index e31405a..536167e 100644 --- a/src/openalpr/textdetection/platemask.h +++ b/src/openalpr/textdetection/platemask.h @@ -24,24 +24,28 @@ #include "pipeline_data.h" #include "textcontours.h" -class PlateMask { -public: - PlateMask(PipelineData* pipeline_data); - virtual ~PlateMask(); - - bool hasPlateMask; - - cv::Mat getMask(); - - void findOuterBoxMask(std::vector contours); - -private: - - PipelineData* pipeline_data; - cv::Mat plateMask; +namespace alpr +{ - -}; + class PlateMask { + public: + PlateMask(PipelineData* pipeline_data); + virtual ~PlateMask(); + bool hasPlateMask; + + cv::Mat getMask(); + + void findOuterBoxMask(std::vector contours); + + private: + + PipelineData* pipeline_data; + cv::Mat plateMask; + + + }; + +} #endif /* OPENALPR_PLATEMASK_H */ diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp index 2b7dd2d..cbff1a6 100644 --- a/src/openalpr/textdetection/textcontours.cpp +++ b/src/openalpr/textdetection/textcontours.cpp @@ -22,117 +22,119 @@ using namespace std; using namespace cv; - -TextContours::TextContours() { - - -} - - -TextContours::TextContours(cv::Mat threshold) { - - load(threshold); -} - - -TextContours::~TextContours() { -} - -void TextContours::load(cv::Mat threshold) { - - Mat tempThreshold(threshold.size(), CV_8U); - threshold.copyTo(tempThreshold); - findContours(tempThreshold, - contours, // a vector of contours - hierarchy, - CV_RETR_TREE, // retrieve all contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours - - for (uint i = 0; i < contours.size(); i++) - goodIndices.push_back(true); - - this->width = threshold.cols; - this->height = threshold.rows; -} - - -uint TextContours::size() { - return contours.size(); -} - - - -int TextContours::getGoodIndicesCount() +namespace alpr { - int count = 0; - for (uint i = 0; i < goodIndices.size(); i++) - { - if (goodIndices[i]) - count++; + + TextContours::TextContours() { + + } - return count; -} + TextContours::TextContours(cv::Mat threshold) { -std::vector TextContours::getIndicesCopy() -{ - vector copyArray; - for (uint i = 0; i < goodIndices.size(); i++) - { - bool val = goodIndices[i]; - copyArray.push_back(goodIndices[i]); + load(threshold); } - - return copyArray; -} -void TextContours::setIndices(std::vector newIndices) -{ - if (newIndices.size() == goodIndices.size()) - { - for (uint i = 0; i < newIndices.size(); i++) - goodIndices[i] = newIndices[i]; + + TextContours::~TextContours() { } - else - { - assert("Invalid set operation on indices"); + + void TextContours::load(cv::Mat threshold) { + + Mat tempThreshold(threshold.size(), CV_8U); + threshold.copyTo(tempThreshold); + findContours(tempThreshold, + contours, // a vector of contours + hierarchy, + CV_RETR_TREE, // retrieve all contours + CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours + + for (uint i = 0; i < contours.size(); i++) + goodIndices.push_back(true); + + this->width = threshold.cols; + this->height = threshold.rows; } -} -Mat TextContours::drawDebugImage() { - Mat img_contours = Mat::zeros(Size(width, height), CV_8U); - - return drawDebugImage(img_contours); -} + uint TextContours::size() { + return contours.size(); + } -Mat TextContours::drawDebugImage(Mat baseImage) { - Mat img_contours(baseImage.size(), CV_8U); - baseImage.copyTo(img_contours); - - cvtColor(img_contours, img_contours, CV_GRAY2RGB); - vector > allowedContours; - for (uint i = 0; i < this->contours.size(); i++) + + int TextContours::getGoodIndicesCount() + { + int count = 0; + for (uint i = 0; i < goodIndices.size(); i++) { - if (this->goodIndices[i]) - allowedContours.push_back(this->contours[i]); + if (goodIndices[i]) + count++; } - drawContours(img_contours, this->contours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 + return count; + } - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 - - return img_contours; + std::vector TextContours::getIndicesCopy() + { + vector copyArray; + for (uint i = 0; i < goodIndices.size(); i++) + { + bool val = goodIndices[i]; + copyArray.push_back(goodIndices[i]); + } + + return copyArray; + } + + void TextContours::setIndices(std::vector newIndices) + { + if (newIndices.size() == goodIndices.size()) + { + for (uint i = 0; i < newIndices.size(); i++) + goodIndices[i] = newIndices[i]; + } + else + { + assert("Invalid set operation on indices"); + } + } + + Mat TextContours::drawDebugImage() { + + Mat img_contours = Mat::zeros(Size(width, height), CV_8U); + + return drawDebugImage(img_contours); + } + + Mat TextContours::drawDebugImage(Mat baseImage) { + Mat img_contours(baseImage.size(), CV_8U); + baseImage.copyTo(img_contours); + + cvtColor(img_contours, img_contours, CV_GRAY2RGB); + + vector > allowedContours; + for (uint i = 0; i < this->contours.size(); i++) + { + if (this->goodIndices[i]) + allowedContours.push_back(this->contours[i]); + } + + drawContours(img_contours, this->contours, + -1, // draw all contours + cv::Scalar(255,0,0), // in blue + 1); // with a thickness of 1 + + drawContours(img_contours, allowedContours, + -1, // draw all contours + cv::Scalar(0,255,0), // in green + 1); // with a thickness of 1 + + + return img_contours; + } + } - - diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h index cf6d6bb..c7d2e45 100644 --- a/src/openalpr/textdetection/textcontours.h +++ b/src/openalpr/textdetection/textcontours.h @@ -23,34 +23,39 @@ #include #include "opencv2/imgproc/imgproc.hpp" -class TextContours { -public: - TextContours(); - TextContours(cv::Mat threshold); - virtual ~TextContours(); - - void load(cv::Mat threshold); - - int width; - int height; - - std::vector goodIndices; - std::vector > contours; - std::vector hierarchy; - - uint size(); - int getGoodIndicesCount(); - - std::vector getIndicesCopy(); - void setIndices(std::vector newIndices); - - cv::Mat drawDebugImage(); - cv::Mat drawDebugImage(cv::Mat baseImage); - -private: - +namespace alpr +{ -}; + class TextContours { + public: + TextContours(); + TextContours(cv::Mat threshold); + virtual ~TextContours(); + + void load(cv::Mat threshold); + + int width; + int height; + + std::vector goodIndices; + std::vector > contours; + std::vector hierarchy; + + uint size(); + int getGoodIndicesCount(); + + std::vector getIndicesCopy(); + void setIndices(std::vector newIndices); + + cv::Mat drawDebugImage(); + cv::Mat drawDebugImage(cv::Mat baseImage); + + private: + + + }; + +} #endif /* TEXTCONTOURS_H */ diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index b0c57af..ec9db71 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -24,84 +24,89 @@ using namespace cv; -TextLine::TextLine(std::vector textArea, std::vector linePolygon) { - std::vector textAreaInts, linePolygonInts; - - for (uint i = 0; i < textArea.size(); i++) - textAreaInts.push_back(Point(round(textArea[i].x), round(textArea[i].y))); - for (uint i = 0; i < linePolygon.size(); i++) - linePolygonInts.push_back(Point(round(linePolygon[i].x), round(linePolygon[i].y))); - - initialize(textAreaInts, linePolygonInts); -} +namespace alpr +{ -TextLine::TextLine(std::vector textArea, std::vector linePolygon) { - initialize(textArea, linePolygon); -} + TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + std::vector textAreaInts, linePolygonInts; - -TextLine::~TextLine() { -} - - - -void TextLine::initialize(std::vector textArea, std::vector linePolygon) { - if (textArea.size() > 0) - { - if (this->textArea.size() > 0) - this->textArea.clear(); - if (this->linePolygon.size() > 0) - this->linePolygon.clear(); - for (uint i = 0; i < textArea.size(); i++) - this->textArea.push_back(textArea[i]); - + textAreaInts.push_back(Point(round(textArea[i].x), round(textArea[i].y))); for (uint i = 0; i < linePolygon.size(); i++) - this->linePolygon.push_back(linePolygon[i]); - - this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); - this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + linePolygonInts.push_back(Point(round(linePolygon[i].x), round(linePolygon[i].y))); - this->charBoxTop = LineSegment(textArea[0].x, textArea[0].y, textArea[1].x, textArea[1].y); - this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); - this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); - this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); - - // Calculate line height - float x = ((float) linePolygon[1].x) / 2; - Point midpoint = Point(x, bottomLine.getPointAt(x)); - Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); - this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); - - // Subtract a pixel since the height is a little overestimated by the bounding box - this->lineHeight = this->lineHeight - 1; - - this->angle = (topLine.angle + bottomLine.angle) / 2; - + initialize(textAreaInts, linePolygonInts); + } + + TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + initialize(textArea, linePolygon); } -} -cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { - cv::Mat debugImage(baseImage.size(), baseImage.type()); - - baseImage.copyTo(debugImage); - - cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR); - - - fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); + TextLine::~TextLine() { + } - fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); - - line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); - line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); - - line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); - line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); - line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); - line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); - - - return debugImage; -} + + + void TextLine::initialize(std::vector textArea, std::vector linePolygon) { + if (textArea.size() > 0) + { + if (this->textArea.size() > 0) + this->textArea.clear(); + if (this->linePolygon.size() > 0) + this->linePolygon.clear(); + + for (uint i = 0; i < textArea.size(); i++) + this->textArea.push_back(textArea[i]); + + for (uint i = 0; i < linePolygon.size(); i++) + this->linePolygon.push_back(linePolygon[i]); + + this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + + this->charBoxTop = LineSegment(textArea[0].x, textArea[0].y, textArea[1].x, textArea[1].y); + this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); + this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); + this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); + + // Calculate line height + float x = ((float) linePolygon[1].x) / 2; + Point midpoint = Point(x, bottomLine.getPointAt(x)); + Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); + this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); + + // Subtract a pixel since the height is a little overestimated by the bounding box + this->lineHeight = this->lineHeight - 1; + + this->angle = (topLine.angle + bottomLine.angle) / 2; + + } + } + + + cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { + cv::Mat debugImage(baseImage.size(), baseImage.type()); + + baseImage.copyTo(debugImage); + + cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR); + + + fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); + + fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); + + line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); + line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); + + line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); + + + return debugImage; + } + +} \ No newline at end of file diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index ec9e6d5..ac7f48a 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -24,31 +24,36 @@ #include "utility.h" #include "opencv2/imgproc/imgproc.hpp" -class TextLine { -public: - TextLine(std::vector textArea, std::vector linePolygon); - TextLine(std::vector textArea, std::vector linePolygon); - virtual ~TextLine(); - - - std::vector linePolygon; - std::vector textArea; - LineSegment topLine; - LineSegment bottomLine; +namespace alpr +{ - LineSegment charBoxTop; - LineSegment charBoxBottom; - LineSegment charBoxLeft; - LineSegment charBoxRight; - - float lineHeight; - float angle; + class TextLine { + public: + TextLine(std::vector textArea, std::vector linePolygon); + TextLine(std::vector textArea, std::vector linePolygon); + virtual ~TextLine(); - cv::Mat drawDebugImage(cv::Mat baseImage); -private: - void initialize(std::vector textArea, std::vector linePolygon); -}; + std::vector linePolygon; + std::vector textArea; + LineSegment topLine; + LineSegment bottomLine; + + LineSegment charBoxTop; + LineSegment charBoxBottom; + LineSegment charBoxLeft; + LineSegment charBoxRight; + + float lineHeight; + float angle; + + cv::Mat drawDebugImage(cv::Mat baseImage); + private: + + void initialize(std::vector textArea, std::vector linePolygon); + }; + +} #endif /* OPENALPR_TEXTLINE_H */ diff --git a/src/openalpr/transformation.cpp b/src/openalpr/transformation.cpp index b3f32af..bda2311 100644 --- a/src/openalpr/transformation.cpp +++ b/src/openalpr/transformation.cpp @@ -22,118 +22,123 @@ using namespace std; using namespace cv; -Transformation::Transformation(Mat bigImage, Mat smallImage, Rect regionInBigImage) { - this->bigImage = bigImage; - this->smallImage = smallImage; - this->regionInBigImage = regionInBigImage; -} - - -Transformation::~Transformation() { -} - -// Re-maps the coordinates from the smallImage to the coordinate space of the bigImage. -vector Transformation::transformSmallPointsToBigImage(vector points) +namespace alpr { - vector floatPoints; - for (unsigned int i = 0; i < points.size(); i++) - floatPoints.push_back(points[i]); - - return transformSmallPointsToBigImage(floatPoints); - -} -// Re-maps the coordinates from the smallImage to the coordinate space of the bigImage. -vector Transformation::transformSmallPointsToBigImage(vector points) -{ - vector bigPoints; - for (uint i = 0; i < points.size(); i++) - { - float bigX = (points[i].x * ((float) regionInBigImage.width / smallImage.cols)); - float bigY = (points[i].y * ((float) regionInBigImage.height / smallImage.rows)); - - bigX = bigX + regionInBigImage.x; - bigY = bigY + regionInBigImage.y; - - bigPoints.push_back(Point2f(bigX, bigY)); + Transformation::Transformation(Mat bigImage, Mat smallImage, Rect regionInBigImage) { + this->bigImage = bigImage; + this->smallImage = smallImage; + this->regionInBigImage = regionInBigImage; } - return bigPoints; -} + Transformation::~Transformation() { + } -Mat Transformation::getTransformationMatrix(vector corners, Size outputImageSize) -{ - // Corners of the destination image - vector quad_pts; - quad_pts.push_back(Point2f(0, 0)); - quad_pts.push_back(Point2f(outputImageSize.width, 0)); - quad_pts.push_back(Point2f(outputImageSize.width, outputImageSize.height)); - quad_pts.push_back(Point2f(0, outputImageSize.height)); - - return getTransformationMatrix(corners, quad_pts); -} - -Mat Transformation::getTransformationMatrix(vector corners, vector outputCorners) -{ - - // Get transformation matrix - Mat transmtx = getPerspectiveTransform(corners, outputCorners); - - return transmtx; -} - - -Mat Transformation::crop(Size outputImageSize, Mat transformationMatrix) -{ - - - Mat deskewed(outputImageSize, this->bigImage.type()); - - // Apply perspective transformation to the image - warpPerspective(this->bigImage, deskewed, transformationMatrix, deskewed.size(), INTER_CUBIC); - - - - - return deskewed; -} - -vector Transformation::remapSmallPointstoCrop(vector smallPoints, cv::Mat transformationMatrix) -{ - vector floatPoints; - for (unsigned int i = 0; i < smallPoints.size(); i++) - floatPoints.push_back(smallPoints[i]); - - return remapSmallPointstoCrop(floatPoints, transformationMatrix); -} - -vector Transformation::remapSmallPointstoCrop(vector smallPoints, cv::Mat transformationMatrix) -{ - vector remappedPoints; - perspectiveTransform(smallPoints, remappedPoints, transformationMatrix); - - return remappedPoints; -} - -Size Transformation::getCropSize(vector areaCorners, Size targetSize) -{ - // Figure out the approximate width/height of the license plate region, so we can maintain the aspect ratio. - LineSegment leftEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[0].x), round(areaCorners[0].y)); - LineSegment rightEdge(round(areaCorners[2].x), round(areaCorners[2].y), round(areaCorners[1].x), round(areaCorners[1].y)); - LineSegment topEdge(round(areaCorners[0].x), round(areaCorners[0].y), round(areaCorners[1].x), round(areaCorners[1].y)); - LineSegment bottomEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[2].x), round(areaCorners[2].y)); - - float w = distanceBetweenPoints(leftEdge.midpoint(), rightEdge.midpoint()); - float h = distanceBetweenPoints(bottomEdge.midpoint(), topEdge.midpoint()); - float aspect = w/h; - int width = targetSize.width; - int height = round(((float) width) / aspect); - if (height > targetSize.height) + // Re-maps the coordinates from the smallImage to the coordinate space of the bigImage. + vector Transformation::transformSmallPointsToBigImage(vector points) { - height = targetSize.height; - width = round(((float) height) * aspect); + vector floatPoints; + for (unsigned int i = 0; i < points.size(); i++) + floatPoints.push_back(points[i]); + + return transformSmallPointsToBigImage(floatPoints); + + } + + // Re-maps the coordinates from the smallImage to the coordinate space of the bigImage. + vector Transformation::transformSmallPointsToBigImage(vector points) + { + vector bigPoints; + for (uint i = 0; i < points.size(); i++) + { + float bigX = (points[i].x * ((float) regionInBigImage.width / smallImage.cols)); + float bigY = (points[i].y * ((float) regionInBigImage.height / smallImage.rows)); + + bigX = bigX + regionInBigImage.x; + bigY = bigY + regionInBigImage.y; + + bigPoints.push_back(Point2f(bigX, bigY)); + } + + return bigPoints; + } + + + Mat Transformation::getTransformationMatrix(vector corners, Size outputImageSize) + { + // Corners of the destination image + vector quad_pts; + quad_pts.push_back(Point2f(0, 0)); + quad_pts.push_back(Point2f(outputImageSize.width, 0)); + quad_pts.push_back(Point2f(outputImageSize.width, outputImageSize.height)); + quad_pts.push_back(Point2f(0, outputImageSize.height)); + + return getTransformationMatrix(corners, quad_pts); + } + + Mat Transformation::getTransformationMatrix(vector corners, vector outputCorners) + { + + // Get transformation matrix + Mat transmtx = getPerspectiveTransform(corners, outputCorners); + + return transmtx; + } + + + Mat Transformation::crop(Size outputImageSize, Mat transformationMatrix) + { + + + Mat deskewed(outputImageSize, this->bigImage.type()); + + // Apply perspective transformation to the image + warpPerspective(this->bigImage, deskewed, transformationMatrix, deskewed.size(), INTER_CUBIC); + + + + + return deskewed; + } + + vector Transformation::remapSmallPointstoCrop(vector smallPoints, cv::Mat transformationMatrix) + { + vector floatPoints; + for (unsigned int i = 0; i < smallPoints.size(); i++) + floatPoints.push_back(smallPoints[i]); + + return remapSmallPointstoCrop(floatPoints, transformationMatrix); + } + + vector Transformation::remapSmallPointstoCrop(vector smallPoints, cv::Mat transformationMatrix) + { + vector remappedPoints; + perspectiveTransform(smallPoints, remappedPoints, transformationMatrix); + + return remappedPoints; + } + + Size Transformation::getCropSize(vector areaCorners, Size targetSize) + { + // Figure out the approximate width/height of the license plate region, so we can maintain the aspect ratio. + LineSegment leftEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[0].x), round(areaCorners[0].y)); + LineSegment rightEdge(round(areaCorners[2].x), round(areaCorners[2].y), round(areaCorners[1].x), round(areaCorners[1].y)); + LineSegment topEdge(round(areaCorners[0].x), round(areaCorners[0].y), round(areaCorners[1].x), round(areaCorners[1].y)); + LineSegment bottomEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[2].x), round(areaCorners[2].y)); + + float w = distanceBetweenPoints(leftEdge.midpoint(), rightEdge.midpoint()); + float h = distanceBetweenPoints(bottomEdge.midpoint(), topEdge.midpoint()); + float aspect = w/h; + int width = targetSize.width; + int height = round(((float) width) / aspect); + if (height > targetSize.height) + { + height = targetSize.height; + width = round(((float) height) * aspect); + } + + return Size(width, height); } - return Size(width, height); } \ No newline at end of file diff --git a/src/openalpr/transformation.h b/src/openalpr/transformation.h index 67bbe97..e5e34ba 100644 --- a/src/openalpr/transformation.h +++ b/src/openalpr/transformation.h @@ -23,29 +23,34 @@ #include "opencv2/imgproc/imgproc.hpp" #include "utility.h" -class Transformation { -public: - Transformation(cv::Mat bigImage, cv::Mat smallImage, cv::Rect regionInBigImage); - virtual ~Transformation(); - - std::vector transformSmallPointsToBigImage(std::vector points); - std::vector transformSmallPointsToBigImage(std::vector points); - - cv::Mat getTransformationMatrix(std::vector corners, cv::Size outputImageSize); - cv::Mat getTransformationMatrix(std::vector corners, std::vector outputCorners); - - cv::Mat crop(cv::Size outputImageSize, cv::Mat transformationMatrix); - std::vector remapSmallPointstoCrop(std::vector smallPoints, cv::Mat transformationMatrix); - std::vector remapSmallPointstoCrop(std::vector smallPoints, cv::Mat transformationMatrix); - - cv::Size getCropSize(std::vector areaCorners, cv::Size targetSize); - -private: - cv::Mat bigImage; - cv::Mat smallImage; - cv::Rect regionInBigImage; - -}; +namespace alpr +{ + + class Transformation { + public: + Transformation(cv::Mat bigImage, cv::Mat smallImage, cv::Rect regionInBigImage); + virtual ~Transformation(); + + std::vector transformSmallPointsToBigImage(std::vector points); + std::vector transformSmallPointsToBigImage(std::vector points); + + cv::Mat getTransformationMatrix(std::vector corners, cv::Size outputImageSize); + cv::Mat getTransformationMatrix(std::vector corners, std::vector outputCorners); + + cv::Mat crop(cv::Size outputImageSize, cv::Mat transformationMatrix); + std::vector remapSmallPointstoCrop(std::vector smallPoints, cv::Mat transformationMatrix); + std::vector remapSmallPointstoCrop(std::vector smallPoints, cv::Mat transformationMatrix); + + cv::Size getCropSize(std::vector areaCorners, cv::Size targetSize); + + private: + cv::Mat bigImage; + cv::Mat smallImage; + cv::Rect regionInBigImage; + + }; + +} #endif /* OPENALPR_TRANSFORMATION_H */ diff --git a/src/openalpr/utility.cpp b/src/openalpr/utility.cpp index 693fdef..4aa844f 100644 --- a/src/openalpr/utility.cpp +++ b/src/openalpr/utility.cpp @@ -24,413 +24,418 @@ using namespace cv; using namespace std; -Rect expandRect(Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY) +namespace alpr { - Rect expandedRegion = Rect(original); - float halfX = round((float) expandXPixels / 2.0); - float halfY = round((float) expandYPixels / 2.0); - expandedRegion.x = expandedRegion.x - halfX; - expandedRegion.width = expandedRegion.width + expandXPixels; - expandedRegion.y = expandedRegion.y - halfY; - expandedRegion.height = expandedRegion.height + expandYPixels; - - if (expandedRegion.x < 0) - expandedRegion.x = 0; - if (expandedRegion.y < 0) - expandedRegion.y = 0; - if (expandedRegion.x + expandedRegion.width > maxX) - expandedRegion.width = maxX - expandedRegion.x; - if (expandedRegion.y + expandedRegion.height > maxY) - expandedRegion.height = maxY - expandedRegion.y; - - return expandedRegion; -} - -Mat drawImageDashboard(vector images, int imageType, uint numColumns) -{ - uint numRows = ceil((float) images.size() / (float) numColumns); - - Mat dashboard(Size(images[0].cols * numColumns, images[0].rows * numRows), imageType); - - for (uint i = 0; i < numColumns * numRows; i++) + Rect expandRect(Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY) { - if (i < images.size()) - images[i].copyTo(dashboard(Rect((i%numColumns) * images[i].cols, floor((float) i/numColumns) * images[i].rows, images[i].cols, images[i].rows))); - else + Rect expandedRegion = Rect(original); + + float halfX = round((float) expandXPixels / 2.0); + float halfY = round((float) expandYPixels / 2.0); + expandedRegion.x = expandedRegion.x - halfX; + expandedRegion.width = expandedRegion.width + expandXPixels; + expandedRegion.y = expandedRegion.y - halfY; + expandedRegion.height = expandedRegion.height + expandYPixels; + + if (expandedRegion.x < 0) + expandedRegion.x = 0; + if (expandedRegion.y < 0) + expandedRegion.y = 0; + if (expandedRegion.x + expandedRegion.width > maxX) + expandedRegion.width = maxX - expandedRegion.x; + if (expandedRegion.y + expandedRegion.height > maxY) + expandedRegion.height = maxY - expandedRegion.y; + + return expandedRegion; + } + + Mat drawImageDashboard(vector images, int imageType, uint numColumns) + { + uint numRows = ceil((float) images.size() / (float) numColumns); + + Mat dashboard(Size(images[0].cols * numColumns, images[0].rows * numRows), imageType); + + for (uint i = 0; i < numColumns * numRows; i++) { - Mat black = Mat::zeros(images[0].size(), imageType); - black.copyTo(dashboard(Rect((i%numColumns) * images[0].cols, floor((float) i/numColumns) * images[0].rows, images[0].cols, images[0].rows))); + if (i < images.size()) + images[i].copyTo(dashboard(Rect((i%numColumns) * images[i].cols, floor((float) i/numColumns) * images[i].rows, images[i].cols, images[i].rows))); + else + { + Mat black = Mat::zeros(images[0].size(), imageType); + black.copyTo(dashboard(Rect((i%numColumns) * images[0].cols, floor((float) i/numColumns) * images[0].rows, images[0].cols, images[0].rows))); + } + } + + return dashboard; + } + + Mat addLabel(Mat input, string label) + { + const int border_size = 1; + const Scalar border_color(0,0,255); + const int extraHeight = 20; + const Scalar bg(222,222,222); + const Scalar fg(0,0,0); + + Rect destinationRect(border_size, extraHeight, input.cols, input.rows); + Mat newImage(Size(input.cols + (border_size), input.rows + extraHeight + (border_size )), input.type()); + input.copyTo(newImage(destinationRect)); + + cout << " Adding label " << label << endl; + if (input.type() == CV_8U) + cvtColor(newImage, newImage, CV_GRAY2BGR); + + rectangle(newImage, Point(0,0), Point(input.cols, extraHeight), bg, CV_FILLED); + putText(newImage, label, Point(5, extraHeight - 5), CV_FONT_HERSHEY_PLAIN , 0.7, fg); + + rectangle(newImage, Point(0,0), Point(newImage.cols - 1, newImage.rows -1), border_color, border_size); + + return newImage; + } + + void drawAndWait(cv::Mat* frame) + { + cv::imshow("Temp Window", *frame); + + while (cv::waitKey(50) == -1) + { + // loop + } + + cv::destroyWindow("Temp Window"); + } + + void displayImage(Config* config, string windowName, cv::Mat frame) + { + if (config->debugShowImages) + { + imshow(windowName, frame); + cv::waitKey(5); } } - return dashboard; -} - -Mat addLabel(Mat input, string label) -{ - const int border_size = 1; - const Scalar border_color(0,0,255); - const int extraHeight = 20; - const Scalar bg(222,222,222); - const Scalar fg(0,0,0); - - Rect destinationRect(border_size, extraHeight, input.cols, input.rows); - Mat newImage(Size(input.cols + (border_size), input.rows + extraHeight + (border_size )), input.type()); - input.copyTo(newImage(destinationRect)); - - cout << " Adding label " << label << endl; - if (input.type() == CV_8U) - cvtColor(newImage, newImage, CV_GRAY2BGR); - - rectangle(newImage, Point(0,0), Point(input.cols, extraHeight), bg, CV_FILLED); - putText(newImage, label, Point(5, extraHeight - 5), CV_FONT_HERSHEY_PLAIN , 0.7, fg); - - rectangle(newImage, Point(0,0), Point(newImage.cols - 1, newImage.rows -1), border_color, border_size); - - return newImage; -} - -void drawAndWait(cv::Mat* frame) -{ - cv::imshow("Temp Window", *frame); - - while (cv::waitKey(50) == -1) + vector produceThresholds(const Mat img_gray, Config* config) { - // loop - } + const int THRESHOLD_COUNT = 3; + //Mat img_equalized = equalizeBrightness(img_gray); - cv::destroyWindow("Temp Window"); -} + timespec startTime; + getTime(&startTime); -void displayImage(Config* config, string windowName, cv::Mat frame) -{ - if (config->debugShowImages) - { - imshow(windowName, frame); - cv::waitKey(5); - } -} + vector thresholds; -vector produceThresholds(const Mat img_gray, Config* config) -{ - const int THRESHOLD_COUNT = 3; - //Mat img_equalized = equalizeBrightness(img_gray); + for (int i = 0; i < THRESHOLD_COUNT; i++) + thresholds.push_back(Mat(img_gray.size(), CV_8U)); - timespec startTime; - getTime(&startTime); + int i = 0; - vector thresholds; + // Adaptive + //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 7, 3); + //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 13, 3); + //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 17, 3); - for (int i = 0; i < THRESHOLD_COUNT; i++) - thresholds.push_back(Mat(img_gray.size(), CV_8U)); + // Wolf + int k = 0, win=18; + //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); + //bitwise_not(thresholds[i-1], thresholds[i-1]); + NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); + bitwise_not(thresholds[i-1], thresholds[i-1]); - int i = 0; + k = 1; + win = 22; + NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); + bitwise_not(thresholds[i-1], thresholds[i-1]); + //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); + //bitwise_not(thresholds[i-1], thresholds[i-1]); - // Adaptive - //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 7, 3); - //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 13, 3); - //adaptiveThreshold(img_gray, thresholds[i++], 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY_INV , 17, 3); + // Sauvola + k = 1; + NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); + bitwise_not(thresholds[i-1], thresholds[i-1]); + //k=2; + //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); + //bitwise_not(thresholds[i-1], thresholds[i-1]); - // Wolf - int k = 0, win=18; - //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); - //bitwise_not(thresholds[i-1], thresholds[i-1]); - NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); - bitwise_not(thresholds[i-1], thresholds[i-1]); - - k = 1; - win = 22; - NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); - bitwise_not(thresholds[i-1], thresholds[i-1]); - //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], WOLFJOLION, win, win, 0.05 + (k * 0.35)); - //bitwise_not(thresholds[i-1], thresholds[i-1]); - - // Sauvola - k = 1; - NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); - bitwise_not(thresholds[i-1], thresholds[i-1]); - //k=2; - //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); - //bitwise_not(thresholds[i-1], thresholds[i-1]); - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << " -- Produce Threshold Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - return thresholds; - //threshold(img_equalized, img_threshold, 100, 255, THRESH_BINARY); -} - -double median(int array[], int arraySize) -{ - if (arraySize == 0) - { - //std::cerr << "Median calculation requested on empty array" << endl; - return 0; - } - - std::sort(&array[0], &array[arraySize]); - return arraySize % 2 ? array[arraySize / 2] : (array[arraySize / 2 - 1] + array[arraySize / 2]) / 2; -} - -Mat equalizeBrightness(Mat img) -{ - // Divide the image by its morphologically closed counterpart - Mat kernel = getStructuringElement(MORPH_ELLIPSE, Size(19,19)); - Mat closed; - morphologyEx(img, closed, MORPH_CLOSE, kernel); - - img.convertTo(img, CV_32FC1); // divide requires floating-point - divide(img, closed, img, 1, CV_32FC1); - normalize(img, img, 0, 255, NORM_MINMAX); - img.convertTo(img, CV_8U); // convert back to unsigned int - - return img; -} - -void drawRotatedRect(Mat* img, RotatedRect rect, Scalar color, int thickness) -{ - Point2f rect_points[4]; - rect.points( rect_points ); - for( int j = 0; j < 4; j++ ) - line( *img, rect_points[j], rect_points[(j+1)%4], color, thickness, 8 ); -} - -void fillMask(Mat img, const Mat mask, Scalar color) -{ - for (int row = 0; row < img.rows; row++) - { - for (int col = 0; col < img.cols; col++) + if (config->debugTiming) { - int m = (int) mask.at(row, col); + timespec endTime; + getTime(&endTime); + cout << " -- Produce Threshold Time: " << diffclock(startTime, endTime) << "ms." << endl; + } - if (m) + return thresholds; + //threshold(img_equalized, img_threshold, 100, 255, THRESH_BINARY); + } + + double median(int array[], int arraySize) + { + if (arraySize == 0) + { + //std::cerr << "Median calculation requested on empty array" << endl; + return 0; + } + + std::sort(&array[0], &array[arraySize]); + return arraySize % 2 ? array[arraySize / 2] : (array[arraySize / 2 - 1] + array[arraySize / 2]) / 2; + } + + Mat equalizeBrightness(Mat img) + { + // Divide the image by its morphologically closed counterpart + Mat kernel = getStructuringElement(MORPH_ELLIPSE, Size(19,19)); + Mat closed; + morphologyEx(img, closed, MORPH_CLOSE, kernel); + + img.convertTo(img, CV_32FC1); // divide requires floating-point + divide(img, closed, img, 1, CV_32FC1); + normalize(img, img, 0, 255, NORM_MINMAX); + img.convertTo(img, CV_8U); // convert back to unsigned int + + return img; + } + + void drawRotatedRect(Mat* img, RotatedRect rect, Scalar color, int thickness) + { + Point2f rect_points[4]; + rect.points( rect_points ); + for( int j = 0; j < 4; j++ ) + line( *img, rect_points[j], rect_points[(j+1)%4], color, thickness, 8 ); + } + + void fillMask(Mat img, const Mat mask, Scalar color) + { + for (int row = 0; row < img.rows; row++) + { + for (int col = 0; col < img.cols; col++) { - for (int z = 0; z < 3; z++) + int m = (int) mask.at(row, col); + + if (m) { - int prevVal = img.at(row, col)[z]; - img.at(row, col)[z] = ((int) color[z]) | prevVal; + for (int z = 0; z < 3; z++) + { + int prevVal = img.at(row, col)[z]; + img.at(row, col)[z] = ((int) color[z]) | prevVal; + } } } } } -} -void drawX(Mat img, Rect rect, Scalar color, int thickness) -{ - Point tl(rect.x, rect.y); - Point tr(rect.x + rect.width, rect.y); - Point bl(rect.x, rect.y + rect.height); - Point br(rect.x + rect.width, rect.y + rect.height); - - line(img, tl, br, color, thickness); - line(img, bl, tr, color, thickness); -} - -double distanceBetweenPoints(Point p1, Point p2) -{ - float asquared = (p2.x - p1.x)*(p2.x - p1.x); - float bsquared = (p2.y - p1.y)*(p2.y - p1.y); - - return sqrt(asquared + bsquared); -} - -float angleBetweenPoints(Point p1, Point p2) -{ - int deltaY = p2.y - p1.y; - int deltaX = p2.x - p1.x; - - return atan2((float) deltaY, (float) deltaX) * (180 / CV_PI); -} - -Size getSizeMaintainingAspect(Mat inputImg, int maxWidth, int maxHeight) -{ - float aspect = ((float) inputImg.cols) / ((float) inputImg.rows); - - if (maxWidth / aspect > maxHeight) + void drawX(Mat img, Rect rect, Scalar color, int thickness) { - return Size(maxHeight * aspect, maxHeight); - } - else - { - return Size(maxWidth, maxWidth / aspect); - } -} + Point tl(rect.x, rect.y); + Point tr(rect.x + rect.width, rect.y); + Point bl(rect.x, rect.y + rect.height); + Point br(rect.x + rect.width, rect.y + rect.height); -LineSegment::LineSegment() -{ - init(0, 0, 0, 0); -} - -LineSegment::LineSegment(Point p1, Point p2) -{ - init(p1.x, p1.y, p2.x, p2.y); -} - -LineSegment::LineSegment(int x1, int y1, int x2, int y2) -{ - init(x1, y1, x2, y2); -} - -void LineSegment::init(int x1, int y1, int x2, int y2) -{ - this->p1 = Point(x1, y1); - this->p2 = Point(x2, y2); - - if (p2.x - p1.x == 0) - this->slope = 0.00000000001; - else - this->slope = (float) (p2.y - p1.y) / (float) (p2.x - p1.x); - - this->length = distanceBetweenPoints(p1, p2); - - this->angle = angleBetweenPoints(p1, p2); -} - -bool LineSegment::isPointBelowLine( Point tp ) -{ - return ((p2.x - p1.x)*(tp.y - p1.y) - (p2.y - p1.y)*(tp.x - p1.x)) > 0; -} - -float LineSegment::getPointAt(float x) -{ - return slope * (x - p2.x) + p2.y; -} - -Point LineSegment::closestPointOnSegmentTo(Point p) -{ - float top = (p.x - p1.x) * (p2.x - p1.x) + (p.y - p1.y)*(p2.y - p1.y); - - float bottom = distanceBetweenPoints(p2, p1); - bottom = bottom * bottom; - - float u = top / bottom; - - float x = p1.x + u * (p2.x - p1.x); - float y = p1.y + u * (p2.y - p1.y); - - return Point(x, y); -} - -Point LineSegment::intersection(LineSegment line) -{ - float c1, c2; - float intersection_X = -1, intersection_Y= -1; - - c1 = p1.y - slope * p1.x; // which is same as y2 - slope * x2 - - c2 = line.p2.y - line.slope * line.p2.x; // which is same as y2 - slope * x2 - - if( (slope - line.slope) == 0) - { - //std::cout << "No Intersection between the lines" << endl; - } - else if (p1.x == p2.x) - { - // Line1 is vertical - return Point(p1.x, line.getPointAt(p1.x)); - } - else if (line.p1.x == line.p2.x) - { - // Line2 is vertical - return Point(line.p1.x, getPointAt(line.p1.x)); - } - else - { - intersection_X = (c2 - c1) / (slope - line.slope); - intersection_Y = slope * intersection_X + c1; + line(img, tl, br, color, thickness); + line(img, bl, tr, color, thickness); } - return Point(intersection_X, intersection_Y); -} - -Point LineSegment::midpoint() -{ - // Handle the case where the line is vertical - if (p1.x == p2.x) + double distanceBetweenPoints(Point p1, Point p2) { - float ydiff = p2.y-p1.y; - float y = p1.y + (ydiff/2); - return Point(p1.x, y); + float asquared = (p2.x - p1.x)*(p2.x - p1.x); + float bsquared = (p2.y - p1.y)*(p2.y - p1.y); + + return sqrt(asquared + bsquared); } - float diff = p2.x - p1.x; - float midX = ((float) p1.x) + (diff / 2); - int midY = getPointAt(midX); - return Point(midX, midY); -} + float angleBetweenPoints(Point p1, Point p2) + { + int deltaY = p2.y - p1.y; + int deltaX = p2.x - p1.x; -LineSegment LineSegment::getParallelLine(float distance) -{ - float diff_x = p2.x - p1.x; - float diff_y = p2.y - p1.y; - float angle = atan2( diff_x, diff_y); - float dist_x = distance * cos(angle); - float dist_y = -distance * sin(angle); + return atan2((float) deltaY, (float) deltaX) * (180 / CV_PI); + } - int offsetX = (int)round(dist_x); - int offsetY = (int)round(dist_y); + Size getSizeMaintainingAspect(Mat inputImg, int maxWidth, int maxHeight) + { + float aspect = ((float) inputImg.cols) / ((float) inputImg.rows); - LineSegment result(p1.x + offsetX, p1.y + offsetY, - p2.x + offsetX, p2.y + offsetY); + if (maxWidth / aspect > maxHeight) + { + return Size(maxHeight * aspect, maxHeight); + } + else + { + return Size(maxWidth, maxWidth / aspect); + } + } - return result; -} + LineSegment::LineSegment() + { + init(0, 0, 0, 0); + } -// Given a contour and a mask, this function determines what percentage of the contour (area) -// is inside the masked area. -float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex) -{ + LineSegment::LineSegment(Point p1, Point p2) + { + init(p1.x, p1.y, p2.x, p2.y); + } + + LineSegment::LineSegment(int x1, int y1, int x2, int y2) + { + init(x1, y1, x2, y2); + } + + void LineSegment::init(int x1, int y1, int x2, int y2) + { + this->p1 = Point(x1, y1); + this->p2 = Point(x2, y2); + + if (p2.x - p1.x == 0) + this->slope = 0.00000000001; + else + this->slope = (float) (p2.y - p1.y) / (float) (p2.x - p1.x); + + this->length = distanceBetweenPoints(p1, p2); + + this->angle = angleBetweenPoints(p1, p2); + } + + bool LineSegment::isPointBelowLine( Point tp ) + { + return ((p2.x - p1.x)*(tp.y - p1.y) - (p2.y - p1.y)*(tp.x - p1.x)) > 0; + } + + float LineSegment::getPointAt(float x) + { + return slope * (x - p2.x) + p2.y; + } + + Point LineSegment::closestPointOnSegmentTo(Point p) + { + float top = (p.x - p1.x) * (p2.x - p1.x) + (p.y - p1.y)*(p2.y - p1.y); + + float bottom = distanceBetweenPoints(p2, p1); + bottom = bottom * bottom; + + float u = top / bottom; + + float x = p1.x + u * (p2.x - p1.x); + float y = p1.y + u * (p2.y - p1.y); + + return Point(x, y); + } + + Point LineSegment::intersection(LineSegment line) + { + float c1, c2; + float intersection_X = -1, intersection_Y= -1; + + c1 = p1.y - slope * p1.x; // which is same as y2 - slope * x2 + + c2 = line.p2.y - line.slope * line.p2.x; // which is same as y2 - slope * x2 + + if( (slope - line.slope) == 0) + { + //std::cout << "No Intersection between the lines" << endl; + } + else if (p1.x == p2.x) + { + // Line1 is vertical + return Point(p1.x, line.getPointAt(p1.x)); + } + else if (line.p1.x == line.p2.x) + { + // Line2 is vertical + return Point(line.p1.x, getPointAt(line.p1.x)); + } + else + { + intersection_X = (c2 - c1) / (slope - line.slope); + intersection_Y = slope * intersection_X + c1; + } + + return Point(intersection_X, intersection_Y); + } + + Point LineSegment::midpoint() + { + // Handle the case where the line is vertical + if (p1.x == p2.x) + { + float ydiff = p2.y-p1.y; + float y = p1.y + (ydiff/2); + return Point(p1.x, y); + } + float diff = p2.x - p1.x; + float midX = ((float) p1.x) + (diff / 2); + int midY = getPointAt(midX); + + return Point(midX, midY); + } + + LineSegment LineSegment::getParallelLine(float distance) + { + float diff_x = p2.x - p1.x; + float diff_y = p2.y - p1.y; + float angle = atan2( diff_x, diff_y); + float dist_x = distance * cos(angle); + float dist_y = -distance * sin(angle); + + int offsetX = (int)round(dist_x); + int offsetY = (int)round(dist_y); + + LineSegment result(p1.x + offsetX, p1.y + offsetY, + p2.x + offsetX, p2.y + offsetY); + + return result; + } + + // Given a contour and a mask, this function determines what percentage of the contour (area) + // is inside the masked area. + float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex) + { + + + Mat innerArea = Mat::zeros(mask.size(), CV_8U); + + + drawContours(innerArea, contours, + contourIndex, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + hierarchy, + 2 + ); + + + int startingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + bitwise_and(innerArea, mask, innerArea); + + int endingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + return ((float) endingPixels) / ((float) startingPixels); + + } + + std::string toString(int value) + { + stringstream ss; + ss << value; + return ss.str(); + } + std::string toString(uint value) + { + return toString((int) value); + } + std::string toString(float value) + { + stringstream ss; + ss << value; + return ss.str(); + } + std::string toString(double value) + { + stringstream ss; + ss << value; + return ss.str(); + } - - Mat innerArea = Mat::zeros(mask.size(), CV_8U); - - - drawContours(innerArea, contours, - contourIndex, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - hierarchy, - 2 - ); - - - int startingPixels = cv::countNonZero(innerArea); - //drawAndWait(&innerArea); - - bitwise_and(innerArea, mask, innerArea); - - int endingPixels = cv::countNonZero(innerArea); - //drawAndWait(&innerArea); - - return ((float) endingPixels) / ((float) startingPixels); - -} - -std::string toString(int value) -{ - stringstream ss; - ss << value; - return ss.str(); -} -std::string toString(uint value) -{ - return toString((int) value); -} -std::string toString(float value) -{ - stringstream ss; - ss << value; - return ss.str(); -} -std::string toString(double value) -{ - stringstream ss; - ss << value; - return ss.str(); } \ No newline at end of file diff --git a/src/openalpr/utility.h b/src/openalpr/utility.h index a2c5b16..148f704 100644 --- a/src/openalpr/utility.h +++ b/src/openalpr/utility.h @@ -33,85 +33,80 @@ #include #include "config.h" -/* -struct LineSegment -{ - float x1; - float y1; - float x2; - float y2; -}; -*/ - -class LineSegment +namespace alpr { - public: - cv::Point p1, p2; - float slope; - float length; - float angle; + class LineSegment + { - // LineSegment(Point point1, Point point2); - LineSegment(); - LineSegment(int x1, int y1, int x2, int y2); - LineSegment(cv::Point p1, cv::Point p2); + public: + cv::Point p1, p2; + float slope; + float length; + float angle; - void init(int x1, int y1, int x2, int y2); + // LineSegment(Point point1, Point point2); + LineSegment(); + LineSegment(int x1, int y1, int x2, int y2); + LineSegment(cv::Point p1, cv::Point p2); - bool isPointBelowLine(cv::Point tp); + void init(int x1, int y1, int x2, int y2); - float getPointAt(float x); + bool isPointBelowLine(cv::Point tp); - cv::Point closestPointOnSegmentTo(cv::Point p); + float getPointAt(float x); - cv::Point intersection(LineSegment line); + cv::Point closestPointOnSegmentTo(cv::Point p); - LineSegment getParallelLine(float distance); + cv::Point intersection(LineSegment line); - cv::Point midpoint(); + LineSegment getParallelLine(float distance); - inline std::string str() - { - std::stringstream ss; - ss << "(" << p1.x << ", " << p1.y << ") : (" << p2.x << ", " << p2.y << ")"; - return ss.str() ; - } + cv::Point midpoint(); -}; + inline std::string str() + { + std::stringstream ss; + ss << "(" << p1.x << ", " << p1.y << ") : (" << p2.x << ", " << p2.y << ")"; + return ss.str() ; + } -double median(int array[], int arraySize); + }; -std::vector produceThresholds(const cv::Mat img_gray, Config* config); + double median(int array[], int arraySize); -cv::Mat drawImageDashboard(std::vector images, int imageType, uint numColumns); + std::vector produceThresholds(const cv::Mat img_gray, Config* config); -void displayImage(Config* config, std::string windowName, cv::Mat frame); -void drawAndWait(cv::Mat* frame); + cv::Mat drawImageDashboard(std::vector images, int imageType, uint numColumns); -double distanceBetweenPoints(cv::Point p1, cv::Point p2); + void displayImage(Config* config, std::string windowName, cv::Mat frame); + void drawAndWait(cv::Mat* frame); -void drawRotatedRect(cv::Mat* img, cv::RotatedRect rect, cv::Scalar color, int thickness); + double distanceBetweenPoints(cv::Point p1, cv::Point p2); -void drawX(cv::Mat img, cv::Rect rect, cv::Scalar color, int thickness); -void fillMask(cv::Mat img, const cv::Mat mask, cv::Scalar color); + void drawRotatedRect(cv::Mat* img, cv::RotatedRect rect, cv::Scalar color, int thickness); -float angleBetweenPoints(cv::Point p1, cv::Point p2); + void drawX(cv::Mat img, cv::Rect rect, cv::Scalar color, int thickness); + void fillMask(cv::Mat img, const cv::Mat mask, cv::Scalar color); -cv::Size getSizeMaintainingAspect(cv::Mat inputImg, int maxWidth, int maxHeight); + float angleBetweenPoints(cv::Point p1, cv::Point p2); -float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex); + cv::Size getSizeMaintainingAspect(cv::Mat inputImg, int maxWidth, int maxHeight); -cv::Mat equalizeBrightness(cv::Mat img); + float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex); -cv::Rect expandRect(cv::Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY); + cv::Mat equalizeBrightness(cv::Mat img); -cv::Mat addLabel(cv::Mat input, std::string label); + cv::Rect expandRect(cv::Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY); + + cv::Mat addLabel(cv::Mat input, std::string label); -std::string toString(int value); -std::string toString(uint value); -std::string toString(float value); -std::string toString(double value); + std::string toString(int value); + std::string toString(uint value); + std::string toString(float value); + std::string toString(double value); + +} #endif // OPENALPR_UTILITY_H diff --git a/src/tests/test_api.cpp b/src/tests/test_api.cpp index a469edf..a9441cd 100644 --- a/src/tests/test_api.cpp +++ b/src/tests/test_api.cpp @@ -14,6 +14,7 @@ using namespace std; +using namespace alpr; TEST_CASE( "JSON Serialization/Deserialization", "[json]" ) { diff --git a/src/tests/test_utility.cpp b/src/tests/test_utility.cpp index acba4f4..de262b0 100644 --- a/src/tests/test_utility.cpp +++ b/src/tests/test_utility.cpp @@ -11,6 +11,7 @@ using namespace std; using namespace cv; +using namespace alpr; diff --git a/src/video/videobuffer.cpp b/src/video/videobuffer.cpp index 8dc13c0..11dae0f 100644 --- a/src/video/videobuffer.cpp +++ b/src/video/videobuffer.cpp @@ -19,6 +19,7 @@ #include "videobuffer.h" +using namespace alpr; void imageCollectionThread(void* arg); void getALPRImages(cv::VideoCapture cap, VideoDispatcher* dispatcher);