From 4e5f82c3fb9015ea00cb0ff8be4a70fc8b8063fa Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 5 Oct 2014 16:44:58 -0400 Subject: [PATCH 01/59] Fixed UUID str output bug for alprd --- src/daemon.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/daemon.cpp b/src/daemon.cpp index 356b27b..82f672f 100644 --- a/src/daemon.cpp +++ b/src/daemon.cpp @@ -266,14 +266,15 @@ void streamRecognitionThread(void* arg) { long epoch_time = getEpochTime(); - std::stringstream uuid; - uuid << tdata->site_id << "-cam" << tdata->camera_id << "-" << epoch_time; - + std::stringstream uuid_ss; + uuid_ss << tdata->site_id << "-cam" << tdata->camera_id << "-" << epoch_time; + std::string uuid = uuid_ss.str(); + // Save the image to disk (using the UUID) if (tdata->output_images) { std::stringstream ss; - ss << tdata->output_image_folder << "/" << uuid.str() << ".jpg"; + ss << tdata->output_image_folder << "/" << uuid << ".jpg"; cv::imwrite(ss.str(), latestFrame); } @@ -283,7 +284,7 @@ void streamRecognitionThread(void* arg) std::string json = alpr.toJson(results, totalProcessingTime, epoch_time); cJSON *root = cJSON_Parse(json.c_str()); - cJSON_AddStringToObject(root, "uuid", uuid.str().c_str()); + cJSON_AddStringToObject(root, "uuid", uuid.c_str()); cJSON_AddNumberToObject(root, "camera_id", tdata->camera_id); cJSON_AddStringToObject(root, "site_id", tdata->site_id.c_str()); cJSON_AddNumberToObject(root, "img_width", latestFrame.cols); From f3ec4fd554f7cb00d5a1400afbc133d838011548 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 6 Oct 2014 22:21:33 -0400 Subject: [PATCH 02/59] Changed aspect --- src/misc_utilities/tagplates.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc_utilities/tagplates.cpp b/src/misc_utilities/tagplates.cpp index 141f449..ba4e21e 100644 --- a/src/misc_utilities/tagplates.cpp +++ b/src/misc_utilities/tagplates.cpp @@ -64,7 +64,7 @@ static int xPos1 = 0; static int yPos1 = 0; static int xPos2 = 0; static int yPos2 = 0; -const float ASPECT_RATIO = 4.33333; +const float ASPECT_RATIO = 1.404; static bool rdragging = false; static int rDragStartX = 0; From 209166d82df94a9304df0c36380681205bf513d9 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 8 Oct 2014 06:04:00 -0400 Subject: [PATCH 03/59] Added file for Continuous Integration --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f4bdb8a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +before_install: + - sudo add-apt-repository ppa:yjwong/opencv2 -y + - sudo add-apt-repository ppa:lyrasis/precise-backports -y + - sudo apt-get update -q + +install: sudo apt-get -y install libopencv-dev libtesseract-dev git cmake build-essential libleptonica-dev liblog4cplus-dev libcurl3-dev beanstalkd + +before_script: + - mkdir -p ./src/build/ + - cd ./src/build/ + - cmake .. +script: + - make + - make classifychars tagplates sortstate benchmark prepcharsfortraining + + From 945354e75d2a5ce3df8b83a74e6152b4c195b55a Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 8 Oct 2014 09:42:35 -0400 Subject: [PATCH 04/59] Moved character analysis to text detection folder --- src/openalpr/CMakeLists.txt | 3 +- src/openalpr/characterregion.h | 2 +- .../{ => textdetection}/characteranalysis.cpp | 0 .../{ => textdetection}/characteranalysis.h | 0 .../textdetection/characteranalysis2l.cpp | 60 +++++++++++++++++++ .../textdetection/characteranalysis2l.h | 38 ++++++++++++ 6 files changed, 101 insertions(+), 2 deletions(-) rename src/openalpr/{ => textdetection}/characteranalysis.cpp (100%) rename src/openalpr/{ => textdetection}/characteranalysis.h (100%) create mode 100644 src/openalpr/textdetection/characteranalysis2l.cpp create mode 100644 src/openalpr/textdetection/characteranalysis2l.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index d979f90..9ca6c59 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -22,7 +22,8 @@ set(lpr_source_files segmentation/verticalhistogram.cpp platecorners.cpp colorfilter.cpp - characteranalysis.cpp + textdetection/characteranalysis.cpp + textdetection/characteranalysis2l.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/characterregion.h b/src/openalpr/characterregion.h index 1890fe9..94a1d90 100644 --- a/src/openalpr/characterregion.h +++ b/src/openalpr/characterregion.h @@ -23,7 +23,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "constants.h" #include "utility.h" -#include "characteranalysis.h" +#include "textdetection/characteranalysis.h" #include "config.h" #include "pipeline_data.h" diff --git a/src/openalpr/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp similarity index 100% rename from src/openalpr/characteranalysis.cpp rename to src/openalpr/textdetection/characteranalysis.cpp diff --git a/src/openalpr/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h similarity index 100% rename from src/openalpr/characteranalysis.h rename to src/openalpr/textdetection/characteranalysis.h diff --git a/src/openalpr/textdetection/characteranalysis2l.cpp b/src/openalpr/textdetection/characteranalysis2l.cpp new file mode 100644 index 0000000..6bfc957 --- /dev/null +++ b/src/openalpr/textdetection/characteranalysis2l.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "characteranalysis2l.h" + +using namespace cv; +using namespace std; + +CharacterAnalysis2L::CharacterAnalysis2L(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; +} + + +CharacterAnalysis2L::~CharacterAnalysis2L() { +} + +void CharacterAnalysis2L::analyze() { + + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, pipeline_data->config); + + std::vector > > allContours; + std::vector > allHierarchy; + + timespec startTime; + getTime(&startTime); + + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + vector > contours; + vector hierarchy; + + Mat tempThreshold(pipeline_data->thresholds[i].size(), CV_8U); + pipeline_data->thresholds[i].copyTo(tempThreshold); + findContours(tempThreshold, + contours, // a vector of contours + hierarchy, + CV_RETR_TREE, // retrieve all contours + CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours + + allContours.push_back(contours); + allHierarchy.push_back(hierarchy); + } +} diff --git a/src/openalpr/textdetection/characteranalysis2l.h b/src/openalpr/textdetection/characteranalysis2l.h new file mode 100644 index 0000000..9efdb26 --- /dev/null +++ b/src/openalpr/textdetection/characteranalysis2l.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_CHARACTERANALYSIS2L_H +#define OPENALPR_CHARACTERANALYSIS2L_H + +#include "utility.h" +#include "config.h" +#include "pipeline_data.h" + +class CharacterAnalysis2L { +public: + CharacterAnalysis2L(PipelineData* pipeline_data); + virtual ~CharacterAnalysis2L(); +private: + + PipelineData* pipeline_data; + void analyze(); +}; + +#endif /* OPENALPR_CHARACTERANALYSIS2L_H */ + From 1903057a10d83635a42e816e4e83792e266603f9 Mon Sep 17 00:00:00 2001 From: Matthew Hill Date: Wed, 8 Oct 2014 22:06:50 -0400 Subject: [PATCH 05/59] Added CI build status --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f51a184..6fd2791 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ Install OpenALPR on Ubuntu 14.04 x64 with the following commands: Compiling ----------- +[![Build Status](https://travis-ci.org/openalpr/openalpr.svg?branch=master)](https://travis-ci.org/openalpr/openalpr) + OpenALPR compiles and runs on Linux, Mac OSX and Windows. OpenALPR requires the following additional libraries: From f190fb3984c350c5491ea377fdc7ddad4bb68d59 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 9 Oct 2014 20:44:58 -0400 Subject: [PATCH 06/59] Removed unused header reference --- src/openalpr/textdetection/characteranalysis.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 35dff6f..7940f93 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -21,7 +21,6 @@ #define OPENALPR_CHARACTERANALYSIS_H #include "opencv2/imgproc/imgproc.hpp" -#include "constants.h" #include "utility.h" #include "config.h" #include "pipeline_data.h" From add77f6b0c7684f1c3ed392f2ef0ff2cb38135d1 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 9 Oct 2014 20:46:26 -0400 Subject: [PATCH 07/59] Split up the plate mask code into its own class --- src/openalpr/CMakeLists.txt | 1 + src/openalpr/characterregion.cpp | 1 - src/openalpr/licenseplatecandidate.cpp | 4 +- src/openalpr/pipeline_data.h | 5 +- .../textdetection/characteranalysis.cpp | 174 +-------------- .../textdetection/characteranalysis.h | 3 +- src/openalpr/textdetection/platemask.cpp | 201 ++++++++++++++++++ src/openalpr/textdetection/platemask.h | 46 ++++ 8 files changed, 267 insertions(+), 168 deletions(-) create mode 100644 src/openalpr/textdetection/platemask.cpp create mode 100644 src/openalpr/textdetection/platemask.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 9ca6c59..fc5fe8b 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -24,6 +24,7 @@ set(lpr_source_files colorfilter.cpp textdetection/characteranalysis.cpp textdetection/characteranalysis2l.cpp + textdetection/platemask.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index 0df17db..6321ffb 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -38,7 +38,6 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) charAnalysis = new CharacterAnalysis(pipeline_data); charAnalysis->analyze(); pipeline_data->plate_inverted = charAnalysis->thresholdsInverted; - pipeline_data->plate_mask = charAnalysis->plateMask; if (this->debug && charAnalysis->linePolygon.size() > 0) { diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index b7a6d60..b89183a 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -56,7 +56,9 @@ void LicensePlateCandidate::recognize() { PlateLines plateLines(config); - plateLines.processImage(pipeline_data->plate_mask, &charRegion, 1.10); + if (pipeline_data->hasPlateBorder) + plateLines.processImage(pipeline_data->plateBorderMask, &charRegion, 1.10); + plateLines.processImage(pipeline_data->crop_gray, &charRegion, 0.9); PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, &charRegion, config); diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 6e21438..14cd045 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -23,7 +23,10 @@ class PipelineData cv::Rect regionOfInterest; cv::Mat crop_gray; - cv::Mat plate_mask; + + bool hasPlateBorder; + cv::Mat plateBorderMask; + std::vector thresholds; std::vector plate_corners; diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index af02c58..9999ff5 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -18,6 +18,7 @@ */ #include "characteranalysis.h" +#include "platemask.h" using namespace cv; using namespace std; @@ -27,7 +28,7 @@ CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) this->pipeline_data = pipeline_data; this->config = pipeline_data->config; - this->hasPlateMask = false; + this->isTwoLine = true; if (this->config->debugCharAnalysis) cout << "Starting CharacterAnalysis identification" << endl; @@ -92,9 +93,13 @@ void CharacterAnalysis::analyze() cout << " -- Character Analysis Filter Time: " << diffclock(startTime, endTime) << "ms." << endl; } - this->plateMask = findOuterBoxMask(); + PlateMask plateMask(pipeline_data); + plateMask.findOuterBoxMask(charSegments, allContours, allHierarchy); + + pipeline_data->hasPlateBorder = plateMask.hasPlateMask; + pipeline_data->plateBorderMask = plateMask.getMask(); - if (hasPlateMask) + if (plateMask.hasPlateMask) { // Filter out bad contours now that we have an outer box mask... for (uint i = 0; i < pipeline_data->thresholds.size(); i++) @@ -195,166 +200,7 @@ int CharacterAnalysis::getGoodIndicesCount(vector goodIndices) return count; } -Mat CharacterAnalysis::findOuterBoxMask() -{ - double min_parent_area = config->templateHeightPx * config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. - int winningIndex = -1; - int winningParentId = -1; - int bestCharCount = 0; - double lowestArea = 99999999999999; - - if (this->config->debugCharAnalysis) - cout << "CharacterAnalysis::findOuterBoxMask" << endl; - - for (uint imgIndex = 0; imgIndex < allContours.size(); imgIndex++) - { - //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); - - int charsRecognized = 0; - int parentId = -1; - bool hasParent = false; - for (uint i = 0; i < charSegments[imgIndex].size(); i++) - { - if (charSegments[imgIndex][i]) charsRecognized++; - if (charSegments[imgIndex][i] && allHierarchy[imgIndex][i][3] != -1) - { - parentId = allHierarchy[imgIndex][i][3]; - hasParent = true; - } - } - - if (charsRecognized == 0) - continue; - - if (hasParent) - { - double boxArea = contourArea(allContours[imgIndex][parentId]); - if (boxArea < min_parent_area) - continue; - - if ((charsRecognized > bestCharCount) || - (charsRecognized == bestCharCount && boxArea < lowestArea)) - //(boxArea < lowestArea) - { - bestCharCount = charsRecognized; - winningIndex = imgIndex; - winningParentId = parentId; - lowestArea = boxArea; - } - } - } - - if (this->config->debugCharAnalysis) - cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; - - if (winningIndex != -1 && bestCharCount >= 3) - { - int longestChildIndex = -1; - double longestChildLength = 0; - // Find the child with the longest permiter/arc length ( just for kicks) - for (uint i = 0; i < allContours[winningIndex].size(); i++) - { - for (uint j = 0; j < allContours[winningIndex].size(); j++) - { - if (allHierarchy[winningIndex][j][3] == winningParentId) - { - double arclength = arcLength(allContours[winningIndex][j], false); - if (arclength > longestChildLength) - { - longestChildIndex = j; - longestChildLength = arclength; - } - } - } - } - - Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - // get rid of the outline by drawing a 1 pixel width black line - drawContours(mask, allContours[winningIndex], - winningParentId, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - allHierarchy[winningIndex], - 0 - ); - - // Morph Open the mask to get rid of any little connectors to non-plate portions - int morph_elem = 2; - int morph_size = 3; - Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - - //morphologyEx( mask, mask, MORPH_CLOSE, element ); - morphologyEx( mask, mask, MORPH_OPEN, element ); - - //morph_size = 1; - //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - //dilate(mask, mask, element); - - // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. - // We'll want to do the contour again and find the larges one so that we remove the clipped portion. - - vector > contoursSecondRound; - - findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - int biggestContourIndex = -1; - double largestArea = 0; - for (uint c = 0; c < contoursSecondRound.size(); c++) - { - double area = contourArea(contoursSecondRound[c]); - if (area > largestArea) - { - biggestContourIndex = c; - largestArea = area; - } - } - - if (biggestContourIndex != -1) - { - mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - vector smoothedMaskPoints; - approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); - - vector > tempvec; - tempvec.push_back(smoothedMaskPoints); - //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); - drawContours(mask, tempvec, - 0, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - allHierarchy[winningIndex], - 0 - ); - } - - if (this->config->debugCharAnalysis) - { - vector debugImgs; - Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); - - debugImgs.push_back(mask); - debugImgs.push_back(pipeline_data->thresholds[winningIndex]); - debugImgs.push_back(debugImgMasked); - - Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); - displayImage(config, "Winning outer box", dashboard); - } - - hasPlateMask = true; - return mask; - } - - hasPlateMask = false; - Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); - bitwise_not(fullMask, fullMask); - return fullMask; -} Mat CharacterAnalysis::getCharacterMask() { @@ -827,13 +673,15 @@ std::vector< bool > CharacterAnalysis::filterByOuterMask(vector< vector< Point > float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; - if (hasPlateMask == false) + if (this->pipeline_data->hasPlateBorder == false) return goodIndices; vector passingIndices; for (uint i = 0; i < goodIndices.size(); i++) passingIndices.push_back(false); + cv::Mat plateMask = pipeline_data->plateBorderMask; + Mat tempMaskedContour = Mat::zeros(plateMask.size(), CV_8U); Mat tempFullContour = Mat::zeros(plateMask.size(), CV_8U); diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 7940f93..e539720 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -32,8 +32,6 @@ class CharacterAnalysis CharacterAnalysis(PipelineData* pipeline_data); virtual ~CharacterAnalysis(); - bool hasPlateMask; - cv::Mat plateMask; cv::Mat bestThreshold; std::vector > bestContours; @@ -52,6 +50,7 @@ class CharacterAnalysis LineSegment charBoxRight; bool thresholdsInverted; + bool isTwoLine; std::vector > > allContours; std::vector > allHierarchy; diff --git a/src/openalpr/textdetection/platemask.cpp b/src/openalpr/textdetection/platemask.cpp new file mode 100644 index 0000000..7828e64 --- /dev/null +++ b/src/openalpr/textdetection/platemask.cpp @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "platemask.h" + +using namespace std; +using namespace cv; + +PlateMask::PlateMask(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; + this->hasPlateMask = false; + +} + + +PlateMask::~PlateMask() { +} + +cv::Mat PlateMask::getMask() { + return this->plateMask; +} + +void PlateMask::findOuterBoxMask(vector > charSegments, + vector > > allContours, + vector > allHierarchy) +{ + double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. + + int winningIndex = -1; + int winningParentId = -1; + int bestCharCount = 0; + double lowestArea = 99999999999999; + + if (pipeline_data->config->debugCharAnalysis) + cout << "CharacterAnalysis::findOuterBoxMask" << endl; + + for (uint imgIndex = 0; imgIndex < allContours.size(); imgIndex++) + { + //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); + + int charsRecognized = 0; + int parentId = -1; + bool hasParent = false; + for (uint i = 0; i < charSegments[imgIndex].size(); i++) + { + if (charSegments[imgIndex][i]) charsRecognized++; + if (charSegments[imgIndex][i] && allHierarchy[imgIndex][i][3] != -1) + { + parentId = allHierarchy[imgIndex][i][3]; + hasParent = true; + } + } + + if (charsRecognized == 0) + continue; + + if (hasParent) + { + double boxArea = contourArea(allContours[imgIndex][parentId]); + if (boxArea < min_parent_area) + continue; + + if ((charsRecognized > bestCharCount) || + (charsRecognized == bestCharCount && boxArea < lowestArea)) + //(boxArea < lowestArea) + { + bestCharCount = charsRecognized; + winningIndex = imgIndex; + winningParentId = parentId; + lowestArea = boxArea; + } + } + } + + if (pipeline_data->config->debugCharAnalysis) + cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; + + if (winningIndex != -1 && bestCharCount >= 3) + { + int longestChildIndex = -1; + double longestChildLength = 0; + // Find the child with the longest permiter/arc length ( just for kicks) + for (uint i = 0; i < allContours[winningIndex].size(); i++) + { + for (uint j = 0; j < allContours[winningIndex].size(); j++) + { + if (allHierarchy[winningIndex][j][3] == winningParentId) + { + double arclength = arcLength(allContours[winningIndex][j], false); + if (arclength > longestChildLength) + { + longestChildIndex = j; + longestChildLength = arclength; + } + } + } + } + + Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + // get rid of the outline by drawing a 1 pixel width black line + drawContours(mask, allContours[winningIndex], + winningParentId, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + allHierarchy[winningIndex], + 0 + ); + + // Morph Open the mask to get rid of any little connectors to non-plate portions + int morph_elem = 2; + int morph_size = 3; + Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + + //morphologyEx( mask, mask, MORPH_CLOSE, element ); + morphologyEx( mask, mask, MORPH_OPEN, element ); + + //morph_size = 1; + //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + //dilate(mask, mask, element); + + // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. + // We'll want to do the contour again and find the larges one so that we remove the clipped portion. + + vector > contoursSecondRound; + + findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + int biggestContourIndex = -1; + double largestArea = 0; + for (uint c = 0; c < contoursSecondRound.size(); c++) + { + double area = contourArea(contoursSecondRound[c]); + if (area > largestArea) + { + biggestContourIndex = c; + largestArea = area; + } + } + + if (biggestContourIndex != -1) + { + mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + vector smoothedMaskPoints; + approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); + + vector > tempvec; + tempvec.push_back(smoothedMaskPoints); + //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); + drawContours(mask, tempvec, + 0, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + allHierarchy[winningIndex], + 0 + ); + } + + if (pipeline_data->config->debugCharAnalysis) + { + vector debugImgs; + Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); + + debugImgs.push_back(mask); + debugImgs.push_back(pipeline_data->thresholds[winningIndex]); + debugImgs.push_back(debugImgMasked); + + Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); + displayImage(pipeline_data->config, "Winning outer box", dashboard); + } + + hasPlateMask = true; + this->plateMask = mask; + } + + hasPlateMask = false; + Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); + bitwise_not(fullMask, fullMask); + + this->plateMask = fullMask; +} diff --git a/src/openalpr/textdetection/platemask.h b/src/openalpr/textdetection/platemask.h new file mode 100644 index 0000000..ef57fc3 --- /dev/null +++ b/src/openalpr/textdetection/platemask.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_PLATEMASK_H +#define OPENALPR_PLATEMASK_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "pipeline_data.h" + +class PlateMask { +public: + PlateMask(PipelineData* pipeline_data); + virtual ~PlateMask(); + + bool hasPlateMask; + + cv::Mat getMask(); + + void findOuterBoxMask(std::vector > charSegments, std::vector > > allContours, std::vector > allHierarchy); + +private: + + PipelineData* pipeline_data; + cv::Mat plateMask; + + +}; + +#endif /* OPENALPR_PLATEMASK_H */ + From d55e66d6659168acd496858b531fba74b545edf9 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 9 Oct 2014 23:23:22 -0400 Subject: [PATCH 08/59] Refactored CharacterAnalysis code to use a class that wraps the raw Contour data --- src/openalpr/CMakeLists.txt | 1 + src/openalpr/characterregion.cpp | 6 +- .../segmentation/charactersegmenter.cpp | 24 +- .../segmentation/charactersegmenter.h | 3 +- .../textdetection/characteranalysis.cpp | 246 ++++++++---------- .../textdetection/characteranalysis.h | 35 +-- src/openalpr/textdetection/platemask.cpp | 30 +-- src/openalpr/textdetection/platemask.h | 3 +- src/openalpr/textdetection/textcontours.cpp | 86 ++++++ src/openalpr/textdetection/textcontours.h | 38 +++ 10 files changed, 286 insertions(+), 186 deletions(-) create mode 100644 src/openalpr/textdetection/textcontours.cpp create mode 100644 src/openalpr/textdetection/textcontours.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index fc5fe8b..659916c 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -25,6 +25,7 @@ set(lpr_source_files textdetection/characteranalysis.cpp textdetection/characteranalysis2l.cpp textdetection/platemask.cpp + textdetection/textcontours.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index 6321ffb..ce47747 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -58,9 +58,9 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) for (uint z = 0; z < charAnalysis->bestContours.size(); z++) { Scalar dcolor(255,0,0); - if (charAnalysis->bestCharSegments[z]) + if (charAnalysis->bestContours.goodIndices[z]) dcolor = Scalar(0,255,0); - drawContours(bestVal, charAnalysis->bestContours, z, dcolor, 1); + drawContours(bestVal, charAnalysis->bestContours.contours, z, dcolor, 1); } tempDash.push_back(bestVal); displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3)); @@ -70,7 +70,7 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) if (charAnalysis->linePolygon.size() > 0) { int confidenceDrainers = 0; - int charSegmentCount = charAnalysis->bestCharSegmentsCount; + int charSegmentCount = charAnalysis->bestContours.getGoodIndicesCount(); if (charSegmentCount == 1) confidenceDrainers += 91; else if (charSegmentCount < 5) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 76b482a..1d1aefc 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -60,11 +60,11 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) vector > allowedContours; for (uint i = 0; i < charAnalysis->bestContours.size(); i++) { - if (charAnalysis->bestCharSegments[i]) - allowedContours.push_back(charAnalysis->bestContours[i]); + if (charAnalysis->bestContours.goodIndices[i]) + allowedContours.push_back(charAnalysis->bestContours.contours[i]); } - drawContours(img_contours, charAnalysis->bestContours, + drawContours(img_contours, charAnalysis->bestContours.contours, -1, // draw all contours cv::Scalar(255,0,0), // in blue 1); // with a thickness of 1 @@ -94,10 +94,10 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) for (uint i = 0; i < charAnalysis->bestContours.size(); i++) { - if (charAnalysis->bestCharSegments[i] == false) + if (charAnalysis->bestContours.goodIndices[i] == false) continue; - Rect mr = boundingRect(charAnalysis->bestContours[i]); + Rect mr = boundingRect(charAnalysis->bestContours.contours[i]); charWidths.push_back(mr.width); charHeights.push_back(mr.height); @@ -106,7 +106,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) float avgCharWidth = median(charWidths.data(), charWidths.size()); float avgCharHeight = median(charHeights.data(), charHeights.size()); - removeSmallContours(pipeline_data->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight); + removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight); // Do the histogram analysis to figure out char regions @@ -116,7 +116,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) vector allHistograms; vector allBoxes; - for (uint i = 0; i < charAnalysis->allContours.size(); i++) + for (uint i = 0; i < charAnalysis->allTextContours.size(); i++) { Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); @@ -443,23 +443,23 @@ vector CharacterSegmenter::get1DHits(Mat img, int yOffset) return hits; } -void CharacterSegmenter::removeSmallContours(vector thresholds, vector > > allContours, float avgCharWidth, float avgCharHeight) +void CharacterSegmenter::removeSmallContours(vector thresholds, vector contours, float avgCharWidth, float avgCharHeight) { //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; for (uint i = 0; i < thresholds.size(); i++) { - for (uint c = 0; c < allContours[i].size(); c++) + for (uint c = 0; c < contours[i].contours.size(); c++) { - if (allContours[i][c].size() == 0) + if (contours[i].contours[c].size() == 0) continue; - Rect mr = boundingRect(allContours[i][c]); + Rect mr = boundingRect(contours[i].contours[c]); if (mr.height < MIN_CONTOUR_HEIGHT) { // Erase it - drawContours(thresholds[i], allContours[i], c, Scalar(0, 0, 0), -1); + drawContours(thresholds[i], contours[i].contours, c, Scalar(0, 0, 0), -1); continue; } } diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 54cb4fc..31dc53b 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -28,6 +28,7 @@ #include "colorfilter.h" #include "verticalhistogram.h" #include "config.h" +#include "textdetection/textcontours.h" //const float MIN_BOX_WIDTH_PX = 4; // 4 pixels @@ -73,7 +74,7 @@ class CharacterSegmenter cv::Mat getCharacterMask(cv::Mat img_threshold, std::vector > contours, std::vector hierarchy, std::vector goodIndices); cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); - void removeSmallContours(std::vector thresholds, std::vector > > allContours, float avgCharWidth, float avgCharHeight); + void removeSmallContours(std::vector thresholds, std::vector contours, float avgCharWidth, float avgCharHeight); cv::Mat getVerticalHistogram(cv::Mat img, cv::Mat mask); std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 9999ff5..426e477 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -18,7 +18,6 @@ */ #include "characteranalysis.h" -#include "platemask.h" using namespace cv; using namespace std; @@ -52,19 +51,9 @@ void CharacterAnalysis::analyze() for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - vector > contours; - vector hierarchy; + TextContours tc(pipeline_data->thresholds[i]); - Mat tempThreshold(pipeline_data->thresholds[i].size(), CV_8U); - pipeline_data->thresholds[i].copyTo(tempThreshold); - findContours(tempThreshold, - contours, // a vector of contours - hierarchy, - CV_RETR_TREE, // retrieve all contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours - - allContours.push_back(contours); - allHierarchy.push_back(hierarchy); + allTextContours.push_back(tc); } if (config->debugTiming) @@ -79,11 +68,10 @@ void CharacterAnalysis::analyze() for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - vector goodIndices = this->filter(pipeline_data->thresholds[i], allContours[i], allHierarchy[i]); - charSegments.push_back(goodIndices); + this->filter(pipeline_data->thresholds[i], allTextContours[i]); if (config->debugCharAnalysis) - cout << "Threshold " << i << " had " << getGoodIndicesCount(goodIndices) << " good indices." << endl; + cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl; } if (config->debugTiming) @@ -94,7 +82,7 @@ void CharacterAnalysis::analyze() } PlateMask plateMask(pipeline_data); - plateMask.findOuterBoxMask(charSegments, allContours, allHierarchy); + plateMask.findOuterBoxMask(allTextContours); pipeline_data->hasPlateBorder = plateMask.hasPlateMask; pipeline_data->plateBorderMask = plateMask.getMask(); @@ -104,7 +92,7 @@ void CharacterAnalysis::analyze() // Filter out bad contours now that we have an outer box mask... for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - charSegments[i] = filterByOuterMask(allContours[i], allHierarchy[i], charSegments[i]); + filterByOuterMask(allTextContours[i]); } } @@ -115,17 +103,14 @@ void CharacterAnalysis::analyze() //vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); //charSegments.push_back(goodIndices); - int segmentCount = getGoodIndicesCount(charSegments[i]); + int segmentCount = allTextContours[i].getGoodIndicesCount(); if (segmentCount > bestFitScore) { bestFitScore = segmentCount; bestFitIndex = i; - bestCharSegments = charSegments[i]; bestThreshold = pipeline_data->thresholds[i]; - bestContours = allContours[i]; - bestHierarchy = allHierarchy[i]; - bestCharSegmentsCount = segmentCount; + bestContours = allTextContours[i]; } } @@ -146,11 +131,11 @@ void CharacterAnalysis::analyze() vector > allowedContours; for (uint i = 0; i < bestContours.size(); i++) { - if (bestCharSegments[i]) - allowedContours.push_back(bestContours[i]); + if (bestContours.goodIndices[i]) + allowedContours.push_back(bestContours.contours[i]); } - drawContours(img_contours, bestContours, + drawContours(img_contours, bestContours.contours, -1, // draw all contours cv::Scalar(255,0,0), // in blue 1); // with a thickness of 1 @@ -165,14 +150,14 @@ void CharacterAnalysis::analyze() //charsegments = this->getPossibleCharRegions(img_threshold, allContours, allHierarchy, STARTING_MIN_HEIGHT + (bestFitIndex * HEIGHT_STEP), STARTING_MAX_HEIGHT + (bestFitIndex * HEIGHT_STEP)); - this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours, bestCharSegments); + this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); if (this->linePolygon.size() > 0) { this->topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); this->bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); //this->charArea = getCharSegmentsBetweenLines(bestThreshold, bestContours, this->linePolygon); - filterBetweenLines(bestThreshold, bestContours, bestHierarchy, linePolygon, bestCharSegments); + filterBetweenLines(bestThreshold, bestContours, linePolygon); this->charArea = getCharArea(); @@ -188,17 +173,6 @@ void CharacterAnalysis::analyze() this->thresholdsInverted = isPlateInverted(); } -int CharacterAnalysis::getGoodIndicesCount(vector goodIndices) -{ - int count = 0; - for (uint i = 0; i < goodIndices.size(); i++) - { - if (goodIndices[i]) - count++; - } - - return count; -} @@ -208,15 +182,15 @@ Mat CharacterAnalysis::getCharacterMask() for (uint i = 0; i < bestContours.size(); i++) { - if (bestCharSegments[i] == false) + if (bestContours.goodIndices[i] == false) continue; - drawContours(charMask, bestContours, + drawContours(charMask, bestContours.contours, i, // draw this contour cv::Scalar(255,255,255), // in CV_FILLED, 8, - bestHierarchy, + bestContours.hierarchy, 1 ); @@ -226,7 +200,7 @@ Mat CharacterAnalysis::getCharacterMask() } // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector CharacterAnalysis::getBestVotedLines(Mat img, vector > contours, vector goodIndices) +vector CharacterAnalysis::getBestVotedLines(Mat img, TextContours textContours) { //if (this->debug) // cout << "CharacterAnalysis::getBestVotedLines" << endl; @@ -235,10 +209,10 @@ vector CharacterAnalysis::getBestVotedLines(Mat img, vector vector charRegions; - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i]) - charRegions.push_back(boundingRect(contours[i])); + if (textContours.goodIndices[i]) + charRegions.push_back(boundingRect(textContours.contours[i])); } // Find the best fit line segment that is parallel with the most char segments @@ -308,7 +282,6 @@ vector CharacterAnalysis::getBestVotedLines(Mat img, vector //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); bottomLines.push_back(LineSegment(x1, y1, x2, y2)); - //drawAndWait(&tempImg); } } @@ -382,65 +355,66 @@ vector CharacterAnalysis::getBestVotedLines(Mat img, vector return bestStripe; } -vector CharacterAnalysis::filter(Mat img, vector > contours, vector hierarchy) +void CharacterAnalysis::filter(Mat img, TextContours& textContours) { static int STARTING_MIN_HEIGHT = round (((float) img.rows) * config->charAnalysisMinPercent); static int STARTING_MAX_HEIGHT = round (((float) img.rows) * (config->charAnalysisMinPercent + config->charAnalysisHeightRange)); static int HEIGHT_STEP = round (((float) img.rows) * config->charAnalysisHeightStepSize); static int NUM_STEPS = config->charAnalysisNumSteps; - vector charSegments; int bestFitScore = -1; + + vector bestIndices; + for (int i = 0; i < NUM_STEPS; i++) { - int goodIndicesCount; + + //vector goodIndices(contours.size()); + for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; - vector goodIndices(contours.size()); - for (uint z = 0; z < goodIndices.size(); z++) goodIndices[z] = true; + this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - goodIndices = this->filterByBoxSize(contours, goodIndices, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - - goodIndicesCount = getGoodIndicesCount(goodIndices); - if ( goodIndicesCount == 0 || goodIndicesCount <= bestFitScore) // Don't bother doing more filtering if we already lost... + int goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - goodIndices = this->filterContourHoles(contours, hierarchy, goodIndices); + + this->filterContourHoles(textContours); - goodIndicesCount = getGoodIndicesCount(goodIndices); - if ( goodIndicesCount == 0 || goodIndicesCount <= bestFitScore) // Don't bother doing more filtering if we already lost... + goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - //goodIndices = this->filterByParentContour( contours, hierarchy, goodIndices); - vector lines = getBestVotedLines(img, contours, goodIndices); - goodIndices = this->filterBetweenLines(img, contours, hierarchy, lines, goodIndices); + + vector lines = getBestVotedLines(img, textContours); + this->filterBetweenLines(img, textContours, lines); - int segmentCount = getGoodIndicesCount(goodIndices); + int segmentCount = textContours.getGoodIndicesCount(); if (segmentCount > bestFitScore) { bestFitScore = segmentCount; - charSegments = goodIndices; + bestIndices = textContours.getIndicesCopy(); } } - return charSegments; + textContours.setIndices(bestIndices); } // Goes through the contours for the plate and picks out possible char segments based on min/max height -vector CharacterAnalysis::filterByBoxSize(vector< vector< Point> > contours, vector goodIndices, int minHeightPx, int maxHeightPx) +void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx) { float idealAspect=config->charWidthMM / config->charHeightMM; float aspecttolerance=0.25; - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices.push_back(false); - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + //Create bounding rect of object - Rect mr= boundingRect(contours[i]); + Rect mr= boundingRect(textContours.contours[i]); float minWidth = mr.height * 0.2; //Crop image @@ -450,27 +424,25 @@ vector CharacterAnalysis::filterByBoxSize(vector< vector< Point> > contour float charAspect= (float)mr.width/(float)mr.height; if (abs(charAspect - idealAspect) < aspecttolerance) - includedIndices[i] = true; + textContours.goodIndices[i] = true; } } - return includedIndices; } -vector< bool > CharacterAnalysis::filterContourHoles(vector< vector< Point > > contours, vector< Vec4i > hierarchy, vector< bool > goodIndices) +void CharacterAnalysis::filterContourHoles(TextContours& textContours) { - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices.push_back(false); - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; - int parentIndex = hierarchy[i][3]; + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + + int parentIndex = textContours.hierarchy[i][3]; - if (parentIndex >= 0 && goodIndices[parentIndex]) + if (parentIndex >= 0 && textContours.goodIndices[parentIndex]) { // this contour is a child of an already identified contour. REMOVE it if (this->config->debugCharAnalysis) @@ -480,31 +452,29 @@ vector< bool > CharacterAnalysis::filterContourHoles(vector< vector< Point > > c } else { - includedIndices[i] = true; + textContours.goodIndices[i] = true; } } - return includedIndices; } // Goes through the contours for the plate and picks out possible char segments based on min/max height // returns a vector of indices corresponding to valid contours -vector CharacterAnalysis::filterByParentContour( vector< vector< Point> > contours, vector hierarchy, vector goodIndices) +void CharacterAnalysis::filterByParentContour( TextContours& textContours) { - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices[j] = false; vector parentIDs; vector votes; - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; + textContours.goodIndices[i] = false; // Set it to not included unless it proves + int voteIndex = -1; - int parentID = hierarchy[i][3]; + int parentID = textContours.hierarchy[i][3]; // check if parentID is already in the lsit for (uint j = 0; j < parentIDs.size(); j++) { @@ -540,35 +510,30 @@ vector CharacterAnalysis::filterByParentContour( vector< vector< Point> > } // Now filter out all the contours with a different parent ID (assuming the totalVotes > 2) - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; if (totalVotes <= 2) { - includedIndices[i] = true; + textContours.goodIndices[i] = true; } - else if (hierarchy[i][3] == winningParentId) + else if (textContours.hierarchy[i][3] == winningParentId) { - includedIndices[i] = true; + textContours.goodIndices[i] = true; } } - return includedIndices; } -vector CharacterAnalysis::filterBetweenLines(Mat img, vector > contours, vector hierarchy, vector outerPolygon, vector goodIndices) +void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector outerPolygon ) { static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices[j] = false; - if (outerPolygon.size() == 0) - return includedIndices; + return; vector validPoints; @@ -587,19 +552,21 @@ vector CharacterAnalysis::filterBetweenLines(Mat img, vector fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); // For each contour, determine if enough of it is between the lines to qualify - for (uint i = 0; i < contours.size(); i++) + for (uint i = 0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves innerArea.setTo(Scalar(0,0,0)); - drawContours(innerArea, contours, + drawContours(innerArea, textContours.contours, i, // draw this contour cv::Scalar(255,255,255), // in CV_FILLED, 8, - hierarchy, + textContours.hierarchy, 0 ); @@ -610,7 +577,7 @@ vector CharacterAnalysis::filterBetweenLines(Mat img, vector CV_RETR_EXTERNAL, // retrieve the external contours CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours ); - double totalArea = contourArea(contours[i]); + double totalArea = contourArea(textContours.contours[i]); double areaBetweenLines = 0; for (uint tempContourIdx = 0; tempContourIdx < tempContours.size(); tempContourIdx++) @@ -635,50 +602,46 @@ vector CharacterAnalysis::filterBetweenLines(Mat img, vector int highPointValue = 999999999; int lowPointIndex = 0; int lowPointValue = 0; - for (uint cidx = 0; cidx < contours[i].size(); cidx++) + for (uint cidx = 0; cidx < textContours.contours[i].size(); cidx++) { - if (contours[i][cidx].y < highPointValue) + if (textContours.contours[i][cidx].y < highPointValue) { highPointIndex = cidx; - highPointValue = contours[i][cidx].y; + highPointValue = textContours.contours[i][cidx].y; } - if (contours[i][cidx].y > lowPointValue) + if (textContours.contours[i][cidx].y > lowPointValue) { lowPointIndex = cidx; - lowPointValue = contours[i][cidx].y; + lowPointValue = textContours.contours[i][cidx].y; } } // Get the absolute distance from the top and bottom lines - Point closestTopPoint = topLine.closestPointOnSegmentTo(contours[i][highPointIndex]); - Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(contours[i][lowPointIndex]); + Point closestTopPoint = topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); + Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); - float absTopDistance = distanceBetweenPoints(closestTopPoint, contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, contours[i][lowPointIndex]); + float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); float maxDistance = lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; if (absTopDistance < maxDistance && absBottomDistance < maxDistance) { - includedIndices[i] = true; + textContours.goodIndices[i] = true; } } - return includedIndices; } -std::vector< bool > CharacterAnalysis::filterByOuterMask(vector< vector< Point > > contours, vector< Vec4i > hierarchy, std::vector< bool > goodIndices) +void CharacterAnalysis::filterByOuterMask(TextContours& textContours) { float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; if (this->pipeline_data->hasPlateBorder == false) - return goodIndices; + return; - vector passingIndices; - for (uint i = 0; i < goodIndices.size(); i++) - passingIndices.push_back(false); cv::Mat plateMask = pipeline_data->plateBorderMask; @@ -688,14 +651,18 @@ std::vector< bool > CharacterAnalysis::filterByOuterMask(vector< vector< Point > int charsInsideMask = 0; int totalChars = 0; - for (uint i=0; i < goodIndices.size(); i++) + vector originalindices; + for (uint i = 0; i < textContours.size(); i++) + originalindices.push_back(textContours.goodIndices[i]); + + for (uint i=0; i < textContours.size(); i++) { - if (goodIndices[i] == false) + if (textContours.goodIndices[i] == false) continue; totalChars++; - drawContours(tempFullContour, contours, i, Scalar(255,255,255), CV_FILLED, 8, hierarchy); + drawContours(tempFullContour, textContours.contours, i, Scalar(255,255,255), CV_FILLED, 8, textContours.hierarchy); bitwise_and(tempFullContour, plateMask, tempMaskedContour); float beforeMaskWhiteness = mean(tempFullContour)[0]; @@ -704,20 +671,25 @@ std::vector< bool > CharacterAnalysis::filterByOuterMask(vector< vector< Point > if (afterMaskWhiteness / beforeMaskWhiteness > MINIMUM_PERCENT_LEFT_AFTER_MASK) { charsInsideMask++; - passingIndices[i] = true; + textContours.goodIndices[i] = true; } } if (totalChars == 0) - return goodIndices; + { + textContours.goodIndices = originalindices; + return; + } // Check to make sure that this is a valid box. If the box is too small (e.g., 1 char is inside, and 3 are outside) // then don't use this to filter. float percentCharsInsideMask = ((float) charsInsideMask) / ((float) totalChars); if (percentCharsInsideMask < MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK) - return goodIndices; + { + textContours.goodIndices = originalindices; + return; + } - return passingIndices; } bool CharacterAnalysis::isPlateInverted() @@ -771,15 +743,15 @@ vector CharacterAnalysis::getCharArea() for (uint i = 0; i < bestContours.size(); i++) { - if (bestCharSegments[i] == false) + if (bestContours.goodIndices[i] == false) continue; - for (uint z = 0; z < bestContours[i].size(); z++) + for (uint z = 0; z < bestContours.contours[i].size(); z++) { - if (bestContours[i][z].x < leftX) - leftX = bestContours[i][z].x; - if (bestContours[i][z].x > rightX) - rightX = bestContours[i][z].x; + if (bestContours.contours[i][z].x < leftX) + leftX = bestContours.contours[i][z].x; + if (bestContours.contours[i][z].x > rightX) + rightX = bestContours.contours[i][z].x; } } diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index e539720..94c0516 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -24,6 +24,8 @@ #include "utility.h" #include "config.h" #include "pipeline_data.h" +#include "textcontours.h" +#include "platemask.h" class CharacterAnalysis { @@ -34,10 +36,12 @@ class CharacterAnalysis cv::Mat bestThreshold; - std::vector > bestContours; - std::vector bestHierarchy; - std::vector bestCharSegments; - int bestCharSegmentsCount; + + TextContours bestContours; + //std::vector > bestContours; + //std::vector bestHierarchy; + //std::vector bestCharSegments; + //int bestCharSegmentsCount; LineSegment topLine; LineSegment bottomLine; @@ -52,9 +56,10 @@ class CharacterAnalysis bool thresholdsInverted; bool isTwoLine; - std::vector > > allContours; - std::vector > allHierarchy; - std::vector > charSegments; + std::vector allTextContours; + //std::vector > > allContours; + //std::vector > allHierarchy; + //std::vector > charSegments; void analyze(); @@ -67,21 +72,19 @@ class CharacterAnalysis cv::Mat findOuterBoxMask( ); bool isPlateInverted(); - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); + void filter(cv::Mat img, TextContours& textContours); - std::vector filterByBoxSize(std::vector > contours, std::vector goodIndices, int minHeightPx, int maxHeightPx); - std::vector filterByParentContour( std::vector< std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterByOuterMask(std::vector > contours, std::vector hierarchy, std::vector goodIndices); + void filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx); + void filterByParentContour( TextContours& textContours ); + void filterContourHoles(TextContours& textContours); + void filterByOuterMask(TextContours& textContours); std::vector getCharArea(); - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - //vector getCharSegmentsBetweenLines(Mat img, vector > contours, vector outerPolygon); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); + std::vector getBestVotedLines(cv::Mat img, TextContours textContours); + void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector outerPolygon ); bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - int getGoodIndicesCount(std::vector goodIndices); }; diff --git a/src/openalpr/textdetection/platemask.cpp b/src/openalpr/textdetection/platemask.cpp index 7828e64..04dabbd 100644 --- a/src/openalpr/textdetection/platemask.cpp +++ b/src/openalpr/textdetection/platemask.cpp @@ -36,9 +36,7 @@ cv::Mat PlateMask::getMask() { return this->plateMask; } -void PlateMask::findOuterBoxMask(vector > charSegments, - vector > > allContours, - vector > allHierarchy) +void PlateMask::findOuterBoxMask( vector contours ) { double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. @@ -50,19 +48,19 @@ void PlateMask::findOuterBoxMask(vector > charSegments, if (pipeline_data->config->debugCharAnalysis) cout << "CharacterAnalysis::findOuterBoxMask" << endl; - for (uint imgIndex = 0; imgIndex < allContours.size(); imgIndex++) + for (uint imgIndex = 0; imgIndex < contours.size(); imgIndex++) { //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); int charsRecognized = 0; int parentId = -1; bool hasParent = false; - for (uint i = 0; i < charSegments[imgIndex].size(); i++) + for (uint i = 0; i < contours[imgIndex].goodIndices.size(); i++) { - if (charSegments[imgIndex][i]) charsRecognized++; - if (charSegments[imgIndex][i] && allHierarchy[imgIndex][i][3] != -1) + if (contours[imgIndex].goodIndices[i]) charsRecognized++; + if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1) { - parentId = allHierarchy[imgIndex][i][3]; + parentId = contours[imgIndex].hierarchy[i][3]; hasParent = true; } } @@ -72,7 +70,7 @@ void PlateMask::findOuterBoxMask(vector > charSegments, if (hasParent) { - double boxArea = contourArea(allContours[imgIndex][parentId]); + double boxArea = contourArea(contours[imgIndex].contours[parentId]); if (boxArea < min_parent_area) continue; @@ -96,13 +94,13 @@ void PlateMask::findOuterBoxMask(vector > charSegments, int longestChildIndex = -1; double longestChildLength = 0; // Find the child with the longest permiter/arc length ( just for kicks) - for (uint i = 0; i < allContours[winningIndex].size(); i++) + for (uint i = 0; i < contours[winningIndex].size(); i++) { - for (uint j = 0; j < allContours[winningIndex].size(); j++) + for (uint j = 0; j < contours[winningIndex].size(); j++) { - if (allHierarchy[winningIndex][j][3] == winningParentId) + if (contours[winningIndex].hierarchy[j][3] == winningParentId) { - double arclength = arcLength(allContours[winningIndex][j], false); + double arclength = arcLength(contours[winningIndex].contours[j], false); if (arclength > longestChildLength) { longestChildIndex = j; @@ -115,12 +113,12 @@ void PlateMask::findOuterBoxMask(vector > charSegments, Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); // get rid of the outline by drawing a 1 pixel width black line - drawContours(mask, allContours[winningIndex], + drawContours(mask, contours[winningIndex].contours, winningParentId, // draw this contour cv::Scalar(255,255,255), // in CV_FILLED, 8, - allHierarchy[winningIndex], + contours[winningIndex].hierarchy, 0 ); @@ -169,7 +167,7 @@ void PlateMask::findOuterBoxMask(vector > charSegments, cv::Scalar(255,255,255), // in CV_FILLED, 8, - allHierarchy[winningIndex], + contours[winningIndex].hierarchy, 0 ); } diff --git a/src/openalpr/textdetection/platemask.h b/src/openalpr/textdetection/platemask.h index ef57fc3..e31405a 100644 --- a/src/openalpr/textdetection/platemask.h +++ b/src/openalpr/textdetection/platemask.h @@ -22,6 +22,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "pipeline_data.h" +#include "textcontours.h" class PlateMask { public: @@ -32,7 +33,7 @@ public: cv::Mat getMask(); - void findOuterBoxMask(std::vector > charSegments, std::vector > > allContours, std::vector > allHierarchy); + void findOuterBoxMask(std::vector contours); private: diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp new file mode 100644 index 0000000..ed53fbd --- /dev/null +++ b/src/openalpr/textdetection/textcontours.cpp @@ -0,0 +1,86 @@ +/* + * File: textcontours.cpp + * Author: mhill + * + * Created on October 9, 2014, 7:40 PM + */ + +#include "textcontours.h" + +using namespace std; +using namespace cv; + + +TextContours::TextContours() { + + +} + + +TextContours::TextContours(cv::Mat threshold) { + + load(threshold); +} + + +TextContours::~TextContours() { +} + +void TextContours::load(cv::Mat threshold) { + + Mat tempThreshold(threshold.size(), CV_8U); + threshold.copyTo(tempThreshold); + findContours(tempThreshold, + contours, // a vector of contours + hierarchy, + CV_RETR_TREE, // retrieve all contours + CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours + + for (uint i = 0; i < contours.size(); i++) + goodIndices.push_back(true); +} + + +uint TextContours::size() { + return contours.size(); +} + + + +int TextContours::getGoodIndicesCount() +{ + int count = 0; + for (uint i = 0; i < goodIndices.size(); i++) + { + if (goodIndices[i]) + count++; + } + + return count; +} + + +std::vector TextContours::getIndicesCopy() +{ + vector copyArray; + for (uint i = 0; i < goodIndices.size(); i++) + { + bool val = goodIndices[i]; + copyArray.push_back(goodIndices[i]); + } + + return copyArray; +} + +void TextContours::setIndices(std::vector newIndices) +{ + if (newIndices.size() == goodIndices.size()) + { + for (uint i = 0; i < newIndices.size(); i++) + goodIndices[i] = newIndices[i]; + } + else + { + assert("Invalid set operation on indices"); + } +} \ No newline at end of file diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h new file mode 100644 index 0000000..bc38067 --- /dev/null +++ b/src/openalpr/textdetection/textcontours.h @@ -0,0 +1,38 @@ +/* + * File: textcontours.h + * Author: mhill + * + * Created on October 9, 2014, 7:40 PM + */ + +#ifndef TEXTCONTOURS_H +#define TEXTCONTOURS_H + +#include +#include "opencv2/imgproc/imgproc.hpp" + +class TextContours { +public: + TextContours(); + TextContours(cv::Mat threshold); + virtual ~TextContours(); + + void load(cv::Mat threshold); + + std::vector goodIndices; + std::vector > contours; + std::vector hierarchy; + + uint size(); + int getGoodIndicesCount(); + + std::vector getIndicesCopy(); + void setIndices(std::vector newIndices); + +private: + + +}; + +#endif /* TEXTCONTOURS_H */ + From c156e8900e1bf89c02637d67cf2cc48ba0b206be Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Fri, 10 Oct 2014 20:21:41 -0400 Subject: [PATCH 09/59] Refactored characteranalysis. Using TextLine class to hold character info. Needed for multiline char support --- src/openalpr/CMakeLists.txt | 1 + src/openalpr/characterregion.cpp | 40 +------------- src/openalpr/characterregion.h | 15 ----- src/openalpr/licenseplatecandidate.cpp | 8 +-- src/openalpr/pipeline_data.h | 2 + src/openalpr/platecorners.cpp | 55 +++++++++---------- src/openalpr/platecorners.h | 5 +- src/openalpr/platelines.cpp | 31 +++++++---- src/openalpr/platelines.h | 6 +- .../textdetection/characteranalysis.cpp | 28 +++++----- .../textdetection/characteranalysis.h | 13 ++--- src/openalpr/textdetection/textcontours.cpp | 24 ++++++-- src/openalpr/textdetection/textcontours.h | 22 ++++++-- src/openalpr/textdetection/textline.cpp | 28 ++++++++++ src/openalpr/textdetection/textline.h | 45 +++++++++++++++ 15 files changed, 188 insertions(+), 135 deletions(-) create mode 100644 src/openalpr/textdetection/textline.cpp create mode 100644 src/openalpr/textdetection/textline.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 659916c..04fc165 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -26,6 +26,7 @@ set(lpr_source_files textdetection/characteranalysis2l.cpp textdetection/platemask.cpp textdetection/textcontours.cpp + textdetection/textline.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index ce47747..e698bdf 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -75,8 +75,9 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) confidenceDrainers += 91; else if (charSegmentCount < 5) confidenceDrainers += (5 - charSegmentCount) * 10; - - int absangle = abs(charAnalysis->topLine.angle); + + // Use the angle for the first line -- assume they'll always be parallel for multi-line plates + int absangle = abs(pipeline_data->textLines[0].topLine.angle); if (absangle > config->maxPlateAngleDegrees) confidenceDrainers += 91; else if (absangle > 1) @@ -102,38 +103,3 @@ CharacterRegion::~CharacterRegion() } -LineSegment CharacterRegion::getTopLine() -{ - return charAnalysis->topLine; -} - -LineSegment CharacterRegion::getBottomLine() -{ - return charAnalysis->bottomLine; -} - -vector CharacterRegion::getCharArea() -{ - return charAnalysis->charArea; -} - -LineSegment CharacterRegion::getCharBoxTop() -{ - return charAnalysis->charBoxTop; -} - -LineSegment CharacterRegion::getCharBoxBottom() -{ - return charAnalysis->charBoxBottom; -} - -LineSegment CharacterRegion::getCharBoxLeft() -{ - return charAnalysis->charBoxLeft; -} - -LineSegment CharacterRegion::getCharBoxRight() -{ - return charAnalysis->charBoxRight; -} - diff --git a/src/openalpr/characterregion.h b/src/openalpr/characterregion.h index 94a1d90..8917d17 100644 --- a/src/openalpr/characterregion.h +++ b/src/openalpr/characterregion.h @@ -39,7 +39,6 @@ class CharacterRegion LineSegment getTopLine(); LineSegment getBottomLine(); - std::vector getCharArea(); LineSegment getCharBoxTop(); LineSegment getCharBoxBottom(); @@ -54,20 +53,6 @@ class CharacterRegion CharacterAnalysis *charAnalysis; cv::Mat findOuterBoxMask(std::vector thresholds, std::vector > > allContours, std::vector > allHierarchy); - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); - std::vector filterByBoxSize(cv::Mat img, std::vector > contours, std::vector goodIndices, float minHeightPx, float maxHeightPx); - std::vector filterByParentContour( std::vector< std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - //vector getCharSegmentsBetweenLines(Mat img, vector > contours, vector outerPolygon); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); - cv::Mat getCharacterMask(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector wrapContours(std::vector > contours); - bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - - int getGoodIndicesCount(std::vector goodIndices); bool isPlateInverted(cv::Mat threshold, std::vector > contours, std::vector hierarchy, std::vector goodIndices); diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index b89183a..f21b87a 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -54,14 +54,14 @@ void LicensePlateCandidate::recognize() if (charRegion.confidence > 10) { - PlateLines plateLines(config); + PlateLines plateLines(pipeline_data); if (pipeline_data->hasPlateBorder) - plateLines.processImage(pipeline_data->plateBorderMask, &charRegion, 1.10); + plateLines.processImage(pipeline_data->plateBorderMask, 1.10); - plateLines.processImage(pipeline_data->crop_gray, &charRegion, 0.9); + plateLines.processImage(pipeline_data->crop_gray, 0.9); - PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, &charRegion, config); + PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, pipeline_data); vector smallPlateCorners = cornerFinder.findPlateCorners(); if (cornerFinder.confidence > 0) diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 14cd045..e92302e 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -5,6 +5,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "utility.h" #include "config.h" +#include "textdetection/textline.h" class PipelineData { @@ -26,6 +27,7 @@ class PipelineData bool hasPlateBorder; cv::Mat plateBorderMask; + std::vector textLines; std::vector thresholds; diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index 94b78f5..d4e1d0c 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -22,26 +22,25 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, CharacterRegion* charRegion, Config* config) +PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) { - this->config = config; + this->pipelineData = pipelineData; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) cout << "PlateCorners constructor" << endl; this->inputImage = inputImage; this->plateLines = plateLines; - this->charRegion = charRegion; this->bestHorizontalScore = 9999999999999; this->bestVerticalScore = 9999999999999; - Point topPoint = charRegion->getTopLine().midpoint(); - Point bottomPoint = charRegion->getBottomLine().closestPointOnSegmentTo(topPoint); + Point topPoint = pipelineData->textLines[0].topLine.midpoint(); + Point bottomPoint = pipelineData->textLines[0].bottomLine.closestPointOnSegmentTo(topPoint); this->charHeight = distanceBetweenPoints(topPoint, bottomPoint); - this->charAngle = angleBetweenPoints(charRegion->getCharArea()[0], charRegion->getCharArea()[1]); + this->charAngle = angleBetweenPoints(pipelineData->textLines[0].textArea[0], pipelineData->textLines[0].textArea[1]); } PlateCorners::~PlateCorners() @@ -50,7 +49,7 @@ PlateCorners::~PlateCorners() vector PlateCorners::findPlateCorners() { - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) cout << "PlateCorners::findPlateCorners" << endl; timespec startTime; @@ -81,21 +80,21 @@ vector PlateCorners::findPlateCorners() } } - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { cout << "Drawing debug stuff..." << endl; Mat imgCorners = Mat(inputImage.size(), inputImage.type()); inputImage.copyTo(imgCorners); for (int i = 0; i < 4; i++) - circle(imgCorners, charRegion->getCharArea()[i], 2, Scalar(0, 0, 0)); + circle(imgCorners, pipelineData->textLines[0].textArea[i], 2, Scalar(0, 0, 0)); line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); line(imgCorners, this->bestBottom.p1, this->bestBottom.p2, Scalar(0, 0, 255), 1, CV_AA); line(imgCorners, this->bestLeft.p1, this->bestLeft.p2, Scalar(255, 0, 0), 1, CV_AA); - displayImage(config, "Winning top/bottom Boundaries", imgCorners); + displayImage(pipelineData->config, "Winning top/bottom Boundaries", imgCorners); } // Check if a left/right edge has been established. @@ -112,7 +111,7 @@ vector PlateCorners::findPlateCorners() corners.push_back(bestBottom.intersection(bestRight)); corners.push_back(bestBottom.intersection(bestLeft)); - if (config->debugTiming) + if (pipelineData->config->debugTiming) { timespec endTime; getTime(&endTime); @@ -129,7 +128,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) LineSegment left; LineSegment right; - float charHeightToPlateWidthRatio = config->plateWidthMM / config->charHeightMM; + float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters float confidenceDiff = 0; @@ -138,8 +137,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (v1 == NO_LINE && v2 == NO_LINE) { //return; - Point centerTop = charRegion->getCharBoxTop().midpoint(); - Point centerBottom = charRegion->getCharBoxBottom().midpoint(); + Point centerTop = pipelineData->textLines[0].charBoxTop.midpoint(); + Point centerBottom = pipelineData->textLines[0].charBoxBottom.midpoint(); LineSegment centerLine = LineSegment(centerBottom.x, centerBottom.y, centerTop.x, centerTop.y); left = centerLine.getParallelLine(idealPixelWidth / 2); @@ -174,11 +173,11 @@ void PlateCorners::scoreVerticals(int v1, int v2) score += missingSegmentPenalty; // Make sure this line is to the left of our license plate letters - if (left.isPointBelowLine(charRegion->getCharBoxLeft().midpoint()) == false) + if (left.isPointBelowLine(pipelineData->textLines[0].charBoxLeft.midpoint()) == false) return; // Make sure this line is to the right of our license plate letters - if (right.isPointBelowLine(charRegion->getCharBoxRight().midpoint())) + if (right.isPointBelowLine(pipelineData->textLines[0].charBoxRight.midpoint())) return; ///////////////////////////////////////////////////////////////////////// @@ -212,8 +211,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) // SCORE the shape wrt character position and height relative to position ////////////////////////////////////////////////////////////////////////// - Point leftMidLinePoint = left.closestPointOnSegmentTo(charRegion->getCharBoxLeft().midpoint()); - Point rightMidLinePoint = right.closestPointOnSegmentTo(charRegion->getCharBoxRight().midpoint()); + Point leftMidLinePoint = left.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxLeft.midpoint()); + Point rightMidLinePoint = right.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxRight.midpoint()); float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); @@ -223,7 +222,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) { float scorecomponent; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { cout << "xx xx Score: charHeight " << this->charHeight << endl; cout << "xx xx Score: idealwidth " << idealPixelWidth << endl; @@ -278,7 +277,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) LineSegment top; LineSegment bottom; - float charHeightToPlateHeightRatio = config->plateHeightMM / config->charHeightMM; + float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; float confidenceDiff = 0; @@ -287,8 +286,8 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (h1 == NO_LINE && h2 == NO_LINE) { // return; - Point centerLeft = charRegion->getCharBoxLeft().midpoint(); - Point centerRight = charRegion->getCharBoxRight().midpoint(); + Point centerLeft = pipelineData->textLines[0].charBoxLeft.midpoint(); + Point centerRight = pipelineData->textLines[0].charBoxRight.midpoint(); LineSegment centerLine = LineSegment(centerLeft.x, centerLeft.y, centerRight.x, centerRight.y); top = centerLine.getParallelLine(idealPixelHeight / 2); @@ -323,11 +322,11 @@ void PlateCorners::scoreHorizontals(int h1, int h2) score += missingSegmentPenalty; // Make sure this line is above our license plate letters - if (top.isPointBelowLine(charRegion->getCharBoxTop().midpoint()) == false) + if (top.isPointBelowLine(pipelineData->textLines[0].charBoxTop.midpoint()) == false) return; // Make sure this line is below our license plate letters - if (bottom.isPointBelowLine(charRegion->getCharBoxBottom().midpoint())) + if (bottom.isPointBelowLine(pipelineData->textLines[0].charBoxBottom.midpoint())) return; // We now have 4 possible lines. Let's put them to the test and score them... @@ -353,7 +352,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // Get the height difference float heightRatio = charHeight / plateHeightPx; - float idealHeightRatio = (config->charHeightMM / config->plateHeightMM); + float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); //if (leftRatio < MIN_CHAR_HEIGHT_RATIO || leftRatio > MAX_CHAR_HEIGHT_RATIO || rightRatio < MIN_CHAR_HEIGHT_RATIO || rightRatio > MAX_CHAR_HEIGHT_RATIO) float heightRatioDiff = abs(heightRatio - idealHeightRatio); // Ideal ratio == ~.45 @@ -373,7 +372,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle ////////////////////////////////////////////////////////////////////////// - Point charAreaMidPoint = charRegion->getCharBoxLeft().midpoint(); + Point charAreaMidPoint = pipelineData->textLines[0].charBoxLeft.midpoint(); Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); @@ -406,7 +405,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) { float scorecomponent; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { cout << "xx xx Score: charHeight " << this->charHeight << endl; cout << "xx xx Score: idealHeight " << idealPixelHeight << endl; diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index c965887..d6a91ac 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -47,7 +47,7 @@ class PlateCorners { public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, CharacterRegion* charRegion, Config* config); + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData); virtual ~PlateCorners(); std::vector findPlateCorners(); @@ -56,7 +56,7 @@ class PlateCorners private: - Config* config; + PipelineData* pipelineData; cv::Mat inputImage; float charHeight; float charAngle; @@ -69,7 +69,6 @@ class PlateCorners LineSegment bestRight; PlateLines* plateLines; - CharacterRegion* charRegion; void scoreHorizontals( int h1, int h2 ); void scoreVerticals( int v1, int v2 ); diff --git a/src/openalpr/platelines.cpp b/src/openalpr/platelines.cpp index e244ca6..1f76134 100644 --- a/src/openalpr/platelines.cpp +++ b/src/openalpr/platelines.cpp @@ -25,10 +25,11 @@ using namespace std; const float MIN_CONFIDENCE = 0.3; -PlateLines::PlateLines(Config* config) +PlateLines::PlateLines(PipelineData* pipelineData) { - this->config = config; - this->debug = config->debugPlateLines; + this->pipelineData = pipelineData; + + this->debug = pipelineData->config->debugPlateLines; if (debug) cout << "PlateLines constructor" << endl; @@ -38,7 +39,7 @@ PlateLines::~PlateLines() { } -void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float sensitivity) +void PlateLines::processImage(Mat inputImage, float sensitivity) { if (this->debug) cout << "PlateLines findLines" << endl; @@ -59,7 +60,6 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45); - int morph_elem = 2; int morph_size = 2; Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); @@ -69,11 +69,18 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float Canny(smoothed, edges, 66, 133); // Create a mask that is dilated based on the detected characters - vector > polygons; - polygons.push_back(charRegion->getCharArea()); + Mat mask = Mat::zeros(inputImage.size(), CV_8U); - fillPoly(mask, polygons, Scalar(255,255,255)); + + for (uint i = 0; i < pipelineData->textLines.size(); i++) + { + vector > polygons; + polygons.push_back(pipelineData->textLines[i].textArea); + fillPoly(mask, polygons, Scalar(255,255,255)); + } + + dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); bitwise_not(mask, mask); @@ -114,10 +121,10 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float images.push_back(debugImgVert); Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1); - displayImage(config, "Hough Lines", dashboard); + displayImage(pipelineData->config, "Hough Lines", dashboard); } - if (config->debugTiming) + if (pipelineData->config->debugTiming) { timespec endTime; getTime(&endTime); @@ -134,8 +141,8 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, b if (this->debug) cout << "PlateLines::getLines" << endl; - static int HORIZONTAL_SENSITIVITY = config->plateLinesSensitivityHorizontal; - static int VERTICAL_SENSITIVITY = config->plateLinesSensitivityVertical; + static int HORIZONTAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityHorizontal; + static int VERTICAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityVertical; vector allLines; vector filteredLines; diff --git a/src/openalpr/platelines.h b/src/openalpr/platelines.h index 872a80e..40a9016 100644 --- a/src/openalpr/platelines.h +++ b/src/openalpr/platelines.h @@ -37,10 +37,10 @@ class PlateLines { public: - PlateLines(Config* config); + PlateLines(PipelineData* pipelineData); virtual ~PlateLines(); - void processImage(cv::Mat img, CharacterRegion* charRegion, float sensitivity=1.0); + void processImage(cv::Mat img, float sensitivity=1.0); std::vector horizontalLines; std::vector verticalLines; @@ -49,7 +49,7 @@ class PlateLines private: - Config* config; + PipelineData* pipelineData; bool debug; cv::Mat customGrayscaleConversion(cv::Mat src); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 426e477..46493dd 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -44,11 +44,13 @@ void CharacterAnalysis::analyze() pipeline_data->clearThresholds(); pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - + timespec startTime; getTime(&startTime); + TextLine textLine; + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { TextContours tc(pipeline_data->thresholds[i]); @@ -148,28 +150,28 @@ void CharacterAnalysis::analyze() displayImage(config, "Matching Contours", img_contours); } - //charsegments = this->getPossibleCharRegions(img_threshold, allContours, allHierarchy, STARTING_MIN_HEIGHT + (bestFitIndex * HEIGHT_STEP), STARTING_MAX_HEIGHT + (bestFitIndex * HEIGHT_STEP)); - this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); if (this->linePolygon.size() > 0) { - this->topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); - this->bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); - //this->charArea = getCharSegmentsBetweenLines(bestThreshold, bestContours, this->linePolygon); + textLine.topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); + textLine.bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); + filterBetweenLines(bestThreshold, bestContours, linePolygon); - this->charArea = getCharArea(); + textLine.textArea = getCharArea(textLine.topLine, textLine.bottomLine); - if (this->charArea.size() > 0) + if (textLine.textArea.size() > 0) { - this->charBoxTop = LineSegment(this->charArea[0].x, this->charArea[0].y, this->charArea[1].x, this->charArea[1].y); - this->charBoxBottom = LineSegment(this->charArea[3].x, this->charArea[3].y, this->charArea[2].x, this->charArea[2].y); - this->charBoxLeft = LineSegment(this->charArea[3].x, this->charArea[3].y, this->charArea[0].x, this->charArea[0].y); - this->charBoxRight = LineSegment(this->charArea[2].x, this->charArea[2].y, this->charArea[1].x, this->charArea[1].y); + textLine.charBoxTop = LineSegment(textLine.textArea[0].x, textLine.textArea[0].y, textLine.textArea[1].x, textLine.textArea[1].y); + textLine.charBoxBottom = LineSegment(textLine.textArea[3].x, textLine.textArea[3].y, textLine.textArea[2].x, textLine.textArea[2].y); + textLine.charBoxLeft = LineSegment(textLine.textArea[3].x, textLine.textArea[3].y, textLine.textArea[0].x, textLine.textArea[0].y); + textLine.charBoxRight = LineSegment(textLine.textArea[2].x, textLine.textArea[2].y, textLine.textArea[1].x, textLine.textArea[1].y); } } + pipeline_data->textLines.push_back(textLine); + this->thresholdsInverted = isPlateInverted(); } @@ -733,7 +735,7 @@ bool CharacterAnalysis::verifySize(Mat r, float minHeightPx, float maxHeightPx) return false; } -vector CharacterAnalysis::getCharArea() +vector CharacterAnalysis::getCharArea(LineSegment topLine, LineSegment bottomLine) { const int MAX = 100000; const int MIN= -1; diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 94c0516..52bbe36 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -43,15 +43,10 @@ class CharacterAnalysis //std::vector bestCharSegments; //int bestCharSegmentsCount; - LineSegment topLine; - LineSegment bottomLine; - std::vector linePolygon; - std::vector charArea; - LineSegment charBoxTop; - LineSegment charBoxBottom; - LineSegment charBoxLeft; - LineSegment charBoxRight; + std::vector linePolygon; + + bool thresholdsInverted; bool isTwoLine; @@ -79,7 +74,7 @@ class CharacterAnalysis void filterContourHoles(TextContours& textContours); void filterByOuterMask(TextContours& textContours); - std::vector getCharArea(); + std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); std::vector getBestVotedLines(cv::Mat img, TextContours textContours); void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector outerPolygon ); diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp index ed53fbd..01a2272 100644 --- a/src/openalpr/textdetection/textcontours.cpp +++ b/src/openalpr/textdetection/textcontours.cpp @@ -1,9 +1,21 @@ -/* - * File: textcontours.cpp - * Author: mhill - * - * Created on October 9, 2014, 7:40 PM - */ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ #include "textcontours.h" diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h index bc38067..428a767 100644 --- a/src/openalpr/textdetection/textcontours.h +++ b/src/openalpr/textdetection/textcontours.h @@ -1,9 +1,21 @@ -/* - * File: textcontours.h - * Author: mhill +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] * - * Created on October 9, 2014, 7:40 PM - */ + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ #ifndef TEXTCONTOURS_H #define TEXTCONTOURS_H diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp new file mode 100644 index 0000000..ccd3b2a --- /dev/null +++ b/src/openalpr/textdetection/textline.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#include "textline.h" + +TextLine::TextLine() { +} + + +TextLine::~TextLine() { +} diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h new file mode 100644 index 0000000..6172c01 --- /dev/null +++ b/src/openalpr/textdetection/textline.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#ifndef OPENALPR_TEXTLINE_H +#define OPENALPR_TEXTLINE_H + +#include "utility.h" + +class TextLine { +public: + TextLine(); + virtual ~TextLine(); + + std::vector textArea; + LineSegment topLine; + LineSegment bottomLine; + + LineSegment charBoxTop; + LineSegment charBoxBottom; + LineSegment charBoxLeft; + LineSegment charBoxRight; + +private: + +}; + +#endif /* OPENALPR_TEXTLINE_H */ + From 97518ecd8945525526c02bdba84e3cc83c706bc9 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 13 Oct 2014 18:19:12 -0400 Subject: [PATCH 10/59] Moved polygon logic to textline --- .../textdetection/characteranalysis.cpp | 18 ++++++------------ src/openalpr/textdetection/characteranalysis.h | 10 ---------- src/openalpr/textdetection/textline.cpp | 12 +++++++++++- src/openalpr/textdetection/textline.h | 2 +- 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 46493dd..aeec953 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -49,7 +49,6 @@ void CharacterAnalysis::analyze() timespec startTime; getTime(&startTime); - TextLine textLine; for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { @@ -154,23 +153,18 @@ void CharacterAnalysis::analyze() if (this->linePolygon.size() > 0) { - textLine.topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); - textLine.bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); + LineSegment topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); + LineSegment bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); filterBetweenLines(bestThreshold, bestContours, linePolygon); - textLine.textArea = getCharArea(textLine.topLine, textLine.bottomLine); + vector textArea = getCharArea(topLine, bottomLine); - if (textLine.textArea.size() > 0) - { - textLine.charBoxTop = LineSegment(textLine.textArea[0].x, textLine.textArea[0].y, textLine.textArea[1].x, textLine.textArea[1].y); - textLine.charBoxBottom = LineSegment(textLine.textArea[3].x, textLine.textArea[3].y, textLine.textArea[2].x, textLine.textArea[2].y); - textLine.charBoxLeft = LineSegment(textLine.textArea[3].x, textLine.textArea[3].y, textLine.textArea[0].x, textLine.textArea[0].y); - textLine.charBoxRight = LineSegment(textLine.textArea[2].x, textLine.textArea[2].y, textLine.textArea[1].x, textLine.textArea[1].y); - } + TextLine textLine(textArea, topLine, bottomLine); + + pipeline_data->textLines.push_back(textLine); } - pipeline_data->textLines.push_back(textLine); this->thresholdsInverted = isPlateInverted(); } diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 52bbe36..3811b05 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -38,23 +38,13 @@ class CharacterAnalysis cv::Mat bestThreshold; TextContours bestContours; - //std::vector > bestContours; - //std::vector bestHierarchy; - //std::vector bestCharSegments; - //int bestCharSegmentsCount; - std::vector linePolygon; - - bool thresholdsInverted; bool isTwoLine; std::vector allTextContours; - //std::vector > > allContours; - //std::vector > allHierarchy; - //std::vector > charSegments; void analyze(); diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index ccd3b2a..d8be290 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -20,7 +20,17 @@ #include "textline.h" -TextLine::TextLine() { +TextLine::TextLine(std::vector textArea, LineSegment topLine, LineSegment bottomLine) { + if (textArea.size() > 0) + { + this->textArea = textArea; + this->topLine = topLine; + this->bottomLine = bottomLine; + this->charBoxTop = LineSegment(textArea[0].x, textArea[0].y, textArea[1].x, textArea[1].y); + this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); + this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); + this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); + } } diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index 6172c01..ac3eeb1 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -25,7 +25,7 @@ class TextLine { public: - TextLine(); + TextLine(std::vector textArea, LineSegment topLine, LineSegment bottomLine); virtual ~TextLine(); std::vector textArea; From 8e83e43ecb222ff313ab3a5af775d3ef179e8061 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 13 Oct 2014 22:43:55 -0400 Subject: [PATCH 11/59] Moved more data members to textline --- src/openalpr/characterregion.cpp | 4 ++-- .../segmentation/charactersegmenter.cpp | 21 ++++++++++--------- .../textdetection/characteranalysis.cpp | 14 +++++++------ .../textdetection/characteranalysis.h | 2 -- src/openalpr/textdetection/textline.cpp | 9 +++++--- src/openalpr/textdetection/textline.h | 3 ++- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index e698bdf..c45fea6 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -39,7 +39,7 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) charAnalysis->analyze(); pipeline_data->plate_inverted = charAnalysis->thresholdsInverted; - if (this->debug && charAnalysis->linePolygon.size() > 0) + if (this->debug && pipeline_data->textLines.size() > 0) { vector tempDash; for (uint z = 0; z < pipeline_data->thresholds.size(); z++) @@ -67,7 +67,7 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) } - if (charAnalysis->linePolygon.size() > 0) + if (pipeline_data->textLines.size() > 0) { int confidenceDrainers = 0; int charSegmentCount = charAnalysis->bestContours.getGoodIndicesCount(); diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 1d1aefc..4797ef6 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -51,7 +51,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); } - if (this->config->debugCharSegmenter && charAnalysis->linePolygon.size() > 0) + if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) { Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); charAnalysis->bestThreshold.copyTo(img_contours); @@ -74,20 +74,21 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) cv::Scalar(0,255,0), // in green 1); // with a thickness of 1 - if (charAnalysis->linePolygon.size() > 0) - { - line(img_contours, charAnalysis->linePolygon[0], charAnalysis->linePolygon[1], Scalar(255, 0, 255), 1); - line(img_contours, charAnalysis->linePolygon[3], charAnalysis->linePolygon[2], Scalar(255, 0, 255), 1); - } + + line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); + line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); + Mat bordered = addLabel(img_contours, "Best Contours"); imgDbgGeneral.push_back(bordered); } - if (charAnalysis->linePolygon.size() > 0) + if (pipeline_data->textLines.size() > 0) { - this->top = LineSegment(charAnalysis->linePolygon[0].x, charAnalysis->linePolygon[0].y, charAnalysis->linePolygon[1].x, charAnalysis->linePolygon[1].y); - this->bottom = LineSegment(charAnalysis->linePolygon[3].x, charAnalysis->linePolygon[3].y, charAnalysis->linePolygon[2].x, charAnalysis->linePolygon[2].y); + this->top = LineSegment(pipeline_data->textLines[0].linePolygon[0].x, pipeline_data->textLines[0].linePolygon[0].y, + pipeline_data->textLines[0].linePolygon[1].x, pipeline_data->textLines[0].linePolygon[1].y); + this->bottom = LineSegment(pipeline_data->textLines[0].linePolygon[3].x, pipeline_data->textLines[0].linePolygon[3].y, + pipeline_data->textLines[0].linePolygon[2].x, pipeline_data->textLines[0].linePolygon[2].y); vector charWidths; vector charHeights; @@ -120,7 +121,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) { Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); - fillConvexPoly(histogramMask, charAnalysis->linePolygon.data(), charAnalysis->linePolygon.size(), Scalar(255,255,255)); + fillConvexPoly(histogramMask, pipeline_data->textLines[0].linePolygon.data(), pipeline_data->textLines[0].linePolygon.size(), Scalar(255,255,255)); VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index aeec953..999746f 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -49,6 +49,7 @@ void CharacterAnalysis::analyze() timespec startTime; getTime(&startTime); + pipeline_data->textLines.clear(); for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { @@ -149,18 +150,19 @@ void CharacterAnalysis::analyze() displayImage(config, "Matching Contours", img_contours); } - this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); + vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); - if (this->linePolygon.size() > 0) + if (linePolygon.size() > 0) { - LineSegment topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); - LineSegment bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); + + LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); filterBetweenLines(bestThreshold, bestContours, linePolygon); - + vector textArea = getCharArea(topLine, bottomLine); - TextLine textLine(textArea, topLine, bottomLine); + TextLine textLine(textArea, linePolygon); pipeline_data->textLines.push_back(textLine); } diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 3811b05..fd26ea8 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -39,8 +39,6 @@ class CharacterAnalysis TextContours bestContours; - std::vector linePolygon; - bool thresholdsInverted; bool isTwoLine; diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index d8be290..0e70d4d 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -20,12 +20,15 @@ #include "textline.h" -TextLine::TextLine(std::vector textArea, LineSegment topLine, LineSegment bottomLine) { +TextLine::TextLine(std::vector textArea, std::vector linePolygon) { if (textArea.size() > 0) { this->textArea = textArea; - this->topLine = topLine; - this->bottomLine = bottomLine; + this->linePolygon = linePolygon; + + this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + this->charBoxTop = LineSegment(textArea[0].x, textArea[0].y, textArea[1].x, textArea[1].y); this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index ac3eeb1..cf5fc86 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -25,9 +25,10 @@ class TextLine { public: - TextLine(std::vector textArea, LineSegment topLine, LineSegment bottomLine); + TextLine(std::vector textArea, std::vector linePolygon); virtual ~TextLine(); + std::vector linePolygon; std::vector textArea; LineSegment topLine; LineSegment bottomLine; From 0d475381e793bb8319b5328603dc34fa5467a9af Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 13 Oct 2014 22:46:46 -0400 Subject: [PATCH 12/59] Removed unused functions from charactersegmenter --- .../segmentation/charactersegmenter.cpp | 63 ------------------- .../segmentation/charactersegmenter.h | 11 ---- 2 files changed, 74 deletions(-) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 4797ef6..1b83628 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -957,69 +957,6 @@ void CharacterSegmenter::filterEdgeBoxes(vector thresholds, const vector > contours; - Mat mask = Mat::zeros(thresholds[i].size(),CV_8U); - rectangle(mask, charRegions[boxidx], Scalar(255,255,255), CV_FILLED); - - bitwise_and(thresholds[i], mask, mask); - findContours(mask, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - //int tallContourIndex = isSkinnyLineInsideBox(thresholds[i], charRegions[boxidx], allContours[i], hierarchy[i], avgCharWidth, avgCharHeight); - float tallestContourHeight = 0; - float fattestContourWidth = 0; - float biggestContourArea = 0; - for (int c = 0; c < contours.size(); c++) - { - Rect r = boundingRect(contours[c]); - if (r.height > tallestContourHeight) - tallestContourHeight = r.height; - if (r.width > fattestContourWidth) - fattestContourWidth = r.width; - float a = r.area(); - if (a > biggestContourArea) - biggestContourArea = a; - } - - float minArea = charRegions[boxidx].area() * MIN_EDGE_CONTOUR_AREA_PCT; - if ((fattestContourWidth < MIN_BOX_WIDTH_PX) || - (tallestContourHeight < MIN_EDGE_CONTOUR_HEIGHT) || - (biggestContourArea < minArea) - ) - { - // Find a good place to MASK this contour. - // for now, just mask the whole thing - if (this->debug) - { - rectangle(imgDbgCleanStages[i], charRegions[boxidx], COLOR_DEBUG_EDGE, 2); - cout << "Edge Filter: threshold " << i << " box " << boxidx << endl; - } - rectangle(thresholds[i], charRegions[boxidx], Scalar(0,0,0), -1); - } - else - { - filteredCharRegions.push_back(charRegions[boxidx]); - } - } - } - */ } int CharacterSegmenter::getLongestBlobLengthBetweenLines(Mat img, int col) diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 31dc53b..319dced 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -63,20 +63,10 @@ class CharacterSegmenter std::vector imgDbgGeneral; std::vector imgDbgCleanStages; - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); - std::vector filterByBoxSize(std::vector< std::vector > contours, std::vector goodIndices, float minHeightPx, float maxHeightPx); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - int getGoodIndicesCount(std::vector goodIndices); - - cv::Mat getCharacterMask(cv::Mat img_threshold, std::vector > contours, std::vector hierarchy, std::vector goodIndices); cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); void removeSmallContours(std::vector thresholds, std::vector contours, float avgCharWidth, float avgCharHeight); - cv::Mat getVerticalHistogram(cv::Mat img, cv::Mat mask); std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); std::vector getBestCharBoxes(cv::Mat img, std::vector charBoxes, float avgCharWidth); std::vector combineCloseBoxes( std::vector charBoxes, float avgCharWidth); @@ -93,7 +83,6 @@ class CharacterSegmenter int isSkinnyLineInsideBox(cv::Mat threshold, cv::Rect box, std::vector > contours, std::vector hierarchy, float avgCharWidth, float avgCharHeight); - std::vector getEncapsulatingLines(cv::Mat img, std::vector > contours, std::vector goodIndices); }; #endif // OPENALPR_CHARACTERSEGMENTER_H From 602b20c88362772ff0bffaa7888f7c99f8dbfe1f Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 15 Oct 2014 13:35:27 -0400 Subject: [PATCH 13/59] Added commented debug option for use when debugging with gdb --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ac8f82..42a0aa6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,6 @@ project(src) - +#set(CMAKE_BUILD_TYPE Debug) cmake_minimum_required (VERSION 2.6) # Set the OpenALPR version in cmake, and also add it as a DEFINE for the code to access From 2e83a057c17c34b615afcdc3a29a84eb498a6a8e Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 16 Oct 2014 21:38:18 -0400 Subject: [PATCH 14/59] Added common utility (getContourAreaPercentInsideMask) to utility class --- src/openalpr/utility.cpp | 32 ++++++++++++++++++++++++++++++++ src/openalpr/utility.h | 2 ++ 2 files changed, 34 insertions(+) diff --git a/src/openalpr/utility.cpp b/src/openalpr/utility.cpp index f76f3cb..693fdef 100644 --- a/src/openalpr/utility.cpp +++ b/src/openalpr/utility.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "utility.h" using namespace cv; @@ -379,6 +381,36 @@ LineSegment LineSegment::getParallelLine(float distance) return result; } +// Given a contour and a mask, this function determines what percentage of the contour (area) +// is inside the masked area. +float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex) +{ + + + Mat innerArea = Mat::zeros(mask.size(), CV_8U); + + + drawContours(innerArea, contours, + contourIndex, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + hierarchy, + 2 + ); + + + int startingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + bitwise_and(innerArea, mask, innerArea); + + int endingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + return ((float) endingPixels) / ((float) startingPixels); + +} std::string toString(int value) { diff --git a/src/openalpr/utility.h b/src/openalpr/utility.h index 4e95fa5..a2c5b16 100644 --- a/src/openalpr/utility.h +++ b/src/openalpr/utility.h @@ -100,6 +100,8 @@ float angleBetweenPoints(cv::Point p1, cv::Point p2); cv::Size getSizeMaintainingAspect(cv::Mat inputImg, int maxWidth, int maxHeight); +float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex); + cv::Mat equalizeBrightness(cv::Mat img); cv::Rect expandRect(cv::Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY); From 1ce39c3f1fa0d262298c4b1abad5e59c83cd955a Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 16 Oct 2014 21:45:56 -0400 Subject: [PATCH 15/59] Added config attribute for multi-line plates --- src/openalpr/config.cpp | 2 ++ src/openalpr/config.h | 2 ++ src/openalpr/licenseplatecandidate.cpp | 1 + src/openalpr/pipeline_data.h | 2 ++ src/openalpr/textdetection/characteranalysis.cpp | 1 - 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openalpr/config.cpp b/src/openalpr/config.cpp index 8935d92..2f30a29 100644 --- a/src/openalpr/config.cpp +++ b/src/openalpr/config.cpp @@ -139,6 +139,8 @@ void Config::loadValues(string country) minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100); minPlateSizeHeightPx = getInt(country, "min_plate_size_height_px", 100); + multiline = getBoolean(country, "multiline", false); + plateWidthMM = getFloat(country, "plate_width_mm", 100); plateHeightMM = getFloat(country, "plate_height_mm", 100); diff --git a/src/openalpr/config.h b/src/openalpr/config.h index 21fb8dc..7477b79 100644 --- a/src/openalpr/config.h +++ b/src/openalpr/config.h @@ -57,6 +57,8 @@ class Config float minPlateSizeWidthPx; float minPlateSizeHeightPx; + bool multiline; + float plateWidthMM; float plateHeightMM; diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index f21b87a..4406718 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -40,6 +40,7 @@ void LicensePlateCandidate::recognize() charSegmenter = NULL; pipeline_data->plate_area_confidence = 0; + pipeline_data->isMultiline = config->multiline; int expandX = round(this->pipeline_data->regionOfInterest.width * 0.20); int expandY = round(this->pipeline_data->regionOfInterest.height * 0.15); diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index e92302e..6c71e8a 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -23,6 +23,8 @@ class PipelineData cv::Mat grayImg; cv::Rect regionOfInterest; + bool isMultiline; + cv::Mat crop_gray; bool hasPlateBorder; diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 999746f..d44bfff 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -27,7 +27,6 @@ CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) this->pipeline_data = pipeline_data; this->config = pipeline_data->config; - this->isTwoLine = true; if (this->config->debugCharAnalysis) cout << "Starting CharacterAnalysis identification" << endl; From 4bfef05b9a5924495442846830eff18d7514c379 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Thu, 16 Oct 2014 21:47:01 -0400 Subject: [PATCH 16/59] Added LineFinder class to help find text lines in plate images --- src/openalpr/CMakeLists.txt | 1 + .../textdetection/characteranalysis.cpp | 75 ++---- .../textdetection/characteranalysis.h | 4 +- src/openalpr/textdetection/linefinder.cpp | 232 ++++++++++++++++++ src/openalpr/textdetection/linefinder.h | 48 ++++ src/openalpr/textdetection/textcontours.cpp | 42 +++- src/openalpr/textdetection/textcontours.h | 6 + 7 files changed, 350 insertions(+), 58 deletions(-) create mode 100644 src/openalpr/textdetection/linefinder.cpp create mode 100644 src/openalpr/textdetection/linefinder.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 04fc165..2cdaaa4 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -27,6 +27,7 @@ set(lpr_source_files textdetection/platemask.cpp textdetection/textcontours.cpp textdetection/textline.cpp + textdetection/linefinder.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index d44bfff..6d9ca4f 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -18,6 +18,7 @@ */ #include "characteranalysis.h" +#include "linefinder.h" using namespace cv; using namespace std; @@ -101,8 +102,6 @@ void CharacterAnalysis::analyze() int bestFitIndex = -1; for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { - //vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); - //charSegments.push_back(goodIndices); int segmentCount = allTextContours[i].getGoodIndicesCount(); @@ -125,31 +124,16 @@ void CharacterAnalysis::analyze() if (this->config->debugCharAnalysis) { - Mat img_contours(bestThreshold.size(), CV_8U); - bestThreshold.copyTo(img_contours); - cvtColor(img_contours, img_contours, CV_GRAY2RGB); - - vector > allowedContours; - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestContours.goodIndices[i]) - allowedContours.push_back(bestContours.contours[i]); - } - - drawContours(img_contours, bestContours.contours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 - - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 + Mat img_contours = bestContours.drawDebugImage(bestThreshold); displayImage(config, "Matching Contours", img_contours); } - vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); + LineFinder lf(pipeline_data); + lf.findLines(pipeline_data->crop_gray, bestContours); + + vector linePolygon; + //vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); if (linePolygon.size() > 0) { @@ -370,7 +354,7 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - + int goodIndices = textContours.getGoodIndicesCount(); if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; @@ -381,8 +365,8 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - vector lines = getBestVotedLines(img, textContours); - this->filterBetweenLines(img, textContours, lines); + //vector lines = getBestVotedLines(img, textContours); + //this->filterBetweenLines(img, textContours, lines); int segmentCount = textContours.getGoodIndicesCount(); @@ -415,11 +399,13 @@ void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeigh float minWidth = mr.height * 0.2; //Crop image - //Mat auxRoi(img, mr); + + //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl; if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) { float charAspect= (float)mr.width/(float)mr.height; + //cout << " -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl; if (abs(charAspect - idealAspect) < aspecttolerance) textContours.goodIndices[i] = true; } @@ -545,7 +531,7 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, // Create a white mask for the area inside the polygon Mat outerMask = Mat::zeros(img.size(), CV_8U); - Mat innerArea(img.size(), CV_8U); + fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); // For each contour, determine if enough of it is between the lines to qualify @@ -556,35 +542,14 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, textContours.goodIndices[i] = false; // Set it to not included unless it proves - innerArea.setTo(Scalar(0,0,0)); + float percentInsideMask = getContourAreaPercentInsideMask(outerMask, + textContours.contours, + textContours.hierarchy, + (int) i); - drawContours(innerArea, textContours.contours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - textContours.hierarchy, - 0 - ); - - bitwise_and(innerArea, outerMask, innerArea); - - vector > tempContours; - findContours(innerArea, tempContours, - CV_RETR_EXTERNAL, // retrieve the external contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours ); - - double totalArea = contourArea(textContours.contours[i]); - double areaBetweenLines = 0; - - for (uint tempContourIdx = 0; tempContourIdx < tempContours.size(); tempContourIdx++) - { - areaBetweenLines += contourArea(tempContours[tempContourIdx]); - } - - - if (areaBetweenLines / totalArea < MIN_AREA_PERCENT_WITHIN_LINES) + + if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) { // Not enough area is inside the lines. continue; diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index fd26ea8..1eb8eab 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -26,6 +26,7 @@ #include "pipeline_data.h" #include "textcontours.h" #include "platemask.h" +#include "linefinder.h" class CharacterAnalysis { @@ -40,7 +41,6 @@ class CharacterAnalysis TextContours bestContours; bool thresholdsInverted; - bool isTwoLine; std::vector allTextContours; @@ -68,7 +68,7 @@ class CharacterAnalysis bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - + }; #endif // OPENALPR_CHARACTERANALYSIS_H diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp new file mode 100644 index 0000000..db30e43 --- /dev/null +++ b/src/openalpr/textdetection/linefinder.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include + +#include "linefinder.h" +#include "utility.h" +#include "pipeline_data.h" + +using namespace std; +using namespace cv; + +LineFinder::LineFinder(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; +} + +LineFinder::~LineFinder() { +} + +vector LineFinder::findLines(Mat image, const TextContours contours) +{ + vector linesFound; + + + cvtColor(image, image, CV_GRAY2BGR); + vector boxes = this->getBoundingBoxes(contours); + + vector tops = this->getCharTops(boxes); + vector bottoms = this->getCharBottoms(boxes); + + for (uint i = 0; i < tops.size(); i++) + { + circle(image, tops[i], 1, Scalar(255, 0, 0), 2); + circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2); + } + + drawAndWait(&image); + + vector bestLine = getBestLine(contours, tops, bottoms); + + if (pipeline_data->isMultiline) + { + // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. + } + + for (uint i = 0; i < contours.goodIndices.size(); i++) + { + + } + + + return linesFound; +} + + +vector LineFinder::getBoundingBoxes(const TextContours contours) { + + vector boxes; + for (uint i = 0; i < contours.goodIndices.size(); i++) + { + if (contours.goodIndices[i] == false) + continue; + + Rect bRect = cv::boundingRect( Mat(contours.contours[i]) ); + + boxes.push_back(bRect); + } + + return boxes; +} + + +vector LineFinder::getCharTops(vector boxes) { + + vector tops; + for (uint i = 0; i < boxes.size(); i++) + { + int x = boxes[i].x + (boxes[i].width / 2); + int y = boxes[i].y; + + tops.push_back(Point(x, y)); + } + + return tops; +} + +vector LineFinder::getCharBottoms(vector boxes) { + + vector bottoms; + for (uint i = 0; i < boxes.size(); i++) + { + int x = boxes[i].x + (boxes[i].width / 2); + int y = boxes[i].y + boxes[i].height; + + bottoms.push_back(Point(x, y)); + } + + return bottoms; +} + + + + + +// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width +vector LineFinder::getBestLine(const TextContours contours, vector tops, vector bottoms) +{ + + + vector bestStripe; + + // Find the best fit line segment that is parallel with the most char segments + if (tops.size() <= 1) + { + // Maybe do something about this later, for now let's just ignore + } + else + { + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < tops.size() - 1; i++) + { + for (uint k = i+1; k < tops.size(); k++) + { + + Point topLeft, topRight; + Point bottomLeft, bottomRight; + if (tops[i].x < tops[k].x) + { + topLeft = tops[i]; + topRight = tops[k]; + bottomLeft = bottoms[i]; + bottomRight = bottoms[k]; + } + else + { + topLeft = tops[k]; + topRight = tops[i]; + bottomLeft = bottoms[k]; + bottomRight = bottoms[i]; + } + + topLines.push_back(LineSegment(topLeft, topRight)); + bottomLines.push_back(LineSegment(bottomLeft, bottomRight)); + + } + } + + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < tops.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(tops[charidx].x); + float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x); + + float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD; + float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD; + float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD; + float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (true) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + drawAndWait(&tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); + } + + return bestStripe; +} diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h new file mode 100644 index 0000000..aa4db39 --- /dev/null +++ b/src/openalpr/textdetection/linefinder.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +// This class finds lines of text given an array of contours + +#ifndef OPENALPR_LINEFINDER_H +#define OPENALPR_LINEFINDER_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "textcontours.h" +#include "textline.h" +#include "pipeline_data.h" + + +class LineFinder { +public: + LineFinder(PipelineData* pipeline_data); + virtual ~LineFinder(); + + std::vector findLines(cv::Mat image, const TextContours contours); +private: + PipelineData* pipeline_data; + + std::vector getBoundingBoxes(const TextContours contours); + std::vector getCharTops(std::vector boxes); + std::vector getCharBottoms(std::vector boxes); + + std::vector getBestLine(const TextContours contours, std::vector tops, std::vector bottoms); +}; + +#endif /* OPENALPR_LINEFINDER_H */ + diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp index 01a2272..2b7dd2d 100644 --- a/src/openalpr/textdetection/textcontours.cpp +++ b/src/openalpr/textdetection/textcontours.cpp @@ -50,6 +50,9 @@ void TextContours::load(cv::Mat threshold) { for (uint i = 0; i < contours.size(); i++) goodIndices.push_back(true); + + this->width = threshold.cols; + this->height = threshold.rows; } @@ -95,4 +98,41 @@ void TextContours::setIndices(std::vector newIndices) { assert("Invalid set operation on indices"); } -} \ No newline at end of file +} + +Mat TextContours::drawDebugImage() { + + Mat img_contours = Mat::zeros(Size(width, height), CV_8U); + + return drawDebugImage(img_contours); +} + +Mat TextContours::drawDebugImage(Mat baseImage) { + Mat img_contours(baseImage.size(), CV_8U); + baseImage.copyTo(img_contours); + + cvtColor(img_contours, img_contours, CV_GRAY2RGB); + + vector > allowedContours; + for (uint i = 0; i < this->contours.size(); i++) + { + if (this->goodIndices[i]) + allowedContours.push_back(this->contours[i]); + } + + drawContours(img_contours, this->contours, + -1, // draw all contours + cv::Scalar(255,0,0), // in blue + 1); // with a thickness of 1 + + drawContours(img_contours, allowedContours, + -1, // draw all contours + cv::Scalar(0,255,0), // in green + 1); // with a thickness of 1 + + + return img_contours; +} + + + diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h index 428a767..cf6d6bb 100644 --- a/src/openalpr/textdetection/textcontours.h +++ b/src/openalpr/textdetection/textcontours.h @@ -31,6 +31,9 @@ public: void load(cv::Mat threshold); + int width; + int height; + std::vector goodIndices; std::vector > contours; std::vector hierarchy; @@ -41,6 +44,9 @@ public: std::vector getIndicesCopy(); void setIndices(std::vector newIndices); + cv::Mat drawDebugImage(); + cv::Mat drawDebugImage(cv::Mat baseImage); + private: From 3964ffa49fd908c9a3102ca1602b394e972f66b8 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Fri, 17 Oct 2014 11:34:13 -0400 Subject: [PATCH 17/59] Added angle checking so that text areas with vertical angles are rejected --- src/openalpr/textdetection/linefinder.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index db30e43..401537a 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -157,8 +157,18 @@ vector LineFinder::getBestLine(const TextContours contours, vector bottomRight = bottoms[i]; } - topLines.push_back(LineSegment(topLeft, topRight)); - bottomLines.push_back(LineSegment(bottomLeft, bottomRight)); + + LineSegment top(topLeft, topRight); + LineSegment bottom(bottomLeft, bottomRight); + + // Only allow lines that have a sane angle + if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(top); + bottomLines.push_back(bottom); + } + } } From 7ad3dedea2f123e120fb60f0880e7ee906c20860 Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 17:01:51 +0100 Subject: [PATCH 18/59] Add SKIP_DAEMON option. Change hardcoded paths to use CMAKE_INSTALL_PREFIX and CMAKE_INSTALL_SYSCONFDIR. --- src/CMakeLists.txt | 38 ++++++++++++++++++++++++++++++------- src/openalpr/CMakeLists.txt | 18 +++++++++--------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ac8f82..d6a2b4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,8 +13,19 @@ add_definitions( -DOPENALPR_MAJOR_VERSION=${OPENALPR_MAJOR_VERSION}) add_definitions( -DOPENALPR_MINOR_VERSION=${OPENALPR_MINOR_VERSION}) add_definitions( -DOPENALPR_PATCH_VERSION=${OPENALPR_PATCH_VERSION}) + SET(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake_modules/") +# TODO: switch to http://www.cmake.org/cmake/help/v2.8.5/cmake.html#module:GNUInstallDirs ? +IF (NOT CMAKE_INSTALL_SYSCONFDIR) + SET(CMAKE_INSTALL_SYSCONFDIR "${CMAKE_INSTALL_PREFIX}/etc") +ENDIF() + +#MESSAGE( STATUS " kkjjkj + +IF (NOT WIN32 AND NOT SKIP_DAEMON) + SET(INSTALL_DAEMON ON) +ENDIF() FIND_PACKAGE( Tesseract REQUIRED ) @@ -64,7 +75,7 @@ TARGET_LINK_LIBRARIES(alpr ) # Compile the alprd library on Unix-based OS -IF (NOT WIN32) +IF (INSTALL_DAEMON) ADD_EXECUTABLE( alprd daemon.cpp daemon/beanstalk.c daemon/beanstalk.cc ) TARGET_LINK_LIBRARIES(alprd @@ -89,12 +100,14 @@ add_subdirectory(openalpr) add_subdirectory(video) -install (TARGETS alpr DESTINATION /usr/bin) -install (TARGETS alprd DESTINATION /usr/bin) -install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION /usr/share/man/man1 COMPONENT doc) -install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION /usr/share/openalpr/) -install (FILES ${CMAKE_SOURCE_DIR}/../config/openalpr.conf DESTINATION /etc/openalpr/ COMPONENT config) -install (FILES ${CMAKE_SOURCE_DIR}/../config/alprd.conf DESTINATION /etc/openalpr/ COMPONENT config) +install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/${BINDIR}) +IF (INSTALL_DAEMON) + install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/${BINDIR}) +ENDIF() +install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1 COMPONENT doc) +install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION ${CMAKE_INSTALL_PREFIX}/share/openalpr) +install (FILES ${CMAKE_SOURCE_DIR}/../config/openalpr.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) +install (FILES ${CMAKE_SOURCE_DIR}/../config/alprd.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) SET(CPACK_PACKAGE_VERSION ${OPENALPR_VERSION}) @@ -119,3 +132,14 @@ SET (CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}_${C SET (CPACK_COMPONENTS_ALL Libraries ApplicationData) INCLUDE(CPack) + +# ---------------------------------------------------------------------------- +# Uninstall target, for "make uninstall" +# http://www.cmake.org/Wiki/CMake_FAQ#Can_I_do_.22make_uninstall.22_with_CMake.3F +# ---------------------------------------------------------------------------- +CONFIGURE_FILE( + "${CMAKE_MODULE_PATH}/templates/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + @ONLY) + +ADD_CUSTOM_TARGET(uninstall COMMAND ${CMAKE_COMMAND} -P "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake") diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index d979f90..9b055aa 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -1,5 +1,5 @@ - +SET(Tesseract_DIR "/Users/alex/work/lp/tesseract-ocr") set(lpr_source_files @@ -39,15 +39,15 @@ add_library(openalpr SHARED ${lpr_source_files} ) set_target_properties(openalpr PROPERTIES SOVERSION ${OPENALPR_MAJOR_VERSION}) - TARGET_LINK_LIBRARIES(openalpr - support - ${OpenCV_LIBS} - ${Tesseract_LIBRARIES} - ) +TARGET_LINK_LIBRARIES(openalpr + support + ${OpenCV_LIBS} + ${Tesseract_LIBRARIES} +) -install (FILES alpr.h DESTINATION /usr/include) -install (TARGETS openalpr DESTINATION /usr/lib) +install (FILES alpr.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include) +install (TARGETS openalpr DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) # Add definition for default config file -add_definitions(-DDEFAULT_CONFIG_FILE="/etc/openalpr/openalpr.conf") +add_definitions(-DDEFAULT_CONFIG_FILE="${CMAKE_INSTALL_SYSCONFDIR}/openalpr/openalpr.conf") From 12832b0070a0fc16314ce174664c1f2be30ea90e Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 17:04:12 +0100 Subject: [PATCH 19/59] Remove unwanted Tesseract_DIR definition. --- src/openalpr/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 9b055aa..db7b88f 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -1,5 +1,4 @@ -SET(Tesseract_DIR "/Users/alex/work/lp/tesseract-ocr") set(lpr_source_files From 9f213e25f17c225440c5f734b0413a0a69628757 Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 17:05:51 +0100 Subject: [PATCH 20/59] Remove usage of BINDIR. --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d6a2b4a..a899a5b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,9 +100,9 @@ add_subdirectory(openalpr) add_subdirectory(video) -install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/${BINDIR}) +install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) IF (INSTALL_DAEMON) - install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/${BINDIR}) + install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) ENDIF() install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1 COMPONENT doc) install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION ${CMAKE_INSTALL_PREFIX}/share/openalpr) From 70797bee6ada29c0f3f9570a9aa3dde1346e3baa Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 17:06:47 +0100 Subject: [PATCH 21/59] Remove unused MESSAGE command --- src/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a899a5b..8f7b215 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,6 @@ add_definitions( -DOPENALPR_MAJOR_VERSION=${OPENALPR_MAJOR_VERSION}) add_definitions( -DOPENALPR_MINOR_VERSION=${OPENALPR_MINOR_VERSION}) add_definitions( -DOPENALPR_PATCH_VERSION=${OPENALPR_PATCH_VERSION}) - SET(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake_modules/") # TODO: switch to http://www.cmake.org/cmake/help/v2.8.5/cmake.html#module:GNUInstallDirs ? @@ -21,8 +20,6 @@ IF (NOT CMAKE_INSTALL_SYSCONFDIR) SET(CMAKE_INSTALL_SYSCONFDIR "${CMAKE_INSTALL_PREFIX}/etc") ENDIF() -#MESSAGE( STATUS " kkjjkj - IF (NOT WIN32 AND NOT SKIP_DAEMON) SET(INSTALL_DAEMON ON) ENDIF() From 57f1afa36b6dab2db28359ed229a5c357344f2b7 Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 17:11:12 +0100 Subject: [PATCH 22/59] Add uninstall target template. --- .../templates/cmake_uninstall.cmake.in | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/cmake_modules/templates/cmake_uninstall.cmake.in diff --git a/src/cmake_modules/templates/cmake_uninstall.cmake.in b/src/cmake_modules/templates/cmake_uninstall.cmake.in new file mode 100644 index 0000000..2037e36 --- /dev/null +++ b/src/cmake_modules/templates/cmake_uninstall.cmake.in @@ -0,0 +1,21 @@ +if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + message(FATAL_ERROR "Cannot find install manifest: @CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") +endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + +file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) +string(REGEX REPLACE "\n" ";" files "${files}") +foreach(file ${files}) + message(STATUS "Uninstalling $ENV{DESTDIR}${file}") + if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + exec_program( + "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + OUTPUT_VARIABLE rm_out + RETURN_VALUE rm_retval + ) + if(NOT "${rm_retval}" STREQUAL 0) + message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") + endif(NOT "${rm_retval}" STREQUAL 0) + else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + message(STATUS "File $ENV{DESTDIR}${file} does not exist.") + endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") +endforeach(file) From 211bb3f514d5e7f4ea85d7de6a846a96f331df8d Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 18:00:03 +0100 Subject: [PATCH 23/59] Change daemon option to WITH_DAEMON, and some logic. --- src/CMakeLists.txt | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8f7b215..34164fe 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,10 +20,16 @@ IF (NOT CMAKE_INSTALL_SYSCONFDIR) SET(CMAKE_INSTALL_SYSCONFDIR "${CMAKE_INSTALL_PREFIX}/etc") ENDIF() -IF (NOT WIN32 AND NOT SKIP_DAEMON) - SET(INSTALL_DAEMON ON) +IF ( NOT DEFINED WITH_DAEMON ) + SET(WITH_DAEMON ON) ENDIF() +IF (WIN32 AND WITH_DAEMON) + MESSAGE(WARNING "Skipping alprd daemon installation, as it is not supported in Windows.") + SET(WITH_DAEMON OFF) +ENDIF() + +MESSAGE(STATUS "WITH_DAEMON: ${WITH_DAEMON}") FIND_PACKAGE( Tesseract REQUIRED ) include_directories(${Tesseract_INCLUDE_DIRS}) @@ -72,7 +78,7 @@ TARGET_LINK_LIBRARIES(alpr ) # Compile the alprd library on Unix-based OS -IF (INSTALL_DAEMON) +IF (WITH_DAEMON) ADD_EXECUTABLE( alprd daemon.cpp daemon/beanstalk.c daemon/beanstalk.cc ) TARGET_LINK_LIBRARIES(alprd @@ -98,13 +104,14 @@ add_subdirectory(video) install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -IF (INSTALL_DAEMON) - install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -ENDIF() install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1 COMPONENT doc) install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION ${CMAKE_INSTALL_PREFIX}/share/openalpr) install (FILES ${CMAKE_SOURCE_DIR}/../config/openalpr.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) -install (FILES ${CMAKE_SOURCE_DIR}/../config/alprd.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) + +IF (WITH_DAEMON) + install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) + install (FILES ${CMAKE_SOURCE_DIR}/../config/alprd.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) +ENDIF() SET(CPACK_PACKAGE_VERSION ${OPENALPR_VERSION}) From f7074ec7e6e985859c609a57ebf9f441972ac32d Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 18:01:36 +0100 Subject: [PATCH 24/59] Remove unwated message. --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 34164fe..9d281b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,6 @@ IF (WIN32 AND WITH_DAEMON) SET(WITH_DAEMON OFF) ENDIF() -MESSAGE(STATUS "WITH_DAEMON: ${WITH_DAEMON}") FIND_PACKAGE( Tesseract REQUIRED ) include_directories(${Tesseract_INCLUDE_DIRS}) From bd512a801464d5dc858afc55fcc06c6d4da71eb5 Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 18:14:42 +0100 Subject: [PATCH 25/59] Set the proper value for runtime_dir in openalpr.conf, based on install prefix. --- config/openalpr.conf | 2 +- src/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/openalpr.conf b/config/openalpr.conf index bc374b7..51b0016 100644 --- a/config/openalpr.conf +++ b/config/openalpr.conf @@ -1,7 +1,7 @@ [common] ; Specify the path to the runtime data directory -runtime_dir = /usr/share/openalpr/runtime_data +runtime_dir = ${CMAKE_INSTALL_PREFIX}/share/openalpr/runtime_data ocr_img_size_percent = 1.33333333 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9d281b6..032bc23 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,7 +105,7 @@ add_subdirectory(video) install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1 COMPONENT doc) install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION ${CMAKE_INSTALL_PREFIX}/share/openalpr) -install (FILES ${CMAKE_SOURCE_DIR}/../config/openalpr.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr COMPONENT config) +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/../config/openalpr.conf ${CMAKE_INSTALL_SYSCONFDIR}/openalpr/openalpr.conf) IF (WITH_DAEMON) install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) From eac02fcfaaa2c46c3f93c7afd0d4ae99df14296a Mon Sep 17 00:00:00 2001 From: twelve17 Date: Sat, 18 Oct 2014 18:31:46 +0100 Subject: [PATCH 26/59] Update how openalpr.conf is generated and ultimately installed. --- config/{openalpr.conf => openalpr.conf.in} | 0 src/CMakeLists.txt | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) rename config/{openalpr.conf => openalpr.conf.in} (100%) diff --git a/config/openalpr.conf b/config/openalpr.conf.in similarity index 100% rename from config/openalpr.conf rename to config/openalpr.conf.in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 032bc23..205dd62 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,7 +105,10 @@ add_subdirectory(video) install (TARGETS alpr DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) install (FILES ${CMAKE_SOURCE_DIR}/../doc/man/alpr.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man1 COMPONENT doc) install (DIRECTORY ${CMAKE_SOURCE_DIR}/../runtime_data DESTINATION ${CMAKE_INSTALL_PREFIX}/share/openalpr) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/../config/openalpr.conf ${CMAKE_INSTALL_SYSCONFDIR}/openalpr/openalpr.conf) + +# set runtime_data to reflect the current CMAKE_INSTALL_PREFIX +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/../config/openalpr.conf.in ${CMAKE_CURRENT_BINARY_DIR}/config/openalpr.conf) +install (FILES ${CMAKE_CURRENT_BINARY_DIR}/config/openalpr.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/openalpr/ COMPONENT config) IF (WITH_DAEMON) install (TARGETS alprd DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) From 2ac6337342286faeac002f4c5d695d602c6e71d1 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sat, 18 Oct 2014 20:13:30 -0400 Subject: [PATCH 27/59] Added logic to LineFinder to find both lines for multiline plates --- .../textdetection/characteranalysis.cpp | 60 ++++--- .../textdetection/characteranalysis.h | 2 +- src/openalpr/textdetection/linefinder.cpp | 158 ++++++++---------- src/openalpr/textdetection/linefinder.h | 19 ++- src/openalpr/textdetection/textline.cpp | 45 ++++- src/openalpr/textdetection/textline.h | 4 + 6 files changed, 164 insertions(+), 124 deletions(-) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 6d9ca4f..f849b72 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -130,18 +130,17 @@ void CharacterAnalysis::analyze() } LineFinder lf(pipeline_data); - lf.findLines(pipeline_data->crop_gray, bestContours); + vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); - vector linePolygon; - //vector linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours); - if (linePolygon.size() > 0) + for (uint i = 0; i < linePolygons.size(); i++) { + vector linePolygon = linePolygons[i]; + + cout << "Polygon: " << linePolygon[0] << " - " << linePolygon[1] << " - " << linePolygon[2] << " - " << linePolygon[3] << endl; LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); - - filterBetweenLines(bestThreshold, bestContours, linePolygon); vector textArea = getCharArea(topLine, bottomLine); @@ -149,7 +148,17 @@ void CharacterAnalysis::analyze() pipeline_data->textLines.push_back(textLine); } + + filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines); + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + cout << "Test1" << endl; + Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold); + + cout << "Test2" << endl; + drawAndWait(&debugImage); + } this->thresholdsInverted = isPlateInverted(); } @@ -510,29 +519,22 @@ void CharacterAnalysis::filterByParentContour( TextContours& textContours) } -void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector outerPolygon ) +void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector textLines ) { static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; - if (outerPolygon.size() == 0) + if (textLines.size() == 0) return; vector validPoints; - // Figure out the line height - LineSegment topLine(outerPolygon[0].x, outerPolygon[0].y, outerPolygon[1].x, outerPolygon[1].y); - LineSegment bottomLine(outerPolygon[3].x, outerPolygon[3].y, outerPolygon[2].x, outerPolygon[2].y); - - float x = ((float) img.cols) / 2; - Point midpoint = Point(x, bottomLine.getPointAt(x)); - Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); - float lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); // Create a white mask for the area inside the polygon Mat outerMask = Mat::zeros(img.size(), CV_8U); - fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); + for (uint i = 0; i < textLines.size(); i++) + fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255)); // For each contour, determine if enough of it is between the lines to qualify for (uint i = 0; i < textContours.size(); i++) @@ -579,19 +581,23 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, } // Get the absolute distance from the top and bottom lines - Point closestTopPoint = topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); - Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); - float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); - - float maxDistance = lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; - - if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + for (uint i = 0; i < textLines.size(); i++) { - textContours.goodIndices[i] = true; - } + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); + float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); + + float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + + if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + { + textContours.goodIndices[i] = true; + } + } + } } diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 1eb8eab..441444c 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -64,7 +64,7 @@ class CharacterAnalysis std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); std::vector getBestVotedLines(cv::Mat img, TextContours textContours); - void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector outerPolygon ); + void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index 401537a..cf6ea87 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -33,35 +33,51 @@ LineFinder::LineFinder(PipelineData* pipeline_data) { LineFinder::~LineFinder() { } -vector LineFinder::findLines(Mat image, const TextContours contours) +vector > LineFinder::findLines(Mat image, const TextContours contours) { - vector linesFound; + vector > linesFound; cvtColor(image, image, CV_GRAY2BGR); - vector boxes = this->getBoundingBoxes(contours); - vector tops = this->getCharTops(boxes); - vector bottoms = this->getCharBottoms(boxes); + vector charPoints; - for (uint i = 0; i < tops.size(); i++) + for (uint i = 0; i < contours.contours.size(); i++) { - circle(image, tops[i], 1, Scalar(255, 0, 0), 2); - circle(image, bottoms[i], 1, Scalar(0, 0, 255), 2); + if (contours.goodIndices[i] == false) + continue; + + charPoints.push_back( CharPointInfo(contours.contours[i], i) ); } - drawAndWait(&image); + vector bestLine = getBestLine(contours, charPoints); - vector bestLine = getBestLine(contours, tops, bottoms); + if (bestLine.size() > 0) + linesFound.push_back(bestLine); if (pipeline_data->isMultiline) { // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. - } - - for (uint i = 0; i < contours.goodIndices.size(); i++) - { + // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it. + vector remainingPoints; + for (uint i = 0; i < charPoints.size(); i++) + { + Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U); + fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255)); + + float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); + + if (percentInside < .85) + { + remainingPoints.push_back(charPoints[i]); + } + } + + vector nextBestLine = getBestLine(contours, remainingPoints); + + if (nextBestLine.size() > 0) + linesFound.push_back(nextBestLine); } @@ -69,64 +85,15 @@ vector LineFinder::findLines(Mat image, const TextContours contours) } -vector LineFinder::getBoundingBoxes(const TextContours contours) { - - vector boxes; - for (uint i = 0; i < contours.goodIndices.size(); i++) - { - if (contours.goodIndices[i] == false) - continue; - - Rect bRect = cv::boundingRect( Mat(contours.contours[i]) ); - - boxes.push_back(bRect); - } - - return boxes; -} - - -vector LineFinder::getCharTops(vector boxes) { - - vector tops; - for (uint i = 0; i < boxes.size(); i++) - { - int x = boxes[i].x + (boxes[i].width / 2); - int y = boxes[i].y; - - tops.push_back(Point(x, y)); - } - - return tops; -} - -vector LineFinder::getCharBottoms(vector boxes) { - - vector bottoms; - for (uint i = 0; i < boxes.size(); i++) - { - int x = boxes[i].x + (boxes[i].width / 2); - int y = boxes[i].y + boxes[i].height; - - bottoms.push_back(Point(x, y)); - } - - return bottoms; -} - - - - - // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector LineFinder::getBestLine(const TextContours contours, vector tops, vector bottoms) +vector LineFinder::getBestLine(const TextContours contours, vector charPoints) { vector bestStripe; // Find the best fit line segment that is parallel with the most char segments - if (tops.size() <= 1) + if (charPoints.size() <= 1) { // Maybe do something about this later, for now let's just ignore } @@ -135,31 +102,26 @@ vector LineFinder::getBestLine(const TextContours contours, vector vector topLines; vector bottomLines; // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < tops.size() - 1; i++) + for (uint i = 0; i < charPoints.size() - 1; i++) { - for (uint k = i+1; k < tops.size(); k++) + for (uint k = i+1; k < charPoints.size(); k++) { - Point topLeft, topRight; - Point bottomLeft, bottomRight; - if (tops[i].x < tops[k].x) + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) { - topLeft = tops[i]; - topRight = tops[k]; - bottomLeft = bottoms[i]; - bottomRight = bottoms[k]; + leftCPIndex = i; + rightCPIndex = k; } else { - topLeft = tops[k]; - topRight = tops[i]; - bottomLeft = bottoms[k]; - bottomRight = bottoms[i]; + leftCPIndex = k; + rightCPIndex = i; } - LineSegment top(topLeft, topRight); - LineSegment bottom(bottomLeft, bottomRight); + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); // Only allow lines that have a sane angle if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && @@ -184,15 +146,15 @@ vector LineFinder::getBestLine(const TextContours contours, vector float SCORING_MAX_THRESHOLD = 1.03; int curScore = 0; - for (uint charidx = 0; charidx < tops.size(); charidx++) + for (uint charidx = 0; charidx < charPoints.size(); charidx++) { - float topYPos = topLines[i].getPointAt(tops[charidx].x); - float botYPos = bottomLines[i].getPointAt(bottoms[charidx].x); + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); - float minTop = tops[charidx].y * SCORING_MIN_THRESHOLD; - float maxTop = tops[charidx].y * SCORING_MAX_THRESHOLD; - float minBot = (bottoms[charidx].y) * SCORING_MIN_THRESHOLD; - float maxBot = (bottoms[charidx].y) * SCORING_MAX_THRESHOLD; + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; if ( (topYPos >= minTop && topYPos <= maxTop) && (botYPos >= minBot && botYPos <= maxBot)) { @@ -202,7 +164,7 @@ vector LineFinder::getBestLine(const TextContours contours, vector //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; //drawAndWait(&tempImg); } - + // Tie goes to the one with longer line segments if ((curScore > bestScore) || (curScore == bestScore && topLines[i].length > bestScoreDistance)) @@ -240,3 +202,23 @@ vector LineFinder::getBestLine(const TextContours contours, vector return bestStripe; } + +CharPointInfo::CharPointInfo(vector contour, int index) { + + + this->contourIndex = index; + + this->boundingBox = cv::boundingRect( Mat(contour) ); + + + int x = boundingBox.x + (boundingBox.width / 2); + int y = boundingBox.y; + + this->top = Point(x, y); + + x = boundingBox.x + (boundingBox.width / 2); + y = boundingBox.y + boundingBox.height; + + this->bottom = Point(x,y); + +} diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h index aa4db39..7b7398f 100644 --- a/src/openalpr/textdetection/linefinder.h +++ b/src/openalpr/textdetection/linefinder.h @@ -27,21 +27,28 @@ #include "textline.h" #include "pipeline_data.h" +class CharPointInfo +{ +public: + CharPointInfo(std::vector contour, int index); + + cv::Rect boundingBox; + cv::Point top; + cv::Point bottom; + int contourIndex; + +}; class LineFinder { public: LineFinder(PipelineData* pipeline_data); virtual ~LineFinder(); - std::vector findLines(cv::Mat image, const TextContours contours); + std::vector > findLines(cv::Mat image, const TextContours contours); private: PipelineData* pipeline_data; - std::vector getBoundingBoxes(const TextContours contours); - std::vector getCharTops(std::vector boxes); - std::vector getCharBottoms(std::vector boxes); - - std::vector getBestLine(const TextContours contours, std::vector tops, std::vector bottoms); + std::vector getBestLine(const TextContours contours, std::vector charPoints); }; #endif /* OPENALPR_LINEFINDER_H */ diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index 0e70d4d..a41ce8c 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -18,13 +18,20 @@ */ +#include + #include "textline.h" +using namespace cv; + TextLine::TextLine(std::vector textArea, std::vector linePolygon) { if (textArea.size() > 0) { - this->textArea = textArea; - this->linePolygon = linePolygon; + for (uint i = 0; i < textArea.size(); i++) + this->textArea.push_back(textArea[i]); + + for (uint i = 0; i < linePolygon.size(); i++) + this->linePolygon.push_back(linePolygon[i]); this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); @@ -33,9 +40,43 @@ TextLine::TextLine(std::vector textArea, std::vector lineP this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); + + + float x = ((float) linePolygon[1].x) / 2; + Point midpoint = Point(x, bottomLine.getPointAt(x)); + Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); + this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); } } TextLine::~TextLine() { } + +cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { + cv::Mat debugImage(baseImage.size(), baseImage.type()); + + baseImage.copyTo(debugImage); + + cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR); + + + fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); + + drawAndWait(&debugImage); + fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); + + drawAndWait(&debugImage); + line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); + line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); + + drawAndWait(&debugImage); + line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); + + drawAndWait(&debugImage); + + return debugImage; +} diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index cf5fc86..5403beb 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -22,6 +22,7 @@ #define OPENALPR_TEXTLINE_H #include "utility.h" +#include "opencv2/imgproc/imgproc.hpp" class TextLine { public: @@ -38,6 +39,9 @@ public: LineSegment charBoxLeft; LineSegment charBoxRight; + float lineHeight; + + cv::Mat drawDebugImage(cv::Mat baseImage); private: }; From f122eee12d950612eb9262c1adeafad825f57e84 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 15:54:04 -0400 Subject: [PATCH 28/59] Added angle to TextLine --- src/openalpr/textdetection/textline.cpp | 9 ++++----- src/openalpr/textdetection/textline.h | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index a41ce8c..90baafb 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -41,11 +41,14 @@ TextLine::TextLine(std::vector textArea, std::vector lineP this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); - + // Calculate line height float x = ((float) linePolygon[1].x) / 2; Point midpoint = Point(x, bottomLine.getPointAt(x)); Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); + + this->angle = (topLine.angle + bottomLine.angle) / 2; + } } @@ -63,20 +66,16 @@ cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); - drawAndWait(&debugImage); fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); - drawAndWait(&debugImage); line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); - drawAndWait(&debugImage); line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); - drawAndWait(&debugImage); return debugImage; } diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index 5403beb..bc0b6f6 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -40,6 +40,7 @@ public: LineSegment charBoxRight; float lineHeight; + float angle; cv::Mat drawDebugImage(cv::Mat baseImage); private: From 2896d2186880070d2a03c00c5d5a8a6a92138494 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 15:54:25 -0400 Subject: [PATCH 29/59] Removed unused function prototype from characterAnalysis --- src/openalpr/textdetection/characteranalysis.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index 441444c..b853072 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -63,7 +63,6 @@ class CharacterAnalysis void filterByOuterMask(TextContours& textContours); std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); - std::vector getBestVotedLines(cv::Mat img, TextContours textContours); void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); From d2bfebe443c58090cfaf29ce62e36b530d494c7c Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 15:54:59 -0400 Subject: [PATCH 30/59] Added multiline support in plate corner detector --- src/openalpr/platecorners.cpp | 218 +++++++++++++---- src/openalpr/platecorners.h | 38 ++- .../segmentation/charactersegmenter.cpp | 1 + .../textdetection/characteranalysis.cpp | 205 ++-------------- src/openalpr/textdetection/linefinder.cpp | 231 ++++++++++-------- 5 files changed, 369 insertions(+), 324 deletions(-) diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index d4e1d0c..76fda4d 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -22,7 +22,8 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) +PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) : + tlc(pipelineData) { this->pipelineData = pipelineData; @@ -35,12 +36,7 @@ PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* this->bestHorizontalScore = 9999999999999; this->bestVerticalScore = 9999999999999; - Point topPoint = pipelineData->textLines[0].topLine.midpoint(); - Point bottomPoint = pipelineData->textLines[0].bottomLine.closestPointOnSegmentTo(topPoint); - this->charHeight = distanceBetweenPoints(topPoint, bottomPoint); - - this->charAngle = angleBetweenPoints(pipelineData->textLines[0].textArea[0], pipelineData->textLines[0].textArea[1]); } PlateCorners::~PlateCorners() @@ -86,8 +82,12 @@ vector PlateCorners::findPlateCorners() Mat imgCorners = Mat(inputImage.size(), inputImage.type()); inputImage.copyTo(imgCorners); - for (int i = 0; i < 4; i++) - circle(imgCorners, pipelineData->textLines[0].textArea[i], 2, Scalar(0, 0, 0)); + + for (uint linenum = 0; linenum < pipelineData->textLines.size(); linenum++) + { + for (int i = 0; i < 4; i++) + circle(imgCorners, pipelineData->textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + } line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); @@ -128,8 +128,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) LineSegment left; LineSegment right; + float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; - float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -137,12 +138,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (v1 == NO_LINE && v2 == NO_LINE) { //return; - Point centerTop = pipelineData->textLines[0].charBoxTop.midpoint(); - Point centerBottom = pipelineData->textLines[0].charBoxBottom.midpoint(); - LineSegment centerLine = LineSegment(centerBottom.x, centerBottom.y, centerTop.x, centerTop.y); - left = centerLine.getParallelLine(idealPixelWidth / 2); - right = centerLine.getParallelLine(-1 * idealPixelWidth / 2 ); + left = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2); + right = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; confidenceDiff += 2; @@ -172,12 +170,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is to the left of our license plate letters - if (left.isPointBelowLine(pipelineData->textLines[0].charBoxLeft.midpoint()) == false) - return; - - // Make sure this line is to the right of our license plate letters - if (right.isPointBelowLine(pipelineData->textLines[0].charBoxRight.midpoint())) + // Make sure that the left and right lines are to the left and right of our text + // area + if (tlc.isLeftOfText(left) < 1 || tlc.isLeftOfText(right) > -1) return; ///////////////////////////////////////////////////////////////////////// @@ -202,7 +197,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) // Score angle difference from detected character box ///////////////////////////////////////////////////////////////////////// - float perpendicularCharAngle = charAngle - 90; + float perpendicularCharAngle = tlc.charAngle - 90; float charanglediff = abs(perpendicularCharAngle - left.angle) + abs(perpendicularCharAngle - right.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -211,8 +206,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) // SCORE the shape wrt character position and height relative to position ////////////////////////////////////////////////////////////////////////// - Point leftMidLinePoint = left.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxLeft.midpoint()); - Point rightMidLinePoint = right.closestPointOnSegmentTo(pipelineData->textLines[0].charBoxRight.midpoint()); + Point leftMidLinePoint = left.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); + Point rightMidLinePoint = right.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); @@ -224,7 +219,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealwidth " << idealPixelWidth << endl; cout << "xx xx Score: v1,v2= " << v1 << "," << v2 << endl; cout << "xx xx Score: Left= " << left.str() << endl; @@ -278,7 +273,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) LineSegment bottom; float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; - float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -286,12 +281,10 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (h1 == NO_LINE && h2 == NO_LINE) { // return; - Point centerLeft = pipelineData->textLines[0].charBoxLeft.midpoint(); - Point centerRight = pipelineData->textLines[0].charBoxRight.midpoint(); - LineSegment centerLine = LineSegment(centerLeft.x, centerLeft.y, centerRight.x, centerRight.y); - top = centerLine.getParallelLine(idealPixelHeight / 2); - bottom = centerLine.getParallelLine(-1 * idealPixelHeight / 2 ); + + top = tlc.centerHorizontalLine.getParallelLine(idealPixelHeight / 2); + bottom = tlc.centerHorizontalLine.getParallelLine(-1 * idealPixelHeight / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; confidenceDiff += 2; @@ -321,14 +314,11 @@ void PlateCorners::scoreHorizontals(int h1, int h2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is above our license plate letters - if (top.isPointBelowLine(pipelineData->textLines[0].charBoxTop.midpoint()) == false) + // Make sure that the top and bottom lines are above and below + // the text area + if (tlc.isAboveText(top) < 1 || tlc.isAboveText(bottom) > -1) return; - - // Make sure this line is below our license plate letters - if (bottom.isPointBelowLine(pipelineData->textLines[0].charBoxBottom.midpoint())) - return; - + // We now have 4 possible lines. Let's put them to the test and score them... ///////////////////////////////////////////////////////////////////////// @@ -351,7 +341,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // Get the height difference - float heightRatio = charHeight / plateHeightPx; + float heightRatio = tlc.charHeight / plateHeightPx; float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); //if (leftRatio < MIN_CHAR_HEIGHT_RATIO || leftRatio > MAX_CHAR_HEIGHT_RATIO || rightRatio < MIN_CHAR_HEIGHT_RATIO || rightRatio > MAX_CHAR_HEIGHT_RATIO) float heightRatioDiff = abs(heightRatio - idealHeightRatio); @@ -372,7 +362,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle ////////////////////////////////////////////////////////////////////////// - Point charAreaMidPoint = pipelineData->textLines[0].charBoxLeft.midpoint(); + Point charAreaMidPoint = tlc.centerVerticalLine.midpoint(); Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); @@ -394,7 +384,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE: the shape for angles matching the character region ////////////////////////////////////////////////////////////// - float charanglediff = abs(charAngle - top.angle) + abs(charAngle - bottom.angle); + float charanglediff = abs(tlc.charAngle - top.angle) + abs(tlc.charAngle - bottom.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -407,7 +397,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealHeight " << idealPixelHeight << endl; cout << "xx xx Score: h1,h2= " << h1 << "," << h2 << endl; cout << "xx xx Score: Top= " << top.str() << endl; @@ -447,3 +437,149 @@ void PlateCorners::scoreHorizontals(int h1, int h2) bestBottom = LineSegment(bottom.p1.x, bottom.p1.y, bottom.p2.x, bottom.p2.y); } } + +TextLineCollection::TextLineCollection(PipelineData* pipelineData) { + + this->pipelineData = pipelineData; + + charHeight = 0; + charAngle = 0; + for (uint i = 0; i < pipelineData->textLines.size(); i++) + { + charHeight += pipelineData->textLines[i].lineHeight; + charAngle += pipelineData->textLines[i].angle; + + } + charHeight = charHeight / pipelineData->textLines.size(); + charAngle = charAngle / pipelineData->textLines.size(); + + this->topCharArea = pipelineData->textLines[0].charBoxTop; + this->bottomCharArea = pipelineData->textLines[0].charBoxBottom; + for (uint i = 1; i < pipelineData->textLines.size(); i++) + { + + if (this->topCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxTop.midpoint()) == false) + this->topCharArea = pipelineData->textLines[i].charBoxTop; + + if (this->bottomCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxBottom.midpoint())) + this->bottomCharArea = pipelineData->textLines[i].charBoxBottom; + + } + + longerSegment = this->bottomCharArea; + shorterSegment = this->topCharArea; + if (this->topCharArea.length > this->bottomCharArea.length) + { + longerSegment = this->topCharArea; + shorterSegment = this->bottomCharArea; + } + + findCenterHorizontal(); + findCenterVertical(); + // Center Vertical Line + + Mat debugImage = Mat::zeros(pipelineData->crop_gray.size(), CV_8U); + line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); + line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); + + drawAndWait(&debugImage); +} + +// Returns 1 for above, 0 for within, and -1 for below +int TextLineCollection::isAboveText(LineSegment line) { + // Test four points (left and right corner of top and bottom line) + + Point topLeft = line.closestPointOnSegmentTo(topCharArea.p1); + Point topRight = line.closestPointOnSegmentTo(topCharArea.p2); + + bool lineIsBelowTop = topCharArea.isPointBelowLine(topLeft) || topCharArea.isPointBelowLine(topRight); + + if (!lineIsBelowTop) + return 1; + + Point bottomLeft = line.closestPointOnSegmentTo(bottomCharArea.p1); + Point bottomRight = line.closestPointOnSegmentTo(bottomCharArea.p2); + + bool lineIsBelowBottom = bottomCharArea.isPointBelowLine(bottomLeft) && + bottomCharArea.isPointBelowLine(bottomRight); + + if (lineIsBelowBottom) + return -1; + + return 0; + +} + +// Returns 1 for left, 0 for within, and -1 for to the right +int TextLineCollection::isLeftOfText(LineSegment line) { + + LineSegment leftSide = LineSegment(bottomCharArea.p1, topCharArea.p1); + + Point topLeft = line.closestPointOnSegmentTo(leftSide.p2); + Point bottomLeft = line.closestPointOnSegmentTo(leftSide.p1); + + bool lineIsAboveLeft = (!leftSide.isPointBelowLine(topLeft)) && (!leftSide.isPointBelowLine(bottomLeft)); + + if (lineIsAboveLeft) + return 1; + + LineSegment rightSide = LineSegment(bottomCharArea.p2, topCharArea.p2); + + Point topRight = line.closestPointOnSegmentTo(rightSide.p2); + Point bottomRight = line.closestPointOnSegmentTo(rightSide.p1); + + + bool lineIsBelowRight = rightSide.isPointBelowLine(topRight) && rightSide.isPointBelowLine(bottomRight); + + if (lineIsBelowRight) + return -1; + + return 0; +} + +void TextLineCollection::findCenterHorizontal() { + // To find the center horizontal line: + // Find the longer of the lines (if multiline) + // Get the nearest point on the bottom-most line for the + // left and right + + + + Point leftP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p1); + Point leftP2 = longerSegment.p1; + LineSegment left = LineSegment(leftP1, leftP2); + + Point leftMidpoint = left.midpoint(); + + + + Point rightP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p2); + Point rightP2 = longerSegment.p2; + LineSegment right = LineSegment(rightP1, rightP2); + + Point rightMidpoint = right.midpoint(); + + this->centerHorizontalLine = LineSegment(leftMidpoint, rightMidpoint); + +} + +void TextLineCollection::findCenterVertical() { + // To find the center vertical line: + // Choose the longest line (if multiline) + // Get the midpoint + // Draw a line up/down using the closest point on the bottom line + + + Point p1 = longerSegment.midpoint(); + + Point p2 = shorterSegment.closestPointOnSegmentTo(p1); + + // Draw bottom to top + if (p1.y < p2.y) + this->centerVerticalLine = LineSegment(p1, p2); + else + this->centerVerticalLine = LineSegment(p2, p1); +} + + + diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index d6a91ac..5fc06df 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -43,11 +43,43 @@ #define SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT 0.05 +class TextLineCollection +{ +public: + TextLineCollection(PipelineData* pipelineData); + + int isLeftOfText(LineSegment line); + int isAboveText(LineSegment line); + + LineSegment centerHorizontalLine; + LineSegment centerVerticalLine; + + float charHeight; + float charAngle; + + + +private: + PipelineData* pipelineData; + + LineSegment topCharArea; + LineSegment bottomCharArea; + + LineSegment longerSegment; + LineSegment shorterSegment; + + cv::Mat textMask; + + void findCenterHorizontal(); + void findCenterVertical(); +}; + class PlateCorners { public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData); + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) ; + virtual ~PlateCorners(); std::vector findPlateCorners(); @@ -58,9 +90,9 @@ class PlateCorners PipelineData* pipelineData; cv::Mat inputImage; - float charHeight; - float charAngle; + TextLineCollection tlc; + float bestHorizontalScore; float bestVerticalScore; LineSegment bestTop; diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 1b83628..3ca2e5c 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -40,6 +40,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); + cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; if (pipeline_data->plate_inverted) bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index f849b72..6d2c0a7 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "characteranalysis.h" #include "linefinder.h" @@ -128,7 +130,7 @@ void CharacterAnalysis::analyze() displayImage(config, "Matching Contours", img_contours); } - + LineFinder lf(pipeline_data); vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); @@ -149,6 +151,8 @@ void CharacterAnalysis::analyze() pipeline_data->textLines.push_back(textLine); } + cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; + filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines); for (uint i = 0; i < pipeline_data->textLines.size(); i++) @@ -160,7 +164,10 @@ void CharacterAnalysis::analyze() drawAndWait(&debugImage); } + cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; + this->thresholdsInverted = isPlateInverted(); + cout << "Plate inverted: " << this->thresholdsInverted << endl; } @@ -189,161 +196,6 @@ Mat CharacterAnalysis::getCharacterMask() return charMask; } -// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector CharacterAnalysis::getBestVotedLines(Mat img, TextContours textContours) -{ - //if (this->debug) - // cout << "CharacterAnalysis::getBestVotedLines" << endl; - - vector bestStripe; - - vector charRegions; - - for (uint i = 0; i < textContours.size(); i++) - { - if (textContours.goodIndices[i]) - charRegions.push_back(boundingRect(textContours.contours[i])); - } - - // Find the best fit line segment that is parallel with the most char segments - if (charRegions.size() <= 1) - { - // Maybe do something about this later, for now let's just ignore - } - else - { - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charRegions.size() - 1; i++) - { - for (uint k = i+1; k < charRegions.size(); k++) - { - //Mat tempImg; - //result.copyTo(tempImg); - - Rect* leftRect; - Rect* rightRect; - if (charRegions[i].x < charRegions[k].x) - { - leftRect = &charRegions[i]; - rightRect = &charRegions[k]; - } - else - { - leftRect = &charRegions[k]; - rightRect = &charRegions[i]; - } - - //rectangle(tempImg, *leftRect, Scalar(0, 255, 0), 2); - //rectangle(tempImg, *rightRect, Scalar(255, 255, 255), 2); - - int x1, y1, x2, y2; - - if (leftRect->y > rightRect->y) // Rising line, use the top left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - else // falling line, use the top right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - y1 = leftRect->y; - y2 = rightRect->y; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - topLines.push_back(LineSegment(x1, y1, x2, y2)); - - if (leftRect->y > rightRect->y) // Rising line, use the bottom right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - else // falling line, use the bottom left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - y1 = leftRect->y + leftRect->height; - y2 = rightRect->y + leftRect->height; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - bottomLines.push_back(LineSegment(x1, y1, x2, y2)); - - } - } - - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charRegions.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charRegions[charidx].x); - float botYPos = bottomLines[i].getPointAt(charRegions[charidx].x); - - float minTop = charRegions[charidx].y * SCORING_MIN_THRESHOLD; - float maxTop = charRegions[charidx].y * SCORING_MAX_THRESHOLD; - float minBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MIN_THRESHOLD; - float maxBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (this->config->debugCharAnalysis) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - //Mat tempImg; - //result.copyTo(tempImg); - //cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - //cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - //displayImage(config, "Lines", tempImg); - } - - //winningLines.push_back(topLines[bestScoreIndex]); - //winningLines.push_back(bottomLines[bestScoreIndex]); - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(img.cols, topLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomRight = Point(img.cols, bottomLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); - } - - return bestStripe; -} void CharacterAnalysis::filter(Mat img, TextContours& textContours) { @@ -374,8 +226,6 @@ void CharacterAnalysis::filter(Mat img, TextContours& textContours) if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... continue; - //vector lines = getBestVotedLines(img, textContours); - //this->filterBetweenLines(img, textContours, lines); int segmentCount = textContours.getGoodIndicesCount(); @@ -554,48 +404,43 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) { // Not enough area is inside the lines. + if (config->debugCharAnalysis) + cout << "Rejecting due to insufficient area" << endl; continue; } + textContours.goodIndices[i] = true; // now check to make sure that the top and bottom of the contour are near enough to the lines // First get the high and low point for the contour // Remember that origin is top-left, so the top Y values are actually closer to 0. - int highPointIndex = 0; - int highPointValue = 999999999; - int lowPointIndex = 0; - int lowPointValue = 0; - for (uint cidx = 0; cidx < textContours.contours[i].size(); cidx++) - { - if (textContours.contours[i][cidx].y < highPointValue) - { - highPointIndex = cidx; - highPointValue = textContours.contours[i][cidx].y; - } - if (textContours.contours[i][cidx].y > lowPointValue) - { - lowPointIndex = cidx; - lowPointValue = textContours.contours[i][cidx].y; - } - } + Rect brect = boundingRect(textContours.contours[i]); + int xmiddle = brect.x + (brect.width / 2); + Point topMiddle = Point(xmiddle, brect.y); + Point botMiddle = Point(xmiddle, brect.y+brect.height); // Get the absolute distance from the top and bottom lines for (uint i = 0; i < textLines.size(); i++) { - Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(textContours.contours[i][highPointIndex]); - Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(textContours.contours[i][lowPointIndex]); + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(topMiddle); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(botMiddle); - float absTopDistance = distanceBetweenPoints(closestTopPoint, textContours.contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, textContours.contours[i][lowPointIndex]); + float absTopDistance = distanceBetweenPoints(closestTopPoint, topMiddle); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, botMiddle); float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + cout << "Distances: " << absTopDistance << " : " << maxDistance << " - " << absBottomDistance << " : " << maxDistance << endl; if (absTopDistance < maxDistance && absBottomDistance < maxDistance) { textContours.goodIndices[i] = true; } + else if (config->debugCharAnalysis) + { + cout << "Rejecting due to top/bottom points that are out of range" << endl; + } } } @@ -663,6 +508,8 @@ void CharacterAnalysis::filterByOuterMask(TextContours& textContours) bool CharacterAnalysis::isPlateInverted() { Mat charMask = getCharacterMask(); + + drawAndWait(&charMask); Scalar meanVal = mean(bestThreshold, charMask)[0]; diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index cf6ea87..6e81abc 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -35,8 +35,9 @@ LineFinder::~LineFinder() { vector > LineFinder::findLines(Mat image, const TextContours contours) { - vector > linesFound; + const float MIN_AREA_TO_IGNORE = 0.65; + vector > linesFound; cvtColor(image, image, CV_GRAY2BGR); @@ -68,7 +69,7 @@ vector > LineFinder::findLines(Mat image, const TextContours conto float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); - if (percentInside < .85) + if (percentInside < MIN_AREA_TO_IGNORE) { remainingPoints.push_back(charPoints[i]); } @@ -88,118 +89,146 @@ vector > LineFinder::findLines(Mat image, const TextContours conto // Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width vector LineFinder::getBestLine(const TextContours contours, vector charPoints) { - - vector bestStripe; - + // Find the best fit line segment that is parallel with the most char segments if (charPoints.size() <= 1) { // Maybe do something about this later, for now let's just ignore + return bestStripe; } - else + + + vector charheights; + for (uint i = 0; i < charPoints.size(); i++) + charheights.push_back(charPoints[i].boundingBox.height); + float medianCharHeight = median(charheights.data(), charheights.size()); + + + + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < charPoints.size() - 1; i++) { - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charPoints.size() - 1; i++) + for (uint k = i+1; k < charPoints.size(); k++) { - for (uint k = i+1; k < charPoints.size(); k++) - { - - int leftCPIndex, rightCPIndex; - if (charPoints[i].top.x < charPoints[k].top.x) - { - leftCPIndex = i; - rightCPIndex = k; - } - else - { - leftCPIndex = k; - rightCPIndex = i; - } - - - LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); - LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); - - // Only allow lines that have a sane angle - if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && - abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) - { - topLines.push_back(top); - bottomLines.push_back(bottom); - } - + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) + { + leftCPIndex = i; + rightCPIndex = k; + } + else + { + leftCPIndex = k; + rightCPIndex = i; + } + + + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); + + + // Only allow lines that have a sane angle +// if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && +// abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) +// { +// topLines.push_back(top); +// bottomLines.push_back(bottom); +// } + + LineSegment parallelBot = top.getParallelLine(medianCharHeight * -1); + LineSegment parallelTop = bottom.getParallelLine(medianCharHeight); + + // Only allow lines that have a sane angle + if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(parallelBot.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(top); + bottomLines.push_back(parallelBot); + } + + // Only allow lines that have a sane angle + if (abs(parallelTop.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(parallelTop); + bottomLines.push_back(bottom); } } - - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charPoints.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); - float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); - - float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; - float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; - float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; - float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (true) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); - cvtColor(tempImg, tempImg, CV_GRAY2BGR); - - cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - drawAndWait(&tempImg); - } - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); } + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < charPoints.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); + + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (bestScore < 0) + return bestStripe; + + if (true) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + drawAndWait(&tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); + + return bestStripe; } From a691d8a6fe89059bb9a5ce243d7a25d76d39b297 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 19:07:00 -0400 Subject: [PATCH 31/59] Using a perspective transformation on CharAnalysis instead of redoing the whole process during segmentation. Much faster and seems slightly more accurate -- will need to benchmark. --- src/openalpr/licenseplatecandidate.cpp | 86 ++++++++++++++++++++----- src/openalpr/licenseplatecandidate.h | 6 +- src/openalpr/textdetection/textline.cpp | 29 ++++++++- src/openalpr/textdetection/textline.h | 3 + 4 files changed, 102 insertions(+), 22 deletions(-) diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index 4406718..0c04cc5 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "licenseplatecandidate.h" using namespace std; @@ -67,14 +69,49 @@ void LicensePlateCandidate::recognize() if (cornerFinder.confidence > 0) { + cout << "Transforming" << endl; + + + Mat originalCrop = pipeline_data->crop_gray; + pipeline_data->plate_corners = transformPointsToOriginalImage(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion, smallPlateCorners); - pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, pipeline_data->plate_corners); + Size outputImageSize = getOutputImageSize(pipeline_data->plate_corners); + Mat transmtx = getTransformationMatrix(pipeline_data->plate_corners, outputImageSize); + pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, outputImageSize, transmtx); + cout << "Size: " << outputImageSize.width << " - " << outputImageSize.height << endl; + + + // Apply a perspective transformation to the TextLine objects + // to match the newly deskewed license plate crop + vector newLines; + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + vector textArea = transformPointsToOriginalImage(this->pipeline_data->grayImg, originalCrop, expandedRegion, + pipeline_data->textLines[i].textArea); + vector linePolygon = transformPointsToOriginalImage(this->pipeline_data->grayImg, originalCrop, expandedRegion, + pipeline_data->textLines[i].linePolygon); + + vector textAreaRemapped; + vector linePolygonRemapped; + + perspectiveTransform(textArea, textAreaRemapped, transmtx); + perspectiveTransform(linePolygon, linePolygonRemapped, transmtx); + + newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + } + + pipeline_data->textLines.clear(); + for (uint i = 0; i < newLines.size(); i++) + pipeline_data->textLines.push_back(newLines[i]); + + + Mat debugImg = pipeline_data->textLines[0].drawDebugImage(pipeline_data->crop_gray); + drawAndWait(&debugImg); + charSegmenter = new CharacterSegmenter(pipeline_data); - //this->recognizedText = ocr->recognizedText; - //strcpy(this->recognizedText, ocr.recognizedText); pipeline_data->plate_area_confidence = 100; } @@ -100,13 +137,9 @@ vector LicensePlateCandidate::transformPointsToOriginalImage(Mat bigIma return cornerPoints; } -Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) +Size LicensePlateCandidate::getOutputImageSize(vector corners) { - - timespec startTime; - getTime(&startTime); - - // Figure out the appoximate width/height of the license plate region, so we can maintain the aspect ratio. + // Figure out the approximate width/height of the license plate region, so we can maintain the aspect ratio. LineSegment leftEdge(round(corners[3].x), round(corners[3].y), round(corners[0].x), round(corners[0].y)); LineSegment rightEdge(round(corners[2].x), round(corners[2].y), round(corners[1].x), round(corners[1].y)); LineSegment topEdge(round(corners[0].x), round(corners[0].y), round(corners[1].x), round(corners[1].y)); @@ -115,7 +148,6 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) float w = distanceBetweenPoints(leftEdge.midpoint(), rightEdge.midpoint()); float h = distanceBetweenPoints(bottomEdge.midpoint(), topEdge.midpoint()); float aspect = w/h; - int width = config->ocrImageWidthPx; int height = round(((float) width) / aspect); if (height > config->ocrImageHeightPx) @@ -123,22 +155,37 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) height = config->ocrImageHeightPx; width = round(((float) height) * aspect); } + + return Size(width, height); +} - Mat deskewed(height, width, this->pipeline_data->grayImg.type()); - +Mat LicensePlateCandidate::getTransformationMatrix(vector corners, Size outputImageSize) +{ // Corners of the destination image vector quad_pts; quad_pts.push_back(Point2f(0, 0)); - quad_pts.push_back(Point2f(deskewed.cols, 0)); - quad_pts.push_back(Point2f(deskewed.cols, deskewed.rows)); - quad_pts.push_back(Point2f(0, deskewed.rows)); + quad_pts.push_back(Point2f(outputImageSize.width, 0)); + quad_pts.push_back(Point2f(outputImageSize.width, outputImageSize.height)); + quad_pts.push_back(Point2f(0, outputImageSize.height)); // Get transformation matrix Mat transmtx = getPerspectiveTransform(corners, quad_pts); - // Apply perspective transformation - warpPerspective(inputImage, deskewed, transmtx, deskewed.size(), INTER_CUBIC); + return transmtx; +} +Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, Size outputImageSize, Mat transformationMatrix) +{ + + timespec startTime; + getTime(&startTime); + + Mat deskewed(outputImageSize, this->pipeline_data->grayImg.type()); + + // Apply perspective transformation to the image + warpPerspective(inputImage, deskewed, transformationMatrix, deskewed.size(), INTER_CUBIC); + + if (config->debugTiming) { timespec endTime; @@ -152,3 +199,8 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) return deskewed; } +//void LicensePlateCandidate::remapTextArea(cv::Mat inputImage, std::vector corners) { +// +//} + + diff --git a/src/openalpr/licenseplatecandidate.h b/src/openalpr/licenseplatecandidate.h index 2ec48ea..63534f5 100644 --- a/src/openalpr/licenseplatecandidate.h +++ b/src/openalpr/licenseplatecandidate.h @@ -59,9 +59,11 @@ class LicensePlateCandidate cv::Mat filterByCharacterHue(std::vector > charRegionContours); std::vector findPlateCorners(cv::Mat inputImage, PlateLines plateLines, CharacterRegion charRegion); // top-left, top-right, bottom-right, bottom-left + cv::Size getOutputImageSize(std::vector corners); std::vector transformPointsToOriginalImage(cv::Mat bigImage, cv::Mat smallImage, cv::Rect region, std::vector corners); - cv::Mat deSkewPlate(cv::Mat inputImage, std::vector corners); - + cv::Mat getTransformationMatrix(std::vector corners, cv::Size outputImageSize); + cv::Mat deSkewPlate(cv::Mat inputImage, cv::Size outputImageSize, cv::Mat transformationMatrix); + }; #endif // OPENALPR_LICENSEPLATECANDIDATE_H diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp index 90baafb..46405e4 100644 --- a/src/openalpr/textdetection/textline.cpp +++ b/src/openalpr/textdetection/textline.cpp @@ -24,9 +24,35 @@ using namespace cv; +TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + std::vector textAreaInts, linePolygonInts; + + for (uint i = 0; i < textArea.size(); i++) + textAreaInts.push_back(Point(round(textArea[i].x), round(textArea[i].y))); + for (uint i = 0; i < linePolygon.size(); i++) + linePolygonInts.push_back(Point(round(linePolygon[i].x), round(linePolygon[i].y))); + + initialize(textAreaInts, linePolygonInts); +} + TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + initialize(textArea, linePolygon); +} + + +TextLine::~TextLine() { +} + + + +void TextLine::initialize(std::vector textArea, std::vector linePolygon) { if (textArea.size() > 0) { + if (this->textArea.size() > 0) + this->textArea.clear(); + if (this->linePolygon.size() > 0) + this->linePolygon.clear(); + for (uint i = 0; i < textArea.size(); i++) this->textArea.push_back(textArea[i]); @@ -53,9 +79,6 @@ TextLine::TextLine(std::vector textArea, std::vector lineP } -TextLine::~TextLine() { -} - cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { cv::Mat debugImage(baseImage.size(), baseImage.type()); diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h index bc0b6f6..ec9e6d5 100644 --- a/src/openalpr/textdetection/textline.h +++ b/src/openalpr/textdetection/textline.h @@ -27,8 +27,10 @@ class TextLine { public: TextLine(std::vector textArea, std::vector linePolygon); + TextLine(std::vector textArea, std::vector linePolygon); virtual ~TextLine(); + std::vector linePolygon; std::vector textArea; LineSegment topLine; @@ -45,6 +47,7 @@ public: cv::Mat drawDebugImage(cv::Mat baseImage); private: + void initialize(std::vector textArea, std::vector linePolygon); }; #endif /* OPENALPR_TEXTLINE_H */ From be99e8219b563e1079444cea899fe64f15b98def Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 19:08:00 -0400 Subject: [PATCH 32/59] Updated segmentation to work with multiline text --- .../segmentation/charactersegmenter.cpp | 149 ++++++++---------- .../segmentation/charactersegmenter.h | 1 - 2 files changed, 68 insertions(+), 82 deletions(-) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 3ca2e5c..7e3f771 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -37,6 +37,8 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) timespec startTime; getTime(&startTime); + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); @@ -44,71 +46,56 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) if (pipeline_data->plate_inverted) bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - charAnalysis = new CharacterAnalysis(pipeline_data); - charAnalysis->analyze(); if (this->config->debugCharSegmenter) { displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); } - if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) +// if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) +// { +// Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); +// charAnalysis->bestThreshold.copyTo(img_contours); +// cvtColor(img_contours, img_contours, CV_GRAY2RGB); +// +// vector > allowedContours; +// for (uint i = 0; i < charAnalysis->bestContours.size(); i++) +// { +// if (charAnalysis->bestContours.goodIndices[i]) +// allowedContours.push_back(charAnalysis->bestContours.contours[i]); +// } +// +// drawContours(img_contours, charAnalysis->bestContours.contours, +// -1, // draw all contours +// cv::Scalar(255,0,0), // in blue +// 1); // with a thickness of 1 +// +// drawContours(img_contours, allowedContours, +// -1, // draw all contours +// cv::Scalar(0,255,0), // in green +// 1); // with a thickness of 1 +// +// +// line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); +// line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); +// +// +// Mat bordered = addLabel(img_contours, "Best Contours"); +// imgDbgGeneral.push_back(bordered); +// } + + for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) { - Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); - charAnalysis->bestThreshold.copyTo(img_contours); - cvtColor(img_contours, img_contours, CV_GRAY2RGB); + this->top = pipeline_data->textLines[lineidx].topLine; + this->bottom = pipeline_data->textLines[lineidx].bottomLine; + - vector > allowedContours; - for (uint i = 0; i < charAnalysis->bestContours.size(); i++) - { - if (charAnalysis->bestContours.goodIndices[i]) - allowedContours.push_back(charAnalysis->bestContours.contours[i]); - } + float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight; + float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM; + float avgCharWidth = avgCharHeight / height_to_width_ratio; + //float avgCharWidth = median(charWidths.data(), charWidths.size()); - drawContours(img_contours, charAnalysis->bestContours.contours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 - - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 - - - line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); - line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); - - - Mat bordered = addLabel(img_contours, "Best Contours"); - imgDbgGeneral.push_back(bordered); - } - - if (pipeline_data->textLines.size() > 0) - { - this->top = LineSegment(pipeline_data->textLines[0].linePolygon[0].x, pipeline_data->textLines[0].linePolygon[0].y, - pipeline_data->textLines[0].linePolygon[1].x, pipeline_data->textLines[0].linePolygon[1].y); - this->bottom = LineSegment(pipeline_data->textLines[0].linePolygon[3].x, pipeline_data->textLines[0].linePolygon[3].y, - pipeline_data->textLines[0].linePolygon[2].x, pipeline_data->textLines[0].linePolygon[2].y); - - vector charWidths; - vector charHeights; - - for (uint i = 0; i < charAnalysis->bestContours.size(); i++) - { - if (charAnalysis->bestContours.goodIndices[i] == false) - continue; - - Rect mr = boundingRect(charAnalysis->bestContours.contours[i]); - - charWidths.push_back(mr.width); - charHeights.push_back(mr.height); - } - - float avgCharWidth = median(charWidths.data(), charWidths.size()); - float avgCharHeight = median(charHeights.data(), charHeights.size()); - - removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight); + //removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight); // Do the histogram analysis to figure out char regions @@ -118,11 +105,11 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) vector allHistograms; vector allBoxes; - for (uint i = 0; i < charAnalysis->allTextContours.size(); i++) + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); - fillConvexPoly(histogramMask, pipeline_data->textLines[0].linePolygon.data(), pipeline_data->textLines[0].linePolygon.size(), Scalar(255,255,255)); + fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255)); VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); @@ -239,7 +226,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) CharacterSegmenter::~CharacterSegmenter() { - delete charAnalysis; + } // Given a histogram and the horizontal line boundaries, respond with an array of boxes where the characters are @@ -445,28 +432,28 @@ vector CharacterSegmenter::get1DHits(Mat img, int yOffset) return hits; } -void CharacterSegmenter::removeSmallContours(vector thresholds, vector contours, float avgCharWidth, float avgCharHeight) -{ - //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks - const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; - - for (uint i = 0; i < thresholds.size(); i++) - { - for (uint c = 0; c < contours[i].contours.size(); c++) - { - if (contours[i].contours[c].size() == 0) - continue; - - Rect mr = boundingRect(contours[i].contours[c]); - if (mr.height < MIN_CONTOUR_HEIGHT) - { - // Erase it - drawContours(thresholds[i], contours[i].contours, c, Scalar(0, 0, 0), -1); - continue; - } - } - } -} +//void CharacterSegmenter::removeSmallContours(vector thresholds, vector contours, float avgCharWidth, float avgCharHeight) +//{ +// //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks +// const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; +// +// for (uint i = 0; i < thresholds.size(); i++) +// { +// for (uint c = 0; c < contours[i].contours.size(); c++) +// { +// if (contours[i].contours[c].size() == 0) +// continue; +// +// Rect mr = boundingRect(contours[i].contours[c]); +// if (mr.height < MIN_CONTOUR_HEIGHT) +// { +// // Erase it +// drawContours(thresholds[i], contours[i].contours, c, Scalar(0, 0, 0), -1); +// continue; +// } +// } +// } +//} vector CharacterSegmenter::combineCloseBoxes( vector charBoxes, float biggestCharWidth) { diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 319dced..5540b76 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -55,7 +55,6 @@ class CharacterSegmenter Config* config; PipelineData* pipeline_data; - CharacterAnalysis* charAnalysis; LineSegment top; LineSegment bottom; From 1d8321f4f2b1d0301e1d24e27d37de731145aaa0 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 19:39:45 -0400 Subject: [PATCH 33/59] Fixed bug on plate corner detection, the parallel line was flipped left to right --- src/openalpr/platecorners.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index 76fda4d..e9e3cdb 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -139,8 +139,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) { //return; - left = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2); - right = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2 ); + left = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2); + right = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; confidenceDiff += 2; From 8c6cd5723df26cc130b943110fe730ac83890260 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 19:39:57 -0400 Subject: [PATCH 34/59] Using filter between lines before adding to TextLine array --- .../textdetection/characteranalysis.cpp | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 6d2c0a7..703b209 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -134,7 +134,7 @@ void CharacterAnalysis::analyze() LineFinder lf(pipeline_data); vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); - + vector tempTextLines; for (uint i = 0; i < linePolygons.size(); i++) { vector linePolygon = linePolygons[i]; @@ -148,15 +148,21 @@ void CharacterAnalysis::analyze() TextLine textLine(textArea, linePolygon); - pipeline_data->textLines.push_back(textLine); + tempTextLines.push_back(textLine); } cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; - filterBetweenLines(bestThreshold, bestContours, pipeline_data->textLines); + filterBetweenLines(bestThreshold, bestContours, tempTextLines); - for (uint i = 0; i < pipeline_data->textLines.size(); i++) + // Now that we've filtered a few more contours, re-do the text area. + for (uint i = 0; i < tempTextLines.size(); i++) { + vector updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); + vector linePolygon = tempTextLines[i].linePolygon; + pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); + + cout << "Test1" << endl; Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold); @@ -392,8 +398,6 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, if (textContours.goodIndices[i] == false) continue; - textContours.goodIndices[i] = false; // Set it to not included unless it proves - float percentInsideMask = getContourAreaPercentInsideMask(outerMask, textContours.contours, textContours.hierarchy, @@ -406,10 +410,11 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, // Not enough area is inside the lines. if (config->debugCharAnalysis) cout << "Rejecting due to insufficient area" << endl; + textContours.goodIndices[i] = false; + continue; } - textContours.goodIndices[i] = true; // now check to make sure that the top and bottom of the contour are near enough to the lines @@ -435,10 +440,13 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, cout << "Distances: " << absTopDistance << " : " << maxDistance << " - " << absBottomDistance << " : " << maxDistance << endl; if (absTopDistance < maxDistance && absBottomDistance < maxDistance) { - textContours.goodIndices[i] = true; + // It's ok, leave it as-is. } - else if (config->debugCharAnalysis) + else { + + textContours.goodIndices[i] = false; + if (config->debugCharAnalysis) cout << "Rejecting due to top/bottom points that are out of range" << endl; } } From 02e9517d83f8c3331666d4f70faf6bf2f1507f55 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 19:54:17 -0400 Subject: [PATCH 35/59] Increased max width for segmentation --- src/openalpr/segmentation/charactersegmenter.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 7e3f771..318e936 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -160,7 +160,6 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; } - //ColorFilter colorFilter(img, charAnalysis->getCharacterMask()); vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], allBoxes, medianCharWidth); if (this->config->debugCharSegmenter) @@ -287,7 +286,7 @@ vector CharacterSegmenter::getHistogramBoxes(VerticalHistogram histogram, vector CharacterSegmenter::getBestCharBoxes(Mat img, vector charBoxes, float avgCharWidth) { - float MAX_SEGMENT_WIDTH = avgCharWidth * 1.55; + float MAX_SEGMENT_WIDTH = avgCharWidth * 1.65; // This histogram is based on how many char boxes (from ALL of the many thresholded images) are covering each column // Makes a sort of histogram from all the previous char boxes. Figures out the best fit from that. From 2a9c0de2e3cb59f64c5243c9e37db447390a0172 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:27:49 -0400 Subject: [PATCH 36/59] Sorting TextLines from top to bottom vertically --- src/openalpr/textdetection/characteranalysis.cpp | 5 +++++ src/openalpr/textdetection/characteranalysis.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index 703b209..d78255a 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -25,6 +25,8 @@ using namespace cv; using namespace std; +bool sort_text_line(TextLine i, TextLine j) { return (i.topLine.p1.y < j.topLine.p1.y); } + CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) { this->pipeline_data = pipeline_data; @@ -155,6 +157,9 @@ void CharacterAnalysis::analyze() filterBetweenLines(bestThreshold, bestContours, tempTextLines); + // Sort the lines from top to bottom. + std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line); + // Now that we've filtered a few more contours, re-do the text area. for (uint i = 0; i < tempTextLines.size(); i++) { diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h index b853072..6223cd8 100644 --- a/src/openalpr/textdetection/characteranalysis.h +++ b/src/openalpr/textdetection/characteranalysis.h @@ -20,6 +20,7 @@ #ifndef OPENALPR_CHARACTERANALYSIS_H #define OPENALPR_CHARACTERANALYSIS_H +#include #include "opencv2/imgproc/imgproc.hpp" #include "utility.h" #include "config.h" From 74b81950d5124d01844878bdfaca98c182bdaf1e Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:28:27 -0400 Subject: [PATCH 37/59] Fixed timing measurements for deskew --- src/openalpr/licenseplatecandidate.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index 0c04cc5..44e1855 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -69,8 +69,10 @@ void LicensePlateCandidate::recognize() if (cornerFinder.confidence > 0) { - cout << "Transforming" << endl; - + + timespec startTime; + getTime(&startTime); + Mat originalCrop = pipeline_data->crop_gray; @@ -110,6 +112,13 @@ void LicensePlateCandidate::recognize() Mat debugImg = pipeline_data->textLines[0].drawDebugImage(pipeline_data->crop_gray); drawAndWait(&debugImg); + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + charSegmenter = new CharacterSegmenter(pipeline_data); @@ -177,8 +186,6 @@ Mat LicensePlateCandidate::getTransformationMatrix(vector corners, Size Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, Size outputImageSize, Mat transformationMatrix) { - timespec startTime; - getTime(&startTime); Mat deskewed(outputImageSize, this->pipeline_data->grayImg.type()); @@ -186,12 +193,6 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, Size outputImageSize, Mat warpPerspective(inputImage, deskewed, transformationMatrix, deskewed.size(), INTER_CUBIC); - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; - } if (this->config->debugGeneral) displayImage(config, "quadrilateral", deskewed); @@ -199,8 +200,5 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, Size outputImageSize, Mat return deskewed; } -//void LicensePlateCandidate::remapTextArea(cv::Mat inputImage, std::vector corners) { -// -//} From 689aef8b850aeaf78b5fb47dd17fcd244a23bf48 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:28:38 -0400 Subject: [PATCH 38/59] Fixed multiline support in character analysis --- .../segmentation/charactersegmenter.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 318e936..9fed3e7 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -84,6 +84,8 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) // imgDbgGeneral.push_back(bordered); // } + + for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) { this->top = pipeline_data->textLines[lineidx].topLine; @@ -104,7 +106,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) vector allHistograms; - vector allBoxes; + vector lineBoxes; for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); @@ -139,16 +141,16 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) } for (uint z = 0; z < charBoxes.size(); z++) - allBoxes.push_back(charBoxes[z]); + lineBoxes.push_back(charBoxes[z]); //drawAndWait(&histogramMask); } float medianCharWidth = avgCharWidth; vector widthValues; // Compute largest char width - for (uint i = 0; i < allBoxes.size(); i++) + for (uint i = 0; i < lineBoxes.size(); i++) { - widthValues.push_back(allBoxes[i].width); + widthValues.push_back(lineBoxes[i].width); } medianCharWidth = median(widthValues.data(), widthValues.size()); @@ -160,7 +162,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; } - vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], allBoxes, medianCharWidth); + vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], lineBoxes, medianCharWidth); if (this->config->debugCharSegmenter) { @@ -182,18 +184,14 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) getTime(&startTime); filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth); - - cleanCharRegions(pipeline_data->thresholds, candidateBoxes); cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes); - //cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes); - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - pipeline_data->charRegions = candidateBoxes; + + for (uint cbox = 0; cbox < candidateBoxes.size(); cbox++) + pipeline_data->charRegions.push_back(candidateBoxes[cbox]); if (config->debugTiming) { @@ -214,6 +212,9 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) displayImage(config, "Segmentation Clean Filters", cleanImgDash); } } + + cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); + drawAndWait(&pipeline_data->thresholds[0]); if (config->debugTiming) { From 9a8ff68cd1420da154c98355b3a3522989d92ed6 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:48:21 -0400 Subject: [PATCH 39/59] Re-enabled inverting plates for white-on-black text --- src/openalpr/segmentation/charactersegmenter.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 9fed3e7..6bad207 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "charactersegmenter.h" using namespace cv; @@ -37,8 +39,11 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) timespec startTime; getTime(&startTime); + if (pipeline_data->plate_inverted) + bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); pipeline_data->clearThresholds(); pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); + medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); From a10e65631cb41a59f1ccee3f58160b73bf5c4eb1 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:50:40 -0400 Subject: [PATCH 40/59] Removed unused code file --- src/openalpr/CMakeLists.txt | 1 - .../textdetection/characteranalysis2l.cpp | 60 ------------------- .../textdetection/characteranalysis2l.h | 38 ------------ 3 files changed, 99 deletions(-) delete mode 100644 src/openalpr/textdetection/characteranalysis2l.cpp delete mode 100644 src/openalpr/textdetection/characteranalysis2l.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 2cdaaa4..ba46e8d 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -23,7 +23,6 @@ set(lpr_source_files platecorners.cpp colorfilter.cpp textdetection/characteranalysis.cpp - textdetection/characteranalysis2l.cpp textdetection/platemask.cpp textdetection/textcontours.cpp textdetection/textline.cpp diff --git a/src/openalpr/textdetection/characteranalysis2l.cpp b/src/openalpr/textdetection/characteranalysis2l.cpp deleted file mode 100644 index 6bfc957..0000000 --- a/src/openalpr/textdetection/characteranalysis2l.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2014 New Designs Unlimited, LLC - * Opensource Automated License Plate Recognition [http://www.openalpr.com] - * - * This file is part of OpenAlpr. - * - * OpenAlpr is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License - * version 3 as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -#include "characteranalysis2l.h" - -using namespace cv; -using namespace std; - -CharacterAnalysis2L::CharacterAnalysis2L(PipelineData* pipeline_data) { - this->pipeline_data = pipeline_data; -} - - -CharacterAnalysis2L::~CharacterAnalysis2L() { -} - -void CharacterAnalysis2L::analyze() { - - pipeline_data->clearThresholds(); - pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, pipeline_data->config); - - std::vector > > allContours; - std::vector > allHierarchy; - - timespec startTime; - getTime(&startTime); - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - vector > contours; - vector hierarchy; - - Mat tempThreshold(pipeline_data->thresholds[i].size(), CV_8U); - pipeline_data->thresholds[i].copyTo(tempThreshold); - findContours(tempThreshold, - contours, // a vector of contours - hierarchy, - CV_RETR_TREE, // retrieve all contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours - - allContours.push_back(contours); - allHierarchy.push_back(hierarchy); - } -} diff --git a/src/openalpr/textdetection/characteranalysis2l.h b/src/openalpr/textdetection/characteranalysis2l.h deleted file mode 100644 index 9efdb26..0000000 --- a/src/openalpr/textdetection/characteranalysis2l.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2014 New Designs Unlimited, LLC - * Opensource Automated License Plate Recognition [http://www.openalpr.com] - * - * This file is part of OpenAlpr. - * - * OpenAlpr is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License - * version 3 as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -#ifndef OPENALPR_CHARACTERANALYSIS2L_H -#define OPENALPR_CHARACTERANALYSIS2L_H - -#include "utility.h" -#include "config.h" -#include "pipeline_data.h" - -class CharacterAnalysis2L { -public: - CharacterAnalysis2L(PipelineData* pipeline_data); - virtual ~CharacterAnalysis2L(); -private: - - PipelineData* pipeline_data; - void analyze(); -}; - -#endif /* OPENALPR_CHARACTERANALYSIS2L_H */ - From 7038d0b9e5df109bef65299fca37abe8c89286ca Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 20:55:28 -0400 Subject: [PATCH 41/59] Removed modal image drawing (used for debugging) --- src/openalpr/licenseplatecandidate.cpp | 2 -- src/openalpr/platecorners.cpp | 13 ++++++++----- src/openalpr/segmentation/charactersegmenter.cpp | 1 - src/openalpr/textdetection/characteranalysis.cpp | 5 ----- src/openalpr/textdetection/linefinder.cpp | 4 ++-- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index 44e1855..62b1a4c 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -109,8 +109,6 @@ void LicensePlateCandidate::recognize() pipeline_data->textLines.push_back(newLines[i]); - Mat debugImg = pipeline_data->textLines[0].drawDebugImage(pipeline_data->crop_gray); - drawAndWait(&debugImg); if (config->debugTiming) { diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index e9e3cdb..52e0e7c 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -478,11 +478,14 @@ TextLineCollection::TextLineCollection(PipelineData* pipelineData) { findCenterVertical(); // Center Vertical Line - Mat debugImage = Mat::zeros(pipelineData->crop_gray.size(), CV_8U); - line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); - line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); - - drawAndWait(&debugImage); + if (pipelineData->config->debugPlateCorners) + { + Mat debugImage = Mat::zeros(pipelineData->crop_gray.size(), CV_8U); + line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); + line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); + + displayImage(pipelineData->config, "Plate Corner Center lines", debugImage); + } } // Returns 1 for above, 0 for within, and -1 for below diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 6bad207..0010eaf 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -219,7 +219,6 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) } cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); - drawAndWait(&pipeline_data->thresholds[0]); if (config->debugTiming) { diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index d78255a..fd6a66b 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -167,12 +167,8 @@ void CharacterAnalysis::analyze() vector linePolygon = tempTextLines[i].linePolygon; pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); - - cout << "Test1" << endl; Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold); - cout << "Test2" << endl; - drawAndWait(&debugImage); } cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; @@ -522,7 +518,6 @@ bool CharacterAnalysis::isPlateInverted() { Mat charMask = getCharacterMask(); - drawAndWait(&charMask); Scalar meanVal = mean(bestThreshold, charMask)[0]; diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp index 6e81abc..3ef2b7f 100644 --- a/src/openalpr/textdetection/linefinder.cpp +++ b/src/openalpr/textdetection/linefinder.cpp @@ -204,7 +204,7 @@ vector LineFinder::getBestLine(const TextContours contours, vectorconfig->debugCharAnalysis) { cout << "The winning score is: " << bestScore << endl; // Draw the winning line segment @@ -215,7 +215,7 @@ vector LineFinder::getBestLine(const TextContours contours, vectorconfig, "Winning lines", tempImg); } Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); From cc4f72ff23dda1a1d4e99befb39d9cc214a698d3 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 21:01:16 -0400 Subject: [PATCH 42/59] Removed misc debugging cout statements --- src/openalpr/licenseplatecandidate.cpp | 3 +-- src/openalpr/segmentation/charactersegmenter.cpp | 4 +++- src/openalpr/textdetection/characteranalysis.cpp | 7 ------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index 62b1a4c..bcee6f8 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -82,8 +82,7 @@ void LicensePlateCandidate::recognize() Mat transmtx = getTransformationMatrix(pipeline_data->plate_corners, outputImageSize); pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, outputImageSize, transmtx); - cout << "Size: " << outputImageSize.width << " - " << outputImageSize.height << endl; - + // Apply a perspective transformation to the TextLine objects // to match the newly deskewed license plate crop diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 0010eaf..6e74e02 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -47,7 +47,9 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); - cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; + if (this->config->debugCharSegmenter) + cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; + if (pipeline_data->plate_inverted) bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index fd6a66b..c81c398 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -141,8 +141,6 @@ void CharacterAnalysis::analyze() { vector linePolygon = linePolygons[i]; - cout << "Polygon: " << linePolygon[0] << " - " << linePolygon[1] << " - " << linePolygon[2] << " - " << linePolygon[3] << endl; - LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); @@ -153,8 +151,6 @@ void CharacterAnalysis::analyze() tempTextLines.push_back(textLine); } - cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; - filterBetweenLines(bestThreshold, bestContours, tempTextLines); // Sort the lines from top to bottom. @@ -171,10 +167,8 @@ void CharacterAnalysis::analyze() } - cout << "Good contours inverted left: " << bestContours.getGoodIndicesCount() << endl; this->thresholdsInverted = isPlateInverted(); - cout << "Plate inverted: " << this->thresholdsInverted << endl; } @@ -438,7 +432,6 @@ void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; - cout << "Distances: " << absTopDistance << " : " << maxDistance << " - " << absBottomDistance << " : " << maxDistance << endl; if (absTopDistance < maxDistance && absBottomDistance < maxDistance) { // It's ok, leave it as-is. From c81c0a344217d8e7f2e8bcdfde3c1a7cd7030a78 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 21:09:18 -0400 Subject: [PATCH 43/59] Removed functions from characterregion header --- src/openalpr/characterregion.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/openalpr/characterregion.h b/src/openalpr/characterregion.h index 8917d17..ba55624 100644 --- a/src/openalpr/characterregion.h +++ b/src/openalpr/characterregion.h @@ -37,13 +37,6 @@ class CharacterRegion int confidence; - LineSegment getTopLine(); - LineSegment getBottomLine(); - - LineSegment getCharBoxTop(); - LineSegment getCharBoxBottom(); - LineSegment getCharBoxLeft(); - LineSegment getCharBoxRight(); protected: From dde072ba409716fa0597b638a89eff388a54373d Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 19 Oct 2014 21:09:29 -0400 Subject: [PATCH 44/59] Fixed benchmarks to compile --- src/misc_utilities/benchmarks/benchmark.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/misc_utilities/benchmarks/benchmark.cpp b/src/misc_utilities/benchmarks/benchmark.cpp index db2103c..25bbaab 100644 --- a/src/misc_utilities/benchmarks/benchmark.cpp +++ b/src/misc_utilities/benchmarks/benchmark.cpp @@ -116,14 +116,15 @@ int main( int argc, const char** argv ) CharacterRegion charRegion(&pipeline_data); - if (abs(charRegion.getTopLine().angle) > 4) + if (pipeline_data.textLines.size() > 0 && + abs(pipeline_data.textLines[0].angle) > 4) { // Rotate image: Mat rotated(frame.size(), frame.type()); Mat rot_mat( 2, 3, CV_32FC1 ); Point center = Point( frame.cols/2, frame.rows/2 ); - rot_mat = getRotationMatrix2D( center, charRegion.getTopLine().angle, 1.0 ); + rot_mat = getRotationMatrix2D( center, pipeline_data.textLines[0].angle, 1.0 ); warpAffine( frame, rotated, rot_mat, frame.size() ); rotated.copyTo(frame); From 1d59aedc0ea3bc100725f0b001a941cda8c95983 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 20 Oct 2014 20:24:28 -0400 Subject: [PATCH 45/59] Don't allow multiline plates with only 1 line --- src/openalpr/characterregion.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index c45fea6..e3e3e1c 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -83,6 +83,10 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) else if (absangle > 1) confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ; + // If a multiline plate has only one line, disqualify + if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2) + confidenceDrainers += 95; + if (confidenceDrainers >= 100) this->confidence=1; else From d48a41da492b32cbdeefec3663657376e6a382b2 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 20 Oct 2014 20:25:39 -0400 Subject: [PATCH 46/59] Do not add lines with no valid characters --- src/openalpr/textdetection/characteranalysis.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp index c81c398..d959ff5 100644 --- a/src/openalpr/textdetection/characteranalysis.cpp +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -161,9 +161,10 @@ void CharacterAnalysis::analyze() { vector updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); vector linePolygon = tempTextLines[i].linePolygon; - pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); - - Mat debugImage = pipeline_data->textLines[i].drawDebugImage(bestThreshold); + if (updatedTextArea.size() > 0 && linePolygon.size() > 0) + { + pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); + } } From 00436255eea4480736dda112929ba86cbed46a6d Mon Sep 17 00:00:00 2001 From: Matthew Hill Date: Mon, 20 Oct 2014 22:04:12 -0400 Subject: [PATCH 47/59] Updated readme removed state info and -r option --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6fd2791..54044e0 100644 --- a/README.md +++ b/README.md @@ -15,18 +15,17 @@ For example, the following output is created by analyzing this image: ![Plate Image](http://www.openalpr.com/images/demoscreenshots/plate3.png "Input image") -The library is told that this is a Missouri (MO) license plate which validates the plate letters against a regional template. ``` -user@linux:~/openalpr$ alpr ./samplecar.png -t mo -r ~/openalpr/runtime_dir/ +user@linux:~/openalpr$ alpr ./samplecar.png plate0: top 10 results -- Processing Time = 58.1879ms. - - PE3R2X confidence: 88.9371 template_match: 1 + - PE3R2X confidence: 88.9371 template_match: 0 - PE32X confidence: 78.1385 template_match: 0 - PE3R2 confidence: 77.5444 template_match: 0 - - PE3R2Y confidence: 76.1448 template_match: 1 + - PE3R2Y confidence: 76.1448 template_match: 0 - P63R2X confidence: 72.9016 template_match: 0 - - FE3R2X confidence: 72.1147 template_match: 1 + - FE3R2X confidence: 72.1147 template_match: 0 - PE32 confidence: 66.7458 template_match: 0 - PE32Y confidence: 65.3462 template_match: 0 - P632X confidence: 62.1031 template_match: 0 From 177f5720ef9f65658c4a8bf41b52f072fadc1305 Mon Sep 17 00:00:00 2001 From: Matthew Hill Date: Mon, 20 Oct 2014 22:23:25 -0400 Subject: [PATCH 48/59] Updated readme with correct help text --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 54044e0..b356474 100644 --- a/README.md +++ b/README.md @@ -38,22 +38,21 @@ Detailed command line usage: ``` user@linux:~/openalpr$ alpr --help -USAGE: +USAGE: - alpr [-t ] [-r ] [-n ] - [--seek ] [-c ] - [--clock] [-d] [-j] [--] [--version] [-h] - + alpr [-c ] [--config ] [-n ] [--seek + ] [-t ] [--clock] [-d] [-j] [--] + [--version] [-h] -Where: +Where: - -t , --template_region - Attempt to match the plate number against a region template (e.g., md - for Maryland, ca for California) + -c , --country + Country code to identify (either us for USA or eu for Europe). + Default=us - -r , --runtime_dir - Path to the OpenAlpr runtime data directory + --config + Path to the openalpr.conf file -n , --topn Max number of possible plate numbers to return. Default=10 @@ -61,12 +60,12 @@ Where: --seek Seek to the specied millisecond in a video file. Default=0 - -c , --country - Country code to identify (either us for USA or eu for Europe). - Default=us + -t , --template_region + Attempt to match the plate number against a region template (e.g., md + for Maryland, ca for California) --clock - Measure/print the total time to process image and all plates. + Measure/print the total time to process image and all plates. Default=off -d, --detect_region @@ -85,10 +84,11 @@ Where: Displays usage information and exits. - (required) Image containing license plates + Image containing license plates OpenAlpr Command Line Utility + ``` From 995e385690addb2d4ea90a56ce8209703f1a33c1 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 20 Oct 2014 23:47:54 -0400 Subject: [PATCH 49/59] Got rid of expanding the region before doing text analysis --- src/openalpr/licenseplatecandidate.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index bcee6f8..9adc533 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -44,11 +44,9 @@ void LicensePlateCandidate::recognize() pipeline_data->plate_area_confidence = 0; pipeline_data->isMultiline = config->multiline; - int expandX = round(this->pipeline_data->regionOfInterest.width * 0.20); - int expandY = round(this->pipeline_data->regionOfInterest.height * 0.15); - // expand box by 15% in all directions - Rect expandedRegion = expandRect( this->pipeline_data->regionOfInterest, expandX, expandY, this->pipeline_data->grayImg.cols, this->pipeline_data->grayImg.rows) ; + Rect expandedRegion = this->pipeline_data->regionOfInterest; + pipeline_data->crop_gray = Mat(this->pipeline_data->grayImg, expandedRegion); resize(pipeline_data->crop_gray, pipeline_data->crop_gray, Size(config->templateWidthPx, config->templateHeightPx)); From b47251e9af0b07a0b73e01db804f2709e7cc6de5 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 20 Oct 2014 23:48:32 -0400 Subject: [PATCH 50/59] Re-added despeckle filter --- .../segmentation/charactersegmenter.cpp | 62 +++++++++++-------- .../segmentation/charactersegmenter.h | 2 +- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 6e74e02..065a5ad 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -44,7 +44,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) pipeline_data->clearThresholds(); pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - + // TODO: Perhaps a bilateral filter would be better here. medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); if (this->config->debugCharSegmenter) @@ -91,20 +91,18 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) // imgDbgGeneral.push_back(bordered); // } - + for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) { this->top = pipeline_data->textLines[lineidx].topLine; this->bottom = pipeline_data->textLines[lineidx].bottomLine; - float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight; float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM; float avgCharWidth = avgCharHeight / height_to_width_ratio; - //float avgCharWidth = median(charWidths.data(), charWidths.size()); - //removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight); + removeSmallContours(pipeline_data->thresholds, avgCharHeight, pipeline_data->textLines[lineidx]); // Do the histogram analysis to figure out char regions @@ -438,28 +436,38 @@ vector CharacterSegmenter::get1DHits(Mat img, int yOffset) return hits; } -//void CharacterSegmenter::removeSmallContours(vector thresholds, vector contours, float avgCharWidth, float avgCharHeight) -//{ -// //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks -// const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; -// -// for (uint i = 0; i < thresholds.size(); i++) -// { -// for (uint c = 0; c < contours[i].contours.size(); c++) -// { -// if (contours[i].contours[c].size() == 0) -// continue; -// -// Rect mr = boundingRect(contours[i].contours[c]); -// if (mr.height < MIN_CONTOUR_HEIGHT) -// { -// // Erase it -// drawContours(thresholds[i], contours[i].contours, c, Scalar(0, 0, 0), -1); -// continue; -// } -// } -// } -//} +void CharacterSegmenter::removeSmallContours(vector thresholds, float avgCharHeight, TextLine textLine) +{ + //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks + const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; + + Mat textLineMask = Mat::zeros(thresholds[0].size(), CV_8U); + fillConvexPoly(textLineMask, textLine.linePolygon.data(), textLine.linePolygon.size(), Scalar(255,255,255)); + + for (uint i = 0; i < thresholds.size(); i++) + { + vector > contours; + vector hierarchy; + Mat thresholdsCopy = Mat::zeros(thresholds[i].size(), thresholds[i].type()); + + thresholds[i].copyTo(thresholdsCopy, textLineMask); + findContours(thresholdsCopy, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); + + for (uint c = 0; c < contours.size(); c++) + { + if (contours[c].size() == 0) + continue; + + Rect mr = boundingRect(contours[c]); + if (mr.height < MIN_CONTOUR_HEIGHT) + { + // Erase it + drawContours(thresholds[i], contours, c, Scalar(0, 0, 0), -1); + continue; + } + } + } +} vector CharacterSegmenter::combineCloseBoxes( vector charBoxes, float biggestCharWidth) { diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 5540b76..b468e62 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -64,7 +64,7 @@ class CharacterSegmenter cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); - void removeSmallContours(std::vector thresholds, std::vector contours, float avgCharWidth, float avgCharHeight); + void removeSmallContours(std::vector thresholds, float avgCharHeight, TextLine textLine); std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); std::vector getBestCharBoxes(cv::Mat img, std::vector charBoxes, float avgCharWidth); From b60806e53d66e00100ad51e6c2fee71e793131ab Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 07:09:01 -0400 Subject: [PATCH 51/59] Added libopencv-video as a dependency --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 205dd62..d3153ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,7 +127,7 @@ SET(CPACK_STRIP_FILES "1") SET (CPACK_DEBIAN_PACKAGE_PRIORITY "optional") SET (CPACK_DEBIAN_PACKAGE_SECTION "video") SET (CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) -SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libzmq1, liblog4cplus-1.0-4, libcurl3, beanstalkd") +SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libopencv-video2.4 (>= 2.4.8), libzmq1, liblog4cplus-1.0-4, libcurl3, beanstalkd") SET (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/../LICENSE") SET (CPACK_PACKAGE_DESCRIPTION "OpenALPR - Open Source Automatic License Plate Recognition") From 0f08ef09bf7ba733650f8e7c5c4eeba8656d206c Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 07:26:01 -0400 Subject: [PATCH 52/59] Added opencv_gpu to requirements --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d3153ec..8cdfdac 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,7 +127,7 @@ SET(CPACK_STRIP_FILES "1") SET (CPACK_DEBIAN_PACKAGE_PRIORITY "optional") SET (CPACK_DEBIAN_PACKAGE_SECTION "video") SET (CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) -SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libopencv-video2.4 (>= 2.4.8), libzmq1, liblog4cplus-1.0-4, libcurl3, beanstalkd") +SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libopencv-video2.4 (>= 2.4.8), libopencv-gpu2.4 (>=2.4.8), libzmq1, liblog4cplus-1.0-4, libcurl3, beanstalkd") SET (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/../LICENSE") SET (CPACK_PACKAGE_DESCRIPTION "OpenALPR - Open Source Automatic License Plate Recognition") From da708c3c71aeb7a0c924d3e32514f494218a81ce Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 17:16:10 -0400 Subject: [PATCH 53/59] Removed libzmq1 as an install requirement for deb package --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8cdfdac..d42ebcd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,7 +127,7 @@ SET(CPACK_STRIP_FILES "1") SET (CPACK_DEBIAN_PACKAGE_PRIORITY "optional") SET (CPACK_DEBIAN_PACKAGE_SECTION "video") SET (CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) -SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libopencv-video2.4 (>= 2.4.8), libopencv-gpu2.4 (>=2.4.8), libzmq1, liblog4cplus-1.0-4, libcurl3, beanstalkd") +SET (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgcc1 (>= 4.1.1), libtesseract3 (>= 3.0.3), libopencv-core2.4 (>= 2.4.8), libopencv-objdetect2.4 (>= 2.4.8), libopencv-highgui2.4 (>= 2.4.8), libopencv-imgproc2.4 (>= 2.4.8), libopencv-flann2.4 (>= 2.4.8), libopencv-features2d2.4 (>= 2.4.8), libopencv-video2.4 (>= 2.4.8), libopencv-gpu2.4 (>=2.4.8), liblog4cplus-1.0-4, libcurl3, beanstalkd") SET (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/../LICENSE") SET (CPACK_PACKAGE_DESCRIPTION "OpenALPR - Open Source Automatic License Plate Recognition") From 9a31bd4f515e4ab32d5f7754b17fd004d99cb921 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 17:30:09 -0400 Subject: [PATCH 54/59] Fixed classify chars with two-line enhancements --- src/misc_utilities/classifychars.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/misc_utilities/classifychars.cpp b/src/misc_utilities/classifychars.cpp index 30b09c0..cacc406 100644 --- a/src/misc_utilities/classifychars.cpp +++ b/src/misc_utilities/classifychars.cpp @@ -134,14 +134,14 @@ int main( int argc, const char** argv ) CharacterRegion regionizer(&pipeline_data); - if (abs(regionizer.getTopLine().angle) > 4) + if (abs(pipeline_data.textLines[0].angle) > 4) { // Rotate image: Mat rotated(frame.size(), frame.type()); Mat rot_mat( 2, 3, CV_32FC1 ); Point center = Point( frame.cols/2, frame.rows/2 ); - rot_mat = getRotationMatrix2D( center, regionizer.getTopLine().angle, 1.0 ); + rot_mat = getRotationMatrix2D( center, pipeline_data.textLines[0].angle, 1.0 ); warpAffine( frame, rotated, rot_mat, frame.size() ); rotated.copyTo(frame); From e9280cc2c4bc36fef1e93912a907b8278622d219 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 17:36:09 -0400 Subject: [PATCH 55/59] Added multiline setting to config template --- config/openalpr.conf.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/openalpr.conf.in b/config/openalpr.conf.in index 51b0016..b7761ff 100644 --- a/config/openalpr.conf.in +++ b/config/openalpr.conf.in @@ -83,6 +83,8 @@ segmentation_max_segment_width_percent_vs_average = 1.35; plate_width_mm = 304.8 plate_height_mm = 152.4 +multiline = 0 + char_height_mm = 70 char_width_mm = 35 char_whitespace_top_mm = 38 @@ -102,6 +104,7 @@ min_plate_size_height_px = 35 ocr_language = lus [eu] +; One-line European style plates ; 35-50; 45-60, 55-70, 65-80, 75-90 char_analysis_min_pct = 0.35 @@ -116,6 +119,8 @@ segmentation_max_segment_width_percent_vs_average = 2.0; plate_width_mm = 520 plate_height_mm = 110 +multiline = 0 + char_height_mm = 80 char_width_mm = 53 char_whitespace_top_mm = 10 From 32793c5991ba1cdd9c72d0c6c42608fba10f3a11 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 18:20:29 -0400 Subject: [PATCH 56/59] Resolves issue #58 - Remove template match from output when template not provided --- src/main.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 556398e..64b52c7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -43,6 +43,7 @@ std::string getJson(Alpr* alpr, std::vector results); bool detectandshow(Alpr* alpr, cv::Mat frame, std::string region, bool writeJson); bool measureProcessingTime = false; +std::string templateRegion; // This boolean is set to false when the user hits terminates (e.g., CTRL+C ) // so we can end infinite loops for things like video processing. @@ -55,7 +56,6 @@ int main( int argc, const char** argv ) bool outputJson = false; int seektoms = 0; bool detectRegion = false; - std::string templateRegion; std::string country; int topn; @@ -303,7 +303,11 @@ bool detectandshow( Alpr* alpr, cv::Mat frame, std::string region, bool writeJso for (int k = 0; k < results[i].topNPlates.size(); k++) { - std::cout << " - " << results[i].topNPlates[k].characters << "\t confidence: " << results[i].topNPlates[k].overall_confidence << "\t template_match: " << results[i].topNPlates[k].matches_template << std::endl; + std::cout << " - " << results[i].topNPlates[k].characters << "\t confidence: " << results[i].topNPlates[k].overall_confidence; + if (templateRegion.size() > 0) + std::cout << "\t template_match: " << results[i].topNPlates[k].matches_template; + + std::cout << std::endl; } } } From 8c365a69774f543197a1d1237792e9331d48dfc6 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Tue, 21 Oct 2014 22:09:07 -0400 Subject: [PATCH 57/59] Fixed bug where epoch ms time was not being returned --- src/openalpr/support/timing.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/openalpr/support/timing.cpp b/src/openalpr/support/timing.cpp index 0b6a9fc..6b16997 100644 --- a/src/openalpr/support/timing.cpp +++ b/src/openalpr/support/timing.cpp @@ -149,7 +149,9 @@ long getEpochTime() { struct timeval tp; gettimeofday(&tp, NULL); - long int ms = tp.tv_sec * 1000 + tp.tv_usec / 1000; + long ms = tp.tv_sec * 1000 + tp.tv_usec / 1000; + + return ms; } #endif From ad9c338af002b58ca7f789ead034804cb1b9cff5 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 22 Oct 2014 21:24:39 -0400 Subject: [PATCH 58/59] Added segfault handler to daemon. Resolves issue #56 --- src/daemon.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/daemon.cpp b/src/daemon.cpp index 82f672f..b18df4f 100644 --- a/src/daemon.cpp +++ b/src/daemon.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "daemon/beanstalk.hpp" #include "video/logging_videobuffer.h" @@ -34,6 +35,7 @@ const std::string BEANSTALK_QUEUE_HOST="127.0.0.1"; const int BEANSTALK_PORT=11300; const std::string BEANSTALK_TUBE_NAME="alprd"; + struct CaptureThreadData { std::string stream_url; @@ -54,12 +56,26 @@ struct UploadThreadData std::string upload_url; }; +void segfault_handler(int sig) { + void *array[10]; + size_t size; + + // get void*'s for all entries on the stack + size = backtrace(array, 10); + + // print out all the frames to stderr + fprintf(stderr, "Error: signal %d:\n", sig); + backtrace_symbols_fd(array, size, STDERR_FILENO); + exit(1); +} + bool daemon_active; static log4cplus::Logger logger; int main( int argc, const char** argv ) { + signal(SIGSEGV, segfault_handler); // install our segfault handler daemon_active = true; bool noDaemon = false; From 60cd9e812440c16e94dfaa867c546ad33168901e Mon Sep 17 00:00:00 2001 From: Matthew Hill Date: Wed, 22 Oct 2014 21:27:21 -0400 Subject: [PATCH 59/59] Removed template output from readme --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b356474..3ec299c 100644 --- a/README.md +++ b/README.md @@ -20,16 +20,16 @@ For example, the following output is created by analyzing this image: user@linux:~/openalpr$ alpr ./samplecar.png plate0: top 10 results -- Processing Time = 58.1879ms. - - PE3R2X confidence: 88.9371 template_match: 0 - - PE32X confidence: 78.1385 template_match: 0 - - PE3R2 confidence: 77.5444 template_match: 0 - - PE3R2Y confidence: 76.1448 template_match: 0 - - P63R2X confidence: 72.9016 template_match: 0 - - FE3R2X confidence: 72.1147 template_match: 0 - - PE32 confidence: 66.7458 template_match: 0 - - PE32Y confidence: 65.3462 template_match: 0 - - P632X confidence: 62.1031 template_match: 0 - - P63R2 confidence: 61.5089 template_match: 0 + - PE3R2X confidence: 88.9371 + - PE32X confidence: 78.1385 + - PE3R2 confidence: 77.5444 + - PE3R2Y confidence: 76.1448 + - P63R2X confidence: 72.9016 + - FE3R2X confidence: 72.1147 + - PE32 confidence: 66.7458 + - PE32Y confidence: 65.3462 + - P632X confidence: 62.1031 + - P63R2 confidence: 61.5089 ```