diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d42ebcd..008398b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,6 @@ project(src) - +#set(CMAKE_BUILD_TYPE Debug) cmake_minimum_required (VERSION 2.6) # Set the OpenALPR version in cmake, and also add it as a DEFINE for the code to access diff --git a/src/misc_utilities/benchmarks/benchmark.cpp b/src/misc_utilities/benchmarks/benchmark.cpp index db2103c..25bbaab 100644 --- a/src/misc_utilities/benchmarks/benchmark.cpp +++ b/src/misc_utilities/benchmarks/benchmark.cpp @@ -116,14 +116,15 @@ int main( int argc, const char** argv ) CharacterRegion charRegion(&pipeline_data); - if (abs(charRegion.getTopLine().angle) > 4) + if (pipeline_data.textLines.size() > 0 && + abs(pipeline_data.textLines[0].angle) > 4) { // Rotate image: Mat rotated(frame.size(), frame.type()); Mat rot_mat( 2, 3, CV_32FC1 ); Point center = Point( frame.cols/2, frame.rows/2 ); - rot_mat = getRotationMatrix2D( center, charRegion.getTopLine().angle, 1.0 ); + rot_mat = getRotationMatrix2D( center, pipeline_data.textLines[0].angle, 1.0 ); warpAffine( frame, rotated, rot_mat, frame.size() ); rotated.copyTo(frame); diff --git a/src/misc_utilities/tagplates.cpp b/src/misc_utilities/tagplates.cpp index 141f449..ba4e21e 100644 --- a/src/misc_utilities/tagplates.cpp +++ b/src/misc_utilities/tagplates.cpp @@ -64,7 +64,7 @@ static int xPos1 = 0; static int yPos1 = 0; static int xPos2 = 0; static int yPos2 = 0; -const float ASPECT_RATIO = 4.33333; +const float ASPECT_RATIO = 1.404; static bool rdragging = false; static int rDragStartX = 0; diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index db7b88f..2467186 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -21,7 +21,11 @@ set(lpr_source_files segmentation/verticalhistogram.cpp platecorners.cpp colorfilter.cpp - characteranalysis.cpp + textdetection/characteranalysis.cpp + textdetection/platemask.cpp + textdetection/textcontours.cpp + textdetection/textline.cpp + textdetection/linefinder.cpp pipeline_data.cpp trex.c cjson.c diff --git a/src/openalpr/characteranalysis.cpp b/src/openalpr/characteranalysis.cpp deleted file mode 100644 index af02c58..0000000 --- a/src/openalpr/characteranalysis.cpp +++ /dev/null @@ -1,952 +0,0 @@ -/* - * Copyright (c) 2014 New Designs Unlimited, LLC - * Opensource Automated License Plate Recognition [http://www.openalpr.com] - * - * This file is part of OpenAlpr. - * - * OpenAlpr is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License - * version 3 as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -#include "characteranalysis.h" - -using namespace cv; -using namespace std; - -CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) -{ - this->pipeline_data = pipeline_data; - this->config = pipeline_data->config; - - this->hasPlateMask = false; - - if (this->config->debugCharAnalysis) - cout << "Starting CharacterAnalysis identification" << endl; - -} - -CharacterAnalysis::~CharacterAnalysis() -{ - -} - -void CharacterAnalysis::analyze() -{ - pipeline_data->clearThresholds(); - pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - - - - timespec startTime; - getTime(&startTime); - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - vector > contours; - vector hierarchy; - - Mat tempThreshold(pipeline_data->thresholds[i].size(), CV_8U); - pipeline_data->thresholds[i].copyTo(tempThreshold); - findContours(tempThreshold, - contours, // a vector of contours - hierarchy, - CV_RETR_TREE, // retrieve all contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours - - allContours.push_back(contours); - allHierarchy.push_back(hierarchy); - } - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << " -- Character Analysis Find Contours Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - //Mat img_equalized = equalizeBrightness(img_gray); - - getTime(&startTime); - - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - vector goodIndices = this->filter(pipeline_data->thresholds[i], allContours[i], allHierarchy[i]); - charSegments.push_back(goodIndices); - - if (config->debugCharAnalysis) - cout << "Threshold " << i << " had " << getGoodIndicesCount(goodIndices) << " good indices." << endl; - } - - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << " -- Character Analysis Filter Time: " << diffclock(startTime, endTime) << "ms." << endl; - } - - this->plateMask = findOuterBoxMask(); - - if (hasPlateMask) - { - // Filter out bad contours now that we have an outer box mask... - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - charSegments[i] = filterByOuterMask(allContours[i], allHierarchy[i], charSegments[i]); - } - } - - int bestFitScore = -1; - int bestFitIndex = -1; - for (uint i = 0; i < pipeline_data->thresholds.size(); i++) - { - //vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); - //charSegments.push_back(goodIndices); - - int segmentCount = getGoodIndicesCount(charSegments[i]); - - if (segmentCount > bestFitScore) - { - bestFitScore = segmentCount; - bestFitIndex = i; - bestCharSegments = charSegments[i]; - bestThreshold = pipeline_data->thresholds[i]; - bestContours = allContours[i]; - bestHierarchy = allHierarchy[i]; - bestCharSegmentsCount = segmentCount; - } - } - - if (this->config->debugCharAnalysis) - cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl; - - if (bestFitScore <= 1) - return; - - //getColorMask(img, allContours, allHierarchy, charSegments); - - if (this->config->debugCharAnalysis) - { - Mat img_contours(bestThreshold.size(), CV_8U); - bestThreshold.copyTo(img_contours); - cvtColor(img_contours, img_contours, CV_GRAY2RGB); - - vector > allowedContours; - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestCharSegments[i]) - allowedContours.push_back(bestContours[i]); - } - - drawContours(img_contours, bestContours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 - - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 - - displayImage(config, "Matching Contours", img_contours); - } - - //charsegments = this->getPossibleCharRegions(img_threshold, allContours, allHierarchy, STARTING_MIN_HEIGHT + (bestFitIndex * HEIGHT_STEP), STARTING_MAX_HEIGHT + (bestFitIndex * HEIGHT_STEP)); - - this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours, bestCharSegments); - - if (this->linePolygon.size() > 0) - { - this->topLine = LineSegment(this->linePolygon[0].x, this->linePolygon[0].y, this->linePolygon[1].x, this->linePolygon[1].y); - this->bottomLine = LineSegment(this->linePolygon[3].x, this->linePolygon[3].y, this->linePolygon[2].x, this->linePolygon[2].y); - //this->charArea = getCharSegmentsBetweenLines(bestThreshold, bestContours, this->linePolygon); - filterBetweenLines(bestThreshold, bestContours, bestHierarchy, linePolygon, bestCharSegments); - - this->charArea = getCharArea(); - - if (this->charArea.size() > 0) - { - this->charBoxTop = LineSegment(this->charArea[0].x, this->charArea[0].y, this->charArea[1].x, this->charArea[1].y); - this->charBoxBottom = LineSegment(this->charArea[3].x, this->charArea[3].y, this->charArea[2].x, this->charArea[2].y); - this->charBoxLeft = LineSegment(this->charArea[3].x, this->charArea[3].y, this->charArea[0].x, this->charArea[0].y); - this->charBoxRight = LineSegment(this->charArea[2].x, this->charArea[2].y, this->charArea[1].x, this->charArea[1].y); - } - } - - this->thresholdsInverted = isPlateInverted(); -} - -int CharacterAnalysis::getGoodIndicesCount(vector goodIndices) -{ - int count = 0; - for (uint i = 0; i < goodIndices.size(); i++) - { - if (goodIndices[i]) - count++; - } - - return count; -} - -Mat CharacterAnalysis::findOuterBoxMask() -{ - double min_parent_area = config->templateHeightPx * config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. - - int winningIndex = -1; - int winningParentId = -1; - int bestCharCount = 0; - double lowestArea = 99999999999999; - - if (this->config->debugCharAnalysis) - cout << "CharacterAnalysis::findOuterBoxMask" << endl; - - for (uint imgIndex = 0; imgIndex < allContours.size(); imgIndex++) - { - //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); - - int charsRecognized = 0; - int parentId = -1; - bool hasParent = false; - for (uint i = 0; i < charSegments[imgIndex].size(); i++) - { - if (charSegments[imgIndex][i]) charsRecognized++; - if (charSegments[imgIndex][i] && allHierarchy[imgIndex][i][3] != -1) - { - parentId = allHierarchy[imgIndex][i][3]; - hasParent = true; - } - } - - if (charsRecognized == 0) - continue; - - if (hasParent) - { - double boxArea = contourArea(allContours[imgIndex][parentId]); - if (boxArea < min_parent_area) - continue; - - if ((charsRecognized > bestCharCount) || - (charsRecognized == bestCharCount && boxArea < lowestArea)) - //(boxArea < lowestArea) - { - bestCharCount = charsRecognized; - winningIndex = imgIndex; - winningParentId = parentId; - lowestArea = boxArea; - } - } - } - - if (this->config->debugCharAnalysis) - cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; - - if (winningIndex != -1 && bestCharCount >= 3) - { - int longestChildIndex = -1; - double longestChildLength = 0; - // Find the child with the longest permiter/arc length ( just for kicks) - for (uint i = 0; i < allContours[winningIndex].size(); i++) - { - for (uint j = 0; j < allContours[winningIndex].size(); j++) - { - if (allHierarchy[winningIndex][j][3] == winningParentId) - { - double arclength = arcLength(allContours[winningIndex][j], false); - if (arclength > longestChildLength) - { - longestChildIndex = j; - longestChildLength = arclength; - } - } - } - } - - Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - // get rid of the outline by drawing a 1 pixel width black line - drawContours(mask, allContours[winningIndex], - winningParentId, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - allHierarchy[winningIndex], - 0 - ); - - // Morph Open the mask to get rid of any little connectors to non-plate portions - int morph_elem = 2; - int morph_size = 3; - Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - - //morphologyEx( mask, mask, MORPH_CLOSE, element ); - morphologyEx( mask, mask, MORPH_OPEN, element ); - - //morph_size = 1; - //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - //dilate(mask, mask, element); - - // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. - // We'll want to do the contour again and find the larges one so that we remove the clipped portion. - - vector > contoursSecondRound; - - findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - int biggestContourIndex = -1; - double largestArea = 0; - for (uint c = 0; c < contoursSecondRound.size(); c++) - { - double area = contourArea(contoursSecondRound[c]); - if (area > largestArea) - { - biggestContourIndex = c; - largestArea = area; - } - } - - if (biggestContourIndex != -1) - { - mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - vector smoothedMaskPoints; - approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); - - vector > tempvec; - tempvec.push_back(smoothedMaskPoints); - //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); - drawContours(mask, tempvec, - 0, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - allHierarchy[winningIndex], - 0 - ); - } - - if (this->config->debugCharAnalysis) - { - vector debugImgs; - Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - - pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); - - debugImgs.push_back(mask); - debugImgs.push_back(pipeline_data->thresholds[winningIndex]); - debugImgs.push_back(debugImgMasked); - - Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); - displayImage(config, "Winning outer box", dashboard); - } - - hasPlateMask = true; - return mask; - } - - hasPlateMask = false; - Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); - bitwise_not(fullMask, fullMask); - return fullMask; -} - -Mat CharacterAnalysis::getCharacterMask() -{ - Mat charMask = Mat::zeros(bestThreshold.size(), CV_8U); - - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestCharSegments[i] == false) - continue; - - drawContours(charMask, bestContours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - bestHierarchy, - 1 - ); - - } - - return charMask; -} - -// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width -vector CharacterAnalysis::getBestVotedLines(Mat img, vector > contours, vector goodIndices) -{ - //if (this->debug) - // cout << "CharacterAnalysis::getBestVotedLines" << endl; - - vector bestStripe; - - vector charRegions; - - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i]) - charRegions.push_back(boundingRect(contours[i])); - } - - // Find the best fit line segment that is parallel with the most char segments - if (charRegions.size() <= 1) - { - // Maybe do something about this later, for now let's just ignore - } - else - { - vector topLines; - vector bottomLines; - // Iterate through each possible char and find all possible lines for the top and bottom of each char segment - for (uint i = 0; i < charRegions.size() - 1; i++) - { - for (uint k = i+1; k < charRegions.size(); k++) - { - //Mat tempImg; - //result.copyTo(tempImg); - - Rect* leftRect; - Rect* rightRect; - if (charRegions[i].x < charRegions[k].x) - { - leftRect = &charRegions[i]; - rightRect = &charRegions[k]; - } - else - { - leftRect = &charRegions[k]; - rightRect = &charRegions[i]; - } - - //rectangle(tempImg, *leftRect, Scalar(0, 255, 0), 2); - //rectangle(tempImg, *rightRect, Scalar(255, 255, 255), 2); - - int x1, y1, x2, y2; - - if (leftRect->y > rightRect->y) // Rising line, use the top left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - else // falling line, use the top right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - y1 = leftRect->y; - y2 = rightRect->y; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - topLines.push_back(LineSegment(x1, y1, x2, y2)); - - if (leftRect->y > rightRect->y) // Rising line, use the bottom right corner of the rect - { - x1 = leftRect->x + leftRect->width; - x2 = rightRect->x + rightRect->width; - } - else // falling line, use the bottom left corner of the rect - { - x1 = leftRect->x; - x2 = rightRect->x; - } - y1 = leftRect->y + leftRect->height; - y2 = rightRect->y + leftRect->height; - - //cv::line(tempImg, Point(x1, y1), Point(x2, y2), Scalar(0, 0, 255)); - bottomLines.push_back(LineSegment(x1, y1, x2, y2)); - - //drawAndWait(&tempImg); - } - } - - int bestScoreIndex = 0; - int bestScore = -1; - int bestScoreDistance = -1; // Line segment distance is used as a tie breaker - - // Now, among all possible lines, find the one that is the best fit - for (uint i = 0; i < topLines.size(); i++) - { - float SCORING_MIN_THRESHOLD = 0.97; - float SCORING_MAX_THRESHOLD = 1.03; - - int curScore = 0; - for (uint charidx = 0; charidx < charRegions.size(); charidx++) - { - float topYPos = topLines[i].getPointAt(charRegions[charidx].x); - float botYPos = bottomLines[i].getPointAt(charRegions[charidx].x); - - float minTop = charRegions[charidx].y * SCORING_MIN_THRESHOLD; - float maxTop = charRegions[charidx].y * SCORING_MAX_THRESHOLD; - float minBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MIN_THRESHOLD; - float maxBot = (charRegions[charidx].y + charRegions[charidx].height) * SCORING_MAX_THRESHOLD; - if ( (topYPos >= minTop && topYPos <= maxTop) && - (botYPos >= minBot && botYPos <= maxBot)) - { - curScore++; - } - - //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; - //drawAndWait(&tempImg); - } - - // Tie goes to the one with longer line segments - if ((curScore > bestScore) || - (curScore == bestScore && topLines[i].length > bestScoreDistance)) - { - bestScore = curScore; - bestScoreIndex = i; - // Just use x distance for now - bestScoreDistance = topLines[i].length; - } - } - - if (this->config->debugCharAnalysis) - { - cout << "The winning score is: " << bestScore << endl; - // Draw the winning line segment - //Mat tempImg; - //result.copyTo(tempImg); - //cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - //cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); - - //displayImage(config, "Lines", tempImg); - } - - //winningLines.push_back(topLines[bestScoreIndex]); - //winningLines.push_back(bottomLines[bestScoreIndex]); - - Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); - Point topRight = Point(img.cols, topLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomRight = Point(img.cols, bottomLines[bestScoreIndex].getPointAt(img.cols)); - Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); - - bestStripe.push_back(topLeft); - bestStripe.push_back(topRight); - bestStripe.push_back(bottomRight); - bestStripe.push_back(bottomLeft); - } - - return bestStripe; -} - -vector CharacterAnalysis::filter(Mat img, vector > contours, vector hierarchy) -{ - static int STARTING_MIN_HEIGHT = round (((float) img.rows) * config->charAnalysisMinPercent); - static int STARTING_MAX_HEIGHT = round (((float) img.rows) * (config->charAnalysisMinPercent + config->charAnalysisHeightRange)); - static int HEIGHT_STEP = round (((float) img.rows) * config->charAnalysisHeightStepSize); - static int NUM_STEPS = config->charAnalysisNumSteps; - - vector charSegments; - int bestFitScore = -1; - for (int i = 0; i < NUM_STEPS; i++) - { - int goodIndicesCount; - - vector goodIndices(contours.size()); - for (uint z = 0; z < goodIndices.size(); z++) goodIndices[z] = true; - - goodIndices = this->filterByBoxSize(contours, goodIndices, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); - - goodIndicesCount = getGoodIndicesCount(goodIndices); - if ( goodIndicesCount == 0 || goodIndicesCount <= bestFitScore) // Don't bother doing more filtering if we already lost... - continue; - goodIndices = this->filterContourHoles(contours, hierarchy, goodIndices); - - goodIndicesCount = getGoodIndicesCount(goodIndices); - if ( goodIndicesCount == 0 || goodIndicesCount <= bestFitScore) // Don't bother doing more filtering if we already lost... - continue; - //goodIndices = this->filterByParentContour( contours, hierarchy, goodIndices); - vector lines = getBestVotedLines(img, contours, goodIndices); - goodIndices = this->filterBetweenLines(img, contours, hierarchy, lines, goodIndices); - - int segmentCount = getGoodIndicesCount(goodIndices); - - if (segmentCount > bestFitScore) - { - bestFitScore = segmentCount; - charSegments = goodIndices; - } - } - - return charSegments; -} - -// Goes through the contours for the plate and picks out possible char segments based on min/max height -vector CharacterAnalysis::filterByBoxSize(vector< vector< Point> > contours, vector goodIndices, int minHeightPx, int maxHeightPx) -{ - float idealAspect=config->charWidthMM / config->charHeightMM; - float aspecttolerance=0.25; - - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices.push_back(false); - - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i] == false) - continue; - - //Create bounding rect of object - Rect mr= boundingRect(contours[i]); - - float minWidth = mr.height * 0.2; - //Crop image - //Mat auxRoi(img, mr); - if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) - { - float charAspect= (float)mr.width/(float)mr.height; - - if (abs(charAspect - idealAspect) < aspecttolerance) - includedIndices[i] = true; - } - } - - return includedIndices; -} - -vector< bool > CharacterAnalysis::filterContourHoles(vector< vector< Point > > contours, vector< Vec4i > hierarchy, vector< bool > goodIndices) -{ - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices.push_back(false); - - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i] == false) - continue; - - int parentIndex = hierarchy[i][3]; - - if (parentIndex >= 0 && goodIndices[parentIndex]) - { - // this contour is a child of an already identified contour. REMOVE it - if (this->config->debugCharAnalysis) - { - cout << "filterContourHoles: contour index: " << i << endl; - } - } - else - { - includedIndices[i] = true; - } - } - - return includedIndices; -} - -// Goes through the contours for the plate and picks out possible char segments based on min/max height -// returns a vector of indices corresponding to valid contours -vector CharacterAnalysis::filterByParentContour( vector< vector< Point> > contours, vector hierarchy, vector goodIndices) -{ - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices[j] = false; - - vector parentIDs; - vector votes; - - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i] == false) - continue; - - int voteIndex = -1; - int parentID = hierarchy[i][3]; - // check if parentID is already in the lsit - for (uint j = 0; j < parentIDs.size(); j++) - { - if (parentIDs[j] == parentID) - { - voteIndex = j; - break; - } - } - if (voteIndex == -1) - { - parentIDs.push_back(parentID); - votes.push_back(1); - } - else - { - votes[voteIndex] = votes[voteIndex] + 1; - } - } - - // Tally up the votes, pick the winner - int totalVotes = 0; - int winningParentId = 0; - int highestVotes = 0; - for (uint i = 0; i < parentIDs.size(); i++) - { - if (votes[i] > highestVotes) - { - winningParentId = parentIDs[i]; - highestVotes = votes[i]; - } - totalVotes += votes[i]; - } - - // Now filter out all the contours with a different parent ID (assuming the totalVotes > 2) - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i] == false) - continue; - - if (totalVotes <= 2) - { - includedIndices[i] = true; - } - else if (hierarchy[i][3] == winningParentId) - { - includedIndices[i] = true; - } - } - - return includedIndices; -} - -vector CharacterAnalysis::filterBetweenLines(Mat img, vector > contours, vector hierarchy, vector outerPolygon, vector goodIndices) -{ - static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; - static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; - - vector includedIndices(contours.size()); - for (uint j = 0; j < contours.size(); j++) - includedIndices[j] = false; - - if (outerPolygon.size() == 0) - return includedIndices; - - vector validPoints; - - // Figure out the line height - LineSegment topLine(outerPolygon[0].x, outerPolygon[0].y, outerPolygon[1].x, outerPolygon[1].y); - LineSegment bottomLine(outerPolygon[3].x, outerPolygon[3].y, outerPolygon[2].x, outerPolygon[2].y); - - float x = ((float) img.cols) / 2; - Point midpoint = Point(x, bottomLine.getPointAt(x)); - Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); - float lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); - - // Create a white mask for the area inside the polygon - Mat outerMask = Mat::zeros(img.size(), CV_8U); - Mat innerArea(img.size(), CV_8U); - fillConvexPoly(outerMask, outerPolygon.data(), outerPolygon.size(), Scalar(255,255,255)); - - // For each contour, determine if enough of it is between the lines to qualify - for (uint i = 0; i < contours.size(); i++) - { - if (goodIndices[i] == false) - continue; - - innerArea.setTo(Scalar(0,0,0)); - - drawContours(innerArea, contours, - i, // draw this contour - cv::Scalar(255,255,255), // in - CV_FILLED, - 8, - hierarchy, - 0 - ); - - bitwise_and(innerArea, outerMask, innerArea); - - vector > tempContours; - findContours(innerArea, tempContours, - CV_RETR_EXTERNAL, // retrieve the external contours - CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours ); - - double totalArea = contourArea(contours[i]); - double areaBetweenLines = 0; - - for (uint tempContourIdx = 0; tempContourIdx < tempContours.size(); tempContourIdx++) - { - areaBetweenLines += contourArea(tempContours[tempContourIdx]); - } - - - - if (areaBetweenLines / totalArea < MIN_AREA_PERCENT_WITHIN_LINES) - { - // Not enough area is inside the lines. - continue; - } - - - // now check to make sure that the top and bottom of the contour are near enough to the lines - - // First get the high and low point for the contour - // Remember that origin is top-left, so the top Y values are actually closer to 0. - int highPointIndex = 0; - int highPointValue = 999999999; - int lowPointIndex = 0; - int lowPointValue = 0; - for (uint cidx = 0; cidx < contours[i].size(); cidx++) - { - if (contours[i][cidx].y < highPointValue) - { - highPointIndex = cidx; - highPointValue = contours[i][cidx].y; - } - if (contours[i][cidx].y > lowPointValue) - { - lowPointIndex = cidx; - lowPointValue = contours[i][cidx].y; - } - } - - // Get the absolute distance from the top and bottom lines - Point closestTopPoint = topLine.closestPointOnSegmentTo(contours[i][highPointIndex]); - Point closestBottomPoint = bottomLine.closestPointOnSegmentTo(contours[i][lowPointIndex]); - - float absTopDistance = distanceBetweenPoints(closestTopPoint, contours[i][highPointIndex]); - float absBottomDistance = distanceBetweenPoints(closestBottomPoint, contours[i][lowPointIndex]); - - float maxDistance = lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; - - if (absTopDistance < maxDistance && absBottomDistance < maxDistance) - { - includedIndices[i] = true; - } - - } - - return includedIndices; -} - -std::vector< bool > CharacterAnalysis::filterByOuterMask(vector< vector< Point > > contours, vector< Vec4i > hierarchy, std::vector< bool > goodIndices) -{ - float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; - float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; - - if (hasPlateMask == false) - return goodIndices; - - vector passingIndices; - for (uint i = 0; i < goodIndices.size(); i++) - passingIndices.push_back(false); - - Mat tempMaskedContour = Mat::zeros(plateMask.size(), CV_8U); - Mat tempFullContour = Mat::zeros(plateMask.size(), CV_8U); - - int charsInsideMask = 0; - int totalChars = 0; - - for (uint i=0; i < goodIndices.size(); i++) - { - if (goodIndices[i] == false) - continue; - - totalChars++; - - drawContours(tempFullContour, contours, i, Scalar(255,255,255), CV_FILLED, 8, hierarchy); - bitwise_and(tempFullContour, plateMask, tempMaskedContour); - - float beforeMaskWhiteness = mean(tempFullContour)[0]; - float afterMaskWhiteness = mean(tempMaskedContour)[0]; - - if (afterMaskWhiteness / beforeMaskWhiteness > MINIMUM_PERCENT_LEFT_AFTER_MASK) - { - charsInsideMask++; - passingIndices[i] = true; - } - } - - if (totalChars == 0) - return goodIndices; - - // Check to make sure that this is a valid box. If the box is too small (e.g., 1 char is inside, and 3 are outside) - // then don't use this to filter. - float percentCharsInsideMask = ((float) charsInsideMask) / ((float) totalChars); - if (percentCharsInsideMask < MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK) - return goodIndices; - - return passingIndices; -} - -bool CharacterAnalysis::isPlateInverted() -{ - Mat charMask = getCharacterMask(); - - Scalar meanVal = mean(bestThreshold, charMask)[0]; - - if (this->config->debugCharAnalysis) - cout << "CharacterAnalysis, plate inverted: MEAN: " << meanVal << " : " << bestThreshold.type() << endl; - - if (meanVal[0] < 100) // Half would be 122.5. Give it a little extra oomf before saying it needs inversion. Most states aren't inverted. - return true; - - return false; -} - -bool CharacterAnalysis::verifySize(Mat r, float minHeightPx, float maxHeightPx) -{ - //Char sizes 45x90 - float aspect=config->charWidthMM / config->charHeightMM; - float charAspect= (float)r.cols/(float)r.rows; - float error=0.35; - //float minHeight=TEMPLATE_PLATE_HEIGHT * .35; - //float maxHeight=TEMPLATE_PLATE_HEIGHT * .65; - //We have a different aspect ratio for number 1, and it can be ~0.2 - float minAspect=0.2; - float maxAspect=aspect+aspect*error; - //area of pixels - float area=countNonZero(r); - //bb area - float bbArea=r.cols*r.rows; - //% of pixel in area - float percPixels=area/bbArea; - - //if(DEBUG) - //cout << "Aspect: "<< aspect << " ["<< minAspect << "," << maxAspect << "] " << "Area "<< percPixels <<" Char aspect " << charAspect << " Height char "<< r.rows << "\n"; - if(percPixels < 0.8 && charAspect > minAspect && charAspect < maxAspect && r.rows >= minHeightPx && r.rows < maxHeightPx) - return true; - else - return false; -} - -vector CharacterAnalysis::getCharArea() -{ - const int MAX = 100000; - const int MIN= -1; - - int leftX = MAX; - int rightX = MIN; - - for (uint i = 0; i < bestContours.size(); i++) - { - if (bestCharSegments[i] == false) - continue; - - for (uint z = 0; z < bestContours[i].size(); z++) - { - if (bestContours[i][z].x < leftX) - leftX = bestContours[i][z].x; - if (bestContours[i][z].x > rightX) - rightX = bestContours[i][z].x; - } - } - - vector charArea; - if (leftX != MAX && rightX != MIN) - { - Point tl(leftX, topLine.getPointAt(leftX)); - Point tr(rightX, topLine.getPointAt(rightX)); - Point br(rightX, bottomLine.getPointAt(rightX)); - Point bl(leftX, bottomLine.getPointAt(leftX)); - charArea.push_back(tl); - charArea.push_back(tr); - charArea.push_back(br); - charArea.push_back(bl); - } - - return charArea; -} diff --git a/src/openalpr/characteranalysis.h b/src/openalpr/characteranalysis.h deleted file mode 100644 index 35dff6f..0000000 --- a/src/openalpr/characteranalysis.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2014 New Designs Unlimited, LLC - * Opensource Automated License Plate Recognition [http://www.openalpr.com] - * - * This file is part of OpenAlpr. - * - * OpenAlpr is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License - * version 3 as published by the Free Software Foundation - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -#ifndef OPENALPR_CHARACTERANALYSIS_H -#define OPENALPR_CHARACTERANALYSIS_H - -#include "opencv2/imgproc/imgproc.hpp" -#include "constants.h" -#include "utility.h" -#include "config.h" -#include "pipeline_data.h" - -class CharacterAnalysis -{ - - public: - CharacterAnalysis(PipelineData* pipeline_data); - virtual ~CharacterAnalysis(); - - bool hasPlateMask; - cv::Mat plateMask; - - cv::Mat bestThreshold; - std::vector > bestContours; - std::vector bestHierarchy; - std::vector bestCharSegments; - int bestCharSegmentsCount; - - LineSegment topLine; - LineSegment bottomLine; - std::vector linePolygon; - std::vector charArea; - - LineSegment charBoxTop; - LineSegment charBoxBottom; - LineSegment charBoxLeft; - LineSegment charBoxRight; - - bool thresholdsInverted; - - std::vector > > allContours; - std::vector > allHierarchy; - std::vector > charSegments; - - void analyze(); - - cv::Mat getCharacterMask(); - - private: - PipelineData* pipeline_data; - Config* config; - - cv::Mat findOuterBoxMask( ); - - bool isPlateInverted(); - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); - - std::vector filterByBoxSize(std::vector > contours, std::vector goodIndices, int minHeightPx, int maxHeightPx); - std::vector filterByParentContour( std::vector< std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterByOuterMask(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector getCharArea(); - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - //vector getCharSegmentsBetweenLines(Mat img, vector > contours, vector outerPolygon); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); - - bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - - int getGoodIndicesCount(std::vector goodIndices); - -}; - -#endif // OPENALPR_CHARACTERANALYSIS_H diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index 0df17db..e3e3e1c 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -38,9 +38,8 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) charAnalysis = new CharacterAnalysis(pipeline_data); charAnalysis->analyze(); pipeline_data->plate_inverted = charAnalysis->thresholdsInverted; - pipeline_data->plate_mask = charAnalysis->plateMask; - if (this->debug && charAnalysis->linePolygon.size() > 0) + if (this->debug && pipeline_data->textLines.size() > 0) { vector tempDash; for (uint z = 0; z < pipeline_data->thresholds.size(); z++) @@ -59,30 +58,35 @@ CharacterRegion::CharacterRegion(PipelineData* pipeline_data) for (uint z = 0; z < charAnalysis->bestContours.size(); z++) { Scalar dcolor(255,0,0); - if (charAnalysis->bestCharSegments[z]) + if (charAnalysis->bestContours.goodIndices[z]) dcolor = Scalar(0,255,0); - drawContours(bestVal, charAnalysis->bestContours, z, dcolor, 1); + drawContours(bestVal, charAnalysis->bestContours.contours, z, dcolor, 1); } tempDash.push_back(bestVal); displayImage(config, "Character Region Step 1 Thresholds", drawImageDashboard(tempDash, bestVal.type(), 3)); } - if (charAnalysis->linePolygon.size() > 0) + if (pipeline_data->textLines.size() > 0) { int confidenceDrainers = 0; - int charSegmentCount = charAnalysis->bestCharSegmentsCount; + int charSegmentCount = charAnalysis->bestContours.getGoodIndicesCount(); if (charSegmentCount == 1) confidenceDrainers += 91; else if (charSegmentCount < 5) confidenceDrainers += (5 - charSegmentCount) * 10; - - int absangle = abs(charAnalysis->topLine.angle); + + // Use the angle for the first line -- assume they'll always be parallel for multi-line plates + int absangle = abs(pipeline_data->textLines[0].topLine.angle); if (absangle > config->maxPlateAngleDegrees) confidenceDrainers += 91; else if (absangle > 1) confidenceDrainers += (config->maxPlateAngleDegrees - absangle) ; + // If a multiline plate has only one line, disqualify + if (pipeline_data->isMultiline && pipeline_data->textLines.size() < 2) + confidenceDrainers += 95; + if (confidenceDrainers >= 100) this->confidence=1; else @@ -103,38 +107,3 @@ CharacterRegion::~CharacterRegion() } -LineSegment CharacterRegion::getTopLine() -{ - return charAnalysis->topLine; -} - -LineSegment CharacterRegion::getBottomLine() -{ - return charAnalysis->bottomLine; -} - -vector CharacterRegion::getCharArea() -{ - return charAnalysis->charArea; -} - -LineSegment CharacterRegion::getCharBoxTop() -{ - return charAnalysis->charBoxTop; -} - -LineSegment CharacterRegion::getCharBoxBottom() -{ - return charAnalysis->charBoxBottom; -} - -LineSegment CharacterRegion::getCharBoxLeft() -{ - return charAnalysis->charBoxLeft; -} - -LineSegment CharacterRegion::getCharBoxRight() -{ - return charAnalysis->charBoxRight; -} - diff --git a/src/openalpr/characterregion.h b/src/openalpr/characterregion.h index 1890fe9..ba55624 100644 --- a/src/openalpr/characterregion.h +++ b/src/openalpr/characterregion.h @@ -23,7 +23,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "constants.h" #include "utility.h" -#include "characteranalysis.h" +#include "textdetection/characteranalysis.h" #include "config.h" #include "pipeline_data.h" @@ -37,14 +37,6 @@ class CharacterRegion int confidence; - LineSegment getTopLine(); - LineSegment getBottomLine(); - std::vector getCharArea(); - - LineSegment getCharBoxTop(); - LineSegment getCharBoxBottom(); - LineSegment getCharBoxLeft(); - LineSegment getCharBoxRight(); protected: @@ -54,20 +46,6 @@ class CharacterRegion CharacterAnalysis *charAnalysis; cv::Mat findOuterBoxMask(std::vector thresholds, std::vector > > allContours, std::vector > allHierarchy); - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); - std::vector filterByBoxSize(cv::Mat img, std::vector > contours, std::vector goodIndices, float minHeightPx, float maxHeightPx); - std::vector filterByParentContour( std::vector< std::vector > contours, std::vector hierarchy, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - //vector getCharSegmentsBetweenLines(Mat img, vector > contours, vector outerPolygon); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); - cv::Mat getCharacterMask(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector wrapContours(std::vector > contours); - bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); - - int getGoodIndicesCount(std::vector goodIndices); bool isPlateInverted(cv::Mat threshold, std::vector > contours, std::vector hierarchy, std::vector goodIndices); diff --git a/src/openalpr/config.cpp b/src/openalpr/config.cpp index 8935d92..2f30a29 100644 --- a/src/openalpr/config.cpp +++ b/src/openalpr/config.cpp @@ -139,6 +139,8 @@ void Config::loadValues(string country) minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100); minPlateSizeHeightPx = getInt(country, "min_plate_size_height_px", 100); + multiline = getBoolean(country, "multiline", false); + plateWidthMM = getFloat(country, "plate_width_mm", 100); plateHeightMM = getFloat(country, "plate_height_mm", 100); diff --git a/src/openalpr/config.h b/src/openalpr/config.h index 21fb8dc..7477b79 100644 --- a/src/openalpr/config.h +++ b/src/openalpr/config.h @@ -57,6 +57,8 @@ class Config float minPlateSizeWidthPx; float minPlateSizeHeightPx; + bool multiline; + float plateWidthMM; float plateHeightMM; diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index b7a6d60..9adc533 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "licenseplatecandidate.h" using namespace std; @@ -40,12 +42,11 @@ void LicensePlateCandidate::recognize() charSegmenter = NULL; pipeline_data->plate_area_confidence = 0; + pipeline_data->isMultiline = config->multiline; - int expandX = round(this->pipeline_data->regionOfInterest.width * 0.20); - int expandY = round(this->pipeline_data->regionOfInterest.height * 0.15); - // expand box by 15% in all directions - Rect expandedRegion = expandRect( this->pipeline_data->regionOfInterest, expandX, expandY, this->pipeline_data->grayImg.cols, this->pipeline_data->grayImg.rows) ; + Rect expandedRegion = this->pipeline_data->regionOfInterest; + pipeline_data->crop_gray = Mat(this->pipeline_data->grayImg, expandedRegion); resize(pipeline_data->crop_gray, pipeline_data->crop_gray, Size(config->templateWidthPx, config->templateHeightPx)); @@ -54,24 +55,67 @@ void LicensePlateCandidate::recognize() if (charRegion.confidence > 10) { - PlateLines plateLines(config); + PlateLines plateLines(pipeline_data); - plateLines.processImage(pipeline_data->plate_mask, &charRegion, 1.10); - plateLines.processImage(pipeline_data->crop_gray, &charRegion, 0.9); + if (pipeline_data->hasPlateBorder) + plateLines.processImage(pipeline_data->plateBorderMask, 1.10); + + plateLines.processImage(pipeline_data->crop_gray, 0.9); - PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, &charRegion, config); + PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, pipeline_data); vector smallPlateCorners = cornerFinder.findPlateCorners(); if (cornerFinder.confidence > 0) { + + timespec startTime; + getTime(&startTime); + + + Mat originalCrop = pipeline_data->crop_gray; + pipeline_data->plate_corners = transformPointsToOriginalImage(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion, smallPlateCorners); - pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, pipeline_data->plate_corners); + Size outputImageSize = getOutputImageSize(pipeline_data->plate_corners); + Mat transmtx = getTransformationMatrix(pipeline_data->plate_corners, outputImageSize); + pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, outputImageSize, transmtx); + + + // Apply a perspective transformation to the TextLine objects + // to match the newly deskewed license plate crop + vector newLines; + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + vector textArea = transformPointsToOriginalImage(this->pipeline_data->grayImg, originalCrop, expandedRegion, + pipeline_data->textLines[i].textArea); + vector linePolygon = transformPointsToOriginalImage(this->pipeline_data->grayImg, originalCrop, expandedRegion, + pipeline_data->textLines[i].linePolygon); + + vector textAreaRemapped; + vector linePolygonRemapped; + + perspectiveTransform(textArea, textAreaRemapped, transmtx); + perspectiveTransform(linePolygon, linePolygonRemapped, transmtx); + + newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + } + + pipeline_data->textLines.clear(); + for (uint i = 0; i < newLines.size(); i++) + pipeline_data->textLines.push_back(newLines[i]); + + + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + charSegmenter = new CharacterSegmenter(pipeline_data); - //this->recognizedText = ocr->recognizedText; - //strcpy(this->recognizedText, ocr.recognizedText); pipeline_data->plate_area_confidence = 100; } @@ -97,13 +141,9 @@ vector LicensePlateCandidate::transformPointsToOriginalImage(Mat bigIma return cornerPoints; } -Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) +Size LicensePlateCandidate::getOutputImageSize(vector corners) { - - timespec startTime; - getTime(&startTime); - - // Figure out the appoximate width/height of the license plate region, so we can maintain the aspect ratio. + // Figure out the approximate width/height of the license plate region, so we can maintain the aspect ratio. LineSegment leftEdge(round(corners[3].x), round(corners[3].y), round(corners[0].x), round(corners[0].y)); LineSegment rightEdge(round(corners[2].x), round(corners[2].y), round(corners[1].x), round(corners[1].y)); LineSegment topEdge(round(corners[0].x), round(corners[0].y), round(corners[1].x), round(corners[1].y)); @@ -112,7 +152,6 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) float w = distanceBetweenPoints(leftEdge.midpoint(), rightEdge.midpoint()); float h = distanceBetweenPoints(bottomEdge.midpoint(), topEdge.midpoint()); float aspect = w/h; - int width = config->ocrImageWidthPx; int height = round(((float) width) / aspect); if (height > config->ocrImageHeightPx) @@ -120,28 +159,35 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) height = config->ocrImageHeightPx; width = round(((float) height) * aspect); } + + return Size(width, height); +} - Mat deskewed(height, width, this->pipeline_data->grayImg.type()); - +Mat LicensePlateCandidate::getTransformationMatrix(vector corners, Size outputImageSize) +{ // Corners of the destination image vector quad_pts; quad_pts.push_back(Point2f(0, 0)); - quad_pts.push_back(Point2f(deskewed.cols, 0)); - quad_pts.push_back(Point2f(deskewed.cols, deskewed.rows)); - quad_pts.push_back(Point2f(0, deskewed.rows)); + quad_pts.push_back(Point2f(outputImageSize.width, 0)); + quad_pts.push_back(Point2f(outputImageSize.width, outputImageSize.height)); + quad_pts.push_back(Point2f(0, outputImageSize.height)); // Get transformation matrix Mat transmtx = getPerspectiveTransform(corners, quad_pts); - // Apply perspective transformation - warpPerspective(inputImage, deskewed, transmtx, deskewed.size(), INTER_CUBIC); + return transmtx; +} - if (config->debugTiming) - { - timespec endTime; - getTime(&endTime); - cout << "deskew Time: " << diffclock(startTime, endTime) << "ms." << endl; - } +Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, Size outputImageSize, Mat transformationMatrix) +{ + + + Mat deskewed(outputImageSize, this->pipeline_data->grayImg.type()); + + // Apply perspective transformation to the image + warpPerspective(inputImage, deskewed, transformationMatrix, deskewed.size(), INTER_CUBIC); + + if (this->config->debugGeneral) displayImage(config, "quadrilateral", deskewed); @@ -149,3 +195,5 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) return deskewed; } + + diff --git a/src/openalpr/licenseplatecandidate.h b/src/openalpr/licenseplatecandidate.h index 2ec48ea..63534f5 100644 --- a/src/openalpr/licenseplatecandidate.h +++ b/src/openalpr/licenseplatecandidate.h @@ -59,9 +59,11 @@ class LicensePlateCandidate cv::Mat filterByCharacterHue(std::vector > charRegionContours); std::vector findPlateCorners(cv::Mat inputImage, PlateLines plateLines, CharacterRegion charRegion); // top-left, top-right, bottom-right, bottom-left + cv::Size getOutputImageSize(std::vector corners); std::vector transformPointsToOriginalImage(cv::Mat bigImage, cv::Mat smallImage, cv::Rect region, std::vector corners); - cv::Mat deSkewPlate(cv::Mat inputImage, std::vector corners); - + cv::Mat getTransformationMatrix(std::vector corners, cv::Size outputImageSize); + cv::Mat deSkewPlate(cv::Mat inputImage, cv::Size outputImageSize, cv::Mat transformationMatrix); + }; #endif // OPENALPR_LICENSEPLATECANDIDATE_H diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 6e21438..6c71e8a 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -5,6 +5,7 @@ #include "opencv2/imgproc/imgproc.hpp" #include "utility.h" #include "config.h" +#include "textdetection/textline.h" class PipelineData { @@ -22,8 +23,14 @@ class PipelineData cv::Mat grayImg; cv::Rect regionOfInterest; + bool isMultiline; + cv::Mat crop_gray; - cv::Mat plate_mask; + + bool hasPlateBorder; + cv::Mat plateBorderMask; + std::vector textLines; + std::vector thresholds; std::vector plate_corners; diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index 94b78f5..52e0e7c 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -22,26 +22,21 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, CharacterRegion* charRegion, Config* config) +PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) : + tlc(pipelineData) { - this->config = config; + this->pipelineData = pipelineData; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) cout << "PlateCorners constructor" << endl; this->inputImage = inputImage; this->plateLines = plateLines; - this->charRegion = charRegion; this->bestHorizontalScore = 9999999999999; this->bestVerticalScore = 9999999999999; - Point topPoint = charRegion->getTopLine().midpoint(); - Point bottomPoint = charRegion->getBottomLine().closestPointOnSegmentTo(topPoint); - this->charHeight = distanceBetweenPoints(topPoint, bottomPoint); - - this->charAngle = angleBetweenPoints(charRegion->getCharArea()[0], charRegion->getCharArea()[1]); } PlateCorners::~PlateCorners() @@ -50,7 +45,7 @@ PlateCorners::~PlateCorners() vector PlateCorners::findPlateCorners() { - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) cout << "PlateCorners::findPlateCorners" << endl; timespec startTime; @@ -81,21 +76,25 @@ vector PlateCorners::findPlateCorners() } } - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { cout << "Drawing debug stuff..." << endl; Mat imgCorners = Mat(inputImage.size(), inputImage.type()); inputImage.copyTo(imgCorners); - for (int i = 0; i < 4; i++) - circle(imgCorners, charRegion->getCharArea()[i], 2, Scalar(0, 0, 0)); + + for (uint linenum = 0; linenum < pipelineData->textLines.size(); linenum++) + { + for (int i = 0; i < 4; i++) + circle(imgCorners, pipelineData->textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + } line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); line(imgCorners, this->bestRight.p1, this->bestRight.p2, Scalar(0, 0, 255), 1, CV_AA); line(imgCorners, this->bestBottom.p1, this->bestBottom.p2, Scalar(0, 0, 255), 1, CV_AA); line(imgCorners, this->bestLeft.p1, this->bestLeft.p2, Scalar(255, 0, 0), 1, CV_AA); - displayImage(config, "Winning top/bottom Boundaries", imgCorners); + displayImage(pipelineData->config, "Winning top/bottom Boundaries", imgCorners); } // Check if a left/right edge has been established. @@ -112,7 +111,7 @@ vector PlateCorners::findPlateCorners() corners.push_back(bestBottom.intersection(bestRight)); corners.push_back(bestBottom.intersection(bestLeft)); - if (config->debugTiming) + if (pipelineData->config->debugTiming) { timespec endTime; getTime(&endTime); @@ -129,8 +128,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) LineSegment left; LineSegment right; - float charHeightToPlateWidthRatio = config->plateWidthMM / config->charHeightMM; - float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + + float charHeightToPlateWidthRatio = pipelineData->config->plateWidthMM / pipelineData->config->charHeightMM; + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -138,12 +138,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) if (v1 == NO_LINE && v2 == NO_LINE) { //return; - Point centerTop = charRegion->getCharBoxTop().midpoint(); - Point centerBottom = charRegion->getCharBoxBottom().midpoint(); - LineSegment centerLine = LineSegment(centerBottom.x, centerBottom.y, centerTop.x, centerTop.y); - left = centerLine.getParallelLine(idealPixelWidth / 2); - right = centerLine.getParallelLine(-1 * idealPixelWidth / 2 ); + left = tlc.centerVerticalLine.getParallelLine(-1 * idealPixelWidth / 2); + right = tlc.centerVerticalLine.getParallelLine(idealPixelWidth / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; confidenceDiff += 2; @@ -173,12 +170,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is to the left of our license plate letters - if (left.isPointBelowLine(charRegion->getCharBoxLeft().midpoint()) == false) - return; - - // Make sure this line is to the right of our license plate letters - if (right.isPointBelowLine(charRegion->getCharBoxRight().midpoint())) + // Make sure that the left and right lines are to the left and right of our text + // area + if (tlc.isLeftOfText(left) < 1 || tlc.isLeftOfText(right) > -1) return; ///////////////////////////////////////////////////////////////////////// @@ -203,7 +197,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) // Score angle difference from detected character box ///////////////////////////////////////////////////////////////////////// - float perpendicularCharAngle = charAngle - 90; + float perpendicularCharAngle = tlc.charAngle - 90; float charanglediff = abs(perpendicularCharAngle - left.angle) + abs(perpendicularCharAngle - right.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -212,8 +206,8 @@ void PlateCorners::scoreVerticals(int v1, int v2) // SCORE the shape wrt character position and height relative to position ////////////////////////////////////////////////////////////////////////// - Point leftMidLinePoint = left.closestPointOnSegmentTo(charRegion->getCharBoxLeft().midpoint()); - Point rightMidLinePoint = right.closestPointOnSegmentTo(charRegion->getCharBoxRight().midpoint()); + Point leftMidLinePoint = left.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); + Point rightMidLinePoint = right.closestPointOnSegmentTo(tlc.centerVerticalLine.midpoint()); float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); @@ -223,9 +217,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) { float scorecomponent; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealwidth " << idealPixelWidth << endl; cout << "xx xx Score: v1,v2= " << v1 << "," << v2 << endl; cout << "xx xx Score: Left= " << left.str() << endl; @@ -278,8 +272,8 @@ void PlateCorners::scoreHorizontals(int h1, int h2) LineSegment top; LineSegment bottom; - float charHeightToPlateHeightRatio = config->plateHeightMM / config->charHeightMM; - float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; + float charHeightToPlateHeightRatio = pipelineData->config->plateHeightMM / pipelineData->config->charHeightMM; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; float confidenceDiff = 0; float missingSegmentPenalty = 0; @@ -287,12 +281,10 @@ void PlateCorners::scoreHorizontals(int h1, int h2) if (h1 == NO_LINE && h2 == NO_LINE) { // return; - Point centerLeft = charRegion->getCharBoxLeft().midpoint(); - Point centerRight = charRegion->getCharBoxRight().midpoint(); - LineSegment centerLine = LineSegment(centerLeft.x, centerLeft.y, centerRight.x, centerRight.y); - top = centerLine.getParallelLine(idealPixelHeight / 2); - bottom = centerLine.getParallelLine(-1 * idealPixelHeight / 2 ); + + top = tlc.centerHorizontalLine.getParallelLine(idealPixelHeight / 2); + bottom = tlc.centerHorizontalLine.getParallelLine(-1 * idealPixelHeight / 2 ); missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; confidenceDiff += 2; @@ -322,14 +314,11 @@ void PlateCorners::scoreHorizontals(int h1, int h2) score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; score += missingSegmentPenalty; - // Make sure this line is above our license plate letters - if (top.isPointBelowLine(charRegion->getCharBoxTop().midpoint()) == false) + // Make sure that the top and bottom lines are above and below + // the text area + if (tlc.isAboveText(top) < 1 || tlc.isAboveText(bottom) > -1) return; - - // Make sure this line is below our license plate letters - if (bottom.isPointBelowLine(charRegion->getCharBoxBottom().midpoint())) - return; - + // We now have 4 possible lines. Let's put them to the test and score them... ///////////////////////////////////////////////////////////////////////// @@ -352,8 +341,8 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // Get the height difference - float heightRatio = charHeight / plateHeightPx; - float idealHeightRatio = (config->charHeightMM / config->plateHeightMM); + float heightRatio = tlc.charHeight / plateHeightPx; + float idealHeightRatio = (pipelineData->config->charHeightMM / pipelineData->config->plateHeightMM); //if (leftRatio < MIN_CHAR_HEIGHT_RATIO || leftRatio > MAX_CHAR_HEIGHT_RATIO || rightRatio < MIN_CHAR_HEIGHT_RATIO || rightRatio > MAX_CHAR_HEIGHT_RATIO) float heightRatioDiff = abs(heightRatio - idealHeightRatio); // Ideal ratio == ~.45 @@ -373,7 +362,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE the middliness of the stuff. We want our top and bottom line to have the characters right towards the middle ////////////////////////////////////////////////////////////////////////// - Point charAreaMidPoint = charRegion->getCharBoxLeft().midpoint(); + Point charAreaMidPoint = tlc.centerVerticalLine.midpoint(); Point topLineSpot = top.closestPointOnSegmentTo(charAreaMidPoint); Point botLineSpot = bottom.closestPointOnSegmentTo(charAreaMidPoint); @@ -395,7 +384,7 @@ void PlateCorners::scoreHorizontals(int h1, int h2) // SCORE: the shape for angles matching the character region ////////////////////////////////////////////////////////////// - float charanglediff = abs(charAngle - top.angle) + abs(charAngle - bottom.angle); + float charanglediff = abs(tlc.charAngle - top.angle) + abs(tlc.charAngle - bottom.angle); score += charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; @@ -406,9 +395,9 @@ void PlateCorners::scoreHorizontals(int h1, int h2) { float scorecomponent; - if (this->config->debugPlateCorners) + if (pipelineData->config->debugPlateCorners) { - cout << "xx xx Score: charHeight " << this->charHeight << endl; + cout << "xx xx Score: charHeight " << tlc.charHeight << endl; cout << "xx xx Score: idealHeight " << idealPixelHeight << endl; cout << "xx xx Score: h1,h2= " << h1 << "," << h2 << endl; cout << "xx xx Score: Top= " << top.str() << endl; @@ -448,3 +437,152 @@ void PlateCorners::scoreHorizontals(int h1, int h2) bestBottom = LineSegment(bottom.p1.x, bottom.p1.y, bottom.p2.x, bottom.p2.y); } } + +TextLineCollection::TextLineCollection(PipelineData* pipelineData) { + + this->pipelineData = pipelineData; + + charHeight = 0; + charAngle = 0; + for (uint i = 0; i < pipelineData->textLines.size(); i++) + { + charHeight += pipelineData->textLines[i].lineHeight; + charAngle += pipelineData->textLines[i].angle; + + } + charHeight = charHeight / pipelineData->textLines.size(); + charAngle = charAngle / pipelineData->textLines.size(); + + this->topCharArea = pipelineData->textLines[0].charBoxTop; + this->bottomCharArea = pipelineData->textLines[0].charBoxBottom; + for (uint i = 1; i < pipelineData->textLines.size(); i++) + { + + if (this->topCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxTop.midpoint()) == false) + this->topCharArea = pipelineData->textLines[i].charBoxTop; + + if (this->bottomCharArea.isPointBelowLine(pipelineData->textLines[i].charBoxBottom.midpoint())) + this->bottomCharArea = pipelineData->textLines[i].charBoxBottom; + + } + + longerSegment = this->bottomCharArea; + shorterSegment = this->topCharArea; + if (this->topCharArea.length > this->bottomCharArea.length) + { + longerSegment = this->topCharArea; + shorterSegment = this->bottomCharArea; + } + + findCenterHorizontal(); + findCenterVertical(); + // Center Vertical Line + + if (pipelineData->config->debugPlateCorners) + { + Mat debugImage = Mat::zeros(pipelineData->crop_gray.size(), CV_8U); + line(debugImage, this->centerHorizontalLine.p1, this->centerHorizontalLine.p2, Scalar(255,255,255), 2); + line(debugImage, this->centerVerticalLine.p1, this->centerVerticalLine.p2, Scalar(255,255,255), 2); + + displayImage(pipelineData->config, "Plate Corner Center lines", debugImage); + } +} + +// Returns 1 for above, 0 for within, and -1 for below +int TextLineCollection::isAboveText(LineSegment line) { + // Test four points (left and right corner of top and bottom line) + + Point topLeft = line.closestPointOnSegmentTo(topCharArea.p1); + Point topRight = line.closestPointOnSegmentTo(topCharArea.p2); + + bool lineIsBelowTop = topCharArea.isPointBelowLine(topLeft) || topCharArea.isPointBelowLine(topRight); + + if (!lineIsBelowTop) + return 1; + + Point bottomLeft = line.closestPointOnSegmentTo(bottomCharArea.p1); + Point bottomRight = line.closestPointOnSegmentTo(bottomCharArea.p2); + + bool lineIsBelowBottom = bottomCharArea.isPointBelowLine(bottomLeft) && + bottomCharArea.isPointBelowLine(bottomRight); + + if (lineIsBelowBottom) + return -1; + + return 0; + +} + +// Returns 1 for left, 0 for within, and -1 for to the right +int TextLineCollection::isLeftOfText(LineSegment line) { + + LineSegment leftSide = LineSegment(bottomCharArea.p1, topCharArea.p1); + + Point topLeft = line.closestPointOnSegmentTo(leftSide.p2); + Point bottomLeft = line.closestPointOnSegmentTo(leftSide.p1); + + bool lineIsAboveLeft = (!leftSide.isPointBelowLine(topLeft)) && (!leftSide.isPointBelowLine(bottomLeft)); + + if (lineIsAboveLeft) + return 1; + + LineSegment rightSide = LineSegment(bottomCharArea.p2, topCharArea.p2); + + Point topRight = line.closestPointOnSegmentTo(rightSide.p2); + Point bottomRight = line.closestPointOnSegmentTo(rightSide.p1); + + + bool lineIsBelowRight = rightSide.isPointBelowLine(topRight) && rightSide.isPointBelowLine(bottomRight); + + if (lineIsBelowRight) + return -1; + + return 0; +} + +void TextLineCollection::findCenterHorizontal() { + // To find the center horizontal line: + // Find the longer of the lines (if multiline) + // Get the nearest point on the bottom-most line for the + // left and right + + + + Point leftP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p1); + Point leftP2 = longerSegment.p1; + LineSegment left = LineSegment(leftP1, leftP2); + + Point leftMidpoint = left.midpoint(); + + + + Point rightP1 = shorterSegment.closestPointOnSegmentTo(longerSegment.p2); + Point rightP2 = longerSegment.p2; + LineSegment right = LineSegment(rightP1, rightP2); + + Point rightMidpoint = right.midpoint(); + + this->centerHorizontalLine = LineSegment(leftMidpoint, rightMidpoint); + +} + +void TextLineCollection::findCenterVertical() { + // To find the center vertical line: + // Choose the longest line (if multiline) + // Get the midpoint + // Draw a line up/down using the closest point on the bottom line + + + Point p1 = longerSegment.midpoint(); + + Point p2 = shorterSegment.closestPointOnSegmentTo(p1); + + // Draw bottom to top + if (p1.y < p2.y) + this->centerVerticalLine = LineSegment(p1, p2); + else + this->centerVerticalLine = LineSegment(p2, p1); +} + + + diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index c965887..5fc06df 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -43,11 +43,43 @@ #define SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT 0.05 +class TextLineCollection +{ +public: + TextLineCollection(PipelineData* pipelineData); + + int isLeftOfText(LineSegment line); + int isAboveText(LineSegment line); + + LineSegment centerHorizontalLine; + LineSegment centerVerticalLine; + + float charHeight; + float charAngle; + + + +private: + PipelineData* pipelineData; + + LineSegment topCharArea; + LineSegment bottomCharArea; + + LineSegment longerSegment; + LineSegment shorterSegment; + + cv::Mat textMask; + + void findCenterHorizontal(); + void findCenterVertical(); +}; + class PlateCorners { public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, CharacterRegion* charRegion, Config* config); + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) ; + virtual ~PlateCorners(); std::vector findPlateCorners(); @@ -56,11 +88,11 @@ class PlateCorners private: - Config* config; + PipelineData* pipelineData; cv::Mat inputImage; - float charHeight; - float charAngle; + TextLineCollection tlc; + float bestHorizontalScore; float bestVerticalScore; LineSegment bestTop; @@ -69,7 +101,6 @@ class PlateCorners LineSegment bestRight; PlateLines* plateLines; - CharacterRegion* charRegion; void scoreHorizontals( int h1, int h2 ); void scoreVerticals( int v1, int v2 ); diff --git a/src/openalpr/platelines.cpp b/src/openalpr/platelines.cpp index e244ca6..1f76134 100644 --- a/src/openalpr/platelines.cpp +++ b/src/openalpr/platelines.cpp @@ -25,10 +25,11 @@ using namespace std; const float MIN_CONFIDENCE = 0.3; -PlateLines::PlateLines(Config* config) +PlateLines::PlateLines(PipelineData* pipelineData) { - this->config = config; - this->debug = config->debugPlateLines; + this->pipelineData = pipelineData; + + this->debug = pipelineData->config->debugPlateLines; if (debug) cout << "PlateLines constructor" << endl; @@ -38,7 +39,7 @@ PlateLines::~PlateLines() { } -void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float sensitivity) +void PlateLines::processImage(Mat inputImage, float sensitivity) { if (this->debug) cout << "PlateLines findLines" << endl; @@ -59,7 +60,6 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float adaptiveBilateralFilter(inputImage, smoothed, Size(3,3), 45, 45); - int morph_elem = 2; int morph_size = 2; Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); @@ -69,11 +69,18 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float Canny(smoothed, edges, 66, 133); // Create a mask that is dilated based on the detected characters - vector > polygons; - polygons.push_back(charRegion->getCharArea()); + Mat mask = Mat::zeros(inputImage.size(), CV_8U); - fillPoly(mask, polygons, Scalar(255,255,255)); + + for (uint i = 0; i < pipelineData->textLines.size(); i++) + { + vector > polygons; + polygons.push_back(pipelineData->textLines[i].textArea); + fillPoly(mask, polygons, Scalar(255,255,255)); + } + + dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); bitwise_not(mask, mask); @@ -114,10 +121,10 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float images.push_back(debugImgVert); Mat dashboard = drawImageDashboard(images, debugImgVert.type(), 1); - displayImage(config, "Hough Lines", dashboard); + displayImage(pipelineData->config, "Hough Lines", dashboard); } - if (config->debugTiming) + if (pipelineData->config->debugTiming) { timespec endTime; getTime(&endTime); @@ -134,8 +141,8 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, b if (this->debug) cout << "PlateLines::getLines" << endl; - static int HORIZONTAL_SENSITIVITY = config->plateLinesSensitivityHorizontal; - static int VERTICAL_SENSITIVITY = config->plateLinesSensitivityVertical; + static int HORIZONTAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityHorizontal; + static int VERTICAL_SENSITIVITY = pipelineData->config->plateLinesSensitivityVertical; vector allLines; vector filteredLines; diff --git a/src/openalpr/platelines.h b/src/openalpr/platelines.h index 872a80e..40a9016 100644 --- a/src/openalpr/platelines.h +++ b/src/openalpr/platelines.h @@ -37,10 +37,10 @@ class PlateLines { public: - PlateLines(Config* config); + PlateLines(PipelineData* pipelineData); virtual ~PlateLines(); - void processImage(cv::Mat img, CharacterRegion* charRegion, float sensitivity=1.0); + void processImage(cv::Mat img, float sensitivity=1.0); std::vector horizontalLines; std::vector verticalLines; @@ -49,7 +49,7 @@ class PlateLines private: - Config* config; + PipelineData* pipelineData; bool debug; cv::Mat customGrayscaleConversion(cv::Mat src); diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index 76b482a..065a5ad 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "charactersegmenter.h" using namespace cv; @@ -37,76 +39,70 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) timespec startTime; getTime(&startTime); - + if (pipeline_data->plate_inverted) + bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); + + // TODO: Perhaps a bilateral filter would be better here. medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); + if (this->config->debugCharSegmenter) + cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl; + if (pipeline_data->plate_inverted) bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - charAnalysis = new CharacterAnalysis(pipeline_data); - charAnalysis->analyze(); if (this->config->debugCharSegmenter) { displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); } - if (this->config->debugCharSegmenter && charAnalysis->linePolygon.size() > 0) +// if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0) +// { +// Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); +// charAnalysis->bestThreshold.copyTo(img_contours); +// cvtColor(img_contours, img_contours, CV_GRAY2RGB); +// +// vector > allowedContours; +// for (uint i = 0; i < charAnalysis->bestContours.size(); i++) +// { +// if (charAnalysis->bestContours.goodIndices[i]) +// allowedContours.push_back(charAnalysis->bestContours.contours[i]); +// } +// +// drawContours(img_contours, charAnalysis->bestContours.contours, +// -1, // draw all contours +// cv::Scalar(255,0,0), // in blue +// 1); // with a thickness of 1 +// +// drawContours(img_contours, allowedContours, +// -1, // draw all contours +// cv::Scalar(0,255,0), // in green +// 1); // with a thickness of 1 +// +// +// line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1); +// line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1); +// +// +// Mat bordered = addLabel(img_contours, "Best Contours"); +// imgDbgGeneral.push_back(bordered); +// } + + + + for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++) { - Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U); - charAnalysis->bestThreshold.copyTo(img_contours); - cvtColor(img_contours, img_contours, CV_GRAY2RGB); + this->top = pipeline_data->textLines[lineidx].topLine; + this->bottom = pipeline_data->textLines[lineidx].bottomLine; + + float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight; + float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM; + float avgCharWidth = avgCharHeight / height_to_width_ratio; - vector > allowedContours; - for (uint i = 0; i < charAnalysis->bestContours.size(); i++) - { - if (charAnalysis->bestCharSegments[i]) - allowedContours.push_back(charAnalysis->bestContours[i]); - } - - drawContours(img_contours, charAnalysis->bestContours, - -1, // draw all contours - cv::Scalar(255,0,0), // in blue - 1); // with a thickness of 1 - - drawContours(img_contours, allowedContours, - -1, // draw all contours - cv::Scalar(0,255,0), // in green - 1); // with a thickness of 1 - - if (charAnalysis->linePolygon.size() > 0) - { - line(img_contours, charAnalysis->linePolygon[0], charAnalysis->linePolygon[1], Scalar(255, 0, 255), 1); - line(img_contours, charAnalysis->linePolygon[3], charAnalysis->linePolygon[2], Scalar(255, 0, 255), 1); - } - - Mat bordered = addLabel(img_contours, "Best Contours"); - imgDbgGeneral.push_back(bordered); - } - - if (charAnalysis->linePolygon.size() > 0) - { - this->top = LineSegment(charAnalysis->linePolygon[0].x, charAnalysis->linePolygon[0].y, charAnalysis->linePolygon[1].x, charAnalysis->linePolygon[1].y); - this->bottom = LineSegment(charAnalysis->linePolygon[3].x, charAnalysis->linePolygon[3].y, charAnalysis->linePolygon[2].x, charAnalysis->linePolygon[2].y); - - vector charWidths; - vector charHeights; - - for (uint i = 0; i < charAnalysis->bestContours.size(); i++) - { - if (charAnalysis->bestCharSegments[i] == false) - continue; - - Rect mr = boundingRect(charAnalysis->bestContours[i]); - - charWidths.push_back(mr.width); - charHeights.push_back(mr.height); - } - - float avgCharWidth = median(charWidths.data(), charWidths.size()); - float avgCharHeight = median(charHeights.data(), charHeights.size()); - - removeSmallContours(pipeline_data->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight); + removeSmallContours(pipeline_data->thresholds, avgCharHeight, pipeline_data->textLines[lineidx]); // Do the histogram analysis to figure out char regions @@ -115,12 +111,12 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) vector allHistograms; - vector allBoxes; - for (uint i = 0; i < charAnalysis->allContours.size(); i++) + vector lineBoxes; + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) { Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); - fillConvexPoly(histogramMask, charAnalysis->linePolygon.data(), charAnalysis->linePolygon.size(), Scalar(255,255,255)); + fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255)); VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); @@ -150,16 +146,16 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) } for (uint z = 0; z < charBoxes.size(); z++) - allBoxes.push_back(charBoxes[z]); + lineBoxes.push_back(charBoxes[z]); //drawAndWait(&histogramMask); } float medianCharWidth = avgCharWidth; vector widthValues; // Compute largest char width - for (uint i = 0; i < allBoxes.size(); i++) + for (uint i = 0; i < lineBoxes.size(); i++) { - widthValues.push_back(allBoxes[i].width); + widthValues.push_back(lineBoxes[i].width); } medianCharWidth = median(widthValues.data(), widthValues.size()); @@ -171,8 +167,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) cout << " -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl; } - //ColorFilter colorFilter(img, charAnalysis->getCharacterMask()); - vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], allBoxes, medianCharWidth); + vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], lineBoxes, medianCharWidth); if (this->config->debugCharSegmenter) { @@ -194,18 +189,14 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) getTime(&startTime); filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth); - - cleanCharRegions(pipeline_data->thresholds, candidateBoxes); cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes); - //cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes); - candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - pipeline_data->charRegions = candidateBoxes; + + for (uint cbox = 0; cbox < candidateBoxes.size(); cbox++) + pipeline_data->charRegions.push_back(candidateBoxes[cbox]); if (config->debugTiming) { @@ -226,6 +217,8 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) displayImage(config, "Segmentation Clean Filters", cleanImgDash); } } + + cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); if (config->debugTiming) { @@ -237,7 +230,7 @@ CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) CharacterSegmenter::~CharacterSegmenter() { - delete charAnalysis; + } // Given a histogram and the horizontal line boundaries, respond with an array of boxes where the characters are @@ -298,7 +291,7 @@ vector CharacterSegmenter::getHistogramBoxes(VerticalHistogram histogram, vector CharacterSegmenter::getBestCharBoxes(Mat img, vector charBoxes, float avgCharWidth) { - float MAX_SEGMENT_WIDTH = avgCharWidth * 1.55; + float MAX_SEGMENT_WIDTH = avgCharWidth * 1.65; // This histogram is based on how many char boxes (from ALL of the many thresholded images) are covering each column // Makes a sort of histogram from all the previous char boxes. Figures out the best fit from that. @@ -443,23 +436,33 @@ vector CharacterSegmenter::get1DHits(Mat img, int yOffset) return hits; } -void CharacterSegmenter::removeSmallContours(vector thresholds, vector > > allContours, float avgCharWidth, float avgCharHeight) +void CharacterSegmenter::removeSmallContours(vector thresholds, float avgCharHeight, TextLine textLine) { //const float MIN_CHAR_AREA = 0.02 * avgCharWidth * avgCharHeight; // To clear out the tiny specks const float MIN_CONTOUR_HEIGHT = 0.3 * avgCharHeight; + Mat textLineMask = Mat::zeros(thresholds[0].size(), CV_8U); + fillConvexPoly(textLineMask, textLine.linePolygon.data(), textLine.linePolygon.size(), Scalar(255,255,255)); + for (uint i = 0; i < thresholds.size(); i++) { - for (uint c = 0; c < allContours[i].size(); c++) + vector > contours; + vector hierarchy; + Mat thresholdsCopy = Mat::zeros(thresholds[i].size(), thresholds[i].type()); + + thresholds[i].copyTo(thresholdsCopy, textLineMask); + findContours(thresholdsCopy, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE); + + for (uint c = 0; c < contours.size(); c++) { - if (allContours[i][c].size() == 0) + if (contours[c].size() == 0) continue; - Rect mr = boundingRect(allContours[i][c]); + Rect mr = boundingRect(contours[c]); if (mr.height < MIN_CONTOUR_HEIGHT) { // Erase it - drawContours(thresholds[i], allContours[i], c, Scalar(0, 0, 0), -1); + drawContours(thresholds[i], contours, c, Scalar(0, 0, 0), -1); continue; } } @@ -956,69 +959,6 @@ void CharacterSegmenter::filterEdgeBoxes(vector thresholds, const vector > contours; - Mat mask = Mat::zeros(thresholds[i].size(),CV_8U); - rectangle(mask, charRegions[boxidx], Scalar(255,255,255), CV_FILLED); - - bitwise_and(thresholds[i], mask, mask); - findContours(mask, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); - //int tallContourIndex = isSkinnyLineInsideBox(thresholds[i], charRegions[boxidx], allContours[i], hierarchy[i], avgCharWidth, avgCharHeight); - float tallestContourHeight = 0; - float fattestContourWidth = 0; - float biggestContourArea = 0; - for (int c = 0; c < contours.size(); c++) - { - Rect r = boundingRect(contours[c]); - if (r.height > tallestContourHeight) - tallestContourHeight = r.height; - if (r.width > fattestContourWidth) - fattestContourWidth = r.width; - float a = r.area(); - if (a > biggestContourArea) - biggestContourArea = a; - } - - float minArea = charRegions[boxidx].area() * MIN_EDGE_CONTOUR_AREA_PCT; - if ((fattestContourWidth < MIN_BOX_WIDTH_PX) || - (tallestContourHeight < MIN_EDGE_CONTOUR_HEIGHT) || - (biggestContourArea < minArea) - ) - { - // Find a good place to MASK this contour. - // for now, just mask the whole thing - if (this->debug) - { - rectangle(imgDbgCleanStages[i], charRegions[boxidx], COLOR_DEBUG_EDGE, 2); - cout << "Edge Filter: threshold " << i << " box " << boxidx << endl; - } - rectangle(thresholds[i], charRegions[boxidx], Scalar(0,0,0), -1); - } - else - { - filteredCharRegions.push_back(charRegions[boxidx]); - } - } - } - */ } int CharacterSegmenter::getLongestBlobLengthBetweenLines(Mat img, int col) diff --git a/src/openalpr/segmentation/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h index 54cb4fc..b468e62 100644 --- a/src/openalpr/segmentation/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -28,6 +28,7 @@ #include "colorfilter.h" #include "verticalhistogram.h" #include "config.h" +#include "textdetection/textcontours.h" //const float MIN_BOX_WIDTH_PX = 4; // 4 pixels @@ -54,7 +55,6 @@ class CharacterSegmenter Config* config; PipelineData* pipeline_data; - CharacterAnalysis* charAnalysis; LineSegment top; LineSegment bottom; @@ -62,20 +62,10 @@ class CharacterSegmenter std::vector imgDbgGeneral; std::vector imgDbgCleanStages; - std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); - std::vector filterByBoxSize(std::vector< std::vector > contours, std::vector goodIndices, float minHeightPx, float maxHeightPx); - std::vector filterBetweenLines(cv::Mat img, std::vector > contours, std::vector hierarchy, std::vector outerPolygon, std::vector goodIndices); - std::vector filterContourHoles(std::vector > contours, std::vector hierarchy, std::vector goodIndices); - - std::vector getBestVotedLines(cv::Mat img, std::vector > contours, std::vector goodIndices); - int getGoodIndicesCount(std::vector goodIndices); - - cv::Mat getCharacterMask(cv::Mat img_threshold, std::vector > contours, std::vector hierarchy, std::vector goodIndices); cv::Mat getCharBoxMask(cv::Mat img_threshold, std::vector charBoxes); - void removeSmallContours(std::vector thresholds, std::vector > > allContours, float avgCharWidth, float avgCharHeight); + void removeSmallContours(std::vector thresholds, float avgCharHeight, TextLine textLine); - cv::Mat getVerticalHistogram(cv::Mat img, cv::Mat mask); std::vector getHistogramBoxes(VerticalHistogram histogram, float avgCharWidth, float avgCharHeight, float* score); std::vector getBestCharBoxes(cv::Mat img, std::vector charBoxes, float avgCharWidth); std::vector combineCloseBoxes( std::vector charBoxes, float avgCharWidth); @@ -92,7 +82,6 @@ class CharacterSegmenter int isSkinnyLineInsideBox(cv::Mat threshold, cv::Rect box, std::vector > contours, std::vector hierarchy, float avgCharWidth, float avgCharHeight); - std::vector getEncapsulatingLines(cv::Mat img, std::vector > contours, std::vector goodIndices); }; #endif // OPENALPR_CHARACTERSEGMENTER_H diff --git a/src/openalpr/textdetection/characteranalysis.cpp b/src/openalpr/textdetection/characteranalysis.cpp new file mode 100644 index 0000000..d959ff5 --- /dev/null +++ b/src/openalpr/textdetection/characteranalysis.cpp @@ -0,0 +1,589 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include + +#include "characteranalysis.h" +#include "linefinder.h" + +using namespace cv; +using namespace std; + +bool sort_text_line(TextLine i, TextLine j) { return (i.topLine.p1.y < j.topLine.p1.y); } + +CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) +{ + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; + + + if (this->config->debugCharAnalysis) + cout << "Starting CharacterAnalysis identification" << endl; + +} + +CharacterAnalysis::~CharacterAnalysis() +{ + +} + +void CharacterAnalysis::analyze() +{ + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); + + + + timespec startTime; + getTime(&startTime); + + pipeline_data->textLines.clear(); + + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + TextContours tc(pipeline_data->thresholds[i]); + + allTextContours.push_back(tc); + } + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << " -- Character Analysis Find Contours Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + //Mat img_equalized = equalizeBrightness(img_gray); + + getTime(&startTime); + + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + this->filter(pipeline_data->thresholds[i], allTextContours[i]); + + if (config->debugCharAnalysis) + cout << "Threshold " << i << " had " << allTextContours[i].getGoodIndicesCount() << " good indices." << endl; + } + + if (config->debugTiming) + { + timespec endTime; + getTime(&endTime); + cout << " -- Character Analysis Filter Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + PlateMask plateMask(pipeline_data); + plateMask.findOuterBoxMask(allTextContours); + + pipeline_data->hasPlateBorder = plateMask.hasPlateMask; + pipeline_data->plateBorderMask = plateMask.getMask(); + + if (plateMask.hasPlateMask) + { + // Filter out bad contours now that we have an outer box mask... + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + filterByOuterMask(allTextContours[i]); + } + } + + int bestFitScore = -1; + int bestFitIndex = -1; + for (uint i = 0; i < pipeline_data->thresholds.size(); i++) + { + + int segmentCount = allTextContours[i].getGoodIndicesCount(); + + if (segmentCount > bestFitScore) + { + bestFitScore = segmentCount; + bestFitIndex = i; + bestThreshold = pipeline_data->thresholds[i]; + bestContours = allTextContours[i]; + } + } + + if (this->config->debugCharAnalysis) + cout << "Best fit score: " << bestFitScore << " Index: " << bestFitIndex << endl; + + if (bestFitScore <= 1) + return; + + //getColorMask(img, allContours, allHierarchy, charSegments); + + if (this->config->debugCharAnalysis) + { + Mat img_contours = bestContours.drawDebugImage(bestThreshold); + + displayImage(config, "Matching Contours", img_contours); + } + + LineFinder lf(pipeline_data); + vector > linePolygons = lf.findLines(pipeline_data->crop_gray, bestContours); + + vector tempTextLines; + for (uint i = 0; i < linePolygons.size(); i++) + { + vector linePolygon = linePolygons[i]; + + LineSegment topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + LineSegment bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + + vector textArea = getCharArea(topLine, bottomLine); + + TextLine textLine(textArea, linePolygon); + + tempTextLines.push_back(textLine); + } + + filterBetweenLines(bestThreshold, bestContours, tempTextLines); + + // Sort the lines from top to bottom. + std::sort(tempTextLines.begin(), tempTextLines.end(), sort_text_line); + + // Now that we've filtered a few more contours, re-do the text area. + for (uint i = 0; i < tempTextLines.size(); i++) + { + vector updatedTextArea = getCharArea(tempTextLines[i].topLine, tempTextLines[i].bottomLine); + vector linePolygon = tempTextLines[i].linePolygon; + if (updatedTextArea.size() > 0 && linePolygon.size() > 0) + { + pipeline_data->textLines.push_back(TextLine(updatedTextArea, linePolygon)); + } + + } + + + this->thresholdsInverted = isPlateInverted(); +} + + + + +Mat CharacterAnalysis::getCharacterMask() +{ + Mat charMask = Mat::zeros(bestThreshold.size(), CV_8U); + + for (uint i = 0; i < bestContours.size(); i++) + { + if (bestContours.goodIndices[i] == false) + continue; + + drawContours(charMask, bestContours.contours, + i, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + bestContours.hierarchy, + 1 + ); + + } + + return charMask; +} + + +void CharacterAnalysis::filter(Mat img, TextContours& textContours) +{ + static int STARTING_MIN_HEIGHT = round (((float) img.rows) * config->charAnalysisMinPercent); + static int STARTING_MAX_HEIGHT = round (((float) img.rows) * (config->charAnalysisMinPercent + config->charAnalysisHeightRange)); + static int HEIGHT_STEP = round (((float) img.rows) * config->charAnalysisHeightStepSize); + static int NUM_STEPS = config->charAnalysisNumSteps; + + int bestFitScore = -1; + + vector bestIndices; + + for (int i = 0; i < NUM_STEPS; i++) + { + + //vector goodIndices(contours.size()); + for (uint z = 0; z < textContours.size(); z++) textContours.goodIndices[z] = true; + + this->filterByBoxSize(textContours, STARTING_MIN_HEIGHT + (i * HEIGHT_STEP), STARTING_MAX_HEIGHT + (i * HEIGHT_STEP)); + + int goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... + continue; + + this->filterContourHoles(textContours); + + goodIndices = textContours.getGoodIndicesCount(); + if ( goodIndices == 0 || goodIndices <= bestFitScore) // Don't bother doing more filtering if we already lost... + continue; + + + int segmentCount = textContours.getGoodIndicesCount(); + + if (segmentCount > bestFitScore) + { + bestFitScore = segmentCount; + bestIndices = textContours.getIndicesCopy(); + } + } + + textContours.setIndices(bestIndices); +} + +// Goes through the contours for the plate and picks out possible char segments based on min/max height +void CharacterAnalysis::filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx) +{ + float idealAspect=config->charWidthMM / config->charHeightMM; + float aspecttolerance=0.25; + + + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + + //Create bounding rect of object + Rect mr= boundingRect(textContours.contours[i]); + + float minWidth = mr.height * 0.2; + //Crop image + + //cout << "Height: " << minHeightPx << " - " << mr.height << " - " << maxHeightPx << " ////// Width: " << mr.width << " - " << minWidth << endl; + if(mr.height >= minHeightPx && mr.height <= maxHeightPx && mr.width > minWidth) + { + float charAspect= (float)mr.width/(float)mr.height; + + //cout << " -- stage 2 aspect: " << abs(charAspect) << " - " << aspecttolerance << endl; + if (abs(charAspect - idealAspect) < aspecttolerance) + textContours.goodIndices[i] = true; + } + } + +} + +void CharacterAnalysis::filterContourHoles(TextContours& textContours) +{ + + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves valid + + int parentIndex = textContours.hierarchy[i][3]; + + if (parentIndex >= 0 && textContours.goodIndices[parentIndex]) + { + // this contour is a child of an already identified contour. REMOVE it + if (this->config->debugCharAnalysis) + { + cout << "filterContourHoles: contour index: " << i << endl; + } + } + else + { + textContours.goodIndices[i] = true; + } + } + +} + +// Goes through the contours for the plate and picks out possible char segments based on min/max height +// returns a vector of indices corresponding to valid contours +void CharacterAnalysis::filterByParentContour( TextContours& textContours) +{ + + vector parentIDs; + vector votes; + + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + textContours.goodIndices[i] = false; // Set it to not included unless it proves + + int voteIndex = -1; + int parentID = textContours.hierarchy[i][3]; + // check if parentID is already in the lsit + for (uint j = 0; j < parentIDs.size(); j++) + { + if (parentIDs[j] == parentID) + { + voteIndex = j; + break; + } + } + if (voteIndex == -1) + { + parentIDs.push_back(parentID); + votes.push_back(1); + } + else + { + votes[voteIndex] = votes[voteIndex] + 1; + } + } + + // Tally up the votes, pick the winner + int totalVotes = 0; + int winningParentId = 0; + int highestVotes = 0; + for (uint i = 0; i < parentIDs.size(); i++) + { + if (votes[i] > highestVotes) + { + winningParentId = parentIDs[i]; + highestVotes = votes[i]; + } + totalVotes += votes[i]; + } + + // Now filter out all the contours with a different parent ID (assuming the totalVotes > 2) + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + if (totalVotes <= 2) + { + textContours.goodIndices[i] = true; + } + else if (textContours.hierarchy[i][3] == winningParentId) + { + textContours.goodIndices[i] = true; + } + } + +} + +void CharacterAnalysis::filterBetweenLines(Mat img, TextContours& textContours, vector textLines ) +{ + static float MIN_AREA_PERCENT_WITHIN_LINES = 0.88; + static float MAX_DISTANCE_PERCENT_FROM_LINES = 0.15; + + if (textLines.size() == 0) + return; + + vector validPoints; + + + // Create a white mask for the area inside the polygon + Mat outerMask = Mat::zeros(img.size(), CV_8U); + + for (uint i = 0; i < textLines.size(); i++) + fillConvexPoly(outerMask, textLines[i].linePolygon.data(), textLines[i].linePolygon.size(), Scalar(255,255,255)); + + // For each contour, determine if enough of it is between the lines to qualify + for (uint i = 0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + float percentInsideMask = getContourAreaPercentInsideMask(outerMask, + textContours.contours, + textContours.hierarchy, + (int) i); + + + + if (percentInsideMask < MIN_AREA_PERCENT_WITHIN_LINES) + { + // Not enough area is inside the lines. + if (config->debugCharAnalysis) + cout << "Rejecting due to insufficient area" << endl; + textContours.goodIndices[i] = false; + + continue; + } + + + // now check to make sure that the top and bottom of the contour are near enough to the lines + + // First get the high and low point for the contour + // Remember that origin is top-left, so the top Y values are actually closer to 0. + Rect brect = boundingRect(textContours.contours[i]); + int xmiddle = brect.x + (brect.width / 2); + Point topMiddle = Point(xmiddle, brect.y); + Point botMiddle = Point(xmiddle, brect.y+brect.height); + + // Get the absolute distance from the top and bottom lines + + for (uint i = 0; i < textLines.size(); i++) + { + Point closestTopPoint = textLines[i].topLine.closestPointOnSegmentTo(topMiddle); + Point closestBottomPoint = textLines[i].bottomLine.closestPointOnSegmentTo(botMiddle); + + float absTopDistance = distanceBetweenPoints(closestTopPoint, topMiddle); + float absBottomDistance = distanceBetweenPoints(closestBottomPoint, botMiddle); + + float maxDistance = textLines[i].lineHeight * MAX_DISTANCE_PERCENT_FROM_LINES; + + if (absTopDistance < maxDistance && absBottomDistance < maxDistance) + { + // It's ok, leave it as-is. + } + else + { + + textContours.goodIndices[i] = false; + if (config->debugCharAnalysis) + cout << "Rejecting due to top/bottom points that are out of range" << endl; + } + } + + } + +} + +void CharacterAnalysis::filterByOuterMask(TextContours& textContours) +{ + float MINIMUM_PERCENT_LEFT_AFTER_MASK = 0.1; + float MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK = 0.6; + + if (this->pipeline_data->hasPlateBorder == false) + return; + + + cv::Mat plateMask = pipeline_data->plateBorderMask; + + Mat tempMaskedContour = Mat::zeros(plateMask.size(), CV_8U); + Mat tempFullContour = Mat::zeros(plateMask.size(), CV_8U); + + int charsInsideMask = 0; + int totalChars = 0; + + vector originalindices; + for (uint i = 0; i < textContours.size(); i++) + originalindices.push_back(textContours.goodIndices[i]); + + for (uint i=0; i < textContours.size(); i++) + { + if (textContours.goodIndices[i] == false) + continue; + + totalChars++; + + drawContours(tempFullContour, textContours.contours, i, Scalar(255,255,255), CV_FILLED, 8, textContours.hierarchy); + bitwise_and(tempFullContour, plateMask, tempMaskedContour); + + float beforeMaskWhiteness = mean(tempFullContour)[0]; + float afterMaskWhiteness = mean(tempMaskedContour)[0]; + + if (afterMaskWhiteness / beforeMaskWhiteness > MINIMUM_PERCENT_LEFT_AFTER_MASK) + { + charsInsideMask++; + textContours.goodIndices[i] = true; + } + } + + if (totalChars == 0) + { + textContours.goodIndices = originalindices; + return; + } + + // Check to make sure that this is a valid box. If the box is too small (e.g., 1 char is inside, and 3 are outside) + // then don't use this to filter. + float percentCharsInsideMask = ((float) charsInsideMask) / ((float) totalChars); + if (percentCharsInsideMask < MINIMUM_PERCENT_OF_CHARS_INSIDE_PLATE_MASK) + { + textContours.goodIndices = originalindices; + return; + } + +} + +bool CharacterAnalysis::isPlateInverted() +{ + Mat charMask = getCharacterMask(); + + + Scalar meanVal = mean(bestThreshold, charMask)[0]; + + if (this->config->debugCharAnalysis) + cout << "CharacterAnalysis, plate inverted: MEAN: " << meanVal << " : " << bestThreshold.type() << endl; + + if (meanVal[0] < 100) // Half would be 122.5. Give it a little extra oomf before saying it needs inversion. Most states aren't inverted. + return true; + + return false; +} + +bool CharacterAnalysis::verifySize(Mat r, float minHeightPx, float maxHeightPx) +{ + //Char sizes 45x90 + float aspect=config->charWidthMM / config->charHeightMM; + float charAspect= (float)r.cols/(float)r.rows; + float error=0.35; + //float minHeight=TEMPLATE_PLATE_HEIGHT * .35; + //float maxHeight=TEMPLATE_PLATE_HEIGHT * .65; + //We have a different aspect ratio for number 1, and it can be ~0.2 + float minAspect=0.2; + float maxAspect=aspect+aspect*error; + //area of pixels + float area=countNonZero(r); + //bb area + float bbArea=r.cols*r.rows; + //% of pixel in area + float percPixels=area/bbArea; + + //if(DEBUG) + //cout << "Aspect: "<< aspect << " ["<< minAspect << "," << maxAspect << "] " << "Area "<< percPixels <<" Char aspect " << charAspect << " Height char "<< r.rows << "\n"; + if(percPixels < 0.8 && charAspect > minAspect && charAspect < maxAspect && r.rows >= minHeightPx && r.rows < maxHeightPx) + return true; + else + return false; +} + +vector CharacterAnalysis::getCharArea(LineSegment topLine, LineSegment bottomLine) +{ + const int MAX = 100000; + const int MIN= -1; + + int leftX = MAX; + int rightX = MIN; + + for (uint i = 0; i < bestContours.size(); i++) + { + if (bestContours.goodIndices[i] == false) + continue; + + for (uint z = 0; z < bestContours.contours[i].size(); z++) + { + if (bestContours.contours[i][z].x < leftX) + leftX = bestContours.contours[i][z].x; + if (bestContours.contours[i][z].x > rightX) + rightX = bestContours.contours[i][z].x; + } + } + + vector charArea; + if (leftX != MAX && rightX != MIN) + { + Point tl(leftX, topLine.getPointAt(leftX)); + Point tr(rightX, topLine.getPointAt(rightX)); + Point br(rightX, bottomLine.getPointAt(rightX)); + Point bl(leftX, bottomLine.getPointAt(leftX)); + charArea.push_back(tl); + charArea.push_back(tr); + charArea.push_back(br); + charArea.push_back(bl); + } + + return charArea; +} diff --git a/src/openalpr/textdetection/characteranalysis.h b/src/openalpr/textdetection/characteranalysis.h new file mode 100644 index 0000000..6223cd8 --- /dev/null +++ b/src/openalpr/textdetection/characteranalysis.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_CHARACTERANALYSIS_H +#define OPENALPR_CHARACTERANALYSIS_H + +#include +#include "opencv2/imgproc/imgproc.hpp" +#include "utility.h" +#include "config.h" +#include "pipeline_data.h" +#include "textcontours.h" +#include "platemask.h" +#include "linefinder.h" + +class CharacterAnalysis +{ + + public: + CharacterAnalysis(PipelineData* pipeline_data); + virtual ~CharacterAnalysis(); + + + cv::Mat bestThreshold; + + TextContours bestContours; + + bool thresholdsInverted; + + std::vector allTextContours; + + void analyze(); + + cv::Mat getCharacterMask(); + + private: + PipelineData* pipeline_data; + Config* config; + + cv::Mat findOuterBoxMask( ); + + bool isPlateInverted(); + void filter(cv::Mat img, TextContours& textContours); + + void filterByBoxSize(TextContours& textContours, int minHeightPx, int maxHeightPx); + void filterByParentContour( TextContours& textContours ); + void filterContourHoles(TextContours& textContours); + void filterByOuterMask(TextContours& textContours); + + std::vector getCharArea(LineSegment topLine, LineSegment bottomLine); + void filterBetweenLines(cv::Mat img, TextContours& textContours, std::vector textLines ); + + bool verifySize(cv::Mat r, float minHeightPx, float maxHeightPx); + + +}; + +#endif // OPENALPR_CHARACTERANALYSIS_H diff --git a/src/openalpr/textdetection/linefinder.cpp b/src/openalpr/textdetection/linefinder.cpp new file mode 100644 index 0000000..3ef2b7f --- /dev/null +++ b/src/openalpr/textdetection/linefinder.cpp @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include + +#include "linefinder.h" +#include "utility.h" +#include "pipeline_data.h" + +using namespace std; +using namespace cv; + +LineFinder::LineFinder(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; +} + +LineFinder::~LineFinder() { +} + +vector > LineFinder::findLines(Mat image, const TextContours contours) +{ + const float MIN_AREA_TO_IGNORE = 0.65; + + vector > linesFound; + + cvtColor(image, image, CV_GRAY2BGR); + + vector charPoints; + + for (uint i = 0; i < contours.contours.size(); i++) + { + if (contours.goodIndices[i] == false) + continue; + + charPoints.push_back( CharPointInfo(contours.contours[i], i) ); + } + + vector bestLine = getBestLine(contours, charPoints); + + if (bestLine.size() > 0) + linesFound.push_back(bestLine); + + if (pipeline_data->isMultiline) + { + // we have a two-line plate. Find the next best line, removing the tops/bottoms from before. + // Create a mask from the bestLine area, and remove all contours with tops that fall inside of it. + + vector remainingPoints; + for (uint i = 0; i < charPoints.size(); i++) + { + Mat mask = Mat::zeros(Size(contours.width, contours.height), CV_8U); + fillConvexPoly(mask, bestLine.data(), bestLine.size(), Scalar(255,255,255)); + + float percentInside = getContourAreaPercentInsideMask(mask, contours.contours, contours.hierarchy, charPoints[i].contourIndex); + + if (percentInside < MIN_AREA_TO_IGNORE) + { + remainingPoints.push_back(charPoints[i]); + } + } + + vector nextBestLine = getBestLine(contours, remainingPoints); + + if (nextBestLine.size() > 0) + linesFound.push_back(nextBestLine); + } + + + return linesFound; +} + + +// Returns a polygon "stripe" across the width of the character region. The lines are voted and the polygon starts at 0 and extends to image width +vector LineFinder::getBestLine(const TextContours contours, vector charPoints) +{ + vector bestStripe; + + // Find the best fit line segment that is parallel with the most char segments + if (charPoints.size() <= 1) + { + // Maybe do something about this later, for now let's just ignore + return bestStripe; + } + + + vector charheights; + for (uint i = 0; i < charPoints.size(); i++) + charheights.push_back(charPoints[i].boundingBox.height); + float medianCharHeight = median(charheights.data(), charheights.size()); + + + + vector topLines; + vector bottomLines; + // Iterate through each possible char and find all possible lines for the top and bottom of each char segment + for (uint i = 0; i < charPoints.size() - 1; i++) + { + for (uint k = i+1; k < charPoints.size(); k++) + { + + int leftCPIndex, rightCPIndex; + if (charPoints[i].top.x < charPoints[k].top.x) + { + leftCPIndex = i; + rightCPIndex = k; + } + else + { + leftCPIndex = k; + rightCPIndex = i; + } + + + LineSegment top(charPoints[leftCPIndex].top, charPoints[rightCPIndex].top); + LineSegment bottom(charPoints[leftCPIndex].bottom, charPoints[rightCPIndex].bottom); + + + // Only allow lines that have a sane angle +// if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && +// abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) +// { +// topLines.push_back(top); +// bottomLines.push_back(bottom); +// } + + LineSegment parallelBot = top.getParallelLine(medianCharHeight * -1); + LineSegment parallelTop = bottom.getParallelLine(medianCharHeight); + + // Only allow lines that have a sane angle + if (abs(top.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(parallelBot.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(top); + bottomLines.push_back(parallelBot); + } + + // Only allow lines that have a sane angle + if (abs(parallelTop.angle) <= pipeline_data->config->maxPlateAngleDegrees && + abs(bottom.angle) <= pipeline_data->config->maxPlateAngleDegrees) + { + topLines.push_back(parallelTop); + bottomLines.push_back(bottom); + } + } + } + + int bestScoreIndex = 0; + int bestScore = -1; + int bestScoreDistance = -1; // Line segment distance is used as a tie breaker + + // Now, among all possible lines, find the one that is the best fit + for (uint i = 0; i < topLines.size(); i++) + { + float SCORING_MIN_THRESHOLD = 0.97; + float SCORING_MAX_THRESHOLD = 1.03; + + int curScore = 0; + for (uint charidx = 0; charidx < charPoints.size(); charidx++) + { + float topYPos = topLines[i].getPointAt(charPoints[charidx].top.x); + float botYPos = bottomLines[i].getPointAt(charPoints[charidx].bottom.x); + + float minTop = charPoints[charidx].top.y * SCORING_MIN_THRESHOLD; + float maxTop = charPoints[charidx].top.y * SCORING_MAX_THRESHOLD; + float minBot = (charPoints[charidx].bottom.y) * SCORING_MIN_THRESHOLD; + float maxBot = (charPoints[charidx].bottom.y) * SCORING_MAX_THRESHOLD; + if ( (topYPos >= minTop && topYPos <= maxTop) && + (botYPos >= minBot && botYPos <= maxBot)) + { + curScore++; + } + + //cout << "Slope: " << topslope << " yPos: " << topYPos << endl; + //drawAndWait(&tempImg); + } + + // Tie goes to the one with longer line segments + if ((curScore > bestScore) || + (curScore == bestScore && topLines[i].length > bestScoreDistance)) + { + bestScore = curScore; + bestScoreIndex = i; + // Just use x distance for now + bestScoreDistance = topLines[i].length; + } + } + + if (bestScore < 0) + return bestStripe; + + if (pipeline_data->config->debugCharAnalysis) + { + cout << "The winning score is: " << bestScore << endl; + // Draw the winning line segment + + Mat tempImg = Mat::zeros(Size(contours.width, contours.height), CV_8U); + cvtColor(tempImg, tempImg, CV_GRAY2BGR); + + cv::line(tempImg, topLines[bestScoreIndex].p1, topLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + cv::line(tempImg, bottomLines[bestScoreIndex].p1, bottomLines[bestScoreIndex].p2, Scalar(0, 0, 255), 2); + + displayImage(pipeline_data->config, "Winning lines", tempImg); + } + + Point topLeft = Point(0, topLines[bestScoreIndex].getPointAt(0) ); + Point topRight = Point(contours.width, topLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomRight = Point(contours.width, bottomLines[bestScoreIndex].getPointAt(contours.width)); + Point bottomLeft = Point(0, bottomLines[bestScoreIndex].getPointAt(0)); + + bestStripe.push_back(topLeft); + bestStripe.push_back(topRight); + bestStripe.push_back(bottomRight); + bestStripe.push_back(bottomLeft); + + + return bestStripe; +} + +CharPointInfo::CharPointInfo(vector contour, int index) { + + + this->contourIndex = index; + + this->boundingBox = cv::boundingRect( Mat(contour) ); + + + int x = boundingBox.x + (boundingBox.width / 2); + int y = boundingBox.y; + + this->top = Point(x, y); + + x = boundingBox.x + (boundingBox.width / 2); + y = boundingBox.y + boundingBox.height; + + this->bottom = Point(x,y); + +} diff --git a/src/openalpr/textdetection/linefinder.h b/src/openalpr/textdetection/linefinder.h new file mode 100644 index 0000000..7b7398f --- /dev/null +++ b/src/openalpr/textdetection/linefinder.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +// This class finds lines of text given an array of contours + +#ifndef OPENALPR_LINEFINDER_H +#define OPENALPR_LINEFINDER_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "textcontours.h" +#include "textline.h" +#include "pipeline_data.h" + +class CharPointInfo +{ +public: + CharPointInfo(std::vector contour, int index); + + cv::Rect boundingBox; + cv::Point top; + cv::Point bottom; + int contourIndex; + +}; + +class LineFinder { +public: + LineFinder(PipelineData* pipeline_data); + virtual ~LineFinder(); + + std::vector > findLines(cv::Mat image, const TextContours contours); +private: + PipelineData* pipeline_data; + + std::vector getBestLine(const TextContours contours, std::vector charPoints); +}; + +#endif /* OPENALPR_LINEFINDER_H */ + diff --git a/src/openalpr/textdetection/platemask.cpp b/src/openalpr/textdetection/platemask.cpp new file mode 100644 index 0000000..04dabbd --- /dev/null +++ b/src/openalpr/textdetection/platemask.cpp @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "platemask.h" + +using namespace std; +using namespace cv; + +PlateMask::PlateMask(PipelineData* pipeline_data) { + this->pipeline_data = pipeline_data; + this->hasPlateMask = false; + +} + + +PlateMask::~PlateMask() { +} + +cv::Mat PlateMask::getMask() { + return this->plateMask; +} + +void PlateMask::findOuterBoxMask( vector contours ) +{ + double min_parent_area = pipeline_data->config->templateHeightPx * pipeline_data->config->templateWidthPx * 0.10; // Needs to be at least 10% of the plate area to be considered. + + int winningIndex = -1; + int winningParentId = -1; + int bestCharCount = 0; + double lowestArea = 99999999999999; + + if (pipeline_data->config->debugCharAnalysis) + cout << "CharacterAnalysis::findOuterBoxMask" << endl; + + for (uint imgIndex = 0; imgIndex < contours.size(); imgIndex++) + { + //vector charContours = filter(thresholds[imgIndex], allContours[imgIndex], allHierarchy[imgIndex]); + + int charsRecognized = 0; + int parentId = -1; + bool hasParent = false; + for (uint i = 0; i < contours[imgIndex].goodIndices.size(); i++) + { + if (contours[imgIndex].goodIndices[i]) charsRecognized++; + if (contours[imgIndex].goodIndices[i] && contours[imgIndex].hierarchy[i][3] != -1) + { + parentId = contours[imgIndex].hierarchy[i][3]; + hasParent = true; + } + } + + if (charsRecognized == 0) + continue; + + if (hasParent) + { + double boxArea = contourArea(contours[imgIndex].contours[parentId]); + if (boxArea < min_parent_area) + continue; + + if ((charsRecognized > bestCharCount) || + (charsRecognized == bestCharCount && boxArea < lowestArea)) + //(boxArea < lowestArea) + { + bestCharCount = charsRecognized; + winningIndex = imgIndex; + winningParentId = parentId; + lowestArea = boxArea; + } + } + } + + if (pipeline_data->config->debugCharAnalysis) + cout << "Winning image index (findOuterBoxMask) is: " << winningIndex << endl; + + if (winningIndex != -1 && bestCharCount >= 3) + { + int longestChildIndex = -1; + double longestChildLength = 0; + // Find the child with the longest permiter/arc length ( just for kicks) + for (uint i = 0; i < contours[winningIndex].size(); i++) + { + for (uint j = 0; j < contours[winningIndex].size(); j++) + { + if (contours[winningIndex].hierarchy[j][3] == winningParentId) + { + double arclength = arcLength(contours[winningIndex].contours[j], false); + if (arclength > longestChildLength) + { + longestChildIndex = j; + longestChildLength = arclength; + } + } + } + } + + Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + // get rid of the outline by drawing a 1 pixel width black line + drawContours(mask, contours[winningIndex].contours, + winningParentId, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + contours[winningIndex].hierarchy, + 0 + ); + + // Morph Open the mask to get rid of any little connectors to non-plate portions + int morph_elem = 2; + int morph_size = 3; + Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + + //morphologyEx( mask, mask, MORPH_CLOSE, element ); + morphologyEx( mask, mask, MORPH_OPEN, element ); + + //morph_size = 1; + //element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); + //dilate(mask, mask, element); + + // Drawing the edge black effectively erodes the image. This may clip off some extra junk from the edges. + // We'll want to do the contour again and find the larges one so that we remove the clipped portion. + + vector > contoursSecondRound; + + findContours(mask, contoursSecondRound, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + int biggestContourIndex = -1; + double largestArea = 0; + for (uint c = 0; c < contoursSecondRound.size(); c++) + { + double area = contourArea(contoursSecondRound[c]); + if (area > largestArea) + { + biggestContourIndex = c; + largestArea = area; + } + } + + if (biggestContourIndex != -1) + { + mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + vector smoothedMaskPoints; + approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); + + vector > tempvec; + tempvec.push_back(smoothedMaskPoints); + //fillPoly(mask, smoothedMaskPoints.data(), smoothedMaskPoints, Scalar(255,255,255)); + drawContours(mask, tempvec, + 0, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + contours[winningIndex].hierarchy, + 0 + ); + } + + if (pipeline_data->config->debugCharAnalysis) + { + vector debugImgs; + Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); + + pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); + + debugImgs.push_back(mask); + debugImgs.push_back(pipeline_data->thresholds[winningIndex]); + debugImgs.push_back(debugImgMasked); + + Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); + displayImage(pipeline_data->config, "Winning outer box", dashboard); + } + + hasPlateMask = true; + this->plateMask = mask; + } + + hasPlateMask = false; + Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); + bitwise_not(fullMask, fullMask); + + this->plateMask = fullMask; +} diff --git a/src/openalpr/textdetection/platemask.h b/src/openalpr/textdetection/platemask.h new file mode 100644 index 0000000..e31405a --- /dev/null +++ b/src/openalpr/textdetection/platemask.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_PLATEMASK_H +#define OPENALPR_PLATEMASK_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "pipeline_data.h" +#include "textcontours.h" + +class PlateMask { +public: + PlateMask(PipelineData* pipeline_data); + virtual ~PlateMask(); + + bool hasPlateMask; + + cv::Mat getMask(); + + void findOuterBoxMask(std::vector contours); + +private: + + PipelineData* pipeline_data; + cv::Mat plateMask; + + +}; + +#endif /* OPENALPR_PLATEMASK_H */ + diff --git a/src/openalpr/textdetection/textcontours.cpp b/src/openalpr/textdetection/textcontours.cpp new file mode 100644 index 0000000..2b7dd2d --- /dev/null +++ b/src/openalpr/textdetection/textcontours.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "textcontours.h" + +using namespace std; +using namespace cv; + + +TextContours::TextContours() { + + +} + + +TextContours::TextContours(cv::Mat threshold) { + + load(threshold); +} + + +TextContours::~TextContours() { +} + +void TextContours::load(cv::Mat threshold) { + + Mat tempThreshold(threshold.size(), CV_8U); + threshold.copyTo(tempThreshold); + findContours(tempThreshold, + contours, // a vector of contours + hierarchy, + CV_RETR_TREE, // retrieve all contours + CV_CHAIN_APPROX_SIMPLE ); // all pixels of each contours + + for (uint i = 0; i < contours.size(); i++) + goodIndices.push_back(true); + + this->width = threshold.cols; + this->height = threshold.rows; +} + + +uint TextContours::size() { + return contours.size(); +} + + + +int TextContours::getGoodIndicesCount() +{ + int count = 0; + for (uint i = 0; i < goodIndices.size(); i++) + { + if (goodIndices[i]) + count++; + } + + return count; +} + + +std::vector TextContours::getIndicesCopy() +{ + vector copyArray; + for (uint i = 0; i < goodIndices.size(); i++) + { + bool val = goodIndices[i]; + copyArray.push_back(goodIndices[i]); + } + + return copyArray; +} + +void TextContours::setIndices(std::vector newIndices) +{ + if (newIndices.size() == goodIndices.size()) + { + for (uint i = 0; i < newIndices.size(); i++) + goodIndices[i] = newIndices[i]; + } + else + { + assert("Invalid set operation on indices"); + } +} + +Mat TextContours::drawDebugImage() { + + Mat img_contours = Mat::zeros(Size(width, height), CV_8U); + + return drawDebugImage(img_contours); +} + +Mat TextContours::drawDebugImage(Mat baseImage) { + Mat img_contours(baseImage.size(), CV_8U); + baseImage.copyTo(img_contours); + + cvtColor(img_contours, img_contours, CV_GRAY2RGB); + + vector > allowedContours; + for (uint i = 0; i < this->contours.size(); i++) + { + if (this->goodIndices[i]) + allowedContours.push_back(this->contours[i]); + } + + drawContours(img_contours, this->contours, + -1, // draw all contours + cv::Scalar(255,0,0), // in blue + 1); // with a thickness of 1 + + drawContours(img_contours, allowedContours, + -1, // draw all contours + cv::Scalar(0,255,0), // in green + 1); // with a thickness of 1 + + + return img_contours; +} + + + diff --git a/src/openalpr/textdetection/textcontours.h b/src/openalpr/textdetection/textcontours.h new file mode 100644 index 0000000..cf6d6bb --- /dev/null +++ b/src/openalpr/textdetection/textcontours.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef TEXTCONTOURS_H +#define TEXTCONTOURS_H + +#include +#include "opencv2/imgproc/imgproc.hpp" + +class TextContours { +public: + TextContours(); + TextContours(cv::Mat threshold); + virtual ~TextContours(); + + void load(cv::Mat threshold); + + int width; + int height; + + std::vector goodIndices; + std::vector > contours; + std::vector hierarchy; + + uint size(); + int getGoodIndicesCount(); + + std::vector getIndicesCopy(); + void setIndices(std::vector newIndices); + + cv::Mat drawDebugImage(); + cv::Mat drawDebugImage(cv::Mat baseImage); + +private: + + +}; + +#endif /* TEXTCONTOURS_H */ + diff --git a/src/openalpr/textdetection/textline.cpp b/src/openalpr/textdetection/textline.cpp new file mode 100644 index 0000000..46405e4 --- /dev/null +++ b/src/openalpr/textdetection/textline.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#include + +#include "textline.h" + +using namespace cv; + +TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + std::vector textAreaInts, linePolygonInts; + + for (uint i = 0; i < textArea.size(); i++) + textAreaInts.push_back(Point(round(textArea[i].x), round(textArea[i].y))); + for (uint i = 0; i < linePolygon.size(); i++) + linePolygonInts.push_back(Point(round(linePolygon[i].x), round(linePolygon[i].y))); + + initialize(textAreaInts, linePolygonInts); +} + +TextLine::TextLine(std::vector textArea, std::vector linePolygon) { + initialize(textArea, linePolygon); +} + + +TextLine::~TextLine() { +} + + + +void TextLine::initialize(std::vector textArea, std::vector linePolygon) { + if (textArea.size() > 0) + { + if (this->textArea.size() > 0) + this->textArea.clear(); + if (this->linePolygon.size() > 0) + this->linePolygon.clear(); + + for (uint i = 0; i < textArea.size(); i++) + this->textArea.push_back(textArea[i]); + + for (uint i = 0; i < linePolygon.size(); i++) + this->linePolygon.push_back(linePolygon[i]); + + this->topLine = LineSegment(linePolygon[0].x, linePolygon[0].y, linePolygon[1].x, linePolygon[1].y); + this->bottomLine = LineSegment(linePolygon[3].x, linePolygon[3].y, linePolygon[2].x, linePolygon[2].y); + + this->charBoxTop = LineSegment(textArea[0].x, textArea[0].y, textArea[1].x, textArea[1].y); + this->charBoxBottom = LineSegment(textArea[3].x, textArea[3].y, textArea[2].x, textArea[2].y); + this->charBoxLeft = LineSegment(textArea[3].x, textArea[3].y, textArea[0].x, textArea[0].y); + this->charBoxRight = LineSegment(textArea[2].x, textArea[2].y, textArea[1].x, textArea[1].y); + + // Calculate line height + float x = ((float) linePolygon[1].x) / 2; + Point midpoint = Point(x, bottomLine.getPointAt(x)); + Point acrossFromMidpoint = topLine.closestPointOnSegmentTo(midpoint); + this->lineHeight = distanceBetweenPoints(midpoint, acrossFromMidpoint); + + this->angle = (topLine.angle + bottomLine.angle) / 2; + + } +} + + +cv::Mat TextLine::drawDebugImage(cv::Mat baseImage) { + cv::Mat debugImage(baseImage.size(), baseImage.type()); + + baseImage.copyTo(debugImage); + + cv::cvtColor(debugImage, debugImage, CV_GRAY2BGR); + + + fillConvexPoly(debugImage, linePolygon.data(), linePolygon.size(), Scalar(0,0,165)); + + fillConvexPoly(debugImage, textArea.data(), textArea.size(), Scalar(125,255,0)); + + line(debugImage, topLine.p1, topLine.p2, Scalar(255,0,0), 1); + line(debugImage, bottomLine.p1, bottomLine.p2, Scalar(255,0,0), 1); + + line(debugImage, charBoxTop.p1, charBoxTop.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxLeft.p1, charBoxLeft.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxRight.p1, charBoxRight.p2, Scalar(0,125,125), 1); + line(debugImage, charBoxBottom.p1, charBoxBottom.p2, Scalar(0,125,125), 1); + + + return debugImage; +} diff --git a/src/openalpr/textdetection/textline.h b/src/openalpr/textdetection/textline.h new file mode 100644 index 0000000..ec9e6d5 --- /dev/null +++ b/src/openalpr/textdetection/textline.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#ifndef OPENALPR_TEXTLINE_H +#define OPENALPR_TEXTLINE_H + +#include "utility.h" +#include "opencv2/imgproc/imgproc.hpp" + +class TextLine { +public: + TextLine(std::vector textArea, std::vector linePolygon); + TextLine(std::vector textArea, std::vector linePolygon); + virtual ~TextLine(); + + + std::vector linePolygon; + std::vector textArea; + LineSegment topLine; + LineSegment bottomLine; + + LineSegment charBoxTop; + LineSegment charBoxBottom; + LineSegment charBoxLeft; + LineSegment charBoxRight; + + float lineHeight; + float angle; + + cv::Mat drawDebugImage(cv::Mat baseImage); +private: + + void initialize(std::vector textArea, std::vector linePolygon); +}; + +#endif /* OPENALPR_TEXTLINE_H */ + diff --git a/src/openalpr/utility.cpp b/src/openalpr/utility.cpp index f76f3cb..693fdef 100644 --- a/src/openalpr/utility.cpp +++ b/src/openalpr/utility.cpp @@ -17,6 +17,8 @@ * along with this program. If not, see . */ +#include + #include "utility.h" using namespace cv; @@ -379,6 +381,36 @@ LineSegment LineSegment::getParallelLine(float distance) return result; } +// Given a contour and a mask, this function determines what percentage of the contour (area) +// is inside the masked area. +float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex) +{ + + + Mat innerArea = Mat::zeros(mask.size(), CV_8U); + + + drawContours(innerArea, contours, + contourIndex, // draw this contour + cv::Scalar(255,255,255), // in + CV_FILLED, + 8, + hierarchy, + 2 + ); + + + int startingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + bitwise_and(innerArea, mask, innerArea); + + int endingPixels = cv::countNonZero(innerArea); + //drawAndWait(&innerArea); + + return ((float) endingPixels) / ((float) startingPixels); + +} std::string toString(int value) { diff --git a/src/openalpr/utility.h b/src/openalpr/utility.h index 4e95fa5..a2c5b16 100644 --- a/src/openalpr/utility.h +++ b/src/openalpr/utility.h @@ -100,6 +100,8 @@ float angleBetweenPoints(cv::Point p1, cv::Point p2); cv::Size getSizeMaintainingAspect(cv::Mat inputImg, int maxWidth, int maxHeight); +float getContourAreaPercentInsideMask(cv::Mat mask, std::vector > contours, std::vector hierarchy, int contourIndex); + cv::Mat equalizeBrightness(cv::Mat img); cv::Rect expandRect(cv::Rect original, int expandXPixels, int expandYPixels, int maxX, int maxY);