diff --git a/src/openalpr/edges/edgefinder.cpp b/src/openalpr/edges/edgefinder.cpp index 436fa3b..c7ae379 100644 --- a/src/openalpr/edges/edgefinder.cpp +++ b/src/openalpr/edges/edgefinder.cpp @@ -18,6 +18,7 @@ */ #include "edgefinder.h" +#include "textlinecollection.h" using namespace std; using namespace cv; @@ -27,6 +28,7 @@ EdgeFinder::EdgeFinder(PipelineData* pipeline_data) { this->pipeline_data = pipeline_data; // First re-crop the area from the original picture knowing the text position + this->confidence = 0; } @@ -34,4 +36,146 @@ EdgeFinder::EdgeFinder(PipelineData* pipeline_data) { EdgeFinder::~EdgeFinder() { } +std::vector EdgeFinder::findEdgeCorners() { + + TextLineCollection tlc(pipeline_data->textLines); + + vector corners; + + // If the character segment is especially small, just expand the existing box + // If it's a nice, long segment, then guess the correct box based on character height/position + if (tlc.longerSegment.length > tlc.charHeight * 3) + { + float charHeightToPlateWidthRatio = pipeline_data->config->plateWidthMM / pipeline_data->config->charHeightMM; + float idealPixelWidth = tlc.charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + + float charHeightToPlateHeightRatio = pipeline_data->config->plateHeightMM / pipeline_data->config->charHeightMM; + float idealPixelHeight = tlc.charHeight * charHeightToPlateHeightRatio; + + + float verticalOffset = (idealPixelHeight * 1.5 / 2); + float horizontalOffset = (idealPixelWidth * 1.25 / 2); + LineSegment topLine = tlc.centerHorizontalLine.getParallelLine(verticalOffset); + LineSegment bottomLine = tlc.centerHorizontalLine.getParallelLine(-1 * verticalOffset); + + LineSegment leftLine = tlc.centerVerticalLine.getParallelLine(-1 * horizontalOffset); + LineSegment rightLine = tlc.centerVerticalLine.getParallelLine(horizontalOffset); + + Point topLeft = topLine.intersection(leftLine); + Point topRight = topLine.intersection(rightLine); + Point botRight = bottomLine.intersection(rightLine); + Point botLeft = bottomLine.intersection(leftLine); + + corners.push_back(topLeft); + corners.push_back(topRight); + corners.push_back(botRight); + corners.push_back(botLeft); + } + else + { + + //cout << "HEYOOO!" << endl; + int expandX = (int) ((float) pipeline_data->crop_gray.cols) * 0.15f; + int expandY = (int) ((float) pipeline_data->crop_gray.rows) * 0.15f; + int w = pipeline_data->crop_gray.cols; + int h = pipeline_data->crop_gray.rows; + + corners.push_back(Point(-1 * expandX, -1 * expandY)); + corners.push_back(Point(expandX + w, -1 * expandY)); + corners.push_back(Point(expandX + w, expandY + h)); + corners.push_back(Point(-1 * expandX, expandY + h)); + +// for (int i = 0; i < 4; i++) +// { +// std::cout << "CORNER: " << corners[i].x << " - " << corners[i].y << std::endl; +// } + } + + + Transformation imgTransform(pipeline_data->grayImg, pipeline_data->crop_gray, pipeline_data->regionOfInterest); + vector remappedCorners = imgTransform.transformSmallPointsToBigImage(corners); + +// cout << "topLeft: " << remappedCorners[0] << endl; +// cout << "topRight: " << remappedCorners[1] << endl; +// cout << "botRight: " << remappedCorners[2] << endl; +// cout << "botLeft: " << remappedCorners[3] << endl; + + Size cropSize = imgTransform.getCropSize(remappedCorners, + Size(pipeline_data->config->templateWidthPx, pipeline_data->config->templateHeightPx)); + + Mat transmtx = imgTransform.getTransformationMatrix(remappedCorners, cropSize); + Mat newCrop = imgTransform.crop(cropSize, transmtx); + +// drawAndWait(&newCrop); + + vector newLines; + for (uint i = 0; i < pipeline_data->textLines.size(); i++) + { + vector textArea = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].textArea); + vector linePolygon = imgTransform.transformSmallPointsToBigImage(pipeline_data->textLines[i].linePolygon); + + vector textAreaRemapped; + vector linePolygonRemapped; + + textAreaRemapped = imgTransform.remapSmallPointstoCrop(textArea, transmtx); + linePolygonRemapped = imgTransform.remapSmallPointstoCrop(linePolygon, transmtx); + + newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped)); + } + + PlateLines plateLines(pipeline_data); + plateLines.processImage(newCrop, newLines, 1.2); + + + PlateCorners cornerFinder(newCrop, &plateLines, pipeline_data, newLines); + vector smallPlateCorners = cornerFinder.findPlateCorners(); + + confidence = cornerFinder.confidence; + +// cout << "topLeft: " << smallPlateCorners[0] << endl; +// cout << "topRight: " << smallPlateCorners[1] << endl; +// cout << "botRight: " << smallPlateCorners[2] << endl; +// cout << "botLeft: " << smallPlateCorners[3] << endl; + +// cvtColor(newCrop, newCrop, CV_GRAY2BGR); +// for (int i = 0; i < 4; i++) +// circle(newCrop, smallPlateCorners[i], 4, Scalar(0,255,0), -1); +// +// drawAndWait(&newCrop); + +// Transformation reTrans(pipeline_data->crop_gray, newCrop, cv::Rect(0,0,pipeline_data->crop_gray.cols, pipeline_data->crop_gray.rows)); +// vector cornersInOriginalCrop = reTrans.transformSmallPointsToBigImage(smallPlateCorners); +// +// for (int i = 0; i < 4; i++) +// circle(pipeline_data->crop_gray, cornersInOriginalCrop[i], 4, Scalar(255,255,255), -1); +// +// drawAndWait(&pipeline_data->crop_gray); + + std::vector imgArea; + imgArea.push_back(Point2f(0, 0)); + imgArea.push_back(Point2f(newCrop.cols, 0)); + imgArea.push_back(Point2f(newCrop.cols, newCrop.rows)); + imgArea.push_back(Point2f(0, newCrop.rows)); + Mat newCropTransmtx = imgTransform.getTransformationMatrix(imgArea, remappedCorners); + + vector cornersInOriginalImg = imgTransform.remapSmallPointstoCrop(smallPlateCorners, newCropTransmtx); + +// cout << "topLeft: " << cornersInOriginalImg[0] << endl; +// cout << "topRight: " << cornersInOriginalImg[1] << endl; +// cout << "botRight: " << cornersInOriginalImg[2] << endl; +// cout << "botLeft: " << cornersInOriginalImg[3] << endl; + +// Mat debugimg; +// cvtColor(pipeline_data->crop_gray, debugimg, CV_GRAY2BGR); +// for (int i = 0; i < 4; i++) +// circle(debugimg, cornersInOriginalImg[i], 4, Scalar(0,255,0), -1); +// drawAndWait(&debugimg); + + +// for (int i = 0; i < 4; i++) +// circle(pipeline_data->colorImg, cornersInOriginalImg[i], 4, Scalar(0,255,0), -1); + + return cornersInOriginalImg; + +} diff --git a/src/openalpr/edges/edgefinder.h b/src/openalpr/edges/edgefinder.h index 609dca8..d0832d2 100644 --- a/src/openalpr/edges/edgefinder.h +++ b/src/openalpr/edges/edgefinder.h @@ -22,12 +22,19 @@ #include "opencv2/imgproc/imgproc.hpp" #include "pipeline_data.h" +#include "transformation.h" +#include "platelines.h" +#include "platecorners.h" class EdgeFinder { public: EdgeFinder(PipelineData* pipeline_data); virtual ~EdgeFinder(); + std::vector findEdgeCorners(); + + float confidence; + private: PipelineData* pipeline_data; diff --git a/src/openalpr/edges/platecorners.cpp b/src/openalpr/edges/platecorners.cpp index 3e9514f..bab31f1 100644 --- a/src/openalpr/edges/platecorners.cpp +++ b/src/openalpr/edges/platecorners.cpp @@ -22,8 +22,8 @@ using namespace cv; using namespace std; -PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) : - tlc(pipelineData->textLines) +PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, vector textLines) : + tlc(textLines) { this->pipelineData = pipelineData; @@ -32,6 +32,7 @@ PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, PipelineData* this->inputImage = inputImage; this->plateLines = plateLines; + this->textLines = textLines; this->bestHorizontalScore = 9999999999999; this->bestVerticalScore = 9999999999999; @@ -83,10 +84,10 @@ vector PlateCorners::findPlateCorners() Mat imgCorners = Mat(inputImage.size(), inputImage.type()); inputImage.copyTo(imgCorners); - for (uint linenum = 0; linenum < pipelineData->textLines.size(); linenum++) + for (uint linenum = 0; linenum < textLines.size(); linenum++) { for (int i = 0; i < 4; i++) - circle(imgCorners, pipelineData->textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); + circle(imgCorners, textLines[linenum].textArea[i], 2, Scalar(0, 0, 0)); } line(imgCorners, this->bestTop.p1, this->bestTop.p2, Scalar(255, 0, 0), 1, CV_AA); diff --git a/src/openalpr/edges/platecorners.h b/src/openalpr/edges/platecorners.h index 3415ee1..80f3458 100644 --- a/src/openalpr/edges/platecorners.h +++ b/src/openalpr/edges/platecorners.h @@ -28,8 +28,8 @@ #define NO_LINE -1 -#define SCORING_MISSING_SEGMENT_PENALTY_VERTICAL 10 -#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 1 +#define SCORING_MISSING_SEGMENT_PENALTY_VERTICAL 20 +#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 10 #define SCORING_BOXINESS_WEIGHT 0.8 #define SCORING_PLATEHEIGHT_WEIGHT 2.2 @@ -47,7 +47,7 @@ class PlateCorners { public: - PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData) ; + PlateCorners(cv::Mat inputImage, PlateLines* plateLines, PipelineData* pipelineData, std::vector textLines) ; virtual ~PlateCorners(); @@ -60,6 +60,7 @@ class PlateCorners PipelineData* pipelineData; cv::Mat inputImage; + std::vector textLines; TextLineCollection tlc; float bestHorizontalScore; diff --git a/src/openalpr/edges/platelines.cpp b/src/openalpr/edges/platelines.cpp index 1f76134..7c37b75 100644 --- a/src/openalpr/edges/platelines.cpp +++ b/src/openalpr/edges/platelines.cpp @@ -39,7 +39,7 @@ PlateLines::~PlateLines() { } -void PlateLines::processImage(Mat inputImage, float sensitivity) +void PlateLines::processImage(Mat inputImage, vector textLines, float sensitivity) { if (this->debug) cout << "PlateLines findLines" << endl; @@ -73,10 +73,10 @@ void PlateLines::processImage(Mat inputImage, float sensitivity) Mat mask = Mat::zeros(inputImage.size(), CV_8U); - for (uint i = 0; i < pipelineData->textLines.size(); i++) + for (uint i = 0; i < textLines.size(); i++) { vector > polygons; - polygons.push_back(pipelineData->textLines[i].textArea); + polygons.push_back(textLines[i].textArea); fillPoly(mask, polygons, Scalar(255,255,255)); } diff --git a/src/openalpr/edges/platelines.h b/src/openalpr/edges/platelines.h index f25be91..4dde7ed 100644 --- a/src/openalpr/edges/platelines.h +++ b/src/openalpr/edges/platelines.h @@ -40,7 +40,7 @@ class PlateLines PlateLines(PipelineData* pipelineData); virtual ~PlateLines(); - void processImage(cv::Mat img, float sensitivity=1.0); + void processImage(cv::Mat img, std::vector textLines, float sensitivity=1.0); std::vector horizontalLines; std::vector verticalLines; diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index e388ed8..c00c4dc 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -57,19 +57,12 @@ void LicensePlateCandidate::recognize() if (textAnalysis.confidence > 10) { - PlateLines plateLines(pipeline_data); - - if (pipeline_data->hasPlateBorder) - plateLines.processImage(pipeline_data->plateBorderMask, 1.10); - - plateLines.processImage(pipeline_data->crop_gray, 0.9); - - PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, pipeline_data); - vector smallPlateCorners = cornerFinder.findPlateCorners(); EdgeFinder edgeFinder(pipeline_data); - if (cornerFinder.confidence > 0) + pipeline_data->plate_corners = edgeFinder.findEdgeCorners(); + + if (edgeFinder.confidence > 0) { timespec startTime; @@ -79,9 +72,9 @@ void LicensePlateCandidate::recognize() Mat originalCrop = pipeline_data->crop_gray; Transformation imgTransform(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion); - pipeline_data->plate_corners = imgTransform.transformSmallPointsToBigImage(smallPlateCorners); - Size cropSize = getCropSize(pipeline_data->plate_corners); + Size cropSize = imgTransform.getCropSize(pipeline_data->plate_corners, + Size(pipeline_data->config->ocrImageWidthPx, pipeline_data->config->ocrImageHeightPx)); Mat transmtx = imgTransform.getTransformationMatrix(pipeline_data->plate_corners, cropSize); pipeline_data->crop_gray = imgTransform.crop(cropSize, transmtx); @@ -130,25 +123,4 @@ void LicensePlateCandidate::recognize() } } -Size LicensePlateCandidate::getCropSize(vector areaCorners) -{ - // Figure out the approximate width/height of the license plate region, so we can maintain the aspect ratio. - LineSegment leftEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[0].x), round(areaCorners[0].y)); - LineSegment rightEdge(round(areaCorners[2].x), round(areaCorners[2].y), round(areaCorners[1].x), round(areaCorners[1].y)); - LineSegment topEdge(round(areaCorners[0].x), round(areaCorners[0].y), round(areaCorners[1].x), round(areaCorners[1].y)); - LineSegment bottomEdge(round(areaCorners[3].x), round(areaCorners[3].y), round(areaCorners[2].x), round(areaCorners[2].y)); - - float w = distanceBetweenPoints(leftEdge.midpoint(), rightEdge.midpoint()); - float h = distanceBetweenPoints(bottomEdge.midpoint(), topEdge.midpoint()); - float aspect = w/h; - int width = config->ocrImageWidthPx; - int height = round(((float) width) / aspect); - if (height > config->ocrImageHeightPx) - { - height = config->ocrImageHeightPx; - width = round(((float) height) * aspect); - } - - return Size(width, height); -}