From 96135dfc56fc2610c0479171e30546129a030034 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sat, 28 Mar 2015 13:13:38 -0400 Subject: [PATCH] Added prewarp to ALPR analysis --- src/openalpr/CMakeLists.txt | 1 + src/openalpr/alpr_impl.cpp | 47 ++++-- src/openalpr/alpr_impl.h | 3 + src/openalpr/config.cpp | 3 + src/openalpr/config.h | 3 + src/openalpr/pipeline_data.cpp | 14 +- src/openalpr/pipeline_data.h | 1 + src/openalpr/prewarp.cpp | 254 +++++++++++++++++++++++++++++++++ src/openalpr/prewarp.h | 56 ++++++++ 9 files changed, 369 insertions(+), 13 deletions(-) create mode 100644 src/openalpr/prewarp.cpp create mode 100644 src/openalpr/prewarp.h diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 9c233a7..30de103 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -25,6 +25,7 @@ set(lpr_source_files edges/textlinecollection.cpp edges/scorekeeper.cpp colorfilter.cpp + prewarp.cpp transformation.cpp textdetection/characteranalysis.cpp textdetection/platemask.cpp diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 970ebad..9492c2f 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -18,6 +18,7 @@ */ #include "alpr_impl.h" +#include "prewarp.h" void plateAnalysisThread(void* arg); @@ -33,6 +34,7 @@ namespace alpr plateDetector = ALPR_NULL_PTR; stateIdentifier = ALPR_NULL_PTR; ocr = ALPR_NULL_PTR; + prewarp = ALPR_NULL_PTR; // Config file or runtime dir not found. Don't process any further. if (config->loaded == false) @@ -47,6 +49,8 @@ namespace alpr setDetectRegion(DEFAULT_DETECT_REGION); this->topN = DEFAULT_TOPN; setDefaultRegion(""); + + prewarp = new PreWarp(config); } AlprImpl::~AlprImpl() @@ -61,6 +65,9 @@ namespace alpr if (ocr != ALPR_NULL_PTR) delete ocr; + + if (prewarp != ALPR_NULL_PTR) + delete prewarp; } bool AlprImpl::isLoaded() @@ -96,26 +103,38 @@ namespace alpr return response; } + // Convert image to grayscale if required + Mat grayImg = img; + if (img.channels() > 2) + cvtColor( img, grayImg, CV_BGR2GRAY ); + + // Prewarp the image and ROIs if configured] + std::vector warpedRegionsOfInterest = regionsOfInterest; + // Warp the image if prewarp is provided + grayImg = prewarp->warpImage(grayImg); + warpedRegionsOfInterest = prewarp->projectRects(regionsOfInterest, grayImg.cols, grayImg.rows, false); + + vector warpedPlateRegions; // Find all the candidate regions if (config->skipDetection == false) { - response.plateRegions = plateDetector->detect(img, regionsOfInterest); + warpedPlateRegions = plateDetector->detect(grayImg, warpedRegionsOfInterest); } else { // They have elected to skip plate detection. Instead, return a list of plate regions // based on their regions of interest - for (unsigned int i = 0; i < regionsOfInterest.size(); i++) + for (unsigned int i = 0; i < warpedRegionsOfInterest.size(); i++) { PlateRegion pr; - pr.rect = cv::Rect(regionsOfInterest[i]); - response.plateRegions.push_back(pr); + pr.rect = cv::Rect(warpedRegionsOfInterest[i]); + warpedPlateRegions.push_back(pr); } } queue plateQueue; - for (unsigned int i = 0; i < response.plateRegions.size(); i++) - plateQueue.push(response.plateRegions[i]); + for (unsigned int i = 0; i < warpedPlateRegions.size(); i++) + plateQueue.push(warpedPlateRegions[i]); int platecount = 0; while(!plateQueue.empty()) @@ -123,7 +142,7 @@ namespace alpr PlateRegion plateRegion = plateQueue.front(); plateQueue.pop(); - PipelineData pipeline_data(img, plateRegion.rect, config); + PipelineData pipeline_data(img, grayImg, plateRegion.rect, config); timespec platestarttime; getTimeMonotonic(&platestarttime); @@ -140,12 +159,16 @@ namespace alpr plateResult.regionConfidence = 0; plateResult.plate_index = platecount++; + // If using prewarp, remap the plate corners to the original image + vector cornerPoints = pipeline_data.plate_corners; + cornerPoints = prewarp->projectPoints(cornerPoints, true); + for (int pointidx = 0; pointidx < 4; pointidx++) { - plateResult.plate_points[pointidx].x = (int) pipeline_data.plate_corners[pointidx].x; - plateResult.plate_points[pointidx].y = (int) pipeline_data.plate_corners[pointidx].y; + plateResult.plate_points[pointidx].x = (int) cornerPoints[pointidx].x; + plateResult.plate_points[pointidx].y = (int) cornerPoints[pointidx].y; } - + if (detectRegion) { stateIdentifier->recognize(&pipeline_data); @@ -225,6 +248,10 @@ namespace alpr } + // Unwarp plate regions if necessary + prewarp->projectPlateRegions(warpedPlateRegions, grayImg.cols, grayImg.rows, true); + response.plateRegions = warpedPlateRegions; + timespec endTime; getTimeMonotonic(&endTime); response.results.total_processing_time_ms = diffclock(startTime, endTime); diff --git a/src/openalpr/alpr_impl.h b/src/openalpr/alpr_impl.h index 4318a2a..a1ed0f9 100644 --- a/src/openalpr/alpr_impl.h +++ b/src/openalpr/alpr_impl.h @@ -32,6 +32,8 @@ #include "detection/detector.h" #include "detection/detectorfactory.h" +#include "prewarp.h" + #include "licenseplatecandidate.h" #include "stateidentifier.h" #include "segmentation/charactersegmenter.h" @@ -96,6 +98,7 @@ namespace alpr Detector* plateDetector; StateIdentifier* stateIdentifier; OCR* ocr; + PreWarp* prewarp; int topN; bool detectRegion; diff --git a/src/openalpr/config.cpp b/src/openalpr/config.cpp index 3b57787..c47c354 100644 --- a/src/openalpr/config.cpp +++ b/src/openalpr/config.cpp @@ -154,6 +154,8 @@ namespace alpr skipDetection = getBoolean("common", "skip_detection", false); + prewarp = getString("common", "prewarp", ""); + maxPlateAngleDegrees = getInt("common", "max_plate_angle_degrees", 15); minPlateSizeWidthPx = getInt(country, "min_plate_size_width_px", 100); @@ -205,6 +207,7 @@ namespace alpr debugGeneral = getBoolean("debug", "general", false); debugTiming = getBoolean("debug", "timing", false); + debugPrewarp = getBoolean("debug", "prewarp", false); debugDetector = getBoolean("debug", "detector", false); debugStateId = getBoolean("debug", "state_id", false); debugPlateLines = getBoolean("debug", "plate_lines", false); diff --git a/src/openalpr/config.h b/src/openalpr/config.h index 520d559..6960aa6 100644 --- a/src/openalpr/config.h +++ b/src/openalpr/config.h @@ -58,6 +58,8 @@ namespace alpr bool skipDetection; + std::string prewarp; + int maxPlateAngleDegrees; float minPlateSizeWidthPx; @@ -106,6 +108,7 @@ namespace alpr bool debugGeneral; bool debugTiming; + bool debugPrewarp; bool debugDetector; bool debugStateId; bool debugPlateLines; diff --git a/src/openalpr/pipeline_data.cpp b/src/openalpr/pipeline_data.cpp index 3d509c6..8936ff4 100644 --- a/src/openalpr/pipeline_data.cpp +++ b/src/openalpr/pipeline_data.cpp @@ -8,15 +8,23 @@ namespace alpr PipelineData::PipelineData(Mat colorImage, Rect regionOfInterest, Config* config) { - this->colorImg = colorImage; + Mat grayImg; if (colorImage.channels() > 2) { - cvtColor(this->colorImg, this->grayImg, CV_BGR2GRAY); + cvtColor(this->colorImg, grayImg, CV_BGR2GRAY); } else { - this->grayImg = colorImage; + grayImg = colorImage; } + + PipelineData(colorImage, grayImg, regionOfInterest, config); + } + + PipelineData::PipelineData(Mat colorImage, Mat grayImg, Rect regionOfInterest, Config* config) + { + this->colorImg = colorImage; + this->grayImg = grayImg; this->regionOfInterest = regionOfInterest; this->config = config; diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 2c09453..1396630 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -15,6 +15,7 @@ namespace alpr public: PipelineData(cv::Mat colorImage, cv::Rect regionOfInterest, Config* config); + PipelineData(cv::Mat colorImage, cv::Mat grayImage, cv::Rect regionOfInterest, Config* config); virtual ~PipelineData(); void clearThresholds(); diff --git a/src/openalpr/prewarp.cpp b/src/openalpr/prewarp.cpp new file mode 100644 index 0000000..93de5b5 --- /dev/null +++ b/src/openalpr/prewarp.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2015 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#include + +#include "prewarp.h" + +using namespace std; +using namespace cv; + +namespace alpr +{ + + PreWarp::PreWarp(Config* config) { + this->config = config; + + string warp_config = config->prewarp; + + // Do a cursory verification based on number of commas + int commacount = count(warp_config.begin(), warp_config.end(), ','); + + if (warp_config.length() < 4) + { + // No config specified. ignore + if (this->config->debugPrewarp) + cout << "No prewarp configuration specified" << endl; + + this->valid = false; + } + else if (commacount != 8) + { + if (this->config->debugPrewarp) + cout << "Invalid prewarp configuration" << endl; + + this->valid = false; + } + else + { + + // Parse the warp_config + int first_comma = warp_config.find(","); + + + string name = warp_config.substr(0, first_comma); + + if (name != "planar") + { + this->valid = false; + } + else + { + stringstream ss(warp_config.substr(first_comma + 1, warp_config.length())); + + ss >> w; + ss.ignore(); + ss >> h; + ss.ignore(); + ss >> rotationx; + ss.ignore(); // Ignore comma + ss >> rotationy; + ss.ignore(); // Ignore comma + ss >> rotationz; + ss.ignore(); // Ignore comma + ss >> dist; + ss.ignore(); // Ignore comma + ss >> panX; + ss.ignore(); // Ignore comma + ss >> panY; + + this->valid = true; + } + + } + } + + + PreWarp::~PreWarp() { + } + + + cv::Mat PreWarp::warpImage(Mat image) { + if (!this->valid) + { + if (this->config->debugPrewarp) + cout << "prewarp skipped due to missing prewarp config" << endl; + return image; + } + + + float width_ratio = w / ((float)image.cols); + float height_ratio = h / ((float)image.rows); + + float rx = rotationx * width_ratio; + float ry = rotationy * width_ratio; + float px = panX / width_ratio; + float py = panY / height_ratio; + + + transform = findTransform(image.cols, image.rows, rx, ry, rotationz, px, py, dist); + + + Mat warped_image; + + warpPerspective(image, warped_image, transform, image.size(), INTER_CUBIC | WARP_INVERSE_MAP); + + + if (this->config->debugPrewarp && this->config->debugShowImages) + { + imshow("Prewarp", warped_image); + } + return warped_image; + } + + // Projects a "region of interest" into the new space + // The rect needs to be converted to points, warped, then converted back into a + // bounding rectangle + vector PreWarp::projectRects(vector rects, int maxWidth, int maxHeight, bool inverse) { + + if (!this->valid) + return rects; + + vector projected_rects; + + for (unsigned int i = 0; i < rects.size(); i++) + { + vector points; + points.push_back(Point(rects[i].x, rects[i].y)); + points.push_back(Point(rects[i].x + rects[i].width, rects[i].y)); + points.push_back(Point(rects[i].x + rects[i].width, rects[i].y + rects[i].height)); + points.push_back(Point(rects[i].x, rects[i].y + rects[i].height)); + + vector projectedPoints = projectPoints(points, inverse); + + Rect projectedRect = boundingRect(projectedPoints); + projectedRect = expandRect(projectedRect, 0, 0, maxWidth, maxHeight); + projected_rects.push_back(projectedRect); + + } + + return projected_rects; + } + + vector PreWarp::projectPoints(vector points, bool inverse) { + + if (!this->valid) + return points; + + vector output; + + if (!inverse) + perspectiveTransform(points, output, transform.inv()); + else + perspectiveTransform(points, output, transform); + + return output; + } + + + void PreWarp::projectPlateRegions(vector& plateRegions, int maxWidth, int maxHeight, bool inverse){ + + if (!this->valid) + return; + + for (unsigned int i = 0; i < plateRegions.size(); i++) + { + vector singleRect; + singleRect.push_back(plateRegions[i].rect); + vector transformedRect = projectRects(singleRect, maxWidth, maxHeight, inverse); + plateRegions[i].rect.x = transformedRect[0].x; + plateRegions[i].rect.y = transformedRect[0].y; + plateRegions[i].rect.width = transformedRect[0].width; + plateRegions[i].rect.height = transformedRect[0].height; + + projectPlateRegions(plateRegions[i].children, maxWidth, maxHeight, inverse); + } + } + + cv::Mat PreWarp::findTransform(float w, float h, + float rotationx, float rotationy, float rotationz, + float panX, float panY, float dist) { + + float alpha = rotationx; + float beta = rotationy; + float gamma = rotationz; + float f = 1.0; + + // Projection 2D -> 3D matrix + Mat A1 = (Mat_(4,3) << + 1, 0, -w/2, + 0, 1, -h/2, + 0, 0, 0, + 0, 0, 1); + + // Camera Intrisecs matrix 3D -> 2D + Mat A2 = (Mat_(3,4) << + f, 0, w/2, 0, + 0, f, h/2, 0, + 0, 0, 1, 0); + + // Rotation matrices around the X axis + Mat Rx = (Mat_(4, 4) << + 1, 0, 0, 0, + 0, cos(alpha), -sin(alpha), 0, + 0, sin(alpha), cos(alpha), 0, + 0, 0, 0, 1); + + // Rotation matrices around the Y axis + Mat Ry = (Mat_(4, 4) << + cos(beta), 0, sin(beta), 0, + 0, 1, 0, 0, + -sin(beta), 0, cos(beta), 0, + 0, 0, 0, 1); + + // Rotation matrices around the Z axis + Mat Rz = (Mat_(4, 4) << + cos(gamma), -sin(gamma), 0, 0, + sin(gamma), cos(gamma), 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 1); + + Mat R = Rx*Ry*Rz; + + // Translation matrix on the Z axis + Mat T = (Mat_(4, 4) << + 1, 0, 0, panX, + 0, 1, 0, panY, + 0, 0, 1, dist, + 0, 0, 0, 1); + + + return A2 * (T * (R * A1)); + + } + + +} + diff --git a/src/openalpr/prewarp.h b/src/openalpr/prewarp.h new file mode 100644 index 0000000..7e6a589 --- /dev/null +++ b/src/openalpr/prewarp.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + + +#ifndef OPENALPR_PREWARP_H +#define OPENALPR_PREWARP_H + +#include "config.h" +#include "utility.h" +#include "opencv2/imgproc/imgproc.hpp" +#include "detection/detector.h" + +namespace alpr +{ + + class PreWarp { + public: + PreWarp(Config* config); + virtual ~PreWarp(); + + cv::Mat warpImage(cv::Mat image); + std::vector projectPoints(std::vector points, bool inverse); + std::vector projectRects(std::vector rects, int maxWidth, int maxHeight, bool inverse); + void projectPlateRegions(std::vector& plateRegions, int maxWidth, int maxHeight, bool inverse); + + bool valid; + + private: + Config* config; + cv::Mat transform; + + float w, h, rotationx, rotationy, rotationz, dist, panX, panY; + + cv::Mat findTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float dist); + }; + +} + +#endif /* OPENALPR_PREWARP_H */ +