diff --git a/config/openalpr.conf b/config/openalpr.conf index 9fb15cf..4284b79 100644 --- a/config/openalpr.conf +++ b/config/openalpr.conf @@ -41,11 +41,11 @@ max_plate_angle_degrees = 15 ocr_min_font_point = 6 ; Minimum OCR confidence percent to consider. -postprocess_min_confidence = 60 +postprocess_min_confidence = 65 ; Any OCR character lower than this will also add an equally likely ; chance that the character is incorrect and will be skipped. Value is a confidence percent -postprocess_confidence_skip_level = 75 +postprocess_confidence_skip_level = 80 ; Reduces the total permutations to consider for scoring. postprocess_max_substitutions = 2 @@ -73,11 +73,11 @@ pause_on_frame = 0 [us] -; 30-50, 40-60, 50-70, 60-80, 70-90 +; 30-50, 40-60, 50-70, 60-80 char_analysis_min_pct = 0.30 char_analysis_height_range = 0.20 char_analysis_height_step_size = 0.10 -char_analysis_height_num_steps = 5 +char_analysis_height_num_steps = 4 segmentation_min_box_width_px = 4 segmentation_min_charheight_percent = 0.5; @@ -106,11 +106,11 @@ ocr_language = lus [eu] -; 35-50; 45-60, 55-70, 65-80, 75-90, 85-100 +; 35-50; 45-60, 55-70, 65-80, 75-90 char_analysis_min_pct = 0.35 char_analysis_height_range = 0.15 char_analysis_height_step_size = 0.10 -char_analysis_height_num_steps = 6 +char_analysis_height_num_steps = 5 segmentation_min_box_width_px = 5 segmentation_min_charheight_percent = 0.4; diff --git a/src/main.cpp b/src/main.cpp index 6d9ca2a..5541d5b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -154,7 +154,7 @@ int main( int argc, const char** argv ) while (cap.read(frame)) { detectandshow(&alpr, frame, "", outputJson); - cv::waitKey(1); + usleep(1000); framenum++; } } @@ -177,7 +177,8 @@ int main( int argc, const char** argv ) detectandshow( &alpr, latestFrame, "", outputJson); } - cv::waitKey(10); + // Sleep 10ms + usleep(10000); } videoBuffer.disconnect(); @@ -205,7 +206,7 @@ int main( int argc, const char** argv ) detectandshow( &alpr, frame, "", outputJson); //create a 1ms delay - cv::waitKey(1); + usleep(1000); framenum++; } } diff --git a/src/misc_utilities/benchmark/benchmark.cpp b/src/misc_utilities/benchmark/benchmark.cpp index 7a4f23b..eb1e3fa 100644 --- a/src/misc_utilities/benchmark/benchmark.cpp +++ b/src/misc_utilities/benchmark/benchmark.cpp @@ -104,6 +104,8 @@ int main( int argc, const char** argv ) plateCoords.y = 0; plateCoords.width = frame.cols; plateCoords.height = frame.rows; + + PipelineData pipeline_data(frame, plateCoords, config); char statecode[3]; statecode[0] = files[i][0]; @@ -111,7 +113,7 @@ int main( int argc, const char** argv ) statecode[2] = '\0'; string statecodestr(statecode); - CharacterRegion charRegion(frame, config); + CharacterRegion charRegion(&pipeline_data); if (abs(charRegion.getTopLine().angle) > 4) { @@ -124,10 +126,11 @@ int main( int argc, const char** argv ) warpAffine( frame, rotated, rot_mat, frame.size() ); rotated.copyTo(frame); + pipeline_data.crop_gray = frame; } - CharacterSegmenter charSegmenter(frame, charRegion.thresholdsInverted(), config); - ocr->performOCR(charSegmenter.getThresholds(), charSegmenter.characters); + CharacterSegmenter charSegmenter(&pipeline_data); + ocr->performOCR(&pipeline_data); ocr->postProcessor->analyze(statecode, 25); cout << files[i] << "," << statecode << "," << ocr->postProcessor->bestChars << endl; @@ -211,27 +214,30 @@ int main( int argc, const char** argv ) for (int z = 0; z < regions.size(); z++) { + + PipelineData pipeline_data(frame, regions[z].rect, &config); + getTime(&startTime); - char temp[5]; - stateIdentifier.recognize(frame, regions[z].rect, temp); + + stateIdentifier.recognize(&pipeline_data); getTime(&endTime); double stateidTime = diffclock(startTime, endTime); cout << "\tRegion " << z << ": State ID time: " << stateidTime << "ms." << endl; stateIdTimes.push_back(stateidTime); getTime(&startTime); - LicensePlateCandidate lp(frame, regions[z].rect, &config); + LicensePlateCandidate lp(&pipeline_data); lp.recognize(); getTime(&endTime); double analysisTime = diffclock(startTime, endTime); cout << "\tRegion " << z << ": Analysis time: " << analysisTime << "ms." << endl; - if (lp.confidence > 10) + if (pipeline_data.plate_area_confidence > 10) { lpAnalysisPositiveTimes.push_back(analysisTime); getTime(&startTime); - ocr.performOCR(lp.charSegmenter->getThresholds(), lp.charSegmenter->characters); + ocr.performOCR(&pipeline_data); getTime(&endTime); double ocrTime = diffclock(startTime, endTime); cout << "\tRegion " << z << ": OCR time: " << ocrTime << "ms." << endl; diff --git a/src/misc_utilities/classifychars.cpp b/src/misc_utilities/classifychars.cpp index e90e35d..ff49541 100644 --- a/src/misc_utilities/classifychars.cpp +++ b/src/misc_utilities/classifychars.cpp @@ -124,13 +124,14 @@ int main( int argc, const char** argv ) imshow ("Original", frame); + PipelineData pipeline_data(frame, Rect(0, 0, frame.cols, frame.rows), &config); char statecode[3]; statecode[0] = files[i][0]; statecode[1] = files[i][1]; statecode[2] = '\0'; string statecodestr(statecode); - CharacterRegion regionizer(frame, &config); + CharacterRegion regionizer(&pipeline_data); if (abs(regionizer.getTopLine().angle) > 4) { @@ -143,28 +144,29 @@ int main( int argc, const char** argv ) warpAffine( frame, rotated, rot_mat, frame.size() ); rotated.copyTo(frame); + pipeline_data.crop_gray = rotated; } - CharacterSegmenter charSegmenter(frame, regionizer.thresholdsInverted(), &config); + CharacterSegmenter charSegmenter(&pipeline_data); //ocr.cleanCharRegions(charSegmenter.thresholds, charSegmenter.characters); - ocr.performOCR(charSegmenter.getThresholds(), charSegmenter.characters); + ocr.performOCR(&pipeline_data); ocr.postProcessor->analyze(statecodestr, 25); cout << "OCR results: " << ocr.postProcessor->bestChars << endl; - vector selectedBoxes(charSegmenter.getThresholds().size()); - for (int z = 0; z < charSegmenter.getThresholds().size(); z++) + vector selectedBoxes(pipeline_data.thresholds.size()); + for (int z = 0; z < pipeline_data.thresholds.size(); z++) selectedBoxes[z] = false; int curDashboardSelection = 0; - vector humanInputs(charSegmenter.characters.size()); + vector humanInputs(pipeline_data.charRegions.size()); - for (int z = 0; z < charSegmenter.characters.size(); z++) + for (int z = 0; z < pipeline_data.charRegions.size(); z++) humanInputs[z] = ' '; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, 0); + showDashboard(pipeline_data.thresholds, selectedBoxes, 0); char waitkey = (char) waitKey(50); @@ -174,50 +176,50 @@ int main( int argc, const char** argv ) { if (curDashboardSelection > 0) curDashboardSelection--; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); } else if (waitkey == RIGHT_ARROW_KEY) // right arrow key { - if (curDashboardSelection < charSegmenter.getThresholds().size() - 1) + if (curDashboardSelection < pipeline_data.thresholds.size() - 1) curDashboardSelection++; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); } else if (waitkey == DOWN_ARROW_KEY) { - if (curDashboardSelection + DASHBOARD_COLUMNS <= charSegmenter.getThresholds().size() - 1) + if (curDashboardSelection + DASHBOARD_COLUMNS <= pipeline_data.thresholds.size() - 1) curDashboardSelection += DASHBOARD_COLUMNS; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); } else if (waitkey == UP_ARROW_KEY) { if (curDashboardSelection - DASHBOARD_COLUMNS >= 0) curDashboardSelection -= DASHBOARD_COLUMNS; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); } else if (waitkey == ENTER_KEY) { - vector tempdata = showCharSelection(charSegmenter.getThresholds()[curDashboardSelection], charSegmenter.characters, statecodestr); - for (int c = 0; c < charSegmenter.characters.size(); c++) + vector tempdata = showCharSelection(pipeline_data.thresholds[curDashboardSelection], pipeline_data.charRegions, statecodestr); + for (int c = 0; c < pipeline_data.charRegions.size(); c++) humanInputs[c] = tempdata[c]; } else if (waitkey == SPACE_KEY) { selectedBoxes[curDashboardSelection] = !selectedBoxes[curDashboardSelection]; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); } else if (waitkey == 's' || waitkey == 'S' || waitkey == 'W') { if (waitkey == 'W') { selectedBoxes[curDashboardSelection] = true; - showDashboard(charSegmenter.getThresholds(), selectedBoxes, curDashboardSelection); + showDashboard(pipeline_data.thresholds, selectedBoxes, curDashboardSelection); const std::string& ocr_str = ocr.postProcessor->bestChars; humanInputs.assign(ocr_str.begin(), ocr_str.end()); } bool somethingSelected = false; bool chardataTagged = false; - for (int c = 0; c < charSegmenter.getThresholds().size(); c++) + for (int c = 0; c < pipeline_data.thresholds.size(); c++) { if (selectedBoxes[c]) { @@ -225,7 +227,7 @@ int main( int argc, const char** argv ) break; } } - for (int c = 0; c < charSegmenter.characters.size(); c++) + for (int c = 0; c < pipeline_data.charRegions.size(); c++) { if (humanInputs[c] != ' ') { @@ -236,18 +238,18 @@ int main( int argc, const char** argv ) // Save if (somethingSelected && chardataTagged) { - for (int c = 0; c < charSegmenter.characters.size(); c++) + for (int c = 0; c < pipeline_data.charRegions.size(); c++) { if (humanInputs[c] == ' ') continue; - for (int t = 0; t < charSegmenter.getThresholds().size(); t++) + for (int t = 0; t < pipeline_data.thresholds.size(); t++) { if (selectedBoxes[t] == false) continue; stringstream filename; - Mat cropped = charSegmenter.getThresholds()[t](charSegmenter.characters[c]); + Mat cropped = pipeline_data.thresholds[t](pipeline_data.charRegions[c]); filename << outDir << "/" << humanInputs[c] << "-" << t << "-" << files[i]; imwrite(filename.str(), cropped); cout << "Writing char image: " << filename.str() << endl; diff --git a/src/misc_utilities/sortstate.cpp b/src/misc_utilities/sortstate.cpp index 68770ef..eff5229 100644 --- a/src/misc_utilities/sortstate.cpp +++ b/src/misc_utilities/sortstate.cpp @@ -79,31 +79,26 @@ int main( int argc, const char** argv ) cout << fullpath << endl; frame = imread( fullpath.c_str() ); - char code[4]; - int confidence = identifier.recognize(frame, code); + PipelineData pipeline_data(frame, Rect(0, 0, frame.cols, frame.rows), &config); + identifier.recognize(&pipeline_data); - if (confidence <= 20) + if (pipeline_data.region_confidence <= 20) { - code[0] = 'z'; - code[1] = 'z'; - confidence = 100; + pipeline_data.region_code = 'zz'; + pipeline_data.region_confidence = 100; } - - //imshow("Plate", frame); - if (confidence > 20) - { - cout << confidence << " : " << code; + else + { + cout << pipeline_data.region_confidence << " : " << pipeline_data.region_code; ostringstream convert; // stream used for the conversion convert << i; // insert the textual representation of 'Number' in the characters in the stream - string copyCommand = "cp \"" + fullpath + "\" " + outDir + code + convert.str() + ".png"; + string copyCommand = "cp \"" + fullpath + "\" " + outDir + pipeline_data.region_code + convert.str() + ".png"; system( copyCommand.c_str() ); waitKey(50); //while ((char) waitKey(50) != 'c') { } } - else - waitKey(50); } } } diff --git a/src/openalpr/CMakeLists.txt b/src/openalpr/CMakeLists.txt index 9046926..b5e2056 100644 --- a/src/openalpr/CMakeLists.txt +++ b/src/openalpr/CMakeLists.txt @@ -16,11 +16,12 @@ set(lpr_source_files binarize_wolf.cpp platelines.cpp characterregion.cpp - charactersegmenter.cpp + segmentation/charactersegmenter.cpp + segmentation/verticalhistogram.cpp platecorners.cpp colorfilter.cpp characteranalysis.cpp - verticalhistogram.cpp + pipeline_data.cpp trex.c cjson.c ) diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 2aa42e6..e04b26f 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -146,7 +146,9 @@ AlprFullDetails AlprImpl::recognizeFullDetails(cv::Mat img) displayImage(config, "Main Image", img); - cv::waitKey(1); + + // Sleep 1ms + usleep(1000); } @@ -186,17 +188,17 @@ void plateAnalysisThread(void* arg) if (dispatcher->config->debugGeneral) cout << "Thread: " << tthread::this_thread::get_id() << " loop " << ++loop_count << endl; - Mat img = dispatcher->getImageCopy(); + PipelineData pipeline_data(dispatcher->getImageCopy(), plateRegion.rect, dispatcher->config); timespec platestarttime; getTime(&platestarttime); - LicensePlateCandidate lp(img, plateRegion.rect, dispatcher->config); + LicensePlateCandidate lp(&pipeline_data); lp.recognize(); - if (lp.confidence <= 10) + if (pipeline_data.plate_area_confidence <= 10) { // Not a valid plate // Check if this plate has any children, if so, send them back up to the dispatcher for processing @@ -213,14 +215,14 @@ void plateAnalysisThread(void* arg) for (int pointidx = 0; pointidx < 4; pointidx++) { - plateResult.plate_points[pointidx].x = (int) lp.plateCorners[pointidx].x; - plateResult.plate_points[pointidx].y = (int) lp.plateCorners[pointidx].y; + plateResult.plate_points[pointidx].x = (int) pipeline_data.plate_corners[pointidx].x; + plateResult.plate_points[pointidx].y = (int) pipeline_data.plate_corners[pointidx].y; } if (dispatcher->detectRegion) { char statecode[4]; - plateResult.regionConfidence = dispatcher->stateIdentifier->recognize(img, plateRegion.rect, statecode); + plateResult.regionConfidence = dispatcher->stateIdentifier->recognize(&pipeline_data); if (plateResult.regionConfidence > 0) { plateResult.region = statecode; @@ -230,7 +232,7 @@ void plateAnalysisThread(void* arg) // Tesseract OCR does not appear to be threadsafe dispatcher->ocrMutex.lock(); - dispatcher->ocr->performOCR(lp.charSegmenter->getThresholds(), lp.charSegmenter->characters); + dispatcher->ocr->performOCR(&pipeline_data); dispatcher->ocr->postProcessor->analyze(plateResult.region, dispatcher->topN); const vector ppResults = dispatcher->ocr->postProcessor->getResults(); dispatcher->ocrMutex.unlock(); diff --git a/src/openalpr/alpr_impl.h b/src/openalpr/alpr_impl.h index 64f8a7b..7263812 100644 --- a/src/openalpr/alpr_impl.h +++ b/src/openalpr/alpr_impl.h @@ -30,13 +30,15 @@ #include "regiondetector.h" #include "licenseplatecandidate.h" #include "stateidentifier.h" -#include "charactersegmenter.h" +#include "segmentation/charactersegmenter.h" #include "ocr.h" #include "constants.h" #include "cjson.h" +#include "pipeline_data.h" + #include diff --git a/src/openalpr/characteranalysis.cpp b/src/openalpr/characteranalysis.cpp index c146504..c70691f 100644 --- a/src/openalpr/characteranalysis.cpp +++ b/src/openalpr/characteranalysis.cpp @@ -22,61 +22,40 @@ using namespace cv; using namespace std; -CharacterAnalysis::CharacterAnalysis(Mat img, Config* config) +CharacterAnalysis::CharacterAnalysis(PipelineData* pipeline_data) { - this->config = config; + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; this->hasPlateMask = false; if (this->config->debugCharAnalysis) cout << "Starting CharacterAnalysis identification" << endl; - if (img.type() != CV_8U) - cvtColor( img, this->img_gray, CV_BGR2GRAY ); - else - { - img_gray = Mat(img.size(), img.type()); - img.copyTo(img_gray); - } } CharacterAnalysis::~CharacterAnalysis() { - for (int i = 0; i < thresholds.size(); i++) - { - thresholds[i].release(); - } - thresholds.clear(); + } void CharacterAnalysis::analyze() { - thresholds = produceThresholds(img_gray, config); + pipeline_data->clearThresholds(); + pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config); - /* - // Morph Close the gray image to make it easier to detect blobs - int morph_elem = 1; - int morph_size = 1; - Mat element = getStructuringElement( morph_elem, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) ); - for (int i = 0; i < thresholds.size(); i++) - { - //morphologyEx( mask, mask, MORPH_CLOSE, element ); - morphologyEx( thresholds[i], thresholds[i], MORPH_OPEN, element ); - //dilate( thresholds[i], thresholds[i], element ); - } - */ timespec startTime; getTime(&startTime); - for (int i = 0; i < thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { vector > contours; vector hierarchy; - Mat tempThreshold(thresholds[i].size(), CV_8U); - thresholds[i].copyTo(tempThreshold); + Mat tempThreshold(pipeline_data->thresholds[i].size(), CV_8U); + pipeline_data->thresholds[i].copyTo(tempThreshold); findContours(tempThreshold, contours, // a vector of contours hierarchy, @@ -97,9 +76,9 @@ void CharacterAnalysis::analyze() getTime(&startTime); - for (int i = 0; i < thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { - vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); + vector goodIndices = this->filter(pipeline_data->thresholds[i], allContours[i], allHierarchy[i]); charSegments.push_back(goodIndices); if (config->debugCharAnalysis) @@ -118,7 +97,7 @@ void CharacterAnalysis::analyze() if (hasPlateMask) { // Filter out bad contours now that we have an outer box mask... - for (int i = 0; i < thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { charSegments[i] = filterByOuterMask(allContours[i], allHierarchy[i], charSegments[i]); } @@ -126,7 +105,7 @@ void CharacterAnalysis::analyze() int bestFitScore = -1; int bestFitIndex = -1; - for (int i = 0; i < thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { //vector goodIndices = this->filter(thresholds[i], allContours[i], allHierarchy[i]); //charSegments.push_back(goodIndices); @@ -138,7 +117,7 @@ void CharacterAnalysis::analyze() bestFitScore = segmentCount; bestFitIndex = i; bestCharSegments = charSegments[i]; - bestThreshold = thresholds[i]; + bestThreshold = pipeline_data->thresholds[i]; bestContours = allContours[i]; bestHierarchy = allHierarchy[i]; bestCharSegmentsCount = segmentCount; @@ -181,7 +160,7 @@ void CharacterAnalysis::analyze() //charsegments = this->getPossibleCharRegions(img_threshold, allContours, allHierarchy, STARTING_MIN_HEIGHT + (bestFitIndex * HEIGHT_STEP), STARTING_MAX_HEIGHT + (bestFitIndex * HEIGHT_STEP)); - this->linePolygon = getBestVotedLines(img_gray, bestContours, bestCharSegments); + this->linePolygon = getBestVotedLines(pipeline_data->crop_gray, bestContours, bestCharSegments); if (this->linePolygon.size() > 0) { @@ -290,7 +269,7 @@ Mat CharacterAnalysis::findOuterBoxMask() } } - Mat mask = Mat::zeros(thresholds[winningIndex].size(), CV_8U); + Mat mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); // get rid of the outline by drawing a 1 pixel width black line drawContours(mask, allContours[winningIndex], @@ -334,7 +313,7 @@ Mat CharacterAnalysis::findOuterBoxMask() if (biggestContourIndex != -1) { - mask = Mat::zeros(thresholds[winningIndex].size(), CV_8U); + mask = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); vector smoothedMaskPoints; approxPolyDP(contoursSecondRound[biggestContourIndex], smoothedMaskPoints, 2, true); @@ -355,12 +334,12 @@ Mat CharacterAnalysis::findOuterBoxMask() if (this->config->debugCharAnalysis) { vector debugImgs; - Mat debugImgMasked = Mat::zeros(thresholds[winningIndex].size(), CV_8U); + Mat debugImgMasked = Mat::zeros(pipeline_data->thresholds[winningIndex].size(), CV_8U); - thresholds[winningIndex].copyTo(debugImgMasked, mask); + pipeline_data->thresholds[winningIndex].copyTo(debugImgMasked, mask); debugImgs.push_back(mask); - debugImgs.push_back(thresholds[winningIndex]); + debugImgs.push_back(pipeline_data->thresholds[winningIndex]); debugImgs.push_back(debugImgMasked); Mat dashboard = drawImageDashboard(debugImgs, CV_8U, 1); @@ -372,7 +351,7 @@ Mat CharacterAnalysis::findOuterBoxMask() } hasPlateMask = false; - Mat fullMask = Mat::zeros(thresholds[0].size(), CV_8U); + Mat fullMask = Mat::zeros(pipeline_data->thresholds[0].size(), CV_8U); bitwise_not(fullMask, fullMask); return fullMask; } diff --git a/src/openalpr/characteranalysis.h b/src/openalpr/characteranalysis.h index 588cb21..92d0c2b 100644 --- a/src/openalpr/characteranalysis.h +++ b/src/openalpr/characteranalysis.h @@ -24,13 +24,13 @@ #include "constants.h" #include "utility.h" #include "config.h" - +#include "pipeline_data.h" class CharacterAnalysis { public: - CharacterAnalysis(cv::Mat img, Config* config); + CharacterAnalysis(PipelineData* pipeline_data); virtual ~CharacterAnalysis(); bool hasPlateMask; @@ -54,7 +54,6 @@ class CharacterAnalysis bool thresholdsInverted; - std::vector thresholds; std::vector > > allContours; std::vector > allHierarchy; std::vector > charSegments; @@ -64,10 +63,9 @@ class CharacterAnalysis cv::Mat getCharacterMask(); private: + PipelineData* pipeline_data; Config* config; - cv::Mat img_gray; - cv::Mat findOuterBoxMask( ); bool isPlateInverted(); diff --git a/src/openalpr/characterregion.cpp b/src/openalpr/characterregion.cpp index 6fc4af6..15106f7 100644 --- a/src/openalpr/characterregion.cpp +++ b/src/openalpr/characterregion.cpp @@ -22,9 +22,9 @@ using namespace cv; using namespace std; -CharacterRegion::CharacterRegion(Mat img, Config* config) +CharacterRegion::CharacterRegion(PipelineData* pipeline_data) { - this->config = config; + this->config = pipeline_data->config; this->debug = config->debugCharRegions; this->confidence = 0; @@ -35,16 +35,18 @@ CharacterRegion::CharacterRegion(Mat img, Config* config) timespec startTime; getTime(&startTime); - charAnalysis = new CharacterAnalysis(img, config); + charAnalysis = new CharacterAnalysis(pipeline_data); charAnalysis->analyze(); + pipeline_data->plate_inverted = charAnalysis->thresholdsInverted; + pipeline_data->plate_mask = charAnalysis->plateMask; if (this->debug && charAnalysis->linePolygon.size() > 0) { vector tempDash; - for (int z = 0; z < charAnalysis->thresholds.size(); z++) + for (int z = 0; z < pipeline_data->thresholds.size(); z++) { - Mat tmp(charAnalysis->thresholds[z].size(), charAnalysis->thresholds[z].type()); - charAnalysis->thresholds[z].copyTo(tmp); + Mat tmp(pipeline_data->thresholds[z].size(), pipeline_data->thresholds[z].type()); + pipeline_data->thresholds[z].copyTo(tmp); cvtColor(tmp, tmp, CV_GRAY2BGR); tempDash.push_back(tmp); @@ -100,10 +102,6 @@ CharacterRegion::~CharacterRegion() delete(charAnalysis); } -Mat CharacterRegion::getPlateMask() -{ - return charAnalysis->plateMask; -} LineSegment CharacterRegion::getTopLine() { @@ -140,7 +138,3 @@ LineSegment CharacterRegion::getCharBoxRight() return charAnalysis->charBoxRight; } -bool CharacterRegion::thresholdsInverted() -{ - return charAnalysis->thresholdsInverted; -} diff --git a/src/openalpr/characterregion.h b/src/openalpr/characterregion.h index 4826bae..ee2a8f0 100644 --- a/src/openalpr/characterregion.h +++ b/src/openalpr/characterregion.h @@ -25,23 +25,20 @@ #include "utility.h" #include "characteranalysis.h" #include "config.h" - +#include "pipeline_data.h" class CharacterRegion { public: - CharacterRegion(cv::Mat img, Config* config); + CharacterRegion(PipelineData* pipeline_data); virtual ~CharacterRegion(); - CharacterAnalysis *charAnalysis; int confidence; - cv::Mat getPlateMask(); LineSegment getTopLine(); LineSegment getBottomLine(); - //vector getLinePolygon(); std::vector getCharArea(); LineSegment getCharBoxTop(); @@ -49,12 +46,12 @@ class CharacterRegion LineSegment getCharBoxLeft(); LineSegment getCharBoxRight(); - bool thresholdsInverted(); protected: Config* config; bool debug; + CharacterAnalysis *charAnalysis; cv::Mat findOuterBoxMask(std::vector thresholds, std::vector > > allContours, std::vector > allHierarchy); std::vector filter(cv::Mat img, std::vector > contours, std::vector hierarchy); diff --git a/src/openalpr/licenseplatecandidate.cpp b/src/openalpr/licenseplatecandidate.cpp index b15fad0..8058a0f 100644 --- a/src/openalpr/licenseplatecandidate.cpp +++ b/src/openalpr/licenseplatecandidate.cpp @@ -22,12 +22,11 @@ using namespace std; using namespace cv; -LicensePlateCandidate::LicensePlateCandidate(Mat frame, Rect regionOfInterest, Config* config) +LicensePlateCandidate::LicensePlateCandidate(PipelineData* pipeline_data) { - this->config = config; + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; - this->frame = frame; - this->plateRegion = regionOfInterest; } LicensePlateCandidate::~LicensePlateCandidate() @@ -40,45 +39,41 @@ void LicensePlateCandidate::recognize() { charSegmenter = NULL; - this->confidence = 0; + pipeline_data->plate_area_confidence = 0; - int expandX = round(this->plateRegion.width * 0.20); - int expandY = round(this->plateRegion.height * 0.15); + int expandX = round(this->pipeline_data->regionOfInterest.width * 0.20); + int expandY = round(this->pipeline_data->regionOfInterest.height * 0.15); // expand box by 15% in all directions - Rect expandedRegion = expandRect( this->plateRegion, expandX, expandY, frame.cols, frame.rows) ; + Rect expandedRegion = expandRect( this->pipeline_data->regionOfInterest, expandX, expandY, this->pipeline_data->grayImg.cols, this->pipeline_data->grayImg.rows) ; - Mat plate_bgr = Mat(frame, expandedRegion); - resize(plate_bgr, plate_bgr, Size(config->templateWidthPx, config->templateHeightPx)); - - Mat plate_gray; - cvtColor(plate_bgr, plate_gray, CV_BGR2GRAY); + pipeline_data->crop_gray = Mat(this->pipeline_data->grayImg, expandedRegion); + resize(pipeline_data->crop_gray, pipeline_data->crop_gray, Size(config->templateWidthPx, config->templateHeightPx)); - CharacterRegion charRegion(plate_bgr, config); + CharacterRegion charRegion(pipeline_data); if (charRegion.confidence > 10) { PlateLines plateLines(config); - //Mat boogedy = charRegion.getPlateMask(); - plateLines.processImage(charRegion.getPlateMask(), &charRegion, 1.10); - plateLines.processImage(plate_gray, &charRegion, 0.9); + plateLines.processImage(pipeline_data->plate_mask, &charRegion, 1.10); + plateLines.processImage(pipeline_data->crop_gray, &charRegion, 0.9); - PlateCorners cornerFinder(plate_bgr, &plateLines, &charRegion, config); + PlateCorners cornerFinder(pipeline_data->crop_gray, &plateLines, &charRegion, config); vector smallPlateCorners = cornerFinder.findPlateCorners(); if (cornerFinder.confidence > 0) { - this->plateCorners = transformPointsToOriginalImage(frame, plate_bgr, expandedRegion, smallPlateCorners); + pipeline_data->plate_corners = transformPointsToOriginalImage(this->pipeline_data->grayImg, pipeline_data->crop_gray, expandedRegion, smallPlateCorners); - this->deskewed = deSkewPlate(frame, this->plateCorners); + pipeline_data->crop_gray = deSkewPlate(this->pipeline_data->grayImg, pipeline_data->plate_corners); - charSegmenter = new CharacterSegmenter(deskewed, charRegion.thresholdsInverted(), config); + charSegmenter = new CharacterSegmenter(pipeline_data); //this->recognizedText = ocr->recognizedText; //strcpy(this->recognizedText, ocr.recognizedText); - this->confidence = 100; + pipeline_data->plate_area_confidence = 100; } charRegion.confidence = 0; } @@ -122,7 +117,7 @@ Mat LicensePlateCandidate::deSkewPlate(Mat inputImage, vector corners) width = round(((float) height) * aspect); } - Mat deskewed(height, width, frame.type()); + Mat deskewed(height, width, this->pipeline_data->grayImg.type()); // Corners of the destination image vector quad_pts; diff --git a/src/openalpr/licenseplatecandidate.h b/src/openalpr/licenseplatecandidate.h index e12be32..9143efe 100644 --- a/src/openalpr/licenseplatecandidate.h +++ b/src/openalpr/licenseplatecandidate.h @@ -31,10 +31,10 @@ #include "constants.h" #include "platelines.h" #include "characterregion.h" -#include "charactersegmenter.h" +#include "segmentation/charactersegmenter.h" #include "platecorners.h" #include "config.h" - +#include "pipeline_data.h" //vector getCharacterRegions(Mat frame, vector regionsOfInterest); //vector getCharSegmentsBetweenLines(Mat img, vector > contours, LineSegment top, LineSegment bottom); @@ -43,24 +43,18 @@ class LicensePlateCandidate { public: - LicensePlateCandidate(cv::Mat frame, cv::Rect regionOfInterest, Config* config); + LicensePlateCandidate(PipelineData* pipeline_data); virtual ~LicensePlateCandidate(); - float confidence; // 0-100 - //vector points; // top-left, top-right, bottom-right, bottom-left - std::vector plateCorners; void recognize(); - cv::Mat deskewed; - CharacterSegmenter* charSegmenter; private: - + PipelineData* pipeline_data; Config* config; - cv::Mat frame; - cv::Rect plateRegion; + CharacterSegmenter* charSegmenter; cv::Mat filterByCharacterHue(std::vector > charRegionContours); std::vector findPlateCorners(cv::Mat inputImage, PlateLines plateLines, CharacterRegion charRegion); // top-left, top-right, bottom-right, bottom-left diff --git a/src/openalpr/ocr.cpp b/src/openalpr/ocr.cpp index 5e43e0d..776ea6b 100644 --- a/src/openalpr/ocr.cpp +++ b/src/openalpr/ocr.cpp @@ -52,7 +52,7 @@ OCR::~OCR() delete tesseract; } -void OCR::performOCR(vector thresholds, vector charRegions) +void OCR::performOCR(PipelineData* pipeline_data) { timespec startTime; getTime(&startTime); @@ -60,18 +60,20 @@ void OCR::performOCR(vector thresholds, vector charRegions) postProcessor->clear(); // Don't waste time on OCR processing if it is impossible to get sufficient characters - if (charRegions.size() < config->postProcessMinCharacters) + if (pipeline_data->charRegions.size() < config->postProcessMinCharacters) return; - for (int i = 0; i < thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { // Make it black text on white background - bitwise_not(thresholds[i], thresholds[i]); - tesseract->SetImage((uchar*) thresholds[i].data, thresholds[i].size().width, thresholds[i].size().height, thresholds[i].channels(), thresholds[i].step1()); + bitwise_not(pipeline_data->thresholds[i], pipeline_data->thresholds[i]); + tesseract->SetImage((uchar*) pipeline_data->thresholds[i].data, + pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height, + pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1()); - for (int j = 0; j < charRegions.size(); j++) + for (int j = 0; j < pipeline_data->charRegions.size(); j++) { - Rect expandedRegion = expandRect( charRegions[j], 2, 2, thresholds[i].cols, thresholds[i].rows) ; + Rect expandedRegion = expandRect( pipeline_data->charRegions[j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ; tesseract->SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height); tesseract->Recognize(NULL); diff --git a/src/openalpr/ocr.h b/src/openalpr/ocr.h index 0c9c080..444cb81 100644 --- a/src/openalpr/ocr.h +++ b/src/openalpr/ocr.h @@ -26,6 +26,7 @@ #include "utility.h" #include "postprocess.h" #include "config.h" +#include "pipeline_data.h" #include "constants.h" #include "opencv2/imgproc/imgproc.hpp" @@ -39,7 +40,7 @@ class OCR OCR(Config* config); virtual ~OCR(); - void performOCR(std::vector thresholds, std::vector charRegions); + void performOCR(PipelineData* pipeline_data); PostProcess* postProcessor; //string recognizedText; diff --git a/src/openalpr/pipeline_data.cpp b/src/openalpr/pipeline_data.cpp new file mode 100644 index 0000000..6aec753 --- /dev/null +++ b/src/openalpr/pipeline_data.cpp @@ -0,0 +1,29 @@ +#include "pipeline_data.h" + +using namespace cv; +using namespace std; + +PipelineData::PipelineData(Mat colorImage, Rect regionOfInterest, Config* config) +{ + this->colorImg = colorImage; + cvtColor(this->colorImg, this->grayImg, CV_BGR2GRAY); + + this->regionOfInterest = regionOfInterest; + this->config = config; + + plate_inverted = false; +} + +PipelineData::~PipelineData() +{ + clearThresholds(); +} + +void PipelineData::clearThresholds() +{ + for (int i = 0; i < thresholds.size(); i++) + { + thresholds[i].release(); + } + thresholds.clear(); +} diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h new file mode 100644 index 0000000..6e21438 --- /dev/null +++ b/src/openalpr/pipeline_data.h @@ -0,0 +1,51 @@ + +#ifndef OPENALPR_PIPELINEDATA_H +#define OPENALPR_PIPELINEDATA_H + +#include "opencv2/imgproc/imgproc.hpp" +#include "utility.h" +#include "config.h" + +class PipelineData +{ + + public: + PipelineData(cv::Mat colorImage, cv::Rect regionOfInterest, Config* config); + virtual ~PipelineData(); + + void clearThresholds(); + + // Inputs + Config* config; + + cv::Mat colorImg; + cv::Mat grayImg; + cv::Rect regionOfInterest; + + cv::Mat crop_gray; + cv::Mat plate_mask; + std::vector thresholds; + + std::vector plate_corners; + + + // Outputs + bool plate_inverted; + + std::string region_code; + float region_confidence; + + + float plate_area_confidence; + + std::vector charRegions; + + + + + // OCR + +}; + + +#endif // OPENALPR_PIPELINEDATA_H \ No newline at end of file diff --git a/src/openalpr/platecorners.cpp b/src/openalpr/platecorners.cpp index 453a930..e497192 100644 --- a/src/openalpr/platecorners.cpp +++ b/src/openalpr/platecorners.cpp @@ -40,10 +40,6 @@ PlateCorners::PlateCorners(Mat inputImage, PlateLines* plateLines, CharacterRegi Point bottomPoint = charRegion->getBottomLine().closestPointOnSegmentTo(topPoint); this->charHeight = distanceBetweenPoints(topPoint, bottomPoint); - //this->charHeight = distanceBetweenPoints(charRegion->getCharArea()[0], charRegion->getCharArea()[3]); - //this->charHeight = this->charHeight - 2; // Adjust since this height is a box around our char. - // Adjust the char height for the difference in size... - //this->charHeight = ((float) inputImage.size().height / (float) TEMPLATE_PLATE_HEIGHT) * this->charHeight; this->charAngle = angleBetweenPoints(charRegion->getCharArea()[0], charRegion->getCharArea()[1]); } @@ -136,6 +132,9 @@ void PlateCorners::scoreVerticals(int v1, int v2) float charHeightToPlateWidthRatio = config->plateWidthMM / config->charHeightMM; float idealPixelWidth = this->charHeight * (charHeightToPlateWidthRatio * 1.03); // Add 3% so we don't clip any characters + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + if (v1 == NO_LINE && v2 == NO_LINE) { //return; @@ -146,25 +145,33 @@ void PlateCorners::scoreVerticals(int v1, int v2) left = centerLine.getParallelLine(idealPixelWidth / 2); right = centerLine.getParallelLine(-1 * idealPixelWidth / 2 ); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL * 2; + confidenceDiff += 2; } else if (v1 != NO_LINE && v2 != NO_LINE) { - left = this->plateLines->verticalLines[v1]; - right = this->plateLines->verticalLines[v2]; + left = this->plateLines->verticalLines[v1].line; + right = this->plateLines->verticalLines[v2].line; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); } else if (v1 == NO_LINE && v2 != NO_LINE) { - right = this->plateLines->verticalLines[v2]; + right = this->plateLines->verticalLines[v2].line; left = right.getParallelLine(idealPixelWidth); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v2].confidence); } else if (v1 != NO_LINE && v2 == NO_LINE) { - left = this->plateLines->verticalLines[v1]; + left = this->plateLines->verticalLines[v1].line; right = left.getParallelLine(-1 * idealPixelWidth); - score += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_VERTICAL; + confidenceDiff += (1.0 - this->plateLines->verticalLines[v1].confidence); } + + score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + score += missingSegmentPenalty; // Make sure this line is to the left of our license plate letters if (left.isPointBelowLine(charRegion->getCharBoxLeft().midpoint()) == false) @@ -201,7 +208,7 @@ void PlateCorners::scoreVerticals(int v1, int v2) float plateDistance = abs(idealPixelWidth - distanceBetweenPoints(leftMidLinePoint, rightMidLinePoint)); - score += plateDistance * SCORING_VERTICALDISTANCE_WEIGHT; + score += plateDistance * SCORING_DISTANCE_WEIGHT_VERTICAL; if (score < this->bestVerticalScore) { @@ -215,7 +222,13 @@ void PlateCorners::scoreVerticals(int v1, int v2) cout << "xx xx Score: Left= " << left.str() << endl; cout << "xx xx Score: Right= " << right.str() << endl; + cout << "Vertical breakdown Score:" << endl; + + cout << " -- Missing Segment Score: " << missingSegmentPenalty << " -- Weight (1.0)" << endl; + scorecomponent = missingSegmentPenalty ; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Boxiness Score: " << verticalAngleDiff << " -- Weight (" << SCORING_BOXINESS_WEIGHT << ")" << endl; scorecomponent = verticalAngleDiff * SCORING_BOXINESS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; @@ -224,10 +237,14 @@ void PlateCorners::scoreVerticals(int v1, int v2) scorecomponent = distanceFromEdge * SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; - cout << " -- Distance Score: " << plateDistance << " -- Weight (" << SCORING_VERTICALDISTANCE_WEIGHT << ")" << endl; - scorecomponent = plateDistance * SCORING_VERTICALDISTANCE_WEIGHT; + cout << " -- Distance Score: " << plateDistance << " -- Weight (" << SCORING_DISTANCE_WEIGHT_VERTICAL << ")" << endl; + scorecomponent = plateDistance * SCORING_DISTANCE_WEIGHT_VERTICAL; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; - + + cout << " -- Plate line confidence Score: " << confidenceDiff << " -- Weight (" << SCORING_LINE_CONFIDENCE_WEIGHT << ")" << endl; + scorecomponent = confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Score: " << score << endl; } @@ -251,6 +268,9 @@ void PlateCorners::scoreHorizontals(int h1, int h2) float charHeightToPlateHeightRatio = config->plateHeightMM / config->charHeightMM; float idealPixelHeight = this->charHeight * charHeightToPlateHeightRatio; + float confidenceDiff = 0; + float missingSegmentPenalty = 0; + if (h1 == NO_LINE && h2 == NO_LINE) { // return; @@ -261,25 +281,33 @@ void PlateCorners::scoreHorizontals(int h1, int h2) top = centerLine.getParallelLine(idealPixelHeight / 2); bottom = centerLine.getParallelLine(-1 * idealPixelHeight / 2 ); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL * 2; + confidenceDiff += 2; } else if (h1 != NO_LINE && h2 != NO_LINE) { - top = this->plateLines->horizontalLines[h1]; - bottom = this->plateLines->horizontalLines[h2]; + top = this->plateLines->horizontalLines[h1].line; + bottom = this->plateLines->horizontalLines[h2].line; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); } else if (h1 == NO_LINE && h2 != NO_LINE) { - bottom = this->plateLines->horizontalLines[h2]; + bottom = this->plateLines->horizontalLines[h2].line; top = bottom.getParallelLine(idealPixelHeight); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h2].confidence); } else if (h1 != NO_LINE && h2 == NO_LINE) { - top = this->plateLines->horizontalLines[h1]; + top = this->plateLines->horizontalLines[h1].line; bottom = top.getParallelLine(-1 * idealPixelHeight); - score += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + missingSegmentPenalty += SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL; + confidenceDiff += (1.0 - this->plateLines->horizontalLines[h1].confidence); } + + score += confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + score += missingSegmentPenalty; // Make sure this line is above our license plate letters if (top.isPointBelowLine(charRegion->getCharBoxTop().midpoint()) == false) @@ -373,7 +401,13 @@ void PlateCorners::scoreHorizontals(int h1, int h2) cout << "xx xx Score: Top= " << top.str() << endl; cout << "xx xx Score: Bottom= " << bottom.str() << endl; + cout << "Horizontal breakdown Score:" << endl; + + cout << " -- Missing Segment Score: " << missingSegmentPenalty << " -- Weight (1.0)" << endl; + scorecomponent = missingSegmentPenalty ; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Boxiness Score: " << horizontalAngleDiff << " -- Weight (" << SCORING_BOXINESS_WEIGHT << ")" << endl; scorecomponent = horizontalAngleDiff * SCORING_BOXINESS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; @@ -390,6 +424,10 @@ void PlateCorners::scoreHorizontals(int h1, int h2) scorecomponent = charanglediff * SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT; cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Plate line confidence Score: " << confidenceDiff << " -- Weight (" << SCORING_LINE_CONFIDENCE_WEIGHT << ")" << endl; + scorecomponent = confidenceDiff * SCORING_LINE_CONFIDENCE_WEIGHT; + cout << " -- -- Score: " << scorecomponent << " = " << scorecomponent / score * 100 << "% of score" << endl; + cout << " -- Score: " << score << endl; } this->bestHorizontalScore = score; diff --git a/src/openalpr/platecorners.h b/src/openalpr/platecorners.h index fe9d32e..c0bf693 100644 --- a/src/openalpr/platecorners.h +++ b/src/openalpr/platecorners.h @@ -30,13 +30,16 @@ #define NO_LINE -1 #define SCORING_MISSING_SEGMENT_PENALTY_VERTICAL 10 -#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 15 +#define SCORING_MISSING_SEGMENT_PENALTY_HORIZONTAL 1 #define SCORING_BOXINESS_WEIGHT 0.8 #define SCORING_PLATEHEIGHT_WEIGHT 2.2 -#define SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT 0.05 +#define SCORING_TOP_BOTTOM_SPACE_VS_CHARHEIGHT_WEIGHT 0.10 #define SCORING_ANGLE_MATCHES_LPCHARS_WEIGHT 1.1 -#define SCORING_VERTICALDISTANCE_WEIGHT 0.1 + +#define SCORING_DISTANCE_WEIGHT_VERTICAL 0.04 + +#define SCORING_LINE_CONFIDENCE_WEIGHT 18.0 #define SCORING_VERTICALDISTANCE_FROMEDGE_WEIGHT 0.05 diff --git a/src/openalpr/platelines.cpp b/src/openalpr/platelines.cpp index 10e13fc..a61e23d 100644 --- a/src/openalpr/platelines.cpp +++ b/src/openalpr/platelines.cpp @@ -22,6 +22,9 @@ using namespace cv; using namespace std; +const float MIN_CONFIDENCE = 0.3; + + PlateLines::PlateLines(Config* config) { this->config = config; @@ -67,20 +70,20 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float // Create a mask that is dilated based on the detected characters vector > polygons; - polygons.push_back(charRegion->charAnalysis->charArea); + polygons.push_back(charRegion->getCharArea()); Mat mask = Mat::zeros(inputImage.size(), CV_8U); fillPoly(mask, polygons, Scalar(255,255,255)); - dilate(mask, mask, element); + dilate(mask, mask, getStructuringElement( 1, Size( 1 + 1, 2*1+1 ), Point( 1, 1 ) )); bitwise_not(mask, mask); // AND canny edges with the character mask bitwise_and(edges, mask, edges); - vector hlines = this->getLines(edges, sensitivity, false); - vector vlines = this->getLines(edges, sensitivity, true); + vector hlines = this->getLines(edges, sensitivity, false); + vector vlines = this->getLines(edges, sensitivity, true); for (int i = 0; i < hlines.size(); i++) this->horizontalLines.push_back(hlines[i]); for (int i = 0; i < vlines.size(); i++) @@ -98,12 +101,12 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float for( size_t i = 0; i < this->horizontalLines.size(); i++ ) { - line( debugImgHoriz, this->horizontalLines[i].p1, this->horizontalLines[i].p2, Scalar(0,0,255), 1, CV_AA); + line( debugImgHoriz, this->horizontalLines[i].line.p1, this->horizontalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } for( size_t i = 0; i < this->verticalLines.size(); i++ ) { - line( debugImgVert, this->verticalLines[i].p1, this->verticalLines[i].p2, Scalar(0,0,255), 1, CV_AA); + line( debugImgVert, this->verticalLines[i].line.p1, this->verticalLines[i].line.p2, Scalar(0,0,255), 1, CV_AA); } vector images; @@ -126,7 +129,7 @@ void PlateLines::processImage(Mat inputImage, CharacterRegion* charRegion, float -vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) +vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, bool vertical) { if (this->debug) cout << "PlateLines::getLines" << endl; @@ -135,7 +138,7 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, static int VERTICAL_SENSITIVITY = config->plateLinesSensitivityVertical; vector allLines; - vector filteredLines; + vector filteredLines; int sensitivity; if (vertical) @@ -176,7 +179,11 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, LineSegment bottom(0, edges.rows, edges.cols, edges.rows); Point p1 = line.intersection(bottom); Point p2 = line.intersection(top); - filteredLines.push_back(LineSegment(p1.x, p1.y, p2.x, p2.y)); + + PlateLine plateLine; + plateLine.line = LineSegment(p1.x, p1.y, p2.x, p2.y); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); } } else @@ -196,7 +203,10 @@ vector PlateLines::getLines(Mat edges, float sensitivityMultiplier, int newY1 = line.getPointAt(0); int newY2 = line.getPointAt(edges.cols); - filteredLines.push_back(LineSegment(0, newY1, edges.cols, newY2)); + PlateLine plateLine; + plateLine.line = LineSegment(0, newY1, edges.cols, newY2); + plateLine.confidence = (1.0 - MIN_CONFIDENCE) * ((float) (allLines.size() - i)) / ((float)allLines.size()) + MIN_CONFIDENCE; + filteredLines.push_back(plateLine); } } } diff --git a/src/openalpr/platelines.h b/src/openalpr/platelines.h index c0fab97..76b9b33 100644 --- a/src/openalpr/platelines.h +++ b/src/openalpr/platelines.h @@ -27,6 +27,11 @@ #include "config.h" #include "characterregion.h" +struct PlateLine +{ + LineSegment line; + float confidence; +}; class PlateLines { @@ -37,18 +42,19 @@ class PlateLines void processImage(cv::Mat img, CharacterRegion* charRegion, float sensitivity=1.0); - std::vector horizontalLines; - std::vector verticalLines; + std::vector horizontalLines; + std::vector verticalLines; std::vector winningCorners; private: + Config* config; bool debug; cv::Mat customGrayscaleConversion(cv::Mat src); void findLines(cv::Mat inputImage); - std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); + std::vector getLines(cv::Mat edges, float sensitivityMultiplier, bool vertical); }; #endif // OPENALPR_PLATELINES_H diff --git a/src/openalpr/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp similarity index 95% rename from src/openalpr/charactersegmenter.cpp rename to src/openalpr/segmentation/charactersegmenter.cpp index 9a07c95..62a99fe 100644 --- a/src/openalpr/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -22,9 +22,10 @@ using namespace cv; using namespace std; -CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* config) +CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data) { - this->config = config; + this->pipeline_data = pipeline_data; + this->config = pipeline_data->config; this->confidence = 0; @@ -36,20 +37,18 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con timespec startTime; getTime(&startTime); - Mat img_gray(img.size(), CV_8U); - cvtColor( img, img_gray, CV_BGR2GRAY ); - medianBlur(img_gray, img_gray, 3); + medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3); - if (invertedColors) - bitwise_not(img_gray, img_gray); + if (pipeline_data->plate_inverted) + bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray); - charAnalysis = new CharacterAnalysis(img_gray, config); + charAnalysis = new CharacterAnalysis(pipeline_data); charAnalysis->analyze(); if (this->config->debugCharSegmenter) { - displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(charAnalysis->thresholds, CV_8U, 3)); + displayImage(config, "CharacterSegmenter Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3)); } if (this->config->debugCharSegmenter && charAnalysis->linePolygon.size() > 0) @@ -107,7 +106,7 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con float avgCharWidth = median(charWidths.data(), charWidths.size()); float avgCharHeight = median(charHeights.data(), charHeights.size()); - removeSmallContours(charAnalysis->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight); + removeSmallContours(pipeline_data->thresholds, charAnalysis->allContours, avgCharWidth, avgCharHeight); // Do the histogram analysis to figure out char regions @@ -119,11 +118,11 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con vector allBoxes; for (int i = 0; i < charAnalysis->allContours.size(); i++) { - Mat histogramMask = Mat::zeros(charAnalysis->thresholds[i].size(), CV_8U); + Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U); fillConvexPoly(histogramMask, charAnalysis->linePolygon.data(), charAnalysis->linePolygon.size(), Scalar(255,255,255)); - VerticalHistogram vertHistogram(charAnalysis->thresholds[i], histogramMask); + VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask); if (this->config->debugCharSegmenter) { @@ -173,16 +172,16 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con } //ColorFilter colorFilter(img, charAnalysis->getCharacterMask()); - vector candidateBoxes = getBestCharBoxes(charAnalysis->thresholds[0], allBoxes, medianCharWidth); + vector candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], allBoxes, medianCharWidth); if (this->config->debugCharSegmenter) { // Setup the dashboard images to show the cleaning filters - for (int i = 0; i < charAnalysis->thresholds.size(); i++) + for (int i = 0; i < pipeline_data->thresholds.size(); i++) { - Mat cleanImg = Mat::zeros(charAnalysis->thresholds[i].size(), charAnalysis->thresholds[i].type()); - Mat boxMask = getCharBoxMask(charAnalysis->thresholds[i], candidateBoxes); - charAnalysis->thresholds[i].copyTo(cleanImg); + Mat cleanImg = Mat::zeros(pipeline_data->thresholds[i].size(), pipeline_data->thresholds[i].type()); + Mat boxMask = getCharBoxMask(pipeline_data->thresholds[i], candidateBoxes); + pipeline_data->thresholds[i].copyTo(cleanImg); bitwise_and(cleanImg, boxMask, cleanImg); cvtColor(cleanImg, cleanImg, CV_GRAY2BGR); @@ -194,19 +193,19 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con getTime(&startTime); - filterEdgeBoxes(charAnalysis->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); + filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight); - candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes); + candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth); - cleanCharRegions(charAnalysis->thresholds, candidateBoxes); - cleanMostlyFullBoxes(charAnalysis->thresholds, candidateBoxes); + cleanCharRegions(pipeline_data->thresholds, candidateBoxes); + cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes); //cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes); - candidateBoxes = filterMostlyEmptyBoxes(charAnalysis->thresholds, candidateBoxes); - this->characters = candidateBoxes; + candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); + pipeline_data->charRegions = candidateBoxes; if (config->debugTiming) { @@ -217,7 +216,7 @@ CharacterSegmenter::CharacterSegmenter(Mat img, bool invertedColors, Config* con if (this->config->debugCharSegmenter) { - Mat imgDash = drawImageDashboard(charAnalysis->thresholds, CV_8U, 3); + Mat imgDash = drawImageDashboard(pipeline_data->thresholds, CV_8U, 3); displayImage(config, "Segmentation after cleaning", imgDash); Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2); @@ -491,7 +490,7 @@ vector CharacterSegmenter::combineCloseBoxes( vector charBoxes, floa newCharBoxes.push_back(bigRect); if (this->config->debugCharSegmenter) { - for (int z = 0; z < charAnalysis->thresholds.size(); z++) + for (int z = 0; z < pipeline_data->thresholds.size(); z++) { Point center(bigRect.x + bigRect.width / 2, bigRect.y + bigRect.height / 2); RotatedRect rrect(center, Size2f(bigRect.width, bigRect.height + (bigRect.height / 2)), 0); @@ -1141,7 +1140,4 @@ Mat CharacterSegmenter::getCharBoxMask(Mat img_threshold, vector charBoxes return mask; } -vector CharacterSegmenter::getThresholds() -{ - return charAnalysis->thresholds; -} + diff --git a/src/openalpr/charactersegmenter.h b/src/openalpr/segmentation/charactersegmenter.h similarity index 96% rename from src/openalpr/charactersegmenter.h rename to src/openalpr/segmentation/charactersegmenter.h index bea17b2..12373fd 100644 --- a/src/openalpr/charactersegmenter.h +++ b/src/openalpr/segmentation/charactersegmenter.h @@ -44,17 +44,16 @@ class CharacterSegmenter { public: - CharacterSegmenter(cv::Mat img, bool invertedColors, Config* config); + CharacterSegmenter(PipelineData* pipeline_data); virtual ~CharacterSegmenter(); - std::vector characters; int confidence; - std::vector getThresholds(); private: Config* config; - + PipelineData* pipeline_data; + CharacterAnalysis* charAnalysis; LineSegment top; diff --git a/src/openalpr/segmentation/segment.cpp b/src/openalpr/segmentation/segment.cpp new file mode 100644 index 0000000..1650cc5 --- /dev/null +++ b/src/openalpr/segmentation/segment.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2013 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "segment.h" + +Segment::Segment(cv::Rect newSegment) +{ + this->segment = newSegment; +} + +Segment::~Segment() +{ + +} + +bool Segment::matches(cv::Rect newSegment) +{ + // Compare the two segments with a given leniency + const float WIDTH_LENIENCY_MIN = 0.25; + const float WIDTH_LENIENCY_MAX = 0.20; + + float left_min = segment.x - (((float)segment.width) * WIDTH_LENIENCY_MIN); + float left_max = segment.x + (((float)segment.width) * WIDTH_LENIENCY_MAX); + float right_min = (segment.x + segment.width) - (((float)segment.width) * WIDTH_LENIENCY_MIN); + float right_max = (segment.x + segment.width) + (((float)segment.width) * WIDTH_LENIENCY_MAX); + + int newSegRight = newSegment.x + newSegment.width; + if (newSegment.x >= left_min && newSegment.x <= left_max && + newSegRight >= right_min && newSegRight <= right_max) + return true; + + return false; +} + diff --git a/src/openalpr/segmentation/segment.h b/src/openalpr/segmentation/segment.h new file mode 100644 index 0000000..7743ab6 --- /dev/null +++ b/src/openalpr/segmentation/segment.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2013 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_SEGMENT_H +#define OPENALPR_SEGMENT_H + +#include +#include + +#include "opencv2/imgproc/imgproc.hpp" + +class Segment +{ + + public: + Segment(cv::Rect newSegment); + virtual ~Segment(); + + cv::Rect segment; + + bool matches(cv::Rect newSegment); + +}; + +#endif // OPENALPR_SEGMENTATIONGROUP_H diff --git a/src/openalpr/segmentation/segmentationgroup.cpp b/src/openalpr/segmentation/segmentationgroup.cpp new file mode 100644 index 0000000..a146364 --- /dev/null +++ b/src/openalpr/segmentation/segmentationgroup.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2013 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#include "segmentationgroup.h" + +SegmentationGroup::SegmentationGroup() +{ + +} + +SegmentationGroup::~SegmentationGroup() +{ + +} + +void SegmentationGroup::add(int segmentID) +{ + this->segmentIDs.push_back(segmentID); +} + +bool SegmentationGroup::equals(SegmentationGroup otherGroup) +{ + if (segmentIDs.size() != otherGroup.segmentIDs.size()) + return false; + + for (int i = 0; i < segmentIDs.size(); i++) + { + if (otherGroup.segmentIDs[i] != segmentIDs[i]) + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/openalpr/segmentation/segmentationgroup.h b/src/openalpr/segmentation/segmentationgroup.h new file mode 100644 index 0000000..d9cc633 --- /dev/null +++ b/src/openalpr/segmentation/segmentationgroup.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2013 New Designs Unlimited, LLC + * Opensource Automated License Plate Recognition [http://www.openalpr.com] + * + * This file is part of OpenAlpr. + * + * OpenAlpr is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License + * version 3 as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . +*/ + +#ifndef OPENALPR_SEGMENTATIONGROUP_H +#define OPENALPR_SEGMENTATIONGROUP_H + +#include +#include + +#include "opencv2/imgproc/imgproc.hpp" + +#include "segment.h" + + +class SegmentationGroup +{ + + public: + SegmentationGroup(); + virtual ~SegmentationGroup(); + + void add(int segmentID); + + std::vector segmentIDs; + + bool equals(SegmentationGroup otherGroup); + + + private: + float strength; // Debuggin purposes -- how many threshold segmentations match this one perfectly + +}; + +#endif // OPENALPR_SEGMENTATIONGROUP_H diff --git a/src/openalpr/verticalhistogram.cpp b/src/openalpr/segmentation/verticalhistogram.cpp similarity index 100% rename from src/openalpr/verticalhistogram.cpp rename to src/openalpr/segmentation/verticalhistogram.cpp diff --git a/src/openalpr/verticalhistogram.h b/src/openalpr/segmentation/verticalhistogram.h similarity index 100% rename from src/openalpr/verticalhistogram.h rename to src/openalpr/segmentation/verticalhistogram.h diff --git a/src/openalpr/stateidentifier.cpp b/src/openalpr/stateidentifier.cpp index dde5611..0a84c91 100644 --- a/src/openalpr/stateidentifier.cpp +++ b/src/openalpr/stateidentifier.cpp @@ -43,26 +43,18 @@ StateIdentifier::~StateIdentifier() delete featureMatcher; } -int StateIdentifier::recognize(Mat img, Rect frame, char* stateCode) -{ - Mat croppedImage = Mat(img, frame); - return this->recognize(croppedImage, stateCode); -} -// Attempts to recognize the plate. Returns a confidence level. Updates teh "stateCode" variable -// with the value of the country/state -int StateIdentifier::recognize(Mat img, char* stateCode) +// Attempts to recognize the plate. Returns a confidence level. Updates the region code and confidence +// If region is found, returns true. +bool StateIdentifier::recognize(PipelineData* pipeline_data) { timespec startTime; getTime(&startTime); - cvtColor(img, img, CV_BGR2GRAY); + Mat plateImg = Mat(pipeline_data->grayImg, pipeline_data->regionOfInterest); - resize(img, img, getSizeMaintainingAspect(img, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); + resize(plateImg, plateImg, getSizeMaintainingAspect(plateImg, config->stateIdImageWidthPx, config->stateIdimageHeightPx)); - Mat plateImg(img.size(), img.type()); - //plateImg = equalizeBrightness(img); - img.copyTo(plateImg); Mat debugImg(plateImg.size(), plateImg.type()); plateImg.copyTo(debugImg); @@ -87,7 +79,11 @@ int StateIdentifier::recognize(Mat img, char* stateCode) if (result.haswinner == false) return 0; - strcpy(stateCode, result.winner.c_str()); - - return result.confidence; + pipeline_data->region_code = result.winner; + pipeline_data->region_confidence = result.confidence; + + if (result.confidence >= 10) + return true; + + return false; } diff --git a/src/openalpr/stateidentifier.h b/src/openalpr/stateidentifier.h index e0c8c0b..3092a67 100644 --- a/src/openalpr/stateidentifier.h +++ b/src/openalpr/stateidentifier.h @@ -25,6 +25,7 @@ #include "featurematcher.h" #include "utility.h" #include "config.h" +#include "pipeline_data.h" class StateIdentifier { @@ -33,8 +34,7 @@ class StateIdentifier StateIdentifier(Config* config); virtual ~StateIdentifier(); - int recognize(cv::Mat img, cv::Rect frame, char* stateCode); - int recognize(cv::Mat img, char* stateCode); + bool recognize(PipelineData* pipeline_data); //int confidence; diff --git a/src/openalpr/utility.cpp b/src/openalpr/utility.cpp index cc40e9c..036317e 100644 --- a/src/openalpr/utility.cpp +++ b/src/openalpr/utility.cpp @@ -109,7 +109,7 @@ void displayImage(Config* config, string windowName, cv::Mat frame) vector produceThresholds(const Mat img_gray, Config* config) { - const int THRESHOLD_COUNT = 4; + const int THRESHOLD_COUNT = 3; //Mat img_equalized = equalizeBrightness(img_gray); timespec startTime; @@ -145,9 +145,9 @@ vector produceThresholds(const Mat img_gray, Config* config) k = 1; NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); bitwise_not(thresholds[i-1], thresholds[i-1]); - k=2; - NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); - bitwise_not(thresholds[i-1], thresholds[i-1]); + //k=2; + //NiblackSauvolaWolfJolion (img_gray, thresholds[i++], SAUVOLA, 12, 12, 0.18 * k); + //bitwise_not(thresholds[i-1], thresholds[i-1]); if (config->debugTiming) {