diff --git a/src/main.cpp b/src/main.cpp index cbdeb15..d4aab0c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -337,7 +337,11 @@ bool detectandshow( Alpr* alpr, cv::Mat frame, std::string region, bool writeJso for (int k = 0; k < results.plates[i].topNPlates.size(); k++) { - std::cout << " - " << results.plates[i].topNPlates[k].characters << "\t confidence: " << results.plates[i].topNPlates[k].overall_confidence; + // Replace the multiline newline character with a dash + std::string no_newline = results.plates[i].topNPlates[k].characters; + std::replace(no_newline.begin(), no_newline.end(), '\n','-'); + + std::cout << " - " << no_newline << "\t confidence: " << results.plates[i].topNPlates[k].overall_confidence; if (templatePattern.size() > 0 || results.plates[i].regionConfidence > 0) std::cout << "\t pattern_match: " << results.plates[i].topNPlates[k].matches_template; diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 943b058..dc5478c 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -347,9 +347,11 @@ namespace alpr for (unsigned int c_idx = 0; c_idx < ppResults[pp].letter_details.size(); c_idx++) { AlprChar character_details; - character_details.character = ppResults[pp].letter_details[c_idx].letter; - character_details.confidence = ppResults[pp].letter_details[c_idx].totalscore; - cv::Rect char_rect = pipeline_data.charRegions[ppResults[pp].letter_details[c_idx].charposition]; + Letter l = ppResults[pp].letter_details[c_idx]; + + character_details.character = l.letter; + character_details.confidence = l.totalscore; + cv::Rect char_rect = pipeline_data.charRegionsFlat[l.charposition]; std::vector charpoints = getCharacterPoints(char_rect, charTransformMatrix ); for (int cpt = 0; cpt < 4; cpt++) character_details.corners[cpt] = charpoints[cpt]; diff --git a/src/openalpr/ocr.cpp b/src/openalpr/ocr.cpp index e05fcec..1bb1ccd 100644 --- a/src/openalpr/ocr.cpp +++ b/src/openalpr/ocr.cpp @@ -61,8 +61,15 @@ namespace alpr postProcessor.clear(); // Don't waste time on OCR processing if it is impossible to get sufficient characters - if (pipeline_data->charRegions.size() < config->postProcessMinCharacters) + int total_char_spaces = 0; + for (unsigned int i = 0; i < pipeline_data->charRegions.size(); i++) + total_char_spaces += pipeline_data->charRegions[i].size(); + if (total_char_spaces < config->postProcessMinCharacters) + { + pipeline_data->disqualify_reason = "Insufficient character boxes detected. No OCR performed."; + pipeline_data->disqualified = true; return; + } for (unsigned int i = 0; i < pipeline_data->thresholds.size(); i++) { @@ -72,62 +79,68 @@ namespace alpr pipeline_data->thresholds[i].size().width, pipeline_data->thresholds[i].size().height, pipeline_data->thresholds[i].channels(), pipeline_data->thresholds[i].step1()); - for (unsigned int j = 0; j < pipeline_data->charRegions.size(); j++) + int absolute_charpos = 0; + for (unsigned int line_idx = 0; line_idx < pipeline_data->charRegions.size(); line_idx++) { - Rect expandedRegion = expandRect( pipeline_data->charRegions[j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ; - - tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height); - tesseract.Recognize(NULL); - - tesseract::ResultIterator* ri = tesseract.GetIterator(); - tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL; - do + for (unsigned int j = 0; j < pipeline_data->charRegions[line_idx].size(); j++) { - const char* symbol = ri->GetUTF8Text(level); - float conf = ri->Confidence(level); - - bool dontcare; - int fontindex = 0; - int pointsize = 0; - const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex); + Rect expandedRegion = expandRect( pipeline_data->charRegions[line_idx][j], 2, 2, pipeline_data->thresholds[i].cols, pipeline_data->thresholds[i].rows) ; - // Ignore NULL pointers, spaces, and characters that are way too small to be valid - if(symbol != 0 && symbol[0] != SPACE_CHAR_CODE && pointsize >= config->ocrMinFontSize) + tesseract.SetRectangle(expandedRegion.x, expandedRegion.y, expandedRegion.width, expandedRegion.height); + tesseract.Recognize(NULL); + + tesseract::ResultIterator* ri = tesseract.GetIterator(); + tesseract::PageIteratorLevel level = tesseract::RIL_SYMBOL; + do { - postProcessor.addLetter(string(symbol), j, conf); + const char* symbol = ri->GetUTF8Text(level); + float conf = ri->Confidence(level); + + bool dontcare; + int fontindex = 0; + int pointsize = 0; + const char* fontName = ri->WordFontAttributes(&dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &dontcare, &pointsize, &fontindex); + + // Ignore NULL pointers, spaces, and characters that are way too small to be valid + if(symbol != 0 && symbol[0] != SPACE_CHAR_CODE && pointsize >= config->ocrMinFontSize) + { + postProcessor.addLetter(string(symbol), line_idx, absolute_charpos, conf); + + if (this->config->debugOcr) + printf("charpos%d line%d: threshold %d: symbol %s, conf: %f font: %s (index %d) size %dpx", absolute_charpos, line_idx, i, symbol, conf, fontName, fontindex, pointsize); + + bool indent = false; + tesseract::ChoiceIterator ci(*ri); + do + { + const char* choice = ci.GetUTF8Text(); + + postProcessor.addLetter(string(choice), line_idx, absolute_charpos, ci.Confidence()); + + if (this->config->debugOcr) + { + if (indent) printf("\t\t "); + printf("\t- "); + printf("%s conf: %f\n", choice, ci.Confidence()); + } + + indent = true; + } + while(ci.Next()); + + } if (this->config->debugOcr) - printf("charpos%d: threshold %d: symbol %s, conf: %f font: %s (index %d) size %dpx", j, i, symbol, conf, fontName, fontindex, pointsize); + printf("---------------------------------------------\n"); - bool indent = false; - tesseract::ChoiceIterator ci(*ri); - do - { - const char* choice = ci.GetUTF8Text(); - - postProcessor.addLetter(string(choice), j, ci.Confidence()); - - //letterScores.addScore(*choice, j, ci.Confidence() - MIN_CONFIDENCE); - if (this->config->debugOcr) - { - if (indent) printf("\t\t "); - printf("\t- "); - printf("%s conf: %f\n", choice, ci.Confidence()); - } - - indent = true; - } - while(ci.Next()); + delete[] symbol; } + while((ri->Next(level))); - if (this->config->debugOcr) - printf("---------------------------------------------\n"); - - delete[] symbol; + delete ri; + + absolute_charpos++; } - while((ri->Next(level))); - - delete ri; } } diff --git a/src/openalpr/pipeline_data.h b/src/openalpr/pipeline_data.h index 4fca140..bc7846a 100644 --- a/src/openalpr/pipeline_data.h +++ b/src/openalpr/pipeline_data.h @@ -58,7 +58,12 @@ namespace alpr ScoreKeeper confidence_weights; - std::vector charRegions; + // Boxes around characters in cropped image + // Each row in a multiline plate is an entry in the vector + std::vector > charRegions; + + // Same data, just not broken down by line + std::vector charRegionsFlat; diff --git a/src/openalpr/postprocess/postprocess.cpp b/src/openalpr/postprocess/postprocess.cpp index 1a66b33..df90399 100644 --- a/src/openalpr/postprocess/postprocess.cpp +++ b/src/openalpr/postprocess/postprocess.cpp @@ -72,17 +72,17 @@ namespace alpr } } - void PostProcess::addLetter(string letter, int charposition, float score) + void PostProcess::addLetter(string letter, int line_index, int charposition, float score) { if (score < config->postProcessMinConfidence) return; - insertLetter(letter, charposition, score); + insertLetter(letter, line_index, charposition, score); if (score < config->postProcessConfidenceSkipLevel) { float adjustedScore = abs(config->postProcessConfidenceSkipLevel - score) + config->postProcessMinConfidence; - insertLetter(SKIP_CHAR, charposition, adjustedScore ); + insertLetter(SKIP_CHAR, line_index, charposition, adjustedScore ); } //if (letter == '0') @@ -91,7 +91,7 @@ namespace alpr //} } - void PostProcess::insertLetter(string letter, int charposition, float score) + void PostProcess::insertLetter(string letter, int line_index, int charposition, float score) { score = score - config->postProcessMinConfidence; @@ -108,6 +108,7 @@ namespace alpr for (int i = 0; i < letters[charposition].size(); i++) { if (letters[charposition][i].letter == letter && + letters[charposition][i].line_index == line_index && letters[charposition][i].charposition == charposition) { existingIndex = i; @@ -118,6 +119,7 @@ namespace alpr if (existingIndex == -1) { Letter newLetter; + newLetter.line_index = line_index; newLetter.charposition = charposition; newLetter.letter = letter; newLetter.occurences = 1; @@ -179,7 +181,7 @@ namespace alpr for (int i = 0; i < letters.size(); i++) { for (int j = 0; j < letters[i].size(); j++) - cout << "PostProcess Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl; + cout << "PostProcess Line " << letters[i][j].line_index << " Letter: " << letters[i][j].charposition << " " << letters[i][j].letter << " -- score: " << letters[i][j].totalscore << " -- occurences: " << letters[i][j].occurences << endl; } } @@ -340,6 +342,7 @@ namespace alpr possibility.matchesTemplate = false; int plate_char_length = 0; + int last_line = 0; for (int i = 0; i < letters.size(); i++) { if (letters[i].size() == 0) @@ -347,6 +350,13 @@ namespace alpr Letter letter = letters[i][letterIndices[i]]; + // Add a "\n" on new lines + if (letter.line_index != last_line) + { + possibility.letters = possibility.letters + "\n"; + } + last_line = letter.line_index; + if (letter.letter != SKIP_CHAR) { possibility.letters = possibility.letters + letter.letter; diff --git a/src/openalpr/postprocess/postprocess.h b/src/openalpr/postprocess/postprocess.h index 055dace..69a9300 100644 --- a/src/openalpr/postprocess/postprocess.h +++ b/src/openalpr/postprocess/postprocess.h @@ -40,6 +40,7 @@ namespace alpr struct Letter { std::string letter; + int line_index; int charposition; float totalscore; int occurences; @@ -62,7 +63,7 @@ namespace alpr PostProcess(Config* config); ~PostProcess(); - void addLetter(std::string letter, int charposition, float score); + void addLetter(std::string letter, int line_index, int charposition, float score); void clear(); void analyze(std::string templateregion, int topn); @@ -82,7 +83,7 @@ namespace alpr void findAllPermutations(std::string templateregion, int topn); bool analyzePermutation(std::vector letterIndices, std::string templateregion, int topn); - void insertLetter(std::string letter, int charPosition, float score); + void insertLetter(std::string letter, int line_index, int charPosition, float score); std::map > rules; diff --git a/src/openalpr/segmentation/charactersegmenter.cpp b/src/openalpr/segmentation/charactersegmenter.cpp index b4c734c..a78d923 100644 --- a/src/openalpr/segmentation/charactersegmenter.cpp +++ b/src/openalpr/segmentation/charactersegmenter.cpp @@ -158,8 +158,9 @@ namespace alpr candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes); - for (unsigned int cbox = 0; cbox < candidateBoxes.size(); cbox++) - pipeline_data->charRegions.push_back(candidateBoxes[cbox]); + pipeline_data->charRegions.push_back(candidateBoxes); + for (unsigned int cboxidx = 0; cboxidx < candidateBoxes.size(); cboxidx++) + pipeline_data->charRegionsFlat.push_back(candidateBoxes[cboxidx]); if (config->debugTiming) { @@ -181,7 +182,13 @@ namespace alpr } } - cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions); + vector all_regions_combined; + for (unsigned int lidx = 0; lidx < pipeline_data->charRegions.size(); lidx++) + { + for (unsigned int boxidx = 0; boxidx < pipeline_data->charRegions[lidx].size(); boxidx++) + all_regions_combined.push_back(pipeline_data->charRegions[lidx][boxidx]); + } + cleanCharRegions(pipeline_data->thresholds, all_regions_combined); if (config->debugTiming) {