From e2a302d8f84d3d4b59ac3d0d16aa5b017b74ff64 Mon Sep 17 00:00:00 2001 From: psaintjust Date: Sun, 28 Jun 2015 10:31:33 -0400 Subject: [PATCH] Refactored post processing to improve accuracy of alternative results --- src/openalpr/alpr_impl.cpp | 47 ++-- src/openalpr/postprocess/postprocess.cpp | 279 +++++++++-------------- src/openalpr/postprocess/postprocess.h | 11 +- 3 files changed, 131 insertions(+), 206 deletions(-) diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 0d8c2c5..ff83ecb 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -194,36 +194,29 @@ namespace alpr for (unsigned int pp = 0; pp < ppResults.size(); pp++) { - if (pp >= topN) - break; - int plate_char_length = utf8::distance(ppResults[pp].letters.begin(), ppResults[pp].letters.end()); - if (plate_char_length >= config->postProcessMinCharacters && - plate_char_length <= config->postProcessMaxCharacters) + // Set our "best plate" match to either the first entry, or the first entry with a postprocessor template match + if (bestPlateIndex == 0 && ppResults[pp].matchesTemplate) + bestPlateIndex = plateResult.topNPlates.size(); + + AlprPlate aplate; + aplate.characters = ppResults[pp].letters; + aplate.overall_confidence = ppResults[pp].totalscore; + aplate.matches_template = ppResults[pp].matchesTemplate; + + // Grab detailed results for each character + for (unsigned int c_idx = 0; c_idx < ppResults[pp].letter_details.size(); c_idx++) { - // Set our "best plate" match to either the first entry, or the first entry with a postprocessor template match - if (bestPlateIndex == 0 && ppResults[pp].matchesTemplate) - bestPlateIndex = plateResult.topNPlates.size(); - - AlprPlate aplate; - aplate.characters = ppResults[pp].letters; - aplate.overall_confidence = ppResults[pp].totalscore; - aplate.matches_template = ppResults[pp].matchesTemplate; - - // Grab detailed results for each character - for (unsigned int c_idx = 0; c_idx < ppResults[pp].letter_details.size(); c_idx++) - { - AlprChar character_details; - character_details.character = ppResults[pp].letter_details[c_idx].letter; - character_details.confidence = ppResults[pp].letter_details[c_idx].totalscore; - cv::Rect char_rect = pipeline_data.charRegions[ppResults[pp].letter_details[c_idx].charposition]; - std::vector charpoints = getCharacterPoints(char_rect, &pipeline_data ); - for (int cpt = 0; cpt < 4; cpt++) - character_details.corners[cpt] = charpoints[cpt]; - aplate.character_details.push_back(character_details); - } - plateResult.topNPlates.push_back(aplate); + AlprChar character_details; + character_details.character = ppResults[pp].letter_details[c_idx].letter; + character_details.confidence = ppResults[pp].letter_details[c_idx].totalscore; + cv::Rect char_rect = pipeline_data.charRegions[ppResults[pp].letter_details[c_idx].charposition]; + std::vector charpoints = getCharacterPoints(char_rect, &pipeline_data ); + for (int cpt = 0; cpt < 4; cpt++) + character_details.corners[cpt] = charpoints[cpt]; + aplate.character_details.push_back(character_details); } + plateResult.topNPlates.push_back(aplate); } diff --git a/src/openalpr/postprocess/postprocess.cpp b/src/openalpr/postprocess/postprocess.cpp index 0b42b29..05a9c7a 100644 --- a/src/openalpr/postprocess/postprocess.cpp +++ b/src/openalpr/postprocess/postprocess.cpp @@ -23,7 +23,6 @@ using namespace std; namespace alpr { - PostProcess::PostProcess(Config* config) { @@ -143,6 +142,7 @@ namespace alpr unknownCharPositions.clear(); unknownCharPositions.resize(0); allPossibilities.clear(); + allPossibilitiesLetters.clear(); //allPossibilities.resize(0); bestChars = ""; @@ -183,68 +183,22 @@ namespace alpr } } - // Prune the letters based on the topN value. - // If our topN value is 3, for example, we can get rid of a lot of low scoring letters - // because it would be impossible for them to be a part of our topN results. - vector maxDepth = getMaxDepth(topn); + timespec permutationStartTime; + getTimeMonotonic(&permutationStartTime); - for (int i = 0; i < letters.size(); i++) - { - for (int k = letters[i].size() - 1; k > maxDepth[i]; k--) - { - letters[i].erase(letters[i].begin() + k); - } - } - - //getTopN(); - vector tmp; - findAllPermutations(tmp, 0, config->postProcessMaxSubstitutions); - - timespec sortStartTime; - getTimeMonotonic(&sortStartTime); - - int numelements = topn; - if (allPossibilities.size() < topn) - numelements = allPossibilities.size() - 1; - - partial_sort( allPossibilities.begin(), allPossibilities.begin() + numelements, allPossibilities.end(), wordCompare ); + findAllPermutations(templateregion, topn); if (config->debugTiming) { - timespec sortEndTime; - getTimeMonotonic(&sortEndTime); - cout << " -- PostProcess Sort Time: " << diffclock(sortStartTime, sortEndTime) << "ms." << endl; + timespec permutationEndTime; + getTimeMonotonic(&permutationEndTime); + cout << " -- PostProcess Permutation Time: " << diffclock(permutationStartTime, permutationEndTime) << "ms." << endl; } - matchesTemplate = false; - - if (templateregion != "") + if (allPossibilities.size() > 0) { - vector regionRules = rules[templateregion]; - for (int i = 0; i < allPossibilities.size(); i++) - { - for (int j = 0; j < regionRules.size(); j++) - { - allPossibilities[i].matchesTemplate = regionRules[j]->match(allPossibilities[i].letters); - if (allPossibilities[i].matchesTemplate) - { - allPossibilities[i].letters = regionRules[j]->filterSkips(allPossibilities[i].letters); - //bestChars = regionRules[j]->filterSkips(allPossibilities[i].letters); - matchesTemplate = true; - break; - } - } - - if (i >= topn - 1) - break; - //if (matchesTemplate || i >= TOP_N - 1) - //break; - } - } - - if (matchesTemplate) - { + bestChars = allPossibilities[0].letters; for (int z = 0; z < allPossibilities.size(); z++) { if (allPossibilities[z].matchesTemplate) @@ -253,15 +207,8 @@ namespace alpr break; } } - } - else - { - bestChars = allPossibilities[0].letters; - } - // Now adjust the confidence scores to a percentage value - if (allPossibilities.size() > 0) - { + // Now adjust the confidence scores to a percentage value float maxPercentScore = calculateMaxConfidenceScore(); float highestRelativeScore = (float) allPossibilities[0].totalscore; @@ -280,9 +227,6 @@ namespace alpr if (allPossibilities[i].letters == bestChars) cout << " <--- "; cout << endl; - - if (i >= topn - 1) - break; } cout << allPossibilities.size() << " total permutations" << endl; } @@ -325,128 +269,117 @@ namespace alpr return totalScore / ((float) numScores); } - // Finds the minimum number of letters to include in the recursive sorting algorithm. - // For example, if I have letters - // A-200 B-100 C-100 - // X-99 Y-95 Z-90 - // Q-55 R-80 - // And my topN value was 3, this would return: - // 0, 1, 1 - // Which represents: - // A-200 B-100 C-100 - // Y-95 Z-90 - vector PostProcess::getMaxDepth(int topn) - { - vector depth; - for (int i = 0; i < letters.size(); i++) - depth.push_back(0); - - int nextLeastDropCharPos = getNextLeastDrop(depth); - while (nextLeastDropCharPos != -1) - { - if (getPermutationCount(depth) >= topn) - break; - - depth[nextLeastDropCharPos] = depth[nextLeastDropCharPos] + 1; - - nextLeastDropCharPos = getNextLeastDrop(depth); - } - - return depth; - } - - int PostProcess::getPermutationCount(vector depth) - { - int permutationCount = 1; - for (int i = 0; i < depth.size(); i++) - { - permutationCount *= (depth[i] + 1); - } - - return permutationCount; - } - - int PostProcess::getNextLeastDrop(vector depth) - { - int nextLeastDropCharPos = -1; - float leastNextDrop = 99999999999; - - for (int i = 0; i < letters.size(); i++) - { - if (depth[i] + 1 >= letters[i].size()) - continue; - - float drop = letters[i][depth[i]].totalscore - letters[i][depth[i]+1].totalscore; - - if (drop < leastNextDrop) - { - nextLeastDropCharPos = i; - leastNextDrop = drop; - } - } - - return nextLeastDropCharPos; - } - const vector PostProcess::getResults() { return this->allPossibilities; } - void PostProcess::findAllPermutations(vector prevletters, int charPos, int substitutionsLeft) + struct PermutationCompare { + bool operator() (pair > &a, pair > &b) + { + return (a.first < b.first); + } + }; + + void PostProcess::findAllPermutations(string templateregion, int topn) { + + // use a priority queue to process permutations in highest scoring order + priority_queue >, vector > >, PermutationCompare> permutations; + set permutationHashes; + + // push the first word onto the queue + float totalscore = 0; + for (int i=0; i 0) + totalscore += letters[i][0].totalscore; + } + vector v(letters.size()); + permutations.push(make_pair(totalscore, v)); + + while (permutations.size() > 0) + { + // get the top permutation and analyze + pair > topPermutation = permutations.top(); + analyzePermutation(topPermutation.second, templateregion, topn); + permutations.pop(); + + if (allPossibilities.size() >= topn) { + break; + } + + // add child permutations to queue + for (int i=0; i= letters[i].size()) + continue; + + pair > childPermutation = topPermutation; + childPermutation.first -= letters[i][topPermutation.second[i]].totalscore - letters[i][topPermutation.second[i] + 1].totalscore; + childPermutation.second[i] += 1; + + // ignore permutations that have already been visited (assume that score is a good hash for permutation) + if (permutationHashes.end() != permutationHashes.find(childPermutation.first)) + continue; + + permutations.push(childPermutation); + permutationHashes.insert(childPermutation.first); + } + } + } + + void PostProcess::analyzePermutation(vector letterIndices, string templateregion, int topn) { - if (substitutionsLeft < 0) + PPResult possibility; + possibility.letters = ""; + possibility.totalscore = 0; + possibility.matchesTemplate = false; + int plate_char_length = 0; + + for (int i = 0; i < letters.size(); i++) + { + if (letters[i].size() == 0) + continue; + + Letter letter = letters[i][letterIndices[i]]; + + if (letter.letter != SKIP_CHAR) + { + possibility.letters = possibility.letters + letter.letter; + possibility.letter_details.push_back(letter); + plate_char_length += 1; + } + possibility.totalscore = possibility.totalscore + letter.totalscore; + } + + // ignore plates that don't fit the length requirements + if (plate_char_length < config->postProcessMinCharacters || + plate_char_length > config->postProcessMaxCharacters) return; - // Add my letter to the chain and recurse - for (int i = 0; i < letters[charPos].size(); i++) + // Apply templates + if (templateregion != "") { - if (charPos == letters.size() - 1) + vector regionRules = rules[templateregion]; + + for (int i = 0; i < regionRules.size(); i++) { - // Last letter, add the word - PPResult possibility; - possibility.letters = ""; - possibility.totalscore = 0; - possibility.matchesTemplate = false; - for (int z = 0; z < prevletters.size(); z++) + possibility.matchesTemplate = regionRules[i]->match(possibility.letters); + if (possibility.matchesTemplate) { - if (prevletters[z].letter != SKIP_CHAR) - { - possibility.letters = possibility.letters + prevletters[z].letter; - possibility.letter_details.push_back(prevletters[z]); - } - possibility.totalscore = possibility.totalscore + prevletters[z].totalscore; + possibility.letters = regionRules[i]->filterSkips(possibility.letters); + break; } - - if (letters[charPos][i].letter != SKIP_CHAR) - { - possibility.letters = possibility.letters + letters[charPos][i].letter; - possibility.letter_details.push_back(letters[charPos][i]); - } - possibility.totalscore = possibility.totalscore +letters[charPos][i].totalscore; - - allPossibilities.push_back(possibility); - } - else - { - prevletters.push_back(letters[charPos][i]); - - float scorePercentDiff = abs( letters[charPos][0].totalscore - letters[charPos][i].totalscore ) / letters[charPos][0].totalscore; - if (i != 0 && letters[charPos][i].letter != SKIP_CHAR && scorePercentDiff > 0.10f ) - findAllPermutations(prevletters, charPos + 1, substitutionsLeft - 1); - else - findAllPermutations(prevletters, charPos + 1, substitutionsLeft); - - prevletters.pop_back(); } } - if (letters[charPos].size() == 0) - { - // No letters for this char position... - // Just pass it along - findAllPermutations(prevletters, charPos + 1, substitutionsLeft); - } + // ignore duplicate words + if (allPossibilitiesLetters.end() != allPossibilitiesLetters.find(possibility.letters)) + return; + + allPossibilities.push_back(possibility); + allPossibilitiesLetters.insert(possibility.letters); } bool wordCompare( const PPResult &left, const PPResult &right ) diff --git a/src/openalpr/postprocess/postprocess.h b/src/openalpr/postprocess/postprocess.h index bc9196a..04ee040 100644 --- a/src/openalpr/postprocess/postprocess.h +++ b/src/openalpr/postprocess/postprocess.h @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include #include "config.h" @@ -76,7 +78,8 @@ namespace alpr private: Config* config; //void getTopN(); - void findAllPermutations(std::vector prevletters, int charPos, int substitutionsLeft); + void findAllPermutations(std::string templateregion, int topn); + void analyzePermutation(std::vector letterIndices, std::string templateregion, int topn); void insertLetter(std::string letter, int charPosition, float score); @@ -88,11 +91,7 @@ namespace alpr std::vector unknownCharPositions; std::vector allPossibilities; - - // Functions used to prune the list of letters (based on topn) to improve performance - std::vector getMaxDepth(int topn); - int getPermutationCount(std::vector depth); - int getNextLeastDrop(std::vector depth); + std::set allPossibilitiesLetters; }; }