From 91091ba71e0061913a1f15c9337b36cd80ed4b64 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 13 Mar 2016 14:02:16 -0400 Subject: [PATCH] Added "analysis_count" config setting --- config/openalpr.conf.in | 4 + src/openalpr/alpr_impl.cpp | 21 ++- src/openalpr/config.cpp | 2 + src/openalpr/config.h | 2 + src/openalpr/prewarp.cpp | 21 ++- src/openalpr/prewarp.h | 5 +- src/openalpr/result_aggregator.cpp | 249 ++++++++++++++++++++++++++--- src/openalpr/result_aggregator.h | 35 +++- 8 files changed, 310 insertions(+), 29 deletions(-) diff --git a/config/openalpr.conf.in b/config/openalpr.conf.in index 71d32fe..fd7b64d 100644 --- a/config/openalpr.conf.in +++ b/config/openalpr.conf.in @@ -49,6 +49,10 @@ must_match_pattern = 0 ; Bypasses plate detection. If this is set to 1, the library assumes that each region provided is a likely plate area. skip_detection = 0 +; OpenALPR can scan the same image multiple times with different randomization. Setting this to a value larger than +; 1 may increase accuracy, but will increase processing time linearly (e.g., analysis_count = 3 is 3x slower) +analysis_count = 1 + ; OpenALPR detects high-contrast plate crops and uses an alternative edge detection technique. Setting this to 0.0 ; would classify ALL images as high-contrast, setting it to 1.0 would classify no images as high-contrast. contrast_detection_threshold = 0.3 diff --git a/src/openalpr/alpr_impl.cpp b/src/openalpr/alpr_impl.cpp index 62c34b9..ce6f659 100644 --- a/src/openalpr/alpr_impl.cpp +++ b/src/openalpr/alpr_impl.cpp @@ -125,22 +125,33 @@ namespace alpr // Iterate through each country provided (typically just one) // and aggregate the results if necessary - ResultAggregator aggregator; + ResultAggregator country_aggregator(MERGE_PICK_BEST, topN, config); for (unsigned int i = 0; i < config->loaded_countries.size(); i++) { if (config->debugGeneral) cout << "Analyzing: " << config->loaded_countries[i] << endl; config->setCountry(config->loaded_countries[i]); - AlprFullDetails sub_results = analyzeSingleCountry(img, grayImg, warpedRegionsOfInterest); - + + // Reapply analysis for each multiple analysis value set in the config, + // make a minor imperceptible tweak to the input image each time + ResultAggregator iter_aggregator(MERGE_COMBINE, topN, config); + for (unsigned int iteration = 0; iteration < 2; iteration++) + { + Mat iteration_image = iter_aggregator.applyImperceptibleChange(grayImg, iteration); + //drawAndWait(iteration_image); + AlprFullDetails iter_results = analyzeSingleCountry(img, iteration_image, warpedRegionsOfInterest); + iter_aggregator.addResults(iter_results); + } + + AlprFullDetails sub_results = iter_aggregator.getAggregateResults(); sub_results.results.epoch_time = start_time; sub_results.results.img_width = img.cols; sub_results.results.img_height = img.rows; - aggregator.addResults(sub_results); + country_aggregator.addResults(sub_results); } - response = aggregator.getAggregateResults(); + response = country_aggregator.getAggregateResults(); timespec endTime; getTimeMonotonic(&endTime); diff --git a/src/openalpr/config.cpp b/src/openalpr/config.cpp index 83f0c3c..612d293 100644 --- a/src/openalpr/config.cpp +++ b/src/openalpr/config.cpp @@ -189,6 +189,8 @@ namespace alpr skipDetection = getBoolean(ini, "", "skip_detection", false); + analysis_count = getInt(ini, "", "analysis_count", 1); + prewarp = getString(ini, "", "prewarp", ""); maxPlateAngleDegrees = getInt(ini, "", "max_plate_angle_degrees", 15); diff --git a/src/openalpr/config.h b/src/openalpr/config.h index a0523c3..5faf926 100644 --- a/src/openalpr/config.h +++ b/src/openalpr/config.h @@ -61,6 +61,8 @@ namespace alpr bool skipDetection; + int analysis_count; + bool auto_invert; bool always_invert; diff --git a/src/openalpr/prewarp.cpp b/src/openalpr/prewarp.cpp index 503feb7..ece8783 100644 --- a/src/openalpr/prewarp.cpp +++ b/src/openalpr/prewarp.cpp @@ -75,6 +75,7 @@ namespace alpr { stringstream ss(prewarp_config.substr(first_comma + 1, prewarp_config.length())); + float w, h, rotationx, rotationy, rotationz, panX, panY, stretchX, dist; ss >> w; ss.ignore(); ss >> h; @@ -93,7 +94,7 @@ namespace alpr ss.ignore(); // Ignore comma ss >> panY; - this->valid = true; + setTransform(w, h, rotationx, rotationy, rotationz, panX, panY, stretchX, dist); } } @@ -110,6 +111,20 @@ namespace alpr PreWarp::~PreWarp() { } + void PreWarp::setTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist) + { + this->w = w; + this->h = h; + this->rotationx = rotationx; + this->rotationy = rotationy; + this->rotationz = rotationz; + this->panX = panX; + this->panY = panY; + this->stretchX = stretchX; + this->dist = dist; + + this->valid = true; + } cv::Mat PreWarp::warpImage(Mat image) { if (!this->valid) @@ -129,7 +144,7 @@ namespace alpr float py = panY / height_ratio; - transform = findTransform(image.cols, image.rows, rx, ry, rotationz, px, py, stretchX, dist); + transform = getTransform(image.cols, image.rows, rx, ry, rotationz, px, py, stretchX, dist); Mat warped_image; @@ -208,7 +223,7 @@ namespace alpr } } - cv::Mat PreWarp::findTransform(float w, float h, + cv::Mat PreWarp::getTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist) { diff --git a/src/openalpr/prewarp.h b/src/openalpr/prewarp.h index d5af164..5dcf217 100644 --- a/src/openalpr/prewarp.h +++ b/src/openalpr/prewarp.h @@ -42,15 +42,18 @@ namespace alpr std::vector projectRects(std::vector rects, int maxWidth, int maxHeight, bool inverse); void projectPlateRegions(std::vector& plateRegions, int maxWidth, int maxHeight, bool inverse); + void setTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist); + bool valid; private: Config* config; cv::Mat transform; + cv::Mat getTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist); + float w, h, rotationx, rotationy, rotationz, stretchX, dist, panX, panY; - cv::Mat findTransform(float w, float h, float rotationx, float rotationy, float rotationz, float panX, float panY, float stretchX, float dist); }; } diff --git a/src/openalpr/result_aggregator.cpp b/src/openalpr/result_aggregator.cpp index 38fe800..050d603 100644 --- a/src/openalpr/result_aggregator.cpp +++ b/src/openalpr/result_aggregator.cpp @@ -25,13 +25,16 @@ using namespace cv; namespace alpr { - ResultAggregator::ResultAggregator() + ResultAggregator::ResultAggregator(ResultMergeStrategy merge_strategy, int topn, Config* config) { - + this->prewarp = new PreWarp(config); + this->merge_strategy = merge_strategy; + this->topn = topn; + this->config = config; } ResultAggregator::~ResultAggregator() { - + delete prewarp; } @@ -39,7 +42,46 @@ namespace alpr { all_results.push_back(full_results); } + + cv::Mat ResultAggregator::applyImperceptibleChange(cv::Mat image, int index) { + + const float WIDTH_HEIGHT = 600; + const float NO_MOVE_WIDTH_DIST = 1.0; + const float NO_PAN_VAL = 0; + float step = 0.000035; + + // Don't warp the first indexed image + if (index == 0) + return image; + + // Use 3 bits to figure out which one is on. Multiply by the modulus of 8 + // 000, 001, 010, 011, 100, 101, 110, 111 + // if 101, then x_rotation and z_rotation are on. + if (index % 8 == 0) + { + // Do something special for the 0s, so they don't repeat + index--; + step = step / 1.5; + } + int multiplier = (index / 8) + 1; + int bitwise_on = index % 8; + float x_rotation = ((bitwise_on & 1) == 1) * multiplier * step; + float y_rotation = ((bitwise_on & 2) == 2) * multiplier * step; + float z_rotation = ((bitwise_on & 4) == 4) * multiplier * step; + + //cout << "Iteration: " << index << ": " << x_rotation << ", " << y_rotation << ", " << z_rotation << endl; + + prewarp->setTransform(WIDTH_HEIGHT, WIDTH_HEIGHT, x_rotation, y_rotation, z_rotation, + NO_PAN_VAL, NO_PAN_VAL, NO_MOVE_WIDTH_DIST, NO_MOVE_WIDTH_DIST); + + return prewarp->warpImage(image); + } + + bool compareScore(const std::pair& firstElem, const std::pair& secondElem) { + return firstElem.first > secondElem.first; + } + AlprFullDetails ResultAggregator::getAggregateResults() { assert(all_results.size() > 0); @@ -68,28 +110,199 @@ namespace alpr vector > clusters = findClusters(); - // Assume we have multiple results, one cluster for each unique train data (e.g., eu, eu2) - - // Now for each cluster of plates, pick the best one - for (unsigned int i = 0; i < clusters.size(); i++) + if (merge_strategy == MERGE_PICK_BEST) { - float best_confidence = 0; - int best_index = 0; - for (unsigned int k = 0; k < clusters[i].size(); k++) - { - if (clusters[i][k].bestPlate.overall_confidence > best_confidence) - { - best_confidence = clusters[i][k].bestPlate.overall_confidence; - best_index = k; - } - } + // Assume we have multiple results, one cluster for each unique train data (e.g., eu, eu2) - response.results.plates.push_back(clusters[i][best_index]); + // Now for each cluster of plates, pick the best one + for (unsigned int i = 0; i < clusters.size(); i++) + { + float best_confidence = 0; + int best_index = 0; + for (unsigned int k = 0; k < clusters[i].size(); k++) + { + if (clusters[i][k].bestPlate.overall_confidence > best_confidence) + { + best_confidence = clusters[i][k].bestPlate.overall_confidence; + best_index = k; + } + } + + response.results.plates.push_back(clusters[i][best_index]); + } + } + else if (merge_strategy == MERGE_COMBINE) + { + // Each cluster is the same plate, just analyzed from a slightly different + // perspective. Merge them together and score them as if they are one + + const float MIN_CONFIDENCE = 75; + + + // Factor in the position of the plate in the topN list, the confidence, and the template match status + // First loop is for clusters of possible plates. If they're in separate clusters, they don't get combined, + // since they are likely separate plates in the same image + for (unsigned int unique_plate_idx = 0; unique_plate_idx < clusters.size(); unique_plate_idx++) + { + std::map score_hash; + + // Second loop is for separate plate results for the same plate + for (unsigned int i = 0; i < clusters[unique_plate_idx].size(); i++) + { + // Third loop is the individual topN results for a single plate result + for (unsigned int j = 0; j < clusters[unique_plate_idx][i].topNPlates.size() && j < topn; j++) + { + AlprPlate plateCandidate = clusters[unique_plate_idx][i].topNPlates[j]; + + if (plateCandidate.overall_confidence < MIN_CONFIDENCE) + continue; + + float score = (plateCandidate.overall_confidence - 60) * 4; + + // Add a bonus for matching the template + if (plateCandidate.matches_template) + score += 150; + + // Add a bonus the higher the plate is to the #1 position + // and how frequently it appears there + float position_score_max_bonus = 65; + float frequency_modifier = ((float) position_score_max_bonus) / topn; + score += position_score_max_bonus - (j * frequency_modifier); + + + if (score_hash.find(plateCandidate.characters) == score_hash.end()) + { + ResultPlateScore newentry; + newentry.plate = plateCandidate; + newentry.score_total = 0; + newentry.count = 0; + score_hash[plateCandidate.characters] = newentry; + } + + score_hash[plateCandidate.characters].score_total += score; + score_hash[plateCandidate.characters].count += 1; + // Use the best confidence value for a particular candidate + if (plateCandidate.overall_confidence > score_hash[plateCandidate.characters].plate.overall_confidence) + score_hash[plateCandidate.characters].plate.overall_confidence = plateCandidate.overall_confidence; + } + } + + // There is a big list of results that have scores. Sort them by top score + std::vector > sorted_results; + std::map::iterator iter; + for (iter = score_hash.begin(); iter != score_hash.end(); iter++) { + std::pair r; + r.second = iter->second; + r.first = iter->second.score_total; + sorted_results.push_back(r); + } + + std::sort(sorted_results.begin(), sorted_results.end(), compareScore); + + // output the sorted list for debugging: + if (config->debugGeneral) + { + cout << "Result Aggregator Scores: " << endl; + cout << " " << std::setw(14) << "Plate Num" + << std::setw(15) << "Score" + << std::setw(10) << "Count" + << std::setw(10) << "Best conf (%)" + << endl; + + for (int r_idx = 0; r_idx < sorted_results.size(); r_idx++) + { + cout << " " << std::setw(14) << sorted_results[r_idx].second.plate.characters + << std::setw(15) << sorted_results[r_idx].second.score_total + << std::setw(10) << sorted_results[r_idx].second.count + << std::setw(10) << sorted_results[r_idx].second.plate.overall_confidence + << endl; + + } + } + + // Figure out the best region for this cluster + ResultRegionScore regionResults = findBestRegion(clusters[unique_plate_idx]); + + AlprPlateResult firstResult = clusters[unique_plate_idx][0]; + AlprPlateResult copyResult; + copyResult.bestPlate = sorted_results[0].second.plate; + copyResult.plate_index = firstResult.plate_index; + copyResult.region = regionResults.region; + copyResult.regionConfidence = regionResults.confidence; + copyResult.processing_time_ms = firstResult.processing_time_ms; + copyResult.requested_topn = firstResult.requested_topn; + for (int p_idx = 0; p_idx < 4; p_idx++) + copyResult.plate_points[p_idx] = firstResult.plate_points[p_idx]; + + for (int i = 0; i < sorted_results.size(); i++) + { + if (i >= topn) + break; + + copyResult.topNPlates.push_back(sorted_results[i].second.plate); + } + + response.results.plates.push_back(copyResult); + + } } return response; } + + ResultRegionScore ResultAggregator::findBestRegion(std::vector cluster) { + const float MIN_REGION_CONFIDENCE = 60; + + std::map score_hash; + std::map score_count; + int max_topn = 10; + + ResultRegionScore response; + response.confidence = 0; + response.region = ""; + + for (unsigned int i = 0; i < cluster.size(); i++) + { + AlprPlateResult plate = cluster[i]; + + if (plate.bestPlate.overall_confidence < MIN_REGION_CONFIDENCE ) + continue; + + float score = (float) plate.regionConfidence; + + if (score_hash.count(plate.region) == 0) + { + score_hash[plate.region] = 0; + score_count[plate.region] = 0; + } + + score_hash[plate.region] = score_hash[plate.region] + score; + score_count[plate.region] = score_count[plate.region] + 1; + } + + float best_score = -1; + std::string best_score_val = ""; + // Now we have a hash that contains all the scores. Iterate and find the best one and return it. + for(std::map::iterator hash_iter=score_hash.begin(); hash_iter!=score_hash.end(); ++hash_iter) { + if (hash_iter->second > best_score) + { + best_score = hash_iter->second; + best_score_val = hash_iter->first; + } + } + + if (best_score > 0) + { + response.confidence = best_score / score_count[best_score_val]; + response.region = best_score_val; + } + + return response; + + } + + // Searches all_plates to find overlapping plates // Returns an array containing "clusters" (overlapping plates) std::vector > ResultAggregator::findClusters() diff --git a/src/openalpr/result_aggregator.h b/src/openalpr/result_aggregator.h index 837cd1f..1026134 100644 --- a/src/openalpr/result_aggregator.h +++ b/src/openalpr/result_aggregator.h @@ -22,7 +22,7 @@ #include "alpr_impl.h" - +#include "prewarp.h" // Runs the analysis for multiple training sets, and aggregates the results into the best matches @@ -37,10 +37,30 @@ struct PlateShapeInfo namespace alpr { + enum ResultMergeStrategy + { + MERGE_COMBINE, // Used when running an analysis multiple times for accuracy improvement. Merges results together + MERGE_PICK_BEST // Used when analyzing multiple countries. Chooses results from one country or the other + }; + + struct ResultPlateScore + { + AlprPlate plate; + float score_total; + int count; + }; + + struct ResultRegionScore + { + std::string region; + float confidence; + + }; + class ResultAggregator { public: - ResultAggregator(); + ResultAggregator(ResultMergeStrategy merge_strategy, int topn, Config* config); virtual ~ResultAggregator(); @@ -48,11 +68,22 @@ namespace alpr AlprFullDetails getAggregateResults(); + cv::Mat applyImperceptibleChange(cv::Mat image, int index); + private: + + int topn; + PreWarp* prewarp; + Config* config; + std::vector all_results; PlateShapeInfo getShapeInfo(AlprPlateResult plate); + ResultMergeStrategy merge_strategy; + + ResultRegionScore findBestRegion(std::vector cluster); + std::vector > findClusters(); int overlaps(AlprPlateResult plate, std::vector > clusters); };