From 8972e6373dfb34d65eea6e01a8b7742dc0391bc5 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Mon, 7 Mar 2016 12:59:30 -0500 Subject: [PATCH] Added high-contrast detection and edge finder --- src/openalpr/edges/edgefinder.cpp | 231 +++++++++++++++++++++++++++--- src/openalpr/edges/edgefinder.h | 8 +- 2 files changed, 217 insertions(+), 22 deletions(-) diff --git a/src/openalpr/edges/edgefinder.cpp b/src/openalpr/edges/edgefinder.cpp index 4e7348f..9c3dd53 100644 --- a/src/openalpr/edges/edgefinder.cpp +++ b/src/openalpr/edges/edgefinder.cpp @@ -40,18 +40,46 @@ namespace alpr std::vector EdgeFinder::findEdgeCorners() { - return normalDetection(); + bool high_contrast = is_high_contrast(pipeline_data->crop_gray); + + vector returnPoints; + + if (high_contrast) + { + // Try a high-contrast pass first. If it doesn't return anything, try a normal detection + returnPoints = detection(true); + } + + if (!high_contrast || returnPoints.size() == 0) + { + returnPoints = detection(false); + } + + return returnPoints; + } - vector EdgeFinder::normalDetection() { - + std::vector EdgeFinder::detection(bool high_contrast) { TextLineCollection tlc(pipeline_data->textLines); vector corners; + // If the character segment is especially small, just expand the existing box // If it's a nice, long segment, then guess the correct box based on character height/position - if (tlc.longerSegment.length > tlc.charHeight * 3) + if (high_contrast) + { + int expandX = (int) ((float) pipeline_data->crop_gray.cols) * 0.5f; + int expandY = (int) ((float) pipeline_data->crop_gray.rows) * 0.5f; + int w = pipeline_data->crop_gray.cols; + int h = pipeline_data->crop_gray.rows; + + corners.push_back(Point(-1 * expandX, -1 * expandY)); + corners.push_back(Point(expandX + w, -1 * expandY)); + corners.push_back(Point(expandX + w, expandY + h)); + corners.push_back(Point(-1 * expandX, expandY + h)); + } + else if (tlc.longerSegment.length > tlc.charHeight * 3) { float charHeightToPlateWidthRatio = pipeline_data->config->plateWidthMM / pipeline_data->config->avgCharHeightMM; @@ -92,10 +120,7 @@ namespace alpr corners.push_back(Point(expandX + w, expandY + h)); corners.push_back(Point(-1 * expandX, expandY + h)); - // for (int i = 0; i < 4; i++) - // { - // std::cout << "CORNER: " << corners[i].x << " - " << corners[i].y << std::endl; - // } + } // Re-crop an image (from the original image) using the new coordinates @@ -124,14 +149,16 @@ namespace alpr newLines.push_back(TextLine(textAreaRemapped, linePolygonRemapped, newCrop.size())); } - // Find the PlateLines for this crop - PlateLines plateLines(pipeline_data); - plateLines.processImage(newCrop, newLines, 1.05); - - // Get the best corners - PlateCorners cornerFinder(newCrop, &plateLines, pipeline_data, newLines); - vector smallPlateCorners = cornerFinder.findPlateCorners(); - + vector smallPlateCorners; + + if (high_contrast) + { + smallPlateCorners = highContrastDetection(newCrop, newLines); + } + else + { + smallPlateCorners = normalDetection(newCrop, newLines); + } // Transform the best corner points back to the original image std::vector imgArea; @@ -141,14 +168,178 @@ namespace alpr imgArea.push_back(Point2f(0, newCrop.rows)); Mat newCropTransmtx = imgTransform.getTransformationMatrix(imgArea, remappedCorners); - vector cornersInOriginalImg = imgTransform.remapSmallPointstoCrop(smallPlateCorners, newCropTransmtx); + vector cornersInOriginalImg; + + if (smallPlateCorners.size() > 0) + cornersInOriginalImg = imgTransform.remapSmallPointstoCrop(smallPlateCorners, newCropTransmtx); return cornersInOriginalImg; } - vector EdgeFinder::highContrastDetection() { - vector cornersInOriginalImg; - return cornersInOriginalImg; + + vector EdgeFinder::normalDetection(Mat newCrop, vector newLines) + { + // Find the PlateLines for this crop + PlateLines plateLines(pipeline_data); + plateLines.processImage(newCrop, newLines, 1.05); + + // Get the best corners + PlateCorners cornerFinder(newCrop, &plateLines, pipeline_data, newLines); + return cornerFinder.findPlateCorners(); + } + + vector EdgeFinder::highContrastDetection(Mat newCrop, vector newLines) { + + + vector smallPlateCorners; + + if (pipeline_data->config->debugGeneral) + std::cout << "Performing high-contrast edge detection" << std::endl; + + // Do a morphology operation. Find the biggest white rectangle that fit most of the char area. + + int morph_size = 3; + Mat closureElement = getStructuringElement( 2, // 0 Rect, 1 cross, 2 ellipse + Size( 2 * morph_size + 1, 2* morph_size + 1 ), + Point( morph_size, morph_size ) ); + + morphologyEx(newCrop, newCrop, MORPH_CLOSE, closureElement); + morphologyEx(newCrop, newCrop, MORPH_OPEN, closureElement); + + Mat thresholded_crop; + threshold(newCrop, thresholded_crop, 80, 255, cv::THRESH_OTSU); + + vector > contours; + findContours(thresholded_crop, contours, RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE ); + + float MIN_AREA = 0.05 * newCrop.cols * newCrop.rows; + for (unsigned int i = 0; i < contours.size(); i++) + { + if (contourArea(contours[i]) < MIN_AREA) + continue; + + vector smoothedPoints; + approxPolyDP(contours[i], smoothedPoints, 1, true); + + RotatedRect rrect = minAreaRect(smoothedPoints); + + Point2f rect_points[4]; + rrect.points(rect_points); + + vector sorted_polygon_points = sortPolygonPoints(rect_points, newCrop.size()); + + float polygon_width = (distanceBetweenPoints(sorted_polygon_points[0], sorted_polygon_points[1]) + + distanceBetweenPoints(sorted_polygon_points[3], sorted_polygon_points[2])) / 2; + float polygon_height = (distanceBetweenPoints(sorted_polygon_points[2], sorted_polygon_points[1]) + + distanceBetweenPoints(sorted_polygon_points[3], sorted_polygon_points[0])) / 2; + // If it touches the edges, disqualify it + + // Create an inner rect, and test to make sure all the points are within it + int x_offset = newCrop.cols * 0.1; + int y_offset = newCrop.rows * 0.1; + Rect insideRect(Point(x_offset, y_offset), Point(newCrop.cols - x_offset, newCrop.rows - y_offset)); + + bool isoutside = false; + for (unsigned int ptidx = 0; ptidx < sorted_polygon_points.size(); ptidx++) + { + if (!insideRect.contains(sorted_polygon_points[ptidx])) + isoutside = true; + } + if (isoutside) + continue; + + // If the center is not centered, disqualify it + float MAX_CLOSENESS_TO_EDGE_PERCENT = 0.2; + if (rrect.center.x < (newCrop.cols * MAX_CLOSENESS_TO_EDGE_PERCENT) || + rrect.center.x > (newCrop.cols - (newCrop.cols * MAX_CLOSENESS_TO_EDGE_PERCENT)) || + rrect.center.y < (newCrop.rows * MAX_CLOSENESS_TO_EDGE_PERCENT) || + rrect.center.y > (newCrop.rows - (newCrop.rows * MAX_CLOSENESS_TO_EDGE_PERCENT))) + { + continue; + } + + // Make sure the aspect ratio is somewhat close to a license plate. + float aspect_ratio = polygon_width / polygon_height; + float ideal_aspect_ratio = pipeline_data->config->plateWidthMM / pipeline_data->config->plateHeightMM; + + float ratio = ideal_aspect_ratio / aspect_ratio; + + if (ratio > 2 || ratio < 0.5) + continue; + + // Make sure that the text line(s) are contained within it + + Rect rect_cover = rrect.boundingRect(); + for (unsigned int linenum = 0; linenum < newLines.size(); linenum++) + { + for (unsigned int r = 0; r < newLines[linenum].textArea.size(); r++) + { + if (!rect_cover.contains(newLines[linenum].textArea[r])) + { + isoutside = true; + break; + } + } + } + if (isoutside) + continue; + + + for (int ridx = 0; ridx < 4; ridx++) + smallPlateCorners.push_back(sorted_polygon_points[ridx]); + + + } + + + + return smallPlateCorners; + } + + + + + bool EdgeFinder::is_high_contrast(const cv::Mat crop) { + + int stride = 2; + + int rows = crop.rows; + int cols = crop.cols / stride; + + timespec startTime; + getTimeMonotonic(&startTime); + // Calculate pixel intensity + float avg_intensity = 0; + for (unsigned int y = 0; y < rows; y++) + { + for (unsigned int x = 0; x < crop.cols; x += stride) + { + avg_intensity = avg_intensity + crop.at(y,x); + } + } + avg_intensity = avg_intensity / (float) (rows * cols * 255); + + // Calculate RMS contrast + float contrast = 0; + for (unsigned int y = 0; y < rows; y++) + { + for (unsigned int x = 0; x < crop.cols; x += stride) + { + contrast += pow( ((crop.at(y,x) / 255.0) - avg_intensity), 2.0f); + } + } + contrast /= ((float) rows) * ((float)cols); + + contrast = pow(contrast, 0.5f); + + if (pipeline_data->config->debugTiming) + { + timespec endTime; + getTimeMonotonic(&endTime); + cout << "High Contrast Detection Time: " << diffclock(startTime, endTime) << "ms." << endl; + } + + return contrast > 0.3; } } \ No newline at end of file diff --git a/src/openalpr/edges/edgefinder.h b/src/openalpr/edges/edgefinder.h index a7dacb0..6feaf7e 100644 --- a/src/openalpr/edges/edgefinder.h +++ b/src/openalpr/edges/edgefinder.h @@ -40,9 +40,13 @@ namespace alpr private: PipelineData* pipeline_data; - std::vector highContrastDetection(); - std::vector normalDetection(); + std::vector detection(bool high_contrast); + std::vector highContrastDetection(cv::Mat newCrop, std::vector newLines); + std::vector normalDetection(cv::Mat newCrop, std::vector newLines); + + + bool is_high_contrast(const cv::Mat crop); }; }