// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "fastdeploy/vision/detection/contrib/rknpu2/postprocessor.h" #include "fastdeploy/vision/utils/utils.h" namespace fastdeploy { namespace vision { namespace detection { RKYOLOPostprocessor::RKYOLOPostprocessor() {} bool RKYOLOPostprocessor::Run(const std::vector& tensors, std::vector* results) { results->resize(tensors[0].shape[0]); for (int num = 0; num < tensors[0].shape[0]; ++num) { int validCount = 0; std::vector filterBoxes; std::vector boxesScore; std::vector classId; for (int i = 0; i < tensors.size(); i++) { auto tensor_shape = tensors[i].shape; auto skip_num = std::accumulate(tensor_shape.begin(), tensor_shape.end(), 1, std::multiplies()); int skip_address = num * skip_num; int stride = strides_[i]; int grid_h = height_ / stride; int grid_w = width_ / stride; int* anchor = &(anchors_.data()[i * 2 * anchor_per_branch_]); if (tensors[i].dtype == FDDataType::INT8 || tensors[i].dtype == FDDataType::UINT8) { auto quantization_info = tensors[i].GetQuantizationInfo(); validCount = validCount + ProcessInt8((int8_t*)tensors[i].Data() + skip_address, anchor, grid_h, grid_w, stride, filterBoxes, boxesScore, classId, conf_threshold_, quantization_info.first, quantization_info.second[0]); } else { FDERROR << "RKYOLO Only Support INT8 Model" << std::endl; } } // no object detect if (validCount <= 0) { FDINFO << "The number of object detect is 0." << std::endl; return true; } std::vector indexArray; for (int i = 0; i < validCount; ++i) { indexArray.push_back(i); } QuickSortIndiceInverse(boxesScore, 0, validCount - 1, indexArray); if (anchor_per_branch_ == 3) { NMS(validCount, filterBoxes, classId, indexArray, nms_threshold_, false); } else if (anchor_per_branch_ == 1) { NMS(validCount, filterBoxes, classId, indexArray, nms_threshold_, true); }else{ FDERROR << "anchor_per_branch_ only support 3 or 1." << std::endl; return false; } int last_count = 0; (*results)[num].Clear(); (*results)[num].Reserve(validCount); /* box valid detect target */ for (int i = 0; i < validCount; ++i) { if (indexArray[i] == -1 || boxesScore[i] < conf_threshold_ || last_count >= obj_num_bbox_max_size) { continue; } int n = indexArray[i]; float x1 = filterBoxes[n * 4 + 0]; float y1 = filterBoxes[n * 4 + 1]; float x2 = x1 + filterBoxes[n * 4 + 2]; float y2 = y1 + filterBoxes[n * 4 + 3]; int id = classId[n]; (*results)[num].boxes.emplace_back(std::array{ (float)((Clamp(x1, 0, width_) - pad_hw_values_[num][1] / 2) / scale_[num]), (float)((Clamp(y1, 0, height_) - pad_hw_values_[num][0] / 2) / scale_[num]), (float)((Clamp(x2, 0, width_) - pad_hw_values_[num][1] / 2) / scale_[num]), (float)((Clamp(y2, 0, height_) - pad_hw_values_[num][0] / 2) / scale_[0])}); (*results)[num].label_ids.push_back(id); (*results)[num].scores.push_back(boxesScore[i]); last_count++; } } return true; } int RKYOLOPostprocessor::ProcessInt8(int8_t* input, int* anchor, int grid_h, int grid_w, int stride, std::vector& boxes, std::vector& boxScores, std::vector& classId, float threshold, int32_t zp, float scale) { int validCount = 0; int grid_len = grid_h * grid_w; float thres = threshold; auto thres_i8 = QntF32ToAffine(thres, zp, scale); for (int a = 0; a < anchor_per_branch_; a++) { for (int i = 0; i < grid_h; i++) { for (int j = 0; j < grid_w; j++) { int8_t box_confidence = input[(prob_box_size * a + 4) * grid_len + i * grid_w + j]; if (box_confidence >= thres_i8) { int offset = (prob_box_size * a) * grid_len + i * grid_w + j; int8_t* in_ptr = input + offset; int8_t maxClassProbs = in_ptr[5 * grid_len]; int maxClassId = 0; for (int k = 1; k < obj_class_num; ++k) { int8_t prob = in_ptr[(5 + k) * grid_len]; if (prob > maxClassProbs) { maxClassId = k; maxClassProbs = prob; } } float box_conf_f32 = DeqntAffineToF32(box_confidence, zp, scale); float class_prob_f32 = DeqntAffineToF32(maxClassProbs, zp, scale); float limit_score = 0; if (anchor_per_branch_ == 1) { limit_score = box_conf_f32 * class_prob_f32; } else { limit_score = class_prob_f32; } //printf("limit score: %f\n", limit_score); if (limit_score > conf_threshold_) { float box_x, box_y, box_w, box_h; if (anchor_per_branch_ == 1) { box_x = DeqntAffineToF32(*in_ptr, zp, scale); box_y = DeqntAffineToF32(in_ptr[grid_len], zp, scale); box_w = DeqntAffineToF32(in_ptr[2 * grid_len], zp, scale); box_h = DeqntAffineToF32(in_ptr[3 * grid_len], zp, scale); box_w = exp(box_w) * stride; box_h = exp(box_h) * stride; } else { box_x = DeqntAffineToF32(*in_ptr, zp, scale) * 2.0 - 0.5; box_y = DeqntAffineToF32(in_ptr[grid_len], zp, scale) * 2.0 - 0.5; box_w = DeqntAffineToF32(in_ptr[2 * grid_len], zp, scale) * 2.0; box_h = DeqntAffineToF32(in_ptr[3 * grid_len], zp, scale) * 2.0; box_w = box_w * box_w; box_h = box_h * box_h; } box_x = (box_x + j) * (float)stride; box_y = (box_y + i) * (float)stride; box_w *= (float)anchor[a * 2]; box_h *= (float)anchor[a * 2 + 1]; box_x -= (box_w / 2.0); box_y -= (box_h / 2.0); boxes.push_back(box_x); boxes.push_back(box_y); boxes.push_back(box_w); boxes.push_back(box_h); boxScores.push_back(box_conf_f32 * class_prob_f32); classId.push_back(maxClassId); validCount++; } } } } } return validCount; } int RKYOLOPostprocessor::QuickSortIndiceInverse(std::vector& input, int left, int right, std::vector& indices) { float key; int key_index; int low = left; int high = right; if (left < right) { key_index = indices[left]; key = input[left]; while (low < high) { while (low < high && input[high] <= key) { high--; } input[low] = input[high]; indices[low] = indices[high]; while (low < high && input[low] >= key) { low++; } input[high] = input[low]; indices[high] = indices[low]; } input[low] = key; indices[low] = key_index; QuickSortIndiceInverse(input, left, low - 1, indices); QuickSortIndiceInverse(input, low + 1, right, indices); } return low; } } // namespace detection } // namespace vision } // namespace fastdeploy