apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-11-02 12:44:20 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

[RKNN2] Fix bugs (#851)

Browse Source 
* 修复picodet格式

* * 修正错误文档
* 修复rknpu2 backend后端的部分错误
* 更新pphumanseg example格式

* * 更新pphumanseg example格式

* * 更新picodet example格式

* * 更新scrfd example格式

* * 更新ppseg rknpu2 python example中的错误

* * 修复代码格式问题

* * 修复代码格式问题

* * 修复代码格式问题

* * 修复代码格式问题

* * 修复代码格式问题

* * 修复代码格式问题

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
Zheng_Bicheng
2022-12-12 15:37:31 +08:00
committed by GitHub
parent af4c28d4ae
commit 188dcedc02
14 changed files with 309 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fastdeploy/vision/detection/contrib/rknpu2/model.h
View File

@@ -35,7 +35,10 @@ class FASTDEPLOY_DECL RKYOLOV5 : public RKYOLO {
valid_cpu_backends = {};
valid_gpu_backends = {};
valid_rknpu_backends = {Backend::RKNPU2};
GetPostprocessor().SetModelType(ModelType::RKYOLOV5);
std::vector<int> anchors = {10, 13, 16, 30, 33, 23, 30, 61, 62,
45, 59, 119, 116, 90, 156, 198, 373, 326};
int anchor_per_branch_ = 3;
GetPostprocessor().SetAnchor(anchors, anchor_per_branch_);
}
virtual std::string ModelName() const { return "RKYOLOV5"; }
@@ -58,7 +61,10 @@ class FASTDEPLOY_DECL RKYOLOV7 : public RKYOLO {
valid_cpu_backends = {};
valid_gpu_backends = {};
valid_rknpu_backends = {Backend::RKNPU2};
GetPostprocessor().SetModelType(ModelType::RKYOLOV7);
std::vector<int> anchors = {12, 16, 19, 36, 40, 28, 36, 75, 76,
55, 72, 146, 142, 110, 192, 243, 459, 401};
int anchor_per_branch_ = 3;
GetPostprocessor().SetAnchor(anchors, anchor_per_branch_);
}
virtual std::string ModelName() const { return "RKYOLOV7"; }
@@ -81,7 +87,10 @@ class FASTDEPLOY_DECL RKYOLOX : public RKYOLO {
valid_cpu_backends = {};
valid_gpu_backends = {};
valid_rknpu_backends = {Backend::RKNPU2};
GetPostprocessor().SetModelType(ModelType::RKYOLOX);
std::vector<int> anchors = {10, 13, 16, 30, 33, 23, 30, 61, 62,
45, 59, 119, 116, 90, 156, 198, 373, 326};
int anchor_per_branch_ = 1;
GetPostprocessor().SetAnchor(anchors, anchor_per_branch_);
}
virtual std::string ModelName() const { return "RKYOLOV7"; }

64
fastdeploy/vision/detection/contrib/rknpu2/postprocessor.cc
View File

@@ -21,32 +21,8 @@ namespace detection {
RKYOLOPostprocessor::RKYOLOPostprocessor() {}
void RKYOLOPostprocessor::SetModelType(ModelType model_type) {
model_type_ = model_type;
if (model_type == RKYOLOV5) {
anchors_ = {10, 13, 16, 30, 33, 23, 30, 61, 62,
45, 59, 119, 116, 90, 156, 198, 373, 326};
anchor_per_branch_ = 3;
} else if (model_type == RKYOLOX) {
anchors_ = {10, 13, 16, 30, 33, 23, 30, 61, 62,
45, 59, 119, 116, 90, 156, 198, 373, 326};
anchor_per_branch_ = 1;
} else if (model_type == RKYOLOV7) {
anchors_ = {12, 16, 19, 36, 40, 28, 36, 75, 76,
55, 72, 146, 142, 110, 192, 243, 459, 401};
anchor_per_branch_ = 3;
} else {
return;
}
}
bool RKYOLOPostprocessor::Run(const std::vector<FDTensor>& tensors,
std::vector<DetectionResult>* results) {
if (model_type_ == ModelType::UNKNOWN) {
FDERROR << "RKYOLO Only Support YOLOV5,YOLOV7,YOLOX" << std::endl;
return false;
}
results->resize(tensors[0].shape[0]);
for (int num = 0; num < tensors[0].shape[0]; ++num) {
int validCount = 0;
@@ -62,13 +38,15 @@ bool RKYOLOPostprocessor::Run(const std::vector<FDTensor>& tensors,
int grid_h = height_ / stride;
int grid_w = width_ / stride;
int* anchor = &(anchors_.data()[i * 2 * anchor_per_branch_]);
if (tensors[i].dtype == FDDataType::INT8 || tensors[i].dtype == FDDataType::UINT8) {
if (tensors[i].dtype == FDDataType::INT8 ||
tensors[i].dtype == FDDataType::UINT8) {
auto quantization_info = tensors[i].GetQuantizationInfo();
validCount = validCount +
ProcessInt8((int8_t*)tensors[i].Data() + skip_address,
anchor, grid_h, grid_w, stride, filterBoxes,
boxesScore, classId, conf_threshold_,
quantization_info.first, quantization_info.second[0]);
validCount =
validCount + ProcessInt8((int8_t*)tensors[i].Data() + skip_address,
anchor, grid_h, grid_w, stride,
filterBoxes, boxesScore, classId,
conf_threshold_, quantization_info.first,
quantization_info.second[0]);
} else {
FDERROR << "RKYOLO Only Support INT8 Model" << std::endl;
}
@@ -87,10 +65,13 @@ bool RKYOLOPostprocessor::Run(const std::vector<FDTensor>& tensors,
QuickSortIndiceInverse(boxesScore, 0, validCount - 1, indexArray);
if (model_type_ == RKYOLOV5 || model_type_ == RKYOLOV7) {
if (anchor_per_branch_ == 3) {
NMS(validCount, filterBoxes, classId, indexArray, nms_threshold_, false);
} else if (model_type_ == RKYOLOX) {
} else if (anchor_per_branch_ == 1) {
NMS(validCount, filterBoxes, classId, indexArray, nms_threshold_, true);
}else{
FDERROR << "anchor_per_branch_ only support 3 or 1." << std::endl;
return false;
}
int last_count = 0;
@@ -110,19 +91,18 @@ bool RKYOLOPostprocessor::Run(const std::vector<FDTensor>& tensors,
float y2 = y1 + filterBoxes[n * 4 + 3];
int id = classId[n];
(*results)[num].boxes.emplace_back(std::array<float, 4>{
(float)((clamp(x1, 0, width_) - pad_hw_values_[num][1] / 2) /
(float)((Clamp(x1, 0, width_) - pad_hw_values_[num][1] / 2) /
scale_[num]),
(float)((clamp(y1, 0, height_) - pad_hw_values_[num][0] / 2) /
(float)((Clamp(y1, 0, height_) - pad_hw_values_[num][0] / 2) /
scale_[num]),
(float)((clamp(x2, 0, width_) - pad_hw_values_[num][1] / 2) /
(float)((Clamp(x2, 0, width_) - pad_hw_values_[num][1] / 2) /
scale_[num]),
(float)((clamp(y2, 0, height_) - pad_hw_values_[num][0] / 2) /
(float)((Clamp(y2, 0, height_) - pad_hw_values_[num][0] / 2) /
scale_[0])});
(*results)[num].label_ids.push_back(id);
(*results)[num].scores.push_back(boxesScore[i]);
last_count++;
}
std::cout << "last_count" << last_count << std::endl;
}
return true;
}
@@ -159,7 +139,7 @@ int RKYOLOPostprocessor::ProcessInt8(int8_t* input, int* anchor, int grid_h,
float box_conf_f32 = DeqntAffineToF32(box_confidence, zp, scale);
float class_prob_f32 = DeqntAffineToF32(maxClassProbs, zp, scale);
float limit_score = 0;
if (model_type_ == RKYOLOX) {
if (anchor_per_branch_ == 1) {
limit_score = box_conf_f32 * class_prob_f32;
} else {
limit_score = class_prob_f32;
@@ -167,7 +147,7 @@ int RKYOLOPostprocessor::ProcessInt8(int8_t* input, int* anchor, int grid_h,
//printf("limit score: %f\n", limit_score);
if (limit_score > conf_threshold_) {
float box_x, box_y, box_w, box_h;
if (model_type_ == RKYOLOX) {
if (anchor_per_branch_ == 1) {
box_x = DeqntAffineToF32(*in_ptr, zp, scale);
box_y = DeqntAffineToF32(in_ptr[grid_len], zp, scale);
box_w = DeqntAffineToF32(in_ptr[2 * grid_len], zp, scale);
@@ -234,6 +214,6 @@ int RKYOLOPostprocessor::QuickSortIndiceInverse(std::vector<float>& input,
return low;
}
} // namespace detection
} // namespace vision
} // namespace fastdeploy
} // namespace detection
} // namespace vision
} // namespace fastdeploy

14
fastdeploy/vision/detection/contrib/rknpu2/postprocessor.h
View File

@@ -13,6 +13,7 @@
// limitations under the License.
#pragma once
#include "fastdeploy/vision/common/processors/transform.h"
#include "fastdeploy/vision/common/result.h"
#include "fastdeploy/vision/detection/contrib/rknpu2/utils.h"
@@ -54,9 +55,6 @@ class FASTDEPLOY_DECL RKYOLOPostprocessor {
/// Get nms_threshold, default 0.45
float GetNMSThreshold() const { return nms_threshold_; }
// Set model_type
void SetModelType(ModelType model_type);
// Set height and weight
void SetHeightAndWeight(int& height, int& width) {
height_ = height;
@@ -69,10 +67,16 @@ class FASTDEPLOY_DECL RKYOLOPostprocessor {
}
// Set scale
void SetScale(std::vector<float> scale) { scale_ = scale; }
void SetScale(std::vector<float> scale) {
scale_ = scale;
}
// Set Anchor
void SetAnchor(std::vector<int> anchors,int anchor_per_branch){
anchors_ = anchors;
anchor_per_branch_ = anchor_per_branch;
};
private:
ModelType model_type_ = ModelType::UNKNOWN;
std::vector<int> anchors_ = {10, 13, 16, 30, 33, 23, 30, 61, 62,
45, 59, 119, 116, 90, 156, 198, 373, 326};
int strides_[3] = {8, 16, 32};

32
fastdeploy/vision/detection/contrib/rknpu2/preprocessor.cc
View File

@@ -57,7 +57,7 @@ void RKYOLOPreprocessor::LetterBox(FDMat* mat) {
resize_w = size_[0];
}
pad_hw_values_.push_back({pad_h,pad_w});
pad_hw_values_.push_back({pad_h, pad_w});
if (std::fabs(scale - 1.0f) > 1e-06) {
Resize::Run(mat, resize_w, resize_h);
@@ -75,17 +75,17 @@ void RKYOLOPreprocessor::LetterBox(FDMat* mat) {
bool RKYOLOPreprocessor::Preprocess(FDMat* mat, FDTensor* output) {
// process after image load
// float ratio = std::min(size_[1] * 1.0f / static_cast<float>(mat->Height()),
// size_[0] * 1.0f / static_cast<float>(mat->Width()));
// if (std::fabs(ratio - 1.0f) > 1e-06) {
// int interp = cv::INTER_AREA;
// if (ratio > 1.0) {
// interp = cv::INTER_LINEAR;
// }
// int resize_h = int(mat->Height() * ratio);
// int resize_w = int(mat->Width() * ratio);
// Resize::Run(mat, resize_w, resize_h, -1, -1, interp);
// }
// float ratio = std::min(size_[1] * 1.0f / static_cast<float>(mat->Height()),
// size_[0] * 1.0f / static_cast<float>(mat->Width()));
// if (std::fabs(ratio - 1.0f) > 1e-06) {
// int interp = cv::INTER_AREA;
// if (ratio > 1.0) {
// interp = cv::INTER_LINEAR;
// }
// int resize_h = int(mat->Height() * ratio);
// int resize_w = int(mat->Width() * ratio);
// Resize::Run(mat, resize_w, resize_h, -1, -1, interp);
// }
// RKYOLO's preprocess steps
// 1. letterbox
@@ -93,7 +93,7 @@ bool RKYOLOPreprocessor::Preprocess(FDMat* mat, FDTensor* output) {
LetterBox(mat);
BGR2RGB::Run(mat);
mat->ShareWithTensor(output);
output->ExpandDim(0); // reshape to n, h, w, c
output->ExpandDim(0); // reshape to n, h, w, c
return true;
}
@@ -122,6 +122,6 @@ bool RKYOLOPreprocessor::Run(std::vector<FDMat>* images,
return true;
}
} // namespace detection
} // namespace vision
} // namespace fastdeploy
} // namespace detection
} // namespace vision
} // namespace fastdeploy

31
fastdeploy/vision/detection/contrib/rknpu2/rkyolo.cc
View File

@@ -1,3 +1,16 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/detection/contrib/rknpu2/rkyolo.h"
namespace fastdeploy {
@@ -26,12 +39,11 @@ bool RKYOLO::Initialize() {
return false;
}
auto size = GetPreprocessor().GetSize();
GetPostprocessor().SetHeightAndWeight(size[0],size[1]);
GetPostprocessor().SetHeightAndWeight(size[0], size[1]);
return true;
}
bool RKYOLO::Predict(const cv::Mat& im,
DetectionResult* result) {
bool RKYOLO::Predict(const cv::Mat& im, DetectionResult* result) {
std::vector<DetectionResult> results;
if (!BatchPredict({im}, &results)) {
return false;
@@ -50,7 +62,8 @@ bool RKYOLO::BatchPredict(const std::vector<cv::Mat>& images,
}
auto pad_hw_values_ = preprocessor_.GetPadHWValues();
postprocessor_.SetPadHWValues(preprocessor_.GetPadHWValues());
std::cout << "preprocessor_ scale_ = " << preprocessor_.GetScale()[0] << std::endl;
std::cout << "preprocessor_ scale_ = " << preprocessor_.GetScale()[0]
<< std::endl;
postprocessor_.SetScale(preprocessor_.GetScale());
reused_input_tensors_[0].name = InputInfoOfRuntime(0).name;
@@ -59,15 +72,15 @@ bool RKYOLO::BatchPredict(const std::vector<cv::Mat>& images,
return false;
}
if (!postprocessor_.Run(reused_output_tensors_, results)) {
FDERROR << "Failed to postprocess the inference results by runtime." << std::endl;
FDERROR << "Failed to postprocess the inference results by runtime."
<< std::endl;
return false;
}
return true;
}
} // namespace detection
} // namespace vision
} // namespace fastdeploy
} // namespace detection
} // namespace vision
} // namespace fastdeploy

64
fastdeploy/vision/detection/contrib/rknpu2/utils.cc
View File

@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "fastdeploy/vision/detection/contrib/rknpu2/utils.h"
float clamp(float val, int min, int max) {
namespace fastdeploy {
namespace vision {
namespace detection {
float Clamp(float val, int min, int max) {
return val > min ? (val < max ? val : max) : min;
}
@@ -35,59 +38,56 @@ float DeqntAffineToF32(int8_t qnt, int32_t zp, float scale) {
return ((float)qnt - (float)zp) * scale;
}
static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1, float ymax1)
{
static float CalculateOverlap(float xmin0, float ymin0, float xmax0,
float ymax0, float xmin1, float ymin1,
float xmax1, float ymax1) {
float w = fmax(0.f, fmin(xmax0, xmax1) - fmax(xmin0, xmin1) + 1.0);
float h = fmax(0.f, fmin(ymax0, ymax1) - fmax(ymin0, ymin1) + 1.0);
float i = w * h;
float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) + (xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i;
float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) +
(xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i;
return u <= 0.f ? 0.f : (i / u);
}
int NMS(int validCount,
std::vector<float> &outputLocations,
std::vector<int> &class_id,
std::vector<int> &order,
float threshold,
bool class_agnostic)
{
int NMS(int valid_count, std::vector<float>& output_locations,
std::vector<int>& class_id, std::vector<int>& order, float threshold,
bool class_agnostic) {
// printf("class_agnostic: %d\n", class_agnostic);
for (int i = 0; i < validCount; ++i)
{
if (order[i] == -1)
{
for (int i = 0; i < valid_count; ++i) {
if (order[i] == -1) {
continue;
}
int n = order[i];
for (int j = i + 1; j < validCount; ++j)
{
for (int j = i + 1; j < valid_count; ++j) {
int m = order[j];
if (m == -1)
{
if (m == -1) {
continue;
}
if (!class_agnostic && class_id[n] != class_id[m]){
if (!class_agnostic && class_id[n] != class_id[m]) {
continue;
}
float xmin0 = outputLocations[n * 4 + 0];
float ymin0 = outputLocations[n * 4 + 1];
float xmax0 = outputLocations[n * 4 + 0] + outputLocations[n * 4 + 2];
float ymax0 = outputLocations[n * 4 + 1] + outputLocations[n * 4 + 3];
float xmin0 = output_locations[n * 4 + 0];
float ymin0 = output_locations[n * 4 + 1];
float xmax0 = output_locations[n * 4 + 0] + output_locations[n * 4 + 2];
float ymax0 = output_locations[n * 4 + 1] + output_locations[n * 4 + 3];
float xmin1 = outputLocations[m * 4 + 0];
float ymin1 = outputLocations[m * 4 + 1];
float xmax1 = outputLocations[m * 4 + 0] + outputLocations[m * 4 + 2];
float ymax1 = outputLocations[m * 4 + 1] + outputLocations[m * 4 + 3];
float xmin1 = output_locations[m * 4 + 0];
float ymin1 = output_locations[m * 4 + 1];
float xmax1 = output_locations[m * 4 + 0] + output_locations[m * 4 + 2];
float ymax1 = output_locations[m * 4 + 1] + output_locations[m * 4 + 3];
float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1);
float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1,
xmax1, ymax1);
if (iou > threshold)
{
if (iou > threshold) {
order[j] = -1;
}
}
}
return 0;
}
}
} // namespace detection
} // namespace vision
} // namespace fastdeploy

13
fastdeploy/vision/detection/contrib/rknpu2/utils.h
View File

@@ -14,13 +14,20 @@
#pragma once
#include <cmath>
#include <vector>
typedef enum { RKYOLOX = 0, RKYOLOV5, RKYOLOV7, UNKNOWN } ModelType;
float clamp(float val, int min, int max);
namespace fastdeploy {
namespace vision {
namespace detection {
float Clamp(float val, int min, int max);
float Sigmoid(float x);
float UnSigmoid(float y);
inline static int32_t __clip(float val, float min, float max);
int8_t QntF32ToAffine(float f32, int32_t zp, float scale);
float DeqntAffineToF32(int8_t qnt, int32_t zp, float scale);
int NMS(int validCount, std::vector<float>& outputLocations,
int NMS(int valid_count, std::vector<float>& output_locations,
std::vector<int>& class_id, std::vector<int>& order, float threshold,
bool class_agnostic);
} // namespace detection
} // namespace vision
} // namespace fastdeploy