[Model] 新增scrfd rknpu2代码 (#504)

* * 新增scrfd rknpu2代码

* * 新增scrfd python代码

* 修正文档

* 修正文档以及部分错误

* 修改文档

* 修复部分错误

* 修复部分错误

* 修复部分错误

* scrfd更新代码

* scrfd更新代码
This commit is contained in:
Zheng_Bicheng
2022-11-10 10:05:15 +08:00
committed by GitHub
parent 3319a5bdb2
commit ca0de42d0b
12 changed files with 395 additions and 11 deletions

View File

@@ -0,0 +1,73 @@
# SCRFD RKNPU2部署模型
- [SCRFD](https://github.com/deepinsight/insightface/tree/17cdeab12a35efcebc2660453a8cbeae96e20950)
- 1[官方库](https://github.com/deepinsight/insightface/)中提供的*.pt通过[导出ONNX模型](#导出ONNX模型)操作后,可进行部署;
- 2开发者基于自己数据训练的SCRFD模型可按照[导出ONNX模型](#导出ONNX模型)后,完成部署。
## 下载预训练ONNX模型
为了方便开发者的测试下面提供了SCRFD导出的各系列模型开发者可直接下载使用。下表中模型的精度来源于源官方库
| 模型 | 大小 | 精度 |
|:---------------------------------------------------------------- |:----- |:----- |
| [SCRFD-500M-kps-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape160x160.onnx) | 2.5MB | - |
| [SCRFD-500M-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape160x160.onnx) | 2.2MB | - |
| [SCRFD-500M-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape320x320.onnx) | 2.5MB | - |
| [SCRFD-500M-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape320x320.onnx) | 2.2MB | - |
| [SCRFD-500M-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape640x640.onnx) | 2.5MB | 90.97% |
| [SCRFD-500M-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape640x640.onnx) | 2.2MB | 90.57% |
| [SCRFD-1G-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape160x160.onnx ) | 2.5MB | - |
| [SCRFD-1G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape320x320.onnx) | 2.5MB | - |
| [SCRFD-1G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape640x640.onnx) | 2.5MB | 92.38% |
| [SCRFD-2.5G-kps-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape160x160.onnx) | 3.2MB | - |
| [SCRFD-2.5G-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape160x160.onnx) | 2.6MB | - |
| [SCRFD-2.5G-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape320x320.onnx) | 3.2MB | - |
| [SCRFD-2.5G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape320x320.onnx) | 2.6MB | - |
| [SCRFD-2.5G-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape640x640.onnx) | 3.2MB | 93.8% |
| [SCRFD-2.5G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape640x640.onnx) | 2.6MB | 93.78% |
| [SCRFD-10G-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape320x320.onnx) | 17MB | - |
| [SCRFD-10G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape320x320.onnx) | 15MB | - |
| [SCRFD-10G-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape640x640.onnx) | 17MB | 95.4% |
| [SCRFD-10G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape640x640.onnx) | 15MB | 95.16% |
| [SCRFD-10G-kps-1280](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape1280x1280.onnx) | 17MB | - |
| [SCRFD-10G-1280](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape1280x1280.onnx) | 15MB | - |
## 导出ONNX模型
```bash
#下载scrfd模型文件
e.g. download from https://onedrive.live.com/?authkey=%21ABbFJx2JMhNjhNA&id=4A83B6B633B029CC%215542&cid=4A83B6B633B029CC
# 安装官方库配置环境,此版本导出环境为:
- 手动配置环境
torch==1.8.0
mmcv==1.3.5
mmdet==2.7.0
- 通过docker配置
docker pull qyjdefdocker/onnx-scrfd-converter:v0.3
# 导出onnx格式文件
- 手动生成
python tools/scrfd2onnx.py configs/scrfd/scrfd_500m.py weights/scrfd_500m.pth --shape 640 --input-img face-xxx.jpg
- docker
docker的onnx目录中已有生成好的onnx文件
```
## ONNX模型转换RKNN模型
```bash
wget https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape640x640.onnx
python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3588/scrfd.yaml
```
## 详细部署文档
- [Python部署](python/README.md)
- [C++部署](cpp/README.md)
## 版本说明
- 本版本文档和代码基于[SCRFD CommitID:17cdeab](https://github.com/deepinsight/insightface/tree/17cdeab12a35efcebc2660453a8cbeae96e20950) 编写

View File

@@ -0,0 +1,36 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
project(rknpu_test)
set(CMAKE_CXX_STANDARD 14)
# 指定下载解压后的fastdeploy库路径
set(FASTDEPLOY_INSTALL_DIR "thirdpartys/fastdeploy-0.7.0")
include(${FASTDEPLOY_INSTALL_DIR}/FastDeployConfig.cmake)
include_directories(${FastDeploy_INCLUDE_DIRS})
add_executable(rknpu_test infer.cc)
target_link_libraries(rknpu_test
${FastDeploy_LIBS}
)
set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/build/install)
install(TARGETS rknpu_test DESTINATION ./)
install(DIRECTORY model DESTINATION ./)
install(DIRECTORY images DESTINATION ./)
file(GLOB FASTDEPLOY_LIBS ${FASTDEPLOY_INSTALL_DIR}/lib/*)
message("${FASTDEPLOY_LIBS}")
install(PROGRAMS ${FASTDEPLOY_LIBS} DESTINATION lib)
file(GLOB ONNXRUNTIME_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/onnxruntime/lib/*)
install(PROGRAMS ${ONNXRUNTIME_LIBS} DESTINATION lib)
install(DIRECTORY ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/opencv/lib DESTINATION ./)
file(GLOB PADDLETOONNX_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/paddle2onnx/lib/*)
install(PROGRAMS ${PADDLETOONNX_LIBS} DESTINATION lib)
file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/RK3588/lib/*)
install(PROGRAMS ${RKNPU2_LIBS} DESTINATION lib)

View File

@@ -0,0 +1,72 @@
# SCRFD C++部署示例
本目录下提供`infer.cc`快速完成SCRFD在NPU加速部署的示例。
在部署前,需确认以下两个步骤:
1. 软硬件环境满足要求
2. 根据开发环境下载预编译部署库或者从头编译FastDeploy仓库
以上步骤请参考[RK2代NPU部署库编译](../../../../../../docs/cn/build_and_install/rknpu2.md)实现
## 生成基本目录文件
该例程由以下几个部分组成
```text
.
├── CMakeLists.txt
├── build # 编译文件夹
├── image # 存放图片的文件夹
├── infer_cpu_npu.cc
├── infer_cpu_npu.h
├── main.cc
├── model # 存放模型文件的文件夹
└── thirdpartys # 存放sdk的文件夹
```
首先需要先生成目录结构
```bash
mkdir build
mkdir images
mkdir model
mkdir thirdpartys
```
## 编译
### 编译并拷贝SDK到thirdpartys文件夹
请参考[RK2代NPU部署库编译](../../../../../../docs/cn/build_and_install/rknpu2.md)仓库编译SDK编译完成后将在build目录下生成
fastdeploy-0.7.0目录请移动它至thirdpartys目录下.
### 拷贝模型文件以及配置文件至model文件夹
在Paddle动态图模型 -> Paddle静态图模型 -> ONNX模型的过程中将生成ONNX文件以及对应的yaml配置文件请将配置文件存放到model文件夹内。
转换为RKNN后的模型文件也需要拷贝至model。
### 准备测试图片至image文件夹
```bash
wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_face_detector_3.jpg
cp test_lite_face_detector_3.jpg ./images
```
### 编译example
```bash
cd build
cmake ..
make -j8
make install
```
## 运行例程
```bash
cd ./build/install
./rknpu_test
```
运行完成可视化结果如下图所示
<img width="640" src="https://user-images.githubusercontent.com/67993288/184301789-1981d065-208f-4a6b-857c-9a0f9a63e0b1.jpg">
- [模型介绍](../../README.md)
- [Python部署](../python/README.md)
- [视觉模型预测结果](../../../../../../docs/api/vision_results/README.md)

View File

@@ -0,0 +1,79 @@
#include <iostream>
#include <string>
#include "fastdeploy/vision.h"
void InferScrfd(const std::string& device = "cpu");
int main() {
InferScrfd("npu");
return 0;
}
fastdeploy::RuntimeOption GetOption(const std::string& device) {
auto option = fastdeploy::RuntimeOption();
if (device == "npu") {
option.UseRKNPU2();
} else {
option.UseCpu();
}
return option;
}
fastdeploy::ModelFormat GetFormat(const std::string& device) {
auto format = fastdeploy::ModelFormat::ONNX;
if (device == "npu") {
format = fastdeploy::ModelFormat::RKNN;
} else {
format = fastdeploy::ModelFormat::ONNX;
}
return format;
}
std::string GetModelPath(std::string& model_path, const std::string& device) {
if (device == "npu") {
model_path += "rknn";
} else {
model_path += "onnx";
}
return model_path;
}
void InferScrfd(const std::string& device) {
std::string model_file =
"./model/scrfd_500m_bnkps_shape640x640_rk3588.";
std::string params_file;
fastdeploy::RuntimeOption option = GetOption(device);
fastdeploy::ModelFormat format = GetFormat(device);
model_file = GetModelPath(model_file, device);
auto model = fastdeploy::vision::facedet::SCRFD(
model_file, params_file, option, format);
if (!model.Initialized()) {
std::cerr << "Failed to initialize." << std::endl;
return;
}
auto image_file =
"./images/test_lite_face_detector_3.jpg";
auto im = cv::imread(image_file);
if (device == "npu") {
model.DisableNormalizeAndPermute();
}
fastdeploy::vision::FaceDetectionResult res;
clock_t start = clock();
if (!model.Predict(&im, &res)) {
std::cerr << "Failed to predict." << std::endl;
return;
}
clock_t end = clock();
auto dur = static_cast<double>(end - start);
printf("InferScrfd use time:%f\n",
(dur / CLOCKS_PER_SEC));
std::cout << res.Str() << std::endl;
auto vis_im = fastdeploy::vision::Visualize::VisFaceDetection(im, res);
cv::imwrite("vis_result.jpg", vis_im);
std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
}

View File

@@ -0,0 +1,32 @@
# SCRFD Python部署示例
在部署前,需确认以下两个步骤
- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/rknpu2.md)
本目录下提供`infer.py`快速完成SCRFD在RKNPU上部署的示例。执行如下脚本即可完成
```bash
# 下载部署示例代码
git clone https://github.com/PaddlePaddle/FastDeploy.git
cd FastDeploy/examples/vision/facedet/scrfd/rknpu2/python
# 下载图片
wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_face_detector_3.jpg
# 推理
python3 infer.py --model_file ./scrfd_500m_bnkps_shape640x640_rk3588.rknn \
--image test_lite_face_detector_3.jpg
```
## 注意事项
RKNPU上对模型的输入要求是使用NHWC格式且图片归一化操作会在转RKNN模型时内嵌到模型中因此我们在使用FastDeploy部署时
需要先调用DisableNormalizePermute(C++)或`disable_normalize_permute(Python),在预处理阶段禁用归一化以及数据格式的转换。
## 其它文档
- [SCRFD 模型介绍](../README.md)
- [SCRFD C++部署](../cpp/README.md)
- [模型预测结果说明](../../../../../../docs/api/vision_results/README.md)
- [转换SCRFD RKNN模型文档](../README.md)

View File

@@ -0,0 +1,58 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import fastdeploy as fd
import cv2
import os
def parse_arguments():
import argparse
import ast
parser = argparse.ArgumentParser()
parser.add_argument(
"--model_file", required=True, help="Path of FaceDet model.")
parser.add_argument(
"--image", type=str, required=True, help="Path of test image file.")
return parser.parse_args()
def build_option(args):
option = fd.RuntimeOption()
option.use_rknpu2()
return option
args = parse_arguments()
# 配置runtime加载模型
runtime_option = build_option(args)
model_file = args.model_file
params_file = ""
model = fd.vision.facedet.SCRFD(
model_file,
params_file,
runtime_option=runtime_option,
model_format=fd.ModelFormat.RKNN)
model.disable_normalize_and_permute()
# 预测图片分割结果
im = cv2.imread(args.image)
result = model.predict(im.copy())
print(result)
# 可视化结果
vis_im = fd.vision.vis_face_detection(im, result)
cv2.imwrite("visualized_result.jpg", vis_im)
print("Visualized result save in ./visualized_result.jpg")

View File

@@ -68,6 +68,7 @@ SCRFD::SCRFD(const std::string& model_file, const std::string& params_file,
} else {
valid_cpu_backends = {Backend::PDINFER, Backend::ORT};
valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
valid_rknpu_backends = {Backend::RKNPU2};
}
runtime_option = custom_option;
runtime_option.model_format = model_format;
@@ -135,6 +136,7 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output,
is_scale_up, stride);
BGR2RGB::Run(mat);
if(!this->disable_normalize_and_permute_){
// Normalize::Run(mat, std::vector<float>(mat->Channels(), 0.0),
// std::vector<float>(mat->Channels(), 1.0));
// Compute `result = mat * alpha + beta` directly by channel
@@ -143,11 +145,13 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output,
std::vector<float> alpha = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f};
std::vector<float> beta = {-127.5f / 128.f, -127.5f / 128.f, -127.5f / 128.f};
Convert::Run(mat, alpha, beta);
HWC2CHW::Run(mat);
Cast::Run(mat, "float");
}
// Record output shape of preprocessed image
(*im_info)["output_shape"] = {static_cast<float>(mat->Height()),
static_cast<float>(mat->Width())};
HWC2CHW::Run(mat);
Cast::Run(mat, "float");
mat->ShareWithTensor(output);
output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c
return true;
@@ -347,7 +351,9 @@ bool SCRFD::Predict(cv::Mat* im, FaceDetectionResult* result,
}
return true;
}
void SCRFD::DisableNormalizeAndPermute(){
this->disable_normalize_and_permute_ = true;
}
} // namespace facedet
} // namespace vision
} // namespace fastdeploy

View File

@@ -89,6 +89,9 @@ class FASTDEPLOY_DECL SCRFD : public FastDeployModel {
*/
unsigned int num_anchors;
/// This function will disable normalize and hwc2chw in preprocessing step.
void DisableNormalizeAndPermute();
private:
bool Initialize();
@@ -117,6 +120,9 @@ class FASTDEPLOY_DECL SCRFD : public FastDeployModel {
} SCRFDPoint;
std::unordered_map<int, std::vector<SCRFDPoint>> center_points_;
// for recording the switch of normalize and hwc2chw
bool disable_normalize_and_permute_ = false;
};
} // namespace facedet
} // namespace vision

View File

@@ -28,6 +28,7 @@ void BindSCRFD(pybind11::module& m) {
self.Predict(&mat, &res, conf_threshold, nms_iou_threshold);
return res;
})
.def("disable_normalize_and_permute",&vision::facedet::SCRFD::DisableNormalizeAndPermute)
.def_readwrite("size", &vision::facedet::SCRFD::size)
.def_readwrite("padding_value", &vision::facedet::SCRFD::padding_value)
.def_readwrite("is_mini_pad", &vision::facedet::SCRFD::is_mini_pad)
@@ -41,6 +42,7 @@ void BindSCRFD(pybind11::module& m) {
.def_readwrite("num_anchors", &vision::facedet::SCRFD::num_anchors)
.def_readwrite("landmarks_per_face",
&vision::facedet::SCRFD::landmarks_per_face);
}
} // namespace fastdeploy

View File

@@ -51,6 +51,12 @@ class SCRFD(FastDeployModel):
return self._model.predict(input_image, conf_threshold,
nms_iou_threshold)
def disable_normalize_and_permute(self):
"""
This function will disable normalize and hwc2chw in preprocessing step.
"""
self._model.disable_normalize_and_permute()
# 一些跟SCRFD模型有关的属性封装
# 多数是预处理相关可通过修改如model.size = [640, 640]改变预处理时resize的大小前提是模型支持
@property

View File

@@ -0,0 +1,7 @@
model_path: ./scrfd_500m_bnkps_shape640x640.onnx
output_folder: ./
target_platform: RK3568
normalize:
mean: [[0.5,0.5,0.5]]
std: [[0.5,0.5,0.5]]
outputs: None

View File

@@ -0,0 +1,7 @@
model_path: ./scrfd_500m_bnkps_shape640x640.onnx
output_folder: ./
target_platform: RK3588
normalize:
mean: [[0.5,0.5,0.5]]
std: [[0.5,0.5,0.5]]
outputs: None