diff --git a/examples/vision/facedet/scrfd/rknpu2/README.md b/examples/vision/facedet/scrfd/rknpu2/README.md new file mode 100644 index 000000000..b5f4e7d8b --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/README.md @@ -0,0 +1,73 @@ +# SCRFD RKNPU2部署模型 + + +- [SCRFD](https://github.com/deepinsight/insightface/tree/17cdeab12a35efcebc2660453a8cbeae96e20950) + - (1)[官方库](https://github.com/deepinsight/insightface/)中提供的*.pt通过[导出ONNX模型](#导出ONNX模型)操作后,可进行部署; + - (2)开发者基于自己数据训练的SCRFD模型,可按照[导出ONNX模型](#导出ONNX模型)后,完成部署。 + +## 下载预训练ONNX模型 + +为了方便开发者的测试,下面提供了SCRFD导出的各系列模型,开发者可直接下载使用。(下表中模型的精度来源于源官方库) +| 模型 | 大小 | 精度 | +|:---------------------------------------------------------------- |:----- |:----- | +| [SCRFD-500M-kps-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape160x160.onnx) | 2.5MB | - | +| [SCRFD-500M-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape160x160.onnx) | 2.2MB | - | +| [SCRFD-500M-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape320x320.onnx) | 2.5MB | - | +| [SCRFD-500M-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape320x320.onnx) | 2.2MB | - | +| [SCRFD-500M-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape640x640.onnx) | 2.5MB | 90.97% | +| [SCRFD-500M-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_shape640x640.onnx) | 2.2MB | 90.57% | +| [SCRFD-1G-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape160x160.onnx ) | 2.5MB | - | +| [SCRFD-1G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape320x320.onnx) | 2.5MB | - | +| [SCRFD-1G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_1g_shape640x640.onnx) | 2.5MB | 92.38% | +| [SCRFD-2.5G-kps-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape160x160.onnx) | 3.2MB | - | +| [SCRFD-2.5G-160](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape160x160.onnx) | 2.6MB | - | +| [SCRFD-2.5G-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape320x320.onnx) | 3.2MB | - | +| [SCRFD-2.5G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape320x320.onnx) | 2.6MB | - | +| [SCRFD-2.5G-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_bnkps_shape640x640.onnx) | 3.2MB | 93.8% | +| [SCRFD-2.5G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_2.5g_shape640x640.onnx) | 2.6MB | 93.78% | +| [SCRFD-10G-kps-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape320x320.onnx) | 17MB | - | +| [SCRFD-10G-320](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape320x320.onnx) | 15MB | - | +| [SCRFD-10G-kps-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape640x640.onnx) | 17MB | 95.4% | +| [SCRFD-10G-640](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape640x640.onnx) | 15MB | 95.16% | +| [SCRFD-10G-kps-1280](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_bnkps_shape1280x1280.onnx) | 17MB | - | +| [SCRFD-10G-1280](https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_10g_shape1280x1280.onnx) | 15MB | - | + +## 导出ONNX模型 + + ```bash + #下载scrfd模型文件 + e.g. download from https://onedrive.live.com/?authkey=%21ABbFJx2JMhNjhNA&id=4A83B6B633B029CC%215542&cid=4A83B6B633B029CC + + # 安装官方库配置环境,此版本导出环境为: + - 手动配置环境 + torch==1.8.0 + mmcv==1.3.5 + mmdet==2.7.0 + + - 通过docker配置 + docker pull qyjdefdocker/onnx-scrfd-converter:v0.3 + + # 导出onnx格式文件 + - 手动生成 + python tools/scrfd2onnx.py configs/scrfd/scrfd_500m.py weights/scrfd_500m.pth --shape 640 --input-img face-xxx.jpg + + - docker + docker的onnx目录中已有生成好的onnx文件 + + ``` + +## ONNX模型转换RKNN模型 +```bash +wget https://bj.bcebos.com/paddlehub/fastdeploy/scrfd_500m_bnkps_shape640x640.onnx +python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3588/scrfd.yaml +``` + +## 详细部署文档 + +- [Python部署](python/README.md) +- [C++部署](cpp/README.md) + + +## 版本说明 + +- 本版本文档和代码基于[SCRFD CommitID:17cdeab](https://github.com/deepinsight/insightface/tree/17cdeab12a35efcebc2660453a8cbeae96e20950) 编写 diff --git a/examples/vision/facedet/scrfd/rknpu2/cpp/CMakeLists.txt b/examples/vision/facedet/scrfd/rknpu2/cpp/CMakeLists.txt new file mode 100644 index 000000000..7e207d8e4 --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/cpp/CMakeLists.txt @@ -0,0 +1,36 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.10) +project(rknpu_test) + +set(CMAKE_CXX_STANDARD 14) + +# 指定下载解压后的fastdeploy库路径 +set(FASTDEPLOY_INSTALL_DIR "thirdpartys/fastdeploy-0.7.0") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeployConfig.cmake) +include_directories(${FastDeploy_INCLUDE_DIRS}) +add_executable(rknpu_test infer.cc) +target_link_libraries(rknpu_test + ${FastDeploy_LIBS} + ) + +set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/build/install) + +install(TARGETS rknpu_test DESTINATION ./) + +install(DIRECTORY model DESTINATION ./) +install(DIRECTORY images DESTINATION ./) + +file(GLOB FASTDEPLOY_LIBS ${FASTDEPLOY_INSTALL_DIR}/lib/*) +message("${FASTDEPLOY_LIBS}") +install(PROGRAMS ${FASTDEPLOY_LIBS} DESTINATION lib) + +file(GLOB ONNXRUNTIME_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/onnxruntime/lib/*) +install(PROGRAMS ${ONNXRUNTIME_LIBS} DESTINATION lib) + +install(DIRECTORY ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/opencv/lib DESTINATION ./) + +file(GLOB PADDLETOONNX_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/paddle2onnx/lib/*) +install(PROGRAMS ${PADDLETOONNX_LIBS} DESTINATION lib) + +file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/RK3588/lib/*) +install(PROGRAMS ${RKNPU2_LIBS} DESTINATION lib) diff --git a/examples/vision/facedet/scrfd/rknpu2/cpp/README.md b/examples/vision/facedet/scrfd/rknpu2/cpp/README.md new file mode 100644 index 000000000..f08a70a69 --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/cpp/README.md @@ -0,0 +1,72 @@ +# SCRFD C++部署示例 + +本目录下提供`infer.cc`快速完成SCRFD在NPU加速部署的示例。 + +在部署前,需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境,下载预编译部署库或者从头编译FastDeploy仓库 + +以上步骤请参考[RK2代NPU部署库编译](../../../../../../docs/cn/build_and_install/rknpu2.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer_cpu_npu.cc +├── infer_cpu_npu.h +├── main.cc +├── model # 存放模型文件的文件夹 +└── thirdpartys # 存放sdk的文件夹 +``` + +首先需要先生成目录结构 +```bash +mkdir build +mkdir images +mkdir model +mkdir thirdpartys +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[RK2代NPU部署库编译](../../../../../../docs/cn/build_and_install/rknpu2.md)仓库编译SDK,编译完成后,将在build目录下生成 +fastdeploy-0.7.0目录,请移动它至thirdpartys目录下. + +### 拷贝模型文件,以及配置文件至model文件夹 +在Paddle动态图模型 -> Paddle静态图模型 -> ONNX模型的过程中,将生成ONNX文件以及对应的yaml配置文件,请将配置文件存放到model文件夹内。 +转换为RKNN后的模型文件也需要拷贝至model。 + +### 准备测试图片至image文件夹 +```bash +wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_face_detector_3.jpg +cp test_lite_face_detector_3.jpg ./images +``` + +### 编译example + +```bash +cd build +cmake .. +make -j8 +make install +``` +## 运行例程 + +```bash +cd ./build/install +./rknpu_test +``` +运行完成可视化结果如下图所示 + + + +- [模型介绍](../../README.md) +- [Python部署](../python/README.md) +- [视觉模型预测结果](../../../../../../docs/api/vision_results/README.md) diff --git a/examples/vision/facedet/scrfd/rknpu2/cpp/infer.cc b/examples/vision/facedet/scrfd/rknpu2/cpp/infer.cc new file mode 100644 index 000000000..a01f1b184 --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/cpp/infer.cc @@ -0,0 +1,79 @@ +#include +#include +#include "fastdeploy/vision.h" + +void InferScrfd(const std::string& device = "cpu"); + +int main() { + InferScrfd("npu"); + return 0; +} + +fastdeploy::RuntimeOption GetOption(const std::string& device) { + auto option = fastdeploy::RuntimeOption(); + if (device == "npu") { + option.UseRKNPU2(); + } else { + option.UseCpu(); + } + return option; +} + +fastdeploy::ModelFormat GetFormat(const std::string& device) { + auto format = fastdeploy::ModelFormat::ONNX; + if (device == "npu") { + format = fastdeploy::ModelFormat::RKNN; + } else { + format = fastdeploy::ModelFormat::ONNX; + } + return format; +} + +std::string GetModelPath(std::string& model_path, const std::string& device) { + if (device == "npu") { + model_path += "rknn"; + } else { + model_path += "onnx"; + } + return model_path; +} + +void InferScrfd(const std::string& device) { + std::string model_file = + "./model/scrfd_500m_bnkps_shape640x640_rk3588."; + std::string params_file; + + fastdeploy::RuntimeOption option = GetOption(device); + fastdeploy::ModelFormat format = GetFormat(device); + model_file = GetModelPath(model_file, device); + auto model = fastdeploy::vision::facedet::SCRFD( + model_file, params_file, option, format); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + auto image_file = + "./images/test_lite_face_detector_3.jpg"; + auto im = cv::imread(image_file); + + if (device == "npu") { + model.DisableNormalizeAndPermute(); + } + + fastdeploy::vision::FaceDetectionResult res; + clock_t start = clock(); + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + clock_t end = clock(); + auto dur = static_cast(end - start); + printf("InferScrfd use time:%f\n", + (dur / CLOCKS_PER_SEC)); + + std::cout << res.Str() << std::endl; + auto vis_im = fastdeploy::vision::Visualize::VisFaceDetection(im, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} \ No newline at end of file diff --git a/examples/vision/facedet/scrfd/rknpu2/python/README.md b/examples/vision/facedet/scrfd/rknpu2/python/README.md new file mode 100644 index 000000000..3c8bc158c --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/python/README.md @@ -0,0 +1,32 @@ +# SCRFD Python部署示例 + +在部署前,需确认以下两个步骤 + +- 1. 软硬件环境满足要求,参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/rknpu2.md) + + +本目录下提供`infer.py`快速完成SCRFD在RKNPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/facedet/scrfd/rknpu2/python + +# 下载图片 +wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_face_detector_3.jpg + +# 推理 +python3 infer.py --model_file ./scrfd_500m_bnkps_shape640x640_rk3588.rknn \ + --image test_lite_face_detector_3.jpg +``` + + +## 注意事项 +RKNPU上对模型的输入要求是使用NHWC格式,且图片归一化操作会在转RKNN模型时,内嵌到模型中,因此我们在使用FastDeploy部署时, +需要先调用DisableNormalizePermute(C++)或`disable_normalize_permute(Python),在预处理阶段禁用归一化以及数据格式的转换。 +## 其它文档 + +- [SCRFD 模型介绍](../README.md) +- [SCRFD C++部署](../cpp/README.md) +- [模型预测结果说明](../../../../../../docs/api/vision_results/README.md) +- [转换SCRFD RKNN模型文档](../README.md) diff --git a/examples/vision/facedet/scrfd/rknpu2/python/infer.py b/examples/vision/facedet/scrfd/rknpu2/python/infer.py new file mode 100644 index 000000000..3b3fc9d83 --- /dev/null +++ b/examples/vision/facedet/scrfd/rknpu2/python/infer.py @@ -0,0 +1,58 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_file", required=True, help="Path of FaceDet model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + option.use_rknpu2() + return option + + +args = parse_arguments() + +# 配置runtime,加载模型 +runtime_option = build_option(args) +model_file = args.model_file +params_file = "" +model = fd.vision.facedet.SCRFD( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.RKNN) + +model.disable_normalize_and_permute() + +# 预测图片分割结果 +im = cv2.imread(args.image) +result = model.predict(im.copy()) +print(result) + +# 可视化结果 +vis_im = fd.vision.vis_face_detection(im, result) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/fastdeploy/vision/facedet/contrib/scrfd.cc b/fastdeploy/vision/facedet/contrib/scrfd.cc index 7d6974410..443b30631 100644 --- a/fastdeploy/vision/facedet/contrib/scrfd.cc +++ b/fastdeploy/vision/facedet/contrib/scrfd.cc @@ -68,6 +68,7 @@ SCRFD::SCRFD(const std::string& model_file, const std::string& params_file, } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + valid_rknpu_backends = {Backend::RKNPU2}; } runtime_option = custom_option; runtime_option.model_format = model_format; @@ -135,19 +136,22 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output, is_scale_up, stride); BGR2RGB::Run(mat); - // Normalize::Run(mat, std::vector(mat->Channels(), 0.0), - // std::vector(mat->Channels(), 1.0)); - // Compute `result = mat * alpha + beta` directly by channel - // Original Repo/tools/scrfd.py: cv2.dnn.blobFromImage(img, 1.0/128, - // input_size, (127.5, 127.5, 127.5), swapRB=True) - std::vector alpha = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; - std::vector beta = {-127.5f / 128.f, -127.5f / 128.f, -127.5f / 128.f}; - Convert::Run(mat, alpha, beta); + if(!this->disable_normalize_and_permute_){ + // Normalize::Run(mat, std::vector(mat->Channels(), 0.0), + // std::vector(mat->Channels(), 1.0)); + // Compute `result = mat * alpha + beta` directly by channel + // Original Repo/tools/scrfd.py: cv2.dnn.blobFromImage(img, 1.0/128, + // input_size, (127.5, 127.5, 127.5), swapRB=True) + std::vector alpha = {1.f / 128.f, 1.f / 128.f, 1.f / 128.f}; + std::vector beta = {-127.5f / 128.f, -127.5f / 128.f, -127.5f / 128.f}; + Convert::Run(mat, alpha, beta); + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + } + // Record output shape of preprocessed image (*im_info)["output_shape"] = {static_cast(mat->Height()), static_cast(mat->Width())}; - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); mat->ShareWithTensor(output); output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c return true; @@ -347,7 +351,9 @@ bool SCRFD::Predict(cv::Mat* im, FaceDetectionResult* result, } return true; } - +void SCRFD::DisableNormalizeAndPermute(){ + this->disable_normalize_and_permute_ = true; +} } // namespace facedet } // namespace vision } // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/facedet/contrib/scrfd.h b/fastdeploy/vision/facedet/contrib/scrfd.h index 38da3af42..f1f06b203 100644 --- a/fastdeploy/vision/facedet/contrib/scrfd.h +++ b/fastdeploy/vision/facedet/contrib/scrfd.h @@ -89,6 +89,9 @@ class FASTDEPLOY_DECL SCRFD : public FastDeployModel { */ unsigned int num_anchors; + /// This function will disable normalize and hwc2chw in preprocessing step. + void DisableNormalizeAndPermute(); + private: bool Initialize(); @@ -117,6 +120,9 @@ class FASTDEPLOY_DECL SCRFD : public FastDeployModel { } SCRFDPoint; std::unordered_map> center_points_; + + // for recording the switch of normalize and hwc2chw + bool disable_normalize_and_permute_ = false; }; } // namespace facedet } // namespace vision diff --git a/fastdeploy/vision/facedet/contrib/scrfd_pybind.cc b/fastdeploy/vision/facedet/contrib/scrfd_pybind.cc index 4f70126d9..6b53f1a7e 100644 --- a/fastdeploy/vision/facedet/contrib/scrfd_pybind.cc +++ b/fastdeploy/vision/facedet/contrib/scrfd_pybind.cc @@ -28,6 +28,7 @@ void BindSCRFD(pybind11::module& m) { self.Predict(&mat, &res, conf_threshold, nms_iou_threshold); return res; }) + .def("disable_normalize_and_permute",&vision::facedet::SCRFD::DisableNormalizeAndPermute) .def_readwrite("size", &vision::facedet::SCRFD::size) .def_readwrite("padding_value", &vision::facedet::SCRFD::padding_value) .def_readwrite("is_mini_pad", &vision::facedet::SCRFD::is_mini_pad) @@ -41,6 +42,7 @@ void BindSCRFD(pybind11::module& m) { .def_readwrite("num_anchors", &vision::facedet::SCRFD::num_anchors) .def_readwrite("landmarks_per_face", &vision::facedet::SCRFD::landmarks_per_face); + } } // namespace fastdeploy diff --git a/python/fastdeploy/vision/facedet/contrib/scrfd.py b/python/fastdeploy/vision/facedet/contrib/scrfd.py index 96171088c..e225e69bc 100644 --- a/python/fastdeploy/vision/facedet/contrib/scrfd.py +++ b/python/fastdeploy/vision/facedet/contrib/scrfd.py @@ -51,6 +51,12 @@ class SCRFD(FastDeployModel): return self._model.predict(input_image, conf_threshold, nms_iou_threshold) + def disable_normalize_and_permute(self): + """ + This function will disable normalize and hwc2chw in preprocessing step. + """ + self._model.disable_normalize_and_permute() + # 一些跟SCRFD模型有关的属性封装 # 多数是预处理相关,可通过修改如model.size = [640, 640]改变预处理时resize的大小(前提是模型支持) @property diff --git a/tools/rknpu2/config/RK3568/scrfd.yaml b/tools/rknpu2/config/RK3568/scrfd.yaml new file mode 100644 index 000000000..dac268b1a --- /dev/null +++ b/tools/rknpu2/config/RK3568/scrfd.yaml @@ -0,0 +1,7 @@ +model_path: ./scrfd_500m_bnkps_shape640x640.onnx +output_folder: ./ +target_platform: RK3568 +normalize: + mean: [[0.5,0.5,0.5]] + std: [[0.5,0.5,0.5]] +outputs: None diff --git a/tools/rknpu2/config/RK3588/scrfd.yaml b/tools/rknpu2/config/RK3588/scrfd.yaml new file mode 100644 index 000000000..f8c289baf --- /dev/null +++ b/tools/rknpu2/config/RK3588/scrfd.yaml @@ -0,0 +1,7 @@ +model_path: ./scrfd_500m_bnkps_shape640x640.onnx +output_folder: ./ +target_platform: RK3588 +normalize: + mean: [[0.5,0.5,0.5]] + std: [[0.5,0.5,0.5]] +outputs: None