[Model] Add Picodet RKNPU2 (#635)

* * 更新picodet cpp代码 * * 更新文档 * 更新picodet cpp example * * 删除无用的debug代码 * 新增python example * * 修改c++代码 * * 修改python代码 * * 修改postprocess代码 * 修复没有scale_factor导致的bug * 修复错误 * 更正代码格式 * 更正代码格式
2025-10-05 08:37:06 +08:00 · 2022-11-21 13:44:34 +08:00
parent 5ca779ee32
commit 3e1fc69a0c
20 changed files with 340 additions and 195 deletions
--- a/examples/vision/detection/paddledetection/rknpu2/README.md
+++ b/examples/vision/detection/paddledetection/rknpu2/README.md
@@ -13,7 +13,9 @@ RKNPU部署模型前需要将Paddle模型转换成RKNN模型，具体步骤如


 ## 模型转换example
-下面以Picodet-npu为例子,教大家如何转换PaddleDetection模型到RKNN模型。
+以下步骤均在Ubuntu电脑上完成，请参考配置文档完成转换模型环境配置。下面以Picodet-s为例子,教大家如何转换PaddleDetection模型到RKNN模型。
+
+### 导出ONNX模型
 ```bash
 # 下载Paddle静态图模型并解压
 wget https://paddledet.bj.bcebos.com/deploy/Inference/picodet_s_416_coco_lcnet.tar
@@ -26,12 +28,89 @@ paddle2onnx --model_dir picodet_s_416_coco_lcnet \
            --save_file picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \
            --enable_dev_version True

+# 固定shape
 python -m paddle2onnx.optimize --input_model picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \
                                --output_model picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \
                                --input_shape_dict "{'image':[1,3,416,416]}"
+```
+
+### 编写模型导出配置文件
+以转化RK3568的RKNN模型为例子，我们需要编辑tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml，来转换ONNX模型到RKNN模型。
+
+**修改normalize参数**
+
+如果你需要在NPU上执行normalize操作，请根据你的模型配置normalize参数，例如:
+```yaml
+model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx
+output_folder: ./picodet_s_416_coco_lcnet
+target_platform: RK3568
+normalize:
+  mean: [[0.485,0.456,0.406],[0,0,0]]
+  std: [[0.229,0.224,0.225],[0.003921,0.003921]]
+outputs: ['tmp_17','p2o.Concat.9']
+```
+
+**修改outputs参数**
+由于Paddle2ONNX版本的不同，转换模型的输出节点名称也有所不同，请使用[Netron](https://netron.app)，并找到以下蓝色方框标记的NonMaxSuppression节点，红色方框的节点名称即为目标名称。
+
+例如，使用Netron可视化后，得到以下图片:
+![](https://user-images.githubusercontent.com/58363586/202728663-4af0b843-d012-4aeb-8a66-626b7b87ca69.png)
+
+找到蓝色方框标记的NonMaxSuppression节点，可以看到红色方框标记的两个节点名称为tmp_17和p2o.Concat.9,因此需要修改outputs参数，修改后如下:
+```yaml
+model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx
+output_folder: ./picodet_s_416_coco_lcnet
+target_platform: RK3568
+normalize: None
+outputs: ['tmp_17','p2o.Concat.9']
+```
+
+### 转换模型
+```bash
+
 # ONNX模型转RKNN模型
 # 转换模型,模型将生成在picodet_s_320_coco_lcnet_non_postprocess目录下
-python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml
+python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml
+```
+
+### 修改模型运行时的配置文件
+
+配置文件中，我们只需要修改**Preprocess**下的**Normalize**和**Permute**.
+
+**删除Permute**
+
+RKNPU只支持NHWC的输入格式，因此需要删除Permute操作.删除后，配置文件Precess部分后如下:
+```yaml
+Preprocess:
+- interp: 2
+  keep_ratio: false
+  target_size:
+  - 416
+  - 416
+  type: Resize
+- is_scale: true
+  mean:
+  - 0.485
+  - 0.456
+  - 0.406
+  std:
+  - 0.229
+  - 0.224
+  - 0.225
+  type: NormalizeImage
+```
+
+**根据模型转换文件决定是否删除Normalize**
+
+RKNPU支持使用NPU进行Normalize操作，如果你在导出模型时配置了Normalize参数，请删除**Normalize**.删除后配置文件Precess部分如下:
+```yaml
+Preprocess:
+- interp: 2
+  keep_ratio: false
+  target_size:
+  - 416
+  - 416
+  type: Resize
 ```

 - [Python部署](./python)
--- a/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt
+++ b/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt
@@ -33,5 +33,5 @@ install(DIRECTORY ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/opencv/lib DESTIN
 file(GLOB PADDLETOONNX_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/paddle2onnx/lib/*)
 install(PROGRAMS ${PADDLETOONNX_LIBS} DESTINATION lib)

-file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/RK3588/lib/*)
+file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/${RKNN2_TARGET_SOC}/lib/*)
 install(PROGRAMS ${RKNPU2_LIBS} DESTINATION lib)
--- a/examples/vision/detection/paddledetection/rknpu2/cpp/README.md
+++ b/examples/vision/detection/paddledetection/rknpu2/cpp/README.md
@@ -62,7 +62,7 @@ make install

 ```bash
 cd ./build/install
-./rknpu_test
+./infer_picodet model/picodet_s_416_coco_lcnet images/000000014439.jpg
 ```


--- a/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc
+++ b/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc
@@ -14,73 +14,53 @@
 #include <iostream>
 #include <string>
 #include "fastdeploy/vision.h"
+#include <sys/time.h>
+double __get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec); }
+void InferPicodet(const std::string& model_dir, const std::string& image_file);

-void InferPicodet(const std::string& device = "cpu");
+int main(int argc, char* argv[]) {
+  if (argc < 3) {
+    std::cout
+        << "Usage: infer_demo path/to/model_dir path/to/image run_option, "
+           "e.g ./infer_model ./picodet_model_dir ./test.jpeg"
+        << std::endl;
+    return -1;
+  }
+
+  InferPicodet(argv[1], argv[2]);

-int main() {
-  InferPicodet("npu");
  return 0;
 }

-fastdeploy::RuntimeOption GetOption(const std::string& device) {
+void InferPicodet(const std::string& model_dir, const std::string& image_file) {
+  struct timeval start_time, stop_time;
+  auto model_file = model_dir + "/picodet_s_416_coco_lcnet_rk3568.rknn";
+  auto params_file = "";
+  auto config_file = model_dir + "/infer_cfg.yml";
+
  auto option = fastdeploy::RuntimeOption();
-  if (device == "npu") {
-    option.UseRKNPU2();
-  } else {
-    option.UseCpu();
-  }
-  return option;
-}
+  option.UseRKNPU2();

-fastdeploy::ModelFormat GetFormat(const std::string& device) {
-  auto format = fastdeploy::ModelFormat::ONNX;
-  if (device == "npu") {
-    format = fastdeploy::ModelFormat::RKNN;
-  } else {
-    format = fastdeploy::ModelFormat::ONNX;
-  }
-  return format;
-}
+  auto format = fastdeploy::ModelFormat::RKNN;

-std::string GetModelPath(std::string& model_path, const std::string& device) {
-  if (device == "npu") {
-    model_path += "rknn";
-  } else {
-    model_path += "onnx";
-  }
-  return model_path;
-}
-
-void InferPicodet(const std::string &device) {
-  std::string model_file = "./model/picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet_rk3588.";
-  std::string params_file;
-  std::string config_file = "./model/picodet_s_416_coco_lcnet/infer_cfg.yml";
-
-  fastdeploy::RuntimeOption option = GetOption(device);
-  fastdeploy::ModelFormat format = GetFormat(device);
-  model_file = GetModelPath(model_file, device);
-  auto model = fastdeploy::vision::detection::RKPicoDet(
+  auto model = fastdeploy::vision::detection::PicoDet(
      model_file, params_file, config_file,option,format);

-  if (!model.Initialized()) {
-    std::cerr << "Failed to initialize." << std::endl;
-    return;
-  }
-  auto image_file = "./images/000000014439.jpg";
+  model.GetPostprocessor().ApplyDecodeAndNMS();
+
  auto im = cv::imread(image_file);

  fastdeploy::vision::DetectionResult res;
-  clock_t start = clock();
+  gettimeofday(&start_time, NULL);
  if (!model.Predict(&im, &res)) {
    std::cerr << "Failed to predict." << std::endl;
    return;
  }
-  clock_t end = clock();
-  auto dur = static_cast<double>(end - start);
-  printf("picodet_npu use time:%f\n", (dur / CLOCKS_PER_SEC));
+  gettimeofday(&stop_time, NULL);
+  printf("infer use %f ms\n", (__get_us(stop_time) - __get_us(start_time)) / 1000);

  std::cout << res.Str() << std::endl;
  auto vis_im = fastdeploy::vision::VisDetection(im, res,0.5);
-  cv::imwrite("picodet_npu_result.jpg", vis_im);
-  std::cout << "Visualized result saved in ./picodet_npu_result.jpg" << std::endl;
+  cv::imwrite("picodet_result.jpg", vis_im);
+  std::cout << "Visualized result saved in ./picodet_result.jpg" << std::endl;
 }
--- a/examples/vision/detection/paddledetection/rknpu2/python/README.md
+++ b/examples/vision/detection/paddledetection/rknpu2/python/README.md
@@ -15,11 +15,11 @@ cd FastDeploy/examples/vision/detection/paddledetection/rknpu2/python
 wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg

 # copy model
-cp -r ./picodet_s_416_coco_npu /path/to/FastDeploy/examples/vision/detection/rknpu2detection/paddledetection/python
+cp -r ./picodet_s_416_coco_lcnet /path/to/FastDeploy/examples/vision/detection/rknpu2detection/paddledetection/python

 # 推理
-python3 infer.py --model_file ./picodet_s_416_coco_npu/picodet_s_416_coco_npu_3588.rknn  \
-                  --config_file ./picodet_s_416_coco_npu/infer_cfg.yml \
+python3 infer.py --model_file ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet_rk3568.rknn  \
+                  --config_file ./picodet_s_416_coco_lcnet/infer_cfg.yml \
                  --image 000000014439.jpg
 ```

--- a/examples/vision/detection/paddledetection/rknpu2/python/infer.py
+++ b/examples/vision/detection/paddledetection/rknpu2/python/infer.py
@@ -28,32 +28,32 @@ def parse_arguments():
    return parser.parse_args()


-def build_option(args):
-    option = fd.RuntimeOption()
-    option.use_rknpu2()
-    return option
+if __name__ == "__main__":
+    args = parse_arguments()

+    model_file = args.model_file
+    params_file = ""
+    config_file = args.config_file

-args = parse_arguments()
+    # 配置runtime，加载模型
+    runtime_option = fd.RuntimeOption()
+    runtime_option.use_rknpu2()

-# 配置runtime，加载模型
-runtime_option = build_option(args)
-model_file = args.model_file
-params_file = ""
-config_file = args.config_file
-model = fd.vision.detection.RKPicoDet(
-    model_file,
-    params_file,
-    config_file,
-    runtime_option=runtime_option,
-    model_format=fd.ModelFormat.RKNN)
+    model = fd.vision.detection.PicoDet(
+        model_file,
+        params_file,
+        config_file,
+        runtime_option=runtime_option,
+        model_format=fd.ModelFormat.RKNN)

-# 预测图片分割结果
-im = cv2.imread(args.image)
-result = model.predict(im.copy())
-print(result)
+    model.postprocessor.apply_decode_and_nms()

-# 可视化结果
-vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5)
-cv2.imwrite("visualized_result.jpg", vis_im)
-print("Visualized result save in ./visualized_result.jpg")
+    # 预测图片分割结果
+    im = cv2.imread(args.image)
+    result = model.predict(im.copy())
+    print(result)
+
+    # 可视化结果
+    vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5)
+    cv2.imwrite("visualized_result.jpg", vis_im)
+    print("Visualized result save in ./visualized_result.jpg")