[Hackthon_4th 177] Support PP-YOLOE-R with BM1684 (#1809)

* first draft

* add robx iou

* add benchmark for ppyoloe_r

* remove trash code

* fix bugs

* add pybind nms rotated option

* add missing head file

* fix bug

* fix bug2

* fix shape bug

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

examples/vision/detection/paddledetection/sophgo/README.md

@@ -6,6 +6,7 @@
 - [PP-YOLOE系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe)
 - [PicoDet系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/picodet)
 - [YOLOV8系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4)
+- [PP-YOLOE-R系列模型](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.6/configs/rotate/ppyoloe_r)
 
 ## 准备PP-YOLOE YOLOV8或者PicoDet部署模型以及转换模型
 

examples/vision/detection/paddledetection/sophgo/cpp/CMakeLists.txt

@@ -13,9 +13,11 @@ include_directories(${FASTDEPLOY_INCS})
 include_directories(${FastDeploy_INCLUDE_DIRS})
 
 add_executable(infer_ppyoloe ${PROJECT_SOURCE_DIR}/infer_ppyoloe.cc)
+add_executable(infer_ppyoloe_r ${PROJECT_SOURCE_DIR}/infer_ppyoloe_r.cc)
 add_executable(infer_picodet ${PROJECT_SOURCE_DIR}/infer_picodet.cc)
 add_executable(infer_yolov8 ${PROJECT_SOURCE_DIR}/infer_yolov8.cc)
 # 添加FastDeploy库依赖
 target_link_libraries(infer_ppyoloe ${FASTDEPLOY_LIBS})
+target_link_libraries(infer_ppyoloe_r ${FASTDEPLOY_LIBS})
 target_link_libraries(infer_picodet ${FASTDEPLOY_LIBS})
 target_link_libraries(infer_yolov8 ${FASTDEPLOY_LIBS})

examples/vision/detection/paddledetection/sophgo/cpp/README.md

@@ -1,6 +1,6 @@
 # PaddleDetection C++部署示例
 
-本目录下提供`infer_ppyoloe.cc`,`infer_yolov8.cc`和`infer_picodet.cc`快速完成PP-YOLOE模型,YOLOV8模型和PicoDet模型在SOPHGO BM1684x板子上加速部署的示例。
+本目录下提供`infer_ppyoloe.cc`,`infer_ppyoloe_r.cc`,`infer_yolov8.cc`和`infer_picodet.cc`快速完成PP-YOLOE模型,PP-YOLOE-R模型,YOLOV8模型和PicoDet模型在SOPHGO BM1684x板子上加速部署的示例。
 
 在部署前，需确认以下两个步骤:
 
@@ -18,6 +18,7 @@
 ├── build  # 编译文件夹
 ├── image  # 存放图片的文件夹
 ├── infer_ppyoloe.cc
+├── infer_ppyoloe_r.cc
 ├── infer_picodet.cc
 ├── infer_yolov8.cc
 └── model  # 存放模型文件的文件夹
@@ -37,6 +38,9 @@
 ```bash
 wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
 cp 000000014439.jpg ./images
+
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/P0861__1.0__1154___824.png
+cp P0861__1.0__1154___824.png ./images
 ```
 
 ### 编译example
@@ -53,6 +57,9 @@ make
 #ppyoloe推理示例
 ./infer_ppyoloe model images/000000014439.jpg
 
+#ppyoloe_r推理示例
+./infer_ppyoloe_r model images/P0861__1.0__1154___824.png
+
 #picodet推理示例
 ./infer_picodet model images/000000014439.jpg
 ```

examples/vision/detection/paddledetection/sophgo/cpp/infer_ppyoloe_r.cc

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <sys/time.h>
+
+#include <iostream>
+#include <string>
+
+#include "fastdeploy/vision.h"
+
+void SophgoInfer(const std::string& model_dir, const std::string& image_file) {
+  auto model_file = model_dir + "/ppyoloe_r_crn_s_3x_dota_1684x_f32.bmodel";
+  auto params_file = "";
+  auto config_file = model_dir + "/infer_cfg.yml";
+
+  auto option = fastdeploy::RuntimeOption();
+  option.UseSophgo();
+  option.UseSophgoBackend();
+
+  auto format = fastdeploy::ModelFormat::SOPHGO;
+  auto model = fastdeploy::vision::detection::PPYOLOER(
+      model_file, params_file, config_file, option, format);
+
+  auto im = cv::imread(image_file);
+
+  fastdeploy::vision::DetectionResult res;
+  if (!model.Predict(&im, &res)) {
+    std::cerr << "Failed to predict." << std::endl;
+    return;
+  }
+
+  std::cout << res.Str() << std::endl;
+  auto vis_im = fastdeploy::vision::VisDetection(im, res, 0.1);
+  cv::imwrite("infer_sophgo.jpg", vis_im);
+  std::cout << "Visualized result saved in ./infer_sophgo.jpg" << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+  if (argc < 3) {
+    std::cout
+        << "Usage: infer_demo path/to/model_dir path/to/image run_option, "
+           "e.g ./infer_model ./model_dir ./test.jpeg"
+        << std::endl;
+    return -1;
+  }
+  SophgoInfer(argv[1], argv[2]);
+  return 0;
+}

examples/vision/detection/paddledetection/sophgo/python/README.md

@@ -13,6 +13,7 @@ cd FastDeploy/examples/vision/detection/paddledetection/sophgo/python
 
 # 下载图片
 wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/P0861__1.0__1154___824.png
 
 # 推理
 # ppyoloe推理示例
@@ -33,12 +34,19 @@ python3 infer_picodet.py --auto False --pp_detect_path '' --model_file model/ppy
 python3 infer_yolov8.py --model_file model/yolov8s_s_300e_coco_1684x_f32.bmodel --config_file model/infer_cfg.yml --image ./000000014439.jpg
 # 运行完成后返回结果如下所示
 可视化结果存储在sophgo_result.jpg中
+
+# ppyoloe_r推理示例
+# 指定--auto True，自动完成模型准备、转换和推理，需要指定PaddleDetection路径
+python3 infer_ppyoloe_r.py --model_file model/ppyoloe_r_crn_s_3x_dota_1684x_f32.bmodel --image P0861__1.0__1154___824.png --config_file model/infer_cfg.yml
+可视化结果存储在sophgo_result_ppyoloe_r.jpg中
 ```
 
 ## 其它文档
 - [PP-YOLOE C++部署](../cpp)
 - [PicoDet C++部署](../cpp)
 - [YOLOV8 C++部署](../cpp)
+- [PP-YOLOE-R C++部署](../cpp)
 - [转换PicoDet SOPHGO模型文档](../README.md)
 - [转换PP-YOLOE SOPHGO模型文档](../README.md)
 - [转换YOLOV8 SOPHGO模型文档](../README.md)
+- [转换PP-YOLOE-R SOPHGO模型文档](../README.md)

examples/vision/detection/paddledetection/sophgo/python/infer_ppyoloe_r.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import fastdeploy as fd
+import cv2
+import os
+from subprocess import run
+from prepare_npz import prepare
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--auto",
+        action="store_true",
+        help="Auto download, convert, compile and infer if True")
+    parser.add_argument(
+        "--pp_detect_path",
+        default='/workspace/PaddleDetection',
+        help="Path of PaddleDetection folder")
+    parser.add_argument(
+        "--model_file", required=True, help="Path of sophgo model.")
+    parser.add_argument("--config_file", required=True, help="Path of config.")
+    parser.add_argument(
+        "--image", type=str, required=True, help="Path of test image file.")
+    return parser.parse_args()
+
+
+def export_model(args):
+    PPDetection_path = args.pp_detect_path
+    export_str = 'python3 tools/export_model.py \
+                -c configs/rotate/ppyoloe_r/ppyoloe_r_crn_s_3x_dota.yml \
+                -output_dir=output_inference \
+                -o weights=https://paddledet.bj.bcebos.com/models/ppyoloe_r_crn_s_3x_dota.pdparams'
+
+    cur_path = os.getcwd()
+    os.chdir(PPDetection_path)
+    print(export_str)
+    run(export_str, shell=True)
+    cp_str = 'cp -r ./output_inference/ppyoloe_crn_s_300e_coco ' + cur_path
+    print(cp_str)
+    run(cp_str, shell=True)
+    os.chdir(cur_path)
+
+
+def paddle2onnx():
+    convert_str = 'paddle2onnx --model_dir ppyoloe_r_crn_s_3x_dota \
+                --model_filename model.pdmodel \
+                --params_filename model.pdiparams \
+                --save_file ppyoloe_r_crn_s_3x_dota.onnx \
+                --enable_dev_version True'
+
+    print(convert_str)
+    run(convert_str, shell=True)
+
+
+def mlir_prepare():
+    mlir_path = os.getenv("MODEL_ZOO_PATH")
+    mlir_path = mlir_path[:-13]
+    regression_path = os.path.join(mlir_path, 'regression')
+    mv_str_list = [
+        'mkdir ppyoloe_r', 'cp -rf ' + os.path.join(
+            regression_path, 'dataset/COCO2017/') + ' ./ppyoloe_r',
+        'cp -rf ' + os.path.join(regression_path, 'image/') + ' ./ppyoloe_r',
+        'cp ppyoloe_r_crn_s_3x_dota.onnx ./ppyoloe_r',
+        'mkdir ./ppyoloe_r/workspace'
+    ]
+    for str in mv_str_list:
+        print(str)
+        run(str, shell=True)
+
+
+def image_prepare():
+    img_str = 'wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/P0861__1.0__1154___824.png'
+    if not os.path.exists('P0861__1.0__1154___824.png'):
+        print(img_str)
+        run(img_str, shell=True)
+    prepare('P0861__1.0__1154___824.png', [640, 640])
+    cp_npz_str = 'cp ./inputs.npz ./ppyoloe_r'
+    print(cp_npz_str)
+    run(cp_npz_str, shell=True)
+
+
+def onnx2mlir():
+    transform_str = 'model_transform.py \
+            --model_name ppyoloe_r_crn_s_3x_dota \
+            --model_def ../ppyoloe_r_crn_s_3x_dota.onnx \
+            --input_shapes [[1,3,1024,1024],[1,2]] \
+            --keep_aspect_ratio \
+            --pixel_format rgb \
+            --mlir ppyoloe_r_crn_s_3x_dota.mlir'
+
+    os.chdir('./ppyoloe_r/workspace')
+    print(transform_str)
+    run(transform_str, shell=True)
+    os.chdir('../../')
+
+
+def mlir2bmodel():
+    deploy_str = 'model_deploy.py \
+            --mlir ppyoloe_r_crn_s_3x_dota.mlir \
+            --quantize F32 \
+            --chip bm1684x \
+            --model ppyoloe_r_crn_s_3x_dota_1684x_f32.bmodel'
+
+    os.chdir('./ppyoloe_r/workspace')
+    print(deploy_str)
+    run(deploy_str, shell=True)
+    os.chdir('../../')
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+
+    if args.auto:
+        export_model(args)
+        paddle2onnx()
+        mlir_prepare()
+        image_prepare()
+        onnx2mlir()
+        mlir2bmodel()
+
+    model_file = './ppyoloe/workspace/ppyoloe_crn_s_300e_coco_1684x_f32.bmodel' if args.auto else args.model_file
+    params_file = ""
+    config_file = './ppyoloe_r_crn_s_3x_dota/infer_cfg.yml' if args.auto else args.config_file
+    image_file = './P0861__1.0__1154___824.png' if args.auto else args.image
+
+    # 配置runtime，加载模型
+    runtime_option = fd.RuntimeOption()
+    runtime_option.use_sophgo()
+
+    model = fd.vision.detection.PPYOLOER(
+        model_file,
+        params_file,
+        config_file,
+        runtime_option=runtime_option,
+        model_format=fd.ModelFormat.SOPHGO)
+
+    # 预测图片分割结果
+    im = cv2.imread(image_file)
+    result = model.predict(im)
+    print(result)
+
+    # 可视化结果
+    vis_im = fd.vision.vis_detection(im, result, score_threshold=0.1)
+    cv2.imwrite("sophgo_result_ppyoloe_r.jpg", vis_im)
+    print("Visualized result save in ./sophgo_result_ppyoloe_r.jpg")