From 34bea7649dd7ac7d10fae63bf7e5777042bf484c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dant=C3=A8s?= <120167928+fnavigate84@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:49:17 +0800 Subject: [PATCH 1/4] [Backend] add sophgo backend (#1015) * Add Sophgo Device add sophgo backend in fastdeploy add resnet50, yolov5s, liteseg examples. * replace sophgo lib with download links; fix model.cc bug * modify CodeStyle * remove unuseful files;change the names of sophgo device and sophgo backend * sophgo support python and add python examples * remove unuseful rows in cmake according pr Co-authored-by: Zilong Xing --- CMakeLists.txt | 13 +- FastDeploy.cmake.in | 2 + cmake/sophgo.cmake | 7 + cmake/summary.cmake | 1 + docs/cn/build_and_install/README.md | 1 + docs/cn/build_and_install/sophgo.md | 78 +++++ docs/en/build_and_install/sophgo.md | 77 +++++ .../paddleclas/sophgo/README.md | 84 +++++ .../paddleclas/sophgo/cpp/CMakeLists.txt | 17 + .../paddleclas/sophgo/cpp/README.md | 61 ++++ .../paddleclas/sophgo/cpp/infer.cc | 61 ++++ .../paddleclas/sophgo/python/README.md | 29 ++ .../paddleclas/sophgo/python/infer.py | 41 +++ .../vision/detection/yolov5/sophgo/README.md | 75 +++++ .../yolov5/sophgo/cpp/CMakeLists.txt | 17 + .../detection/yolov5/sophgo/cpp/README.md | 56 ++++ .../detection/yolov5/sophgo/cpp/infer.cc | 61 ++++ .../detection/yolov5/sophgo/python/README.md | 46 +++ .../detection/yolov5/sophgo/python/infer.py | 40 +++ .../segmentation/paddleseg/sophgo/README.md | 89 ++++++ .../paddleseg/sophgo/cpp/CMakeLists.txt | 17 + .../paddleseg/sophgo/cpp/README.md | 56 ++++ .../paddleseg/sophgo/cpp/infer.cc | 71 +++++ .../paddleseg/sophgo/python/README.md | 26 ++ .../paddleseg/sophgo/python/infer.py | 45 +++ fastdeploy/backends/sophgo/sophgo_backend.cc | 290 ++++++++++++++++++ fastdeploy/backends/sophgo/sophgo_backend.h | 75 +++++ fastdeploy/backends/sophgo/sophgo_config.h | 0 fastdeploy/core/fd_type.cc | 12 +- fastdeploy/core/fd_type.h | 4 +- fastdeploy/fastdeploy_model.cc | 32 ++ fastdeploy/fastdeploy_model.h | 4 + fastdeploy/pybind/runtime.cc | 6 +- fastdeploy/runtime.cc | 65 +++- fastdeploy/runtime.h | 8 + .../vision/classification/ppcls/model.cc | 5 +- .../vision/detection/contrib/yolov5/yolov5.cc | 2 + fastdeploy/vision/detection/ppdet/model.h | 1 + fastdeploy/vision/segmentation/ppseg/model.cc | 10 +- python/fastdeploy/runtime.py | 5 + python/setup.py | 2 + 41 files changed, 1583 insertions(+), 9 deletions(-) create mode 100644 cmake/sophgo.cmake create mode 100644 docs/cn/build_and_install/sophgo.md create mode 100644 docs/en/build_and_install/sophgo.md create mode 100644 examples/vision/classification/paddleclas/sophgo/README.md create mode 100644 examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt create mode 100644 examples/vision/classification/paddleclas/sophgo/cpp/README.md create mode 100644 examples/vision/classification/paddleclas/sophgo/cpp/infer.cc create mode 100644 examples/vision/classification/paddleclas/sophgo/python/README.md create mode 100644 examples/vision/classification/paddleclas/sophgo/python/infer.py create mode 100644 examples/vision/detection/yolov5/sophgo/README.md create mode 100644 examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt create mode 100644 examples/vision/detection/yolov5/sophgo/cpp/README.md create mode 100644 examples/vision/detection/yolov5/sophgo/cpp/infer.cc create mode 100644 examples/vision/detection/yolov5/sophgo/python/README.md create mode 100644 examples/vision/detection/yolov5/sophgo/python/infer.py create mode 100644 examples/vision/segmentation/paddleseg/sophgo/README.md create mode 100644 examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt create mode 100644 examples/vision/segmentation/paddleseg/sophgo/cpp/README.md create mode 100644 examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc create mode 100644 examples/vision/segmentation/paddleseg/sophgo/python/README.md create mode 100644 examples/vision/segmentation/paddleseg/sophgo/python/infer.py create mode 100644 fastdeploy/backends/sophgo/sophgo_backend.cc create mode 100644 fastdeploy/backends/sophgo/sophgo_backend.h create mode 100644 fastdeploy/backends/sophgo/sophgo_config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d4fe36fa3..42bc600bb 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ PROJECT(fastdeploy C CXX) CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + option(CSRCS_DIR_NAME "Name of source code directory") option(LIBRARY_NAME "Name of build library name") option(PY_LIBRARY_NAME "Name of build python library name") @@ -60,6 +61,7 @@ option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF) option(ENABLE_POROS_BACKEND "Whether to enable poros backend." OFF) option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF) option(ENABLE_RKNPU2_BACKEND "Whether to enable RKNPU2 backend." OFF) +option(ENABLE_SOPHGO_BACKEND "Whether to enable SOPHON backend." OFF) option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF) option(ENABLE_VISION "Whether to enable vision models usage." OFF) option(ENABLE_TEXT "Whether to enable text models usage." OFF) @@ -194,6 +196,7 @@ file(GLOB_RECURSE DEPLOY_POROS_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fast file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cpp) file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) file(GLOB_RECURSE DEPLOY_RKNPU2_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/rknpu/rknpu2/*.cc) +file(GLOB_RECURSE DEPLOY_SOPHGO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/sophgo/*.cc) file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) file(GLOB_RECURSE DEPLOY_ENCRYPTION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/encryption/*.cc) @@ -201,8 +204,7 @@ file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/f file(GLOB_RECURSE DEPLOY_VISION_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cu) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) - -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS} ${DEPLOY_ENCRYPTION_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS} ${DEPLOY_SOPHGO_SRCS} ${DEPLOY_ENCRYPTION_SRCS}) set(DEPEND_LIBS "") @@ -266,6 +268,13 @@ if(ENABLE_RKNPU2_BACKEND) list(APPEND DEPEND_LIBS ${RKNN_RT_LIB}) endif() +if(ENABLE_SOPHGO_BACKEND) + add_definitions(-DENABLE_SOPHGO_BACKEND) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_SOPHGO_SRCS}) + include(${PROJECT_SOURCE_DIR}/cmake/sophgo.cmake) + list(APPEND DEPEND_LIBS ${SOPHGO_RT_LIB}) +endif() + if(ENABLE_POROS_BACKEND) set(CMAKE_CXX_STANDARD 14) add_definitions(-DENABLE_POROS_BACKEND) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 812caae54..6ba0b4307 100755 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -3,6 +3,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.8) set(WITH_GPU @WITH_GPU@) set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@) set(ENABLE_RKNPU2_BACKEND @ENABLE_RKNPU2_BACKEND@) +set(ENABLE_SOPHGO_BACKEND @ENABLE_SOPHGO_BACKEND@) set(ENABLE_LITE_BACKEND @ENABLE_LITE_BACKEND@) set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@) set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@) @@ -271,6 +272,7 @@ message(STATUS " CXX flags : ${CMAKE_CXX_FLAGS}") message(STATUS " WITH_GPU : ${WITH_GPU}") message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_RKNPU2_BACKEND : ${ENABLE_RKNPU2_BACKEND}") +message(STATUS " ENABLE_SOPHGO_BACKEND : ${ENABLE_SOPHGO_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") message(STATUS " ENABLE_POROS_BACKEND : ${ENABLE_POROS_BACKEND}") message(STATUS " ENABLE_OPENVINO_BACKEND : ${ENABLE_OPENVINO_BACKEND}") diff --git a/cmake/sophgo.cmake b/cmake/sophgo.cmake new file mode 100644 index 000000000..d51d61368 --- /dev/null +++ b/cmake/sophgo.cmake @@ -0,0 +1,7 @@ +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +find_package(libsophon REQUIRED) +message(${LIBSOPHON_LIB_DIRS}) +include_directories(${LIBSOPHON_INCLUDE_DIRS}) +message(${LIBSOPHON_LIB_DIRS}) +set(SOPHGO_RT_LIB ${LIBSOPHON_LIB_DIRS}/libbmrt.so) \ No newline at end of file diff --git a/cmake/summary.cmake b/cmake/summary.cmake index 38fb35530..9c3b1981c 100755 --- a/cmake/summary.cmake +++ b/cmake/summary.cmake @@ -32,6 +32,7 @@ function(fastdeploy_summary) message(STATUS " Paddle2ONNX version : ${PADDLE2ONNX_VERSION}") message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_RKNPU2_BACKEND : ${ENABLE_RKNPU2_BACKEND}") + message(STATUS " ENABLE_SOPHGO_BACKEND : ${ENABLE_SOPHGO_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") message(STATUS " ENABLE_LITE_BACKEND : ${ENABLE_LITE_BACKEND}") message(STATUS " ENABLE_POROS_BACKEND : ${ENABLE_POROS_BACKEND}") diff --git a/docs/cn/build_and_install/README.md b/docs/cn/build_and_install/README.md index 838a9f8ec..5175f50d9 100755 --- a/docs/cn/build_and_install/README.md +++ b/docs/cn/build_and_install/README.md @@ -26,6 +26,7 @@ | ENABLE_PADDLE_BACKEND | 默认OFF，是否编译集成Paddle Inference后端(CPU/GPU上推荐打开) | | ENABLE_LITE_BACKEND | 默认OFF，是否编译集成Paddle Lite后端(编译Android库时需要设置为ON) | | ENABLE_RKNPU2_BACKEND | 默认OFF，是否编译集成RKNPU2后端(RK3588/RK3568/RK3566上推荐打开) | +| ENABLE_SOPHGO_BACKEND | 默认OFF，是否编译集成SOPHGO后端, 当在SOPHGO TPU上部署时，需要设置为ON | | WITH_ASCEND | 默认OFF，当在华为昇腾NPU上部署时, 需要设置为ON | | WITH_KUNLUNXIN | 默认OFF，当在昆仑芯XPU上部署时，需设置为ON | | WITH_TIMVX | 默认OFF，需要在RV1126/RV1109/A311D上部署时，需设置为ON | diff --git a/docs/cn/build_and_install/sophgo.md b/docs/cn/build_and_install/sophgo.md new file mode 100644 index 000000000..f27432e71 --- /dev/null +++ b/docs/cn/build_and_install/sophgo.md @@ -0,0 +1,78 @@ +# SOPHGO 部署库编译 + +## SOPHGO 环境准备 +SOPHGO支持linux下进行编译,系统为Debian/Ubuntu +安装包由三个文件构成 +- [sophon-driver\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/KWqbmEcKp) +- [sophon-libsophon\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/PlvlBXhWY) +- [sophon-libsophon-dev\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/zTErLlpS7) + +其中“$arch”为当前机器的硬件架构，使用以下命令可以获取当前的服务器arch： +```shell +uname -m +``` +通常x86_64 机器对应的硬件架构为amd64,arm64 机器对应的硬件架构为 arm64: +```text +- sophon-driver_0.4.2_$arch.deb +- sophon-libsophon_0.4.2_$arch.deb +- sophon-libsophon-dev_0.4.2_$arch.deb +``` + +其中:sophon-driver 包含了 PCIe 加速卡驱动;sophon-libsophon 包含了运行时环境(库文 +件、工具等);sophon-libsophon-dev 包含了开发环境(头文件等)。如果只是在部署环境上安 +装,则不需要安装 sophon-libsophon-dev。 +可以通过如下步骤安装: +```shell +#安装依赖库,只需要执行一次: +sudo apt install dkms libncurses5 +#安装 libsophon: +sudo dpkg -i sophon-*.deb +#在终端执行如下命令,或者登出再登入当前用户后即可使用 bm-smi 等命令: +source /etc/profile +``` +安装位置为： +```text +/opt/sophon/ +├── driver-0.4.2 +├── libsophon-0.4.2 +| ├──bin +| ├──data +| ├──include +| └──lib +└── libsophon-current->/opt/sophon/libsophon-0.4.2 +``` + +## C++ SDK编译安装 +搭建好编译环境之后，编译命令如下： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy +mkdir build && cd build + +# CMake configuration with Ascend +cmake -DENABLE_SOPHGO_BACKEND=ON \ + -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-sophgo \ + -DENABLE_VISION=ON \ + .. + +# Build FastDeploy Ascend C++ SDK +make -j8 +make install +``` +编译完成之后，会在当前的build目录下生成 fastdeploy-sophgo 目录，编译完成。 + +## Python FastDeploy 库编译 +搭建好编译环境之后，编译命令如下： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/python +export ENABLE_SOPHGO_BACKEND=ON +export ENABLE_VISION=ON + +python setup.py build +python setup.py bdist_wheel + +#编译完成后,请用户自行安装当前目录的dist文件夹内的whl包. +``` diff --git a/docs/en/build_and_install/sophgo.md b/docs/en/build_and_install/sophgo.md new file mode 100644 index 000000000..08d18122c --- /dev/null +++ b/docs/en/build_and_install/sophgo.md @@ -0,0 +1,77 @@ + +# How to Build SOPHGO Deployment Environment + +## SOPHGO Environment Preparation +SOPHGO supports compilation on linux, using Debian/Ubuntu as an example +The installation package consists of three files +- [sophon-driver\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/KWqbmEcKp) +- [sophon-libsophon\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/PlvlBXhWY) +- [sophon-libsophon-dev\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/zTErLlpS7) + +$arch indicates the hardware architecture of the current machine. Run the following command to obtain the current server arch: +```shell +uname -m +``` +Generally, the hardware architecture of x86_64 machines is amd64, so the hardware architecture is arm64: +```text +- sophon-driver_0.4.2_$arch.deb +- sophon-libsophon_0.4.2_$arch.deb +- sophon-libsophon-dev_0.4.2_$arch.deb +``` + +sophon-driver contains PCIe acceleration card drivers; sophon-libsophon contains the runtime environment (librarys, tools, etc); sophon-libsophon-dev contains the development environment (header files, etc.). If you install packages only on a deployment environment, you do not need to install sophon-libsophon-dev. +You can perform the following steps to install: +```shell +#To install a dependency library, you only need to do this once: +sudo apt install dkms libncurses5 +#install libsophon: +sudo dpkg -i sophon-*.deb +#Run the following command on the terminal, log out and then log in the current user to use commands such as bm-smi: +source /etc/profile +``` +The position of installation:： +```text +/opt/sophon/ +├── driver-0.4.2 +├── libsophon-0.4.2 +| ├──bin +| ├──data +| ├──include +| └──lib +└── libsophon-current->/opt/sophon/libsophon-0.4.2 +``` + +## How to Build and Install C++ SDK +After setting up the compilation environment, the compilation command is as follows： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy +mkdir build && cd build + +# CMake configuration with Ascend +cmake -DENABLE_SOPHGO_BACKEND=ON \ + -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-sophgo \ + -DENABLE_VISION=ON \ + .. + +# Build FastDeploy Ascend C++ SDK +make -j8 +make install +``` +When the compilation is complete, the fastdeploy-sophgo directory is created in the current build directory, indicating that the FastDeploy library has been compiled. + +## Compiling Python FastDeploy Libraries +After setting up the compilation environment, the compilation command is as follows： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/python +export ENABLE_SOPHGO_BACKEND=ON +export ENABLE_VISION=ON + +python setup.py build +python setup.py bdist_wheel + +#After the compilation is complete, please install the whl package in the dist folder of the current directory. +``` diff --git a/examples/vision/classification/paddleclas/sophgo/README.md b/examples/vision/classification/paddleclas/sophgo/README.md new file mode 100644 index 000000000..32bb3bfbf --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/README.md @@ -0,0 +1,84 @@ +# PaddleDetection SOPHGO部署示例 + +## 支持模型列表 + +目前FastDeploy支持的如下模型的部署[ResNet系列模型](https://github.com/PaddlePaddle/PaddleClas/blob/release/2.4/docs/zh_CN/models/ResNet_and_vd.md) + +## 准备ResNet部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- Paddle动态图模型转换为ONNX模型，请参考[Paddle2ONNX模型转换](https://github.com/PaddlePaddle/Paddle2ONNX/tree/develop/model_zoo/classification) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir)。 + +## 模型转换example + +下面以[ResNet50_vd](https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz)为例子,教大家如何转换Paddle模型到SOPHGO-TPU模型。 + +## 导出ONNX模型 + +### 下载Paddle ResNet50_vd静态图模型并解压 +```shell +wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz +tar xvf ResNet50_vd_infer.tgz +``` + +### 静态图转ONNX模型，注意，这里的save_file请和压缩包名对齐 +```shell +paddle2onnx --model_dir ResNet50_vd_infer \ + --model_filename inference.pdmodel \ + --params_filename inference.pdiparams \ + --save_file ResNet50_vd_infer.onnx \ + --enable_dev_version True +``` +### 导出bmodel模型 + +以转化BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir ResNet50_vd_infer && cd ResNet50_vd_infer + +# 在该文件中放入测试图片，同时将上一步转换好的ResNet50_vd_infer.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +# 放入onnx模型文件ResNet50_vd_infer.onnx + +mkdir workspace && cd workspace + +# 将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name ResNet50_vd_infer \ + --model_def ../ResNet50_vd_infer.onnx \ + --input_shapes [[1,3,224,224]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names save_infer_model/scale_0.tmp_1 \ + --test_input ../image/dog.jpg \ + --test_result ResNet50_vd_infer_top_outputs.npz \ + --mlir ResNet50_vd_infer.mlir + +# 将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir ResNet50_vd_infer.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input ResNet50_vd_infer_in_f32.npz \ + --test_reference ResNet50_vd_infer_top_outputs.npz \ + --model ResNet50_vd_infer_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型ResNet50_vd_infer_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt b/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/README.md b/examples/vision/classification/paddleclas/sophgo/cpp/README.md new file mode 100644 index 000000000..7edfd2c94 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/README.md @@ -0,0 +1,61 @@ +# PaddleClas C++部署示例 + +本目录下提供`infer.cc`快速完成ResNet50_vd模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +├── preprocess_config.yaml #示例前处理配置文件 +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 +将前处理配置文件也拷贝到model中 +```bash +cp preprocess_config.yaml ./model +``` + +### 准备测试图片至image文件夹 +```bash +wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg +cp ILSVRC2012_val_00000010.jpeg ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/ILSVRC2012_val_00000010.jpeg +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc b/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc new file mode 100644 index 000000000..b9281ffb0 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "resnet50_1684x_f32.bmodel"; + auto params_file = model_dir + sep + ""; + auto config_file = model_dir + sep + "preprocess_config.yaml"; + + fastdeploy::RuntimeOption option; + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + auto model = fastdeploy::vision::classification::PaddleClasModel( + model_file, params_file, config_file, option, model_format); + + assert(model.Initialized()); + + auto im = cv::imread(image_file); + + fastdeploy::vision::ClassifyResult res; + if (!model.Predict(im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout << "Usage: infer_demo path/to/model " + "path/to/image " + "run_option, " + "e.g ./infer_demo ./bmodel ./test.jpeg" + << std::endl; + return -1; + } + + std::string model_dir = argv[1]; + std::string test_image = argv[2]; + InitAndInfer(model_dir, test_image); + return 0; +} diff --git a/examples/vision/classification/paddleclas/sophgo/python/README.md b/examples/vision/classification/paddleclas/sophgo/python/README.md new file mode 100644 index 000000000..f495e5830 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/python/README.md @@ -0,0 +1,29 @@ +# PaddleClas Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 ResNet50_vd 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/classification/paddleclas/sophgo/python + +# 下载图片 +wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg + +# 推理 +python3 infer.py --model_file ./bmodel/resnet50_1684x_f32.bmodel --config_file ResNet50_vd_infer/inference_cls.yaml --image ILSVRC2012_val_00000010.jpeg + +# 运行完成后返回结果如下所示 +ClassifyResult( +label_ids: 153, +scores: 0.684570, +) +``` + +## 其它文档 +- [ResNet50_vd C++部署](../cpp) +- [转换ResNet50_vd SOPHGO模型文档](../README.md) diff --git a/examples/vision/classification/paddleclas/sophgo/python/infer.py b/examples/vision/classification/paddleclas/sophgo/python/infer.py new file mode 100644 index 000000000..5bc84789e --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/python/infer.py @@ -0,0 +1,41 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--config_file", required=True, help="Path of config file.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--topk", type=int, default=1, help="Return topk results.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" +config_file = args.config_file + +model = fd.vision.classification.PaddleClasModel( + model_file, + params_file, + config_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im = cv2.imread(args.image) +result = model.predict(im, args.topk) +print(result) diff --git a/examples/vision/detection/yolov5/sophgo/README.md b/examples/vision/detection/yolov5/sophgo/README.md new file mode 100644 index 000000000..d4fa4f7a8 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/README.md @@ -0,0 +1,75 @@ +# YOLOv5 SOPHGO部署示例 + +## 支持模型列表 + +YOLOv5 v6.0部署模型实现来自[YOLOv5](https://github.com/ultralytics/yolov5/tree/v6.0),和[基于COCO的预训练模型](https://github.com/ultralytics/yolov5/releases/tag/v6.0) + +## 准备YOLOv5部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- 下载预训练ONNX模型，请参考[YOLOv5准备部署模型](https://github.com/PaddlePaddle/FastDeploy/tree/develop/examples/vision/detection/yolov5) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir) + +## 模型转换example + +下面以YOLOv5s为例子,教大家如何转换ONNX模型到SOPHGO-TPU模型 + +## 下载YOLOv5s模型 + +### 下载ONNX YOLOv5s静态图模型 +```shell +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx + +``` +### 导出bmodel模型 + +以转化BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir YOLOv5s && cd YOLOv5s + +# 在该文件中放入测试图片，同时将上一步下载的yolov5s.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +# 放入onnx模型文件yolov5s.onnx + +mkdir workspace && cd workspace + +# 将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name yolov5s \ + --model_def ../yolov5s.onnx \ + --input_shapes [[1,3,640,640]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names output,350,498,646 \ + --test_input ../image/dog.jpg \ + --test_result yolov5s_top_outputs.npz \ + --mlir yolov5s.mlir + +# 将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir yolov5s.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input yolov5s_in_f32.npz \ + --test_reference yolov5s_top_outputs.npz \ + --model yolov5s_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型yolov5s_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt b/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/README.md b/examples/vision/detection/yolov5/sophgo/cpp/README.md new file mode 100644 index 000000000..e313da855 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/README.md @@ -0,0 +1,56 @@ +# YOLOv5 C++部署示例 + +本目录下提供`infer.cc`快速完成yolov5s模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 + +### 准备测试图片至image文件夹 +```bash +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg +cp 000000014439.jpg ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/000000014439.jpg +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/infer.cc b/examples/vision/detection/yolov5/sophgo/cpp/infer.cc new file mode 100644 index 000000000..f1f63bcdc --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/infer.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "yolov5s_1684x_f32.bmodel"; + auto params_file = model_dir + sep + ""; + + fastdeploy::RuntimeOption option; + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + + auto model = fastdeploy::vision::detection::YOLOv5( + model_file, params_file, option, model_format); + + assert(model.Initialized()); + + auto im = cv::imread(image_file); + + fastdeploy::vision::DetectionResult res; + if (!model.Predict(im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout << "Usage: infer_demo path/to/model " + "path/to/image " + "run_option, " + "e.g ./infer_demo ./model ./test.jpeg" + << std::endl; + return -1; + } + + std::string model_dir = argv[1]; + std::string test_image = argv[2]; + InitAndInfer(model_dir, test_image); + return 0; +} diff --git a/examples/vision/detection/yolov5/sophgo/python/README.md b/examples/vision/detection/yolov5/sophgo/python/README.md new file mode 100644 index 000000000..ccf8ed7e8 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/python/README.md @@ -0,0 +1,46 @@ +# YOLOv5 Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 YOLOv5 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/sophgo/python + +# 下载图片 +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg + +# 推理 +python3 infer.py --model_file ./bmodel/yolov5s_1684x_f32.bmodel --image 000000014439.jpg + +# 运行完成后返回结果如下所示 +DetectionResult: [xmin, ymin, xmax, ymax, score, label_id] +268.480255,81.053055, 298.694794, 169.439026, 0.896569, 0 +104.731163,45.661972, 127.583824, 93.449387, 0.869531, 0 +378.909363,39.750137, 395.608643, 84.243454, 0.868430, 0 +158.552979,80.361511, 199.185760, 168.181915, 0.842988, 0 +414.375305,90.948090, 506.321899, 280.405182, 0.835842, 0 +364.003448,56.608932, 381.978607, 115.968216, 0.815136, 0 +351.725128,42.635330, 366.910309, 98.048386, 0.808936, 0 +505.888306,114.366791, 593.124878, 275.995270, 0.801361, 0 +327.708618,38.363693, 346.849915, 80.893021, 0.794725, 0 +583.493408,114.532883, 612.354614, 175.873535, 0.760649, 0 +186.470657,44.941360, 199.664505, 61.037643, 0.632591, 0 +169.615891,48.014603, 178.141556, 60.888596, 0.613938, 0 +25.810200,117.199692, 59.888783, 152.850128, 0.590614, 0 +352.145294,46.712723, 381.946075, 106.752151, 0.505329, 0 +1.875000,150.734375, 37.968750, 173.781250, 0.404573, 24 +464.657288,15.901413, 472.512939, 34.116409, 0.346033, 0 +64.625000,135.171875, 84.500000, 154.406250, 0.332831, 24 +57.812500,151.234375, 103.000000, 174.156250, 0.332566, 24 +165.906250,88.609375, 527.906250, 339.953125, 0.259424, 33 +101.406250,152.562500, 118.890625, 169.140625, 0.253891, 24 +``` + +## 其它文档 +- [YOLOv5 C++部署](../cpp) +- [转换YOLOv5 SOPHGO模型文档](../README.md) diff --git a/examples/vision/detection/yolov5/sophgo/python/infer.py b/examples/vision/detection/yolov5/sophgo/python/infer.py new file mode 100644 index 000000000..d1ea190c5 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/python/infer.py @@ -0,0 +1,40 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" + +model = fd.vision.detection.YOLOv5( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im = cv2.imread(args.image) +result = model.predict(im) +print(result) + +# 预测结果可视化 +vis_im = fd.vision.vis_detection(im, result) +cv2.imwrite("sophgo_result.jpg", vis_im) +print("Visualized result save in ./sophgo_result.jpg") diff --git a/examples/vision/segmentation/paddleseg/sophgo/README.md b/examples/vision/segmentation/paddleseg/sophgo/README.md new file mode 100644 index 000000000..afebe3451 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/README.md @@ -0,0 +1,89 @@ +# PaddleSeg C++部署示例 + +## 支持模型列表 + +- PP-LiteSeg部署模型实现来自[PaddleSeg PP-LiteSeg系列模型](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.6/configs/pp_liteseg/README.md) + +## 准备PP-LiteSeg部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- 下载Paddle模型[PP-LiteSeg-B(STDC2)-cityscapes-without-argmax](https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz) +- Pddle模型转换为ONNX模型，请参考[Paddle2ONNX](https://github.com/PaddlePaddle/Paddle2ONNX) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir) + +## 模型转换example + +下面以[PP-LiteSeg-B(STDC2)-cityscapes-without-argmax](https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz)为例子,教大家如何转换Paddle模型到SOPHGO-TPU模型 + +### 下载PP-LiteSeg-B(STDC2)-cityscapes-without-argmax模型,并转换为ONNX模型 +```shell +https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz +tar xvf PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz + +# 修改PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer模型的输入shape，由动态输入变成固定输入 +python paddle_infer_shape.py --model_dir PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --save_dir pp_liteseg_fix \ + --input_shape_dict="{'x':[1,3,512,512]}" + +#将固定输入的Paddle模型转换成ONNX模型 +paddle2onnx --model_dir pp_liteseg_fix \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --save_file pp_liteseg.onnx \ + --enable_dev_version True +``` + +### 导出bmodel模型 + +以转换BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir pp_liteseg && cd pp_liteseg + +#在该文件中放入测试图片，同时将上一步转换的pp_liteseg.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +#放入onnx模型文件pp_liteseg.onnx + +mkdir workspace && cd workspace + +#将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name pp_liteseg \ + --model_def ../pp_liteseg.onnx \ + --input_shapes [[1,3,512,512]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names bilinear_interp_v2_6.tmp_0 \ + --test_input ../image/dog.jpg \ + --test_result pp_liteseg_top_outputs.npz \ + --mlir pp_liteseg.mlir + +#将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir pp_liteseg.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input pp_liteseg_in_f32.npz \ + --test_reference pp_liteseg_top_outputs.npz \ + --model pp_liteseg_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型pp_liteseg_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt b/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md b/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md new file mode 100644 index 000000000..dac3ed565 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md @@ -0,0 +1,56 @@ +# PaddleSeg C++部署示例 + +本目录下提供`infer.cc`快速完成pp_liteseg模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 + +### 准备测试图片至image文件夹 +```bash +wget https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_demo.png +cp cityscapes_demo.png ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/cityscapes_demo.png +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc b/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc new file mode 100644 index 000000000..934ab648c --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include "fastdeploy/vision.h" + +void SophgoInfer(const std::string& model_dir, const std::string& image_file) { + std::string model_file = model_dir + "/pp_liteseg_1684x_f32.bmodel"; + std::string params_file; + std::string config_file = model_dir + "/deploy.yaml"; + auto option = fastdeploy::RuntimeOption(); + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + + auto model = fastdeploy::vision::segmentation::PaddleSegModel( + model_file, params_file, config_file, option, model_format); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + //model.GetPreprocessor().DisableNormalizeAndPermute(); + + fastdeploy::TimeCounter tc; + tc.Start(); + auto im_org = cv::imread(image_file); + + //the input of bmodel should be fixed + int new_width = 512; + int new_height = 512; + cv::Mat im; + cv::resize(im_org, im, cv::Size(new_width, new_height), cv::INTER_LINEAR); + + fastdeploy::vision::SegmentationResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + tc.End(); + tc.PrintInfo("PPSeg in Sophgo"); + + cv::imwrite("infer_sophgo.jpg", vis_im); + std::cout + << "Visualized result saved in ./infer_sophgo.jpg" + << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./bmodel ./test.jpeg" + << std::endl; + return -1; + } + + SophgoInfer(argv[1], argv[2]); + return 0; +} + diff --git a/examples/vision/segmentation/paddleseg/sophgo/python/README.md b/examples/vision/segmentation/paddleseg/sophgo/python/README.md new file mode 100644 index 000000000..e04ad28c4 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/python/README.md @@ -0,0 +1,26 @@ +# PaddleSeg Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 pp_liteseg 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/segmentation/paddleseg/sophgo/python + +# 下载图片 +wget https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_demo.png + +# 推理 +python3 infer.py --model_file ./bmodel/pp_liteseg_1684x_f32.bmodel --config_file ./bmodel/deploy.yaml --image cityscapes_demo.png + +# 运行完成后返回结果如下所示 +运行结果保存在sophgo_img.png中 +``` + +## 其它文档 +- [pp_liteseg C++部署](../cpp) +- [转换 pp_liteseg SOPHGO模型文档](../README.md) diff --git a/examples/vision/segmentation/paddleseg/sophgo/python/infer.py b/examples/vision/segmentation/paddleseg/sophgo/python/infer.py new file mode 100644 index 000000000..1b294da60 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/python/infer.py @@ -0,0 +1,45 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--config_file", required=True, help="Path of config file.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" +config_file = args.config_file + +model = fd.vision.segmentation.PaddleSegModel( + model_file, + params_file, + config_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im_org = cv2.imread(args.image) +#bmodel 是静态模型，模型输入固定，这里设置为[512, 512] +im = cv2.resize(im_org, [512, 512], interpolation=cv2.INTER_LINEAR) +result = model.predict(im) +print(result) + +# 预测结果可视化 +vis_im = fd.vision.vis_segmentation(im, result, weight=0.5) +cv2.imwrite("sophgo_img.png", vis_im) diff --git a/fastdeploy/backends/sophgo/sophgo_backend.cc b/fastdeploy/backends/sophgo/sophgo_backend.cc new file mode 100644 index 000000000..c4a75ce2d --- /dev/null +++ b/fastdeploy/backends/sophgo/sophgo_backend.cc @@ -0,0 +1,290 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/backends/sophgo/sophgo_backend.h" +#include +namespace fastdeploy { + SophgoBackend::~SophgoBackend() { + + bm_dev_free(handle_); + +} +/*************************************************************** + * @name GetSDKAndDeviceVersion + * @brief get Sophgo sdk and device version + * @param None + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::GetSDKAndDeviceVersion() { + + return true; +} + +/*************************************************************** + * @name BuildOption + * @brief save option + * @param SOPHGOTPU2BackendOption + * @note None + ***************************************************************/ +void SophgoBackend::BuildOption(const SophgoBackendOption& option) { +// this->option_ = option; + // save cpu_name +// this->option_.cpu_name = option.cpu_name; +} + +/*************************************************************** + * @name InitFromSophgo + * @brief Initialize Sophgo model + * @param model_file: Binary data for the Sophgo model. + * params_file: None + * option: config + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::InitFromSophgo(const std::string& model_file, + const SophgoBackendOption& option) { + + // LoadModel + if (!this->LoadModel((char*)model_file.data())) { + FDERROR << "load model failed" << std::endl; + return false; + } + + // GetSDKAndDeviceVersion + if (!this->GetSDKAndDeviceVersion()) { + FDERROR << "get SDK and device version failed" << std::endl; + return false; + } + + // BuildOption + this->BuildOption(option); + + // GetModelInputOutputInfos + if (!this->GetModelInputOutputInfos()) { + FDERROR << "get model input output infos failed" << std::endl; + return false; + } + + return true; +} + +/*************************************************************** + * @name LoadModel + * @brief read Sophgo bmodel + * @param model: Binary data for the Sophgo model. + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::LoadModel(void* model) { + unsigned int card_num = 0; + bm_status_t status = bm_get_card_num(&card_num); + status = bm_dev_request(&handle_, 0); + p_bmrt_ = bmrt_create(handle_); + assert(NULL != p_bmrt_); + + bool load_status = bmrt_load_bmodel(p_bmrt_, (char*)model); + assert(load_status); + + int network_num = bmrt_get_network_number(p_bmrt_); + + const char **net_names = NULL; + bmrt_get_network_names(p_bmrt_, &net_names); + net_name_ = net_names[0]; + free(net_names); + + net_info_ = bmrt_get_network_info(p_bmrt_, net_name_.c_str()); + assert(NULL != net_info_); + + return true; +} + +/*************************************************************** + * @name GetModelInputOutputInfos + * @brief Get the detailed input and output infos of Model + * @param None + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::GetModelInputOutputInfos() { + inputs_desc_.resize(net_info_->input_num); + bm_shape_t* input_shapes = net_info_->stages->input_shapes; + for(int idx=0; idxinput_num; idx++){ + std::string temp_name = (net_info_->input_names)[idx]; + std::vector temp_shape{}; + temp_shape.resize(input_shapes[idx].num_dims); + for(int i=0; iinput_dtypes; + //SophgoType to FDDataType + FDDataType temp_dtype = SophgoTensorTypeToFDDataType(*input_dtypes); + TensorInfo temp_input_info = {temp_name, temp_shape, temp_dtype}; + inputs_desc_[idx] = temp_input_info; + } + + outputs_desc_.resize(net_info_->output_num); + bm_shape_t* output_shapes = net_info_->stages->output_shapes; + for(int idx=0; idxoutput_num; idx++){ + std::string temp_name1 = (net_info_->output_names)[idx]; + std::vector temp_shape1{}; + temp_shape1.resize(output_shapes[idx].num_dims); + for(int i=0; ioutput_dtypes; + //SophgoType to FDDataType + FDDataType temp_dtype1 = SophgoTensorTypeToFDDataType(*output_dtypes); + TensorInfo temp_output_info = {temp_name1, temp_shape1, temp_dtype1}; + outputs_desc_[idx] = temp_output_info; + } + return true; +} + +TensorInfo SophgoBackend::GetInputInfo(int index) { + FDASSERT(index < NumInputs(), + "The index: %d should less than the number of inputs: %d.", index, + NumInputs()) + return inputs_desc_[index]; +} + +std::vector SophgoBackend::GetInputInfos() { return inputs_desc_; } + +TensorInfo SophgoBackend::GetOutputInfo(int index) { + FDASSERT(index < NumOutputs(), + "The index: %d should less than the number of outputs %d.", index, + NumOutputs()) + return outputs_desc_[index]; +} + +std::vector SophgoBackend::GetOutputInfos() { return outputs_desc_; } + +bool SophgoBackend::Infer(std::vector& inputs, + std::vector* outputs, + bool copy_to_fd) { + int input_size = inputs.size(); + assert(input_size != 0); + assert(input_size == NumInputs()); + bm_tensor_t input_tensors[input_size]; + bm_status_t status = BM_SUCCESS; + + bm_data_type_t* input_dtypes = net_info_->input_dtypes; + for(int i=0;imax_input_bytes[i]); + assert(BM_SUCCESS == status); + input_tensors[i].dtype = input_dtypes[i]; + input_tensors[i].st_mode = BM_STORE_1N; + input_tensors[i].shape = *(net_info_->stages[i].input_shapes); + unsigned int input_byte = bmrt_tensor_bytesize(&input_tensors[i]); + bm_memcpy_s2d_partial(handle_, input_tensors[i].device_mem, (void *)inputs[i].Data(), + bmrt_tensor_bytesize(&input_tensors[i])); + } + + int output_size = NumOutputs(); + bm_tensor_t output_tensors[output_size]; + for(int i=0;imax_output_bytes[i]); + assert(BM_SUCCESS == status); + } + + bool launch_status = bmrt_launch_tensor_ex(p_bmrt_, net_name_.c_str(), input_tensors, net_info_->input_num, + output_tensors, net_info_->output_num, true, false); + assert(launch_status); + status = bm_thread_sync(handle_); + assert(status == BM_SUCCESS); + + outputs->resize(outputs_desc_.size()); + bm_data_type_t* output_dtypes = net_info_->output_dtypes; + for(int i=0;i temp_shape; + temp_shape.resize(outputs_desc_[i].shape.size()); + for (int j = 0; j < outputs_desc_[i].shape.size(); ++j) { + temp_shape[j] = outputs_desc_[i].shape[j]; + } + (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, outputs_desc_[i].name); + + memcpy((*outputs)[i].MutableData(), temp_out, (*outputs)[i].Nbytes()); + free(temp_out); + } + + return true; +} + +/*************************************************************** + * @name SophgoTensorTypeToFDDataType + * @brief Change SophgoTensorType To FDDataType + * @param bm_data_type_t + * @return None + * @note None + ***************************************************************/ +FDDataType SophgoBackend::SophgoTensorTypeToFDDataType(bm_data_type_t type) { + if (type == BM_FLOAT16) { + return FDDataType::FP32; + } + if (type == BM_FLOAT32) { + return FDDataType::FP32; + } + if (type == BM_INT8) { + return FDDataType::INT8; + } + if (type == BM_INT16) { + return FDDataType::INT16; + } + if (type == BM_INT32) { + return FDDataType::INT32; + } + if (type == BM_UINT8) { + return FDDataType::UINT8; + } + FDERROR << "FDDataType don't support this type" << std::endl; + return FDDataType::UNKNOWN1; +} + +/*************************************************************** + * @name FDDataTypeToSophgoTensorType + * @brief Change FDDataType To SophgoTensorType + * @param FDDataType + * @return None + * @note None + ***************************************************************/ +// Sophgo_tensor_type +bm_data_type_t SophgoBackend::FDDataTypeToSophgoTensorType(fastdeploy::FDDataType type) { + if (type == FDDataType::FP16) { + return BM_FLOAT16; + } + if (type == FDDataType::FP32) { + return BM_FLOAT32; + } + if (type == FDDataType::INT8) { + return BM_INT8; + } + if (type == FDDataType::INT16) { + return BM_INT16; + } + if (type == FDDataType::INT32) { + return BM_INT32; + } + if (type == FDDataType::UINT8) { + return BM_UINT8; + } + FDERROR << "Sophgo_tensor_type don't support this type" << std::endl; + return BM_FLOAT32; +} + +} diff --git a/fastdeploy/backends/sophgo/sophgo_backend.h b/fastdeploy/backends/sophgo/sophgo_backend.h new file mode 100644 index 000000000..8007bfd13 --- /dev/null +++ b/fastdeploy/backends/sophgo/sophgo_backend.h @@ -0,0 +1,75 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/backends/backend.h" +#include "fastdeploy/core/fd_tensor.h" +#include "bmruntime_interface.h" // NOLINT +#include "bmlib_runtime.h" // NOLINT +#include "fastdeploy/backends/sophgo/sophgo_config.h" +#include +#include +#include +#include +#include + +namespace fastdeploy { +struct SophgoBackendOption{ +}; + +class SophgoBackend : public BaseBackend { + public: + SophgoBackend() = default; + virtual ~SophgoBackend(); + bool LoadModel(void* model); + bool GetSDKAndDeviceVersion(); + bool GetModelInputOutputInfos(); + void BuildOption(const SophgoBackendOption& option); + bool InitFromSophgo(const std::string& model_file, + const SophgoBackendOption& option = SophgoBackendOption()); + + int NumInputs() const override { + return static_cast(inputs_desc_.size()); + } + + int NumOutputs() const override { + return static_cast(outputs_desc_.size()); + } + + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + bool Infer(std::vector& inputs, + std::vector* outputs, + bool copy_to_fd = true) override; + + private: + std::vector inputs_desc_; + std::vector outputs_desc_; + std::string net_name_; + + bm_handle_t handle_; + void * p_bmrt_ = nullptr; + + bool infer_init = false; + + const bm_net_info_t* net_info_ = nullptr; + + // SophgoTPU2BackendOption option_; + + static FDDataType SophgoTensorTypeToFDDataType(bm_data_type_t type); + static bm_data_type_t FDDataTypeToSophgoTensorType(FDDataType type); +}; +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/backends/sophgo/sophgo_config.h b/fastdeploy/backends/sophgo/sophgo_config.h new file mode 100644 index 000000000..e69de29bb diff --git a/fastdeploy/core/fd_type.cc b/fastdeploy/core/fd_type.cc index 21e3d30c3..420e03ff7 100755 --- a/fastdeploy/core/fd_type.cc +++ b/fastdeploy/core/fd_type.cc @@ -56,6 +56,9 @@ std::string Str(const Device& d) { case Device::RKNPU: out = "Device::RKNPU"; break; + case Device::SOPHGOTPUD: + out = "Device::SOPHGOTPUD"; + break; case Device::IPU: out = "Device::IPU"; break; @@ -85,6 +88,9 @@ std::ostream& operator<<(std::ostream& out,const Device& d){ case Device::RKNPU: out << "Device::RKNPU"; break; + case Device::SOPHGOTPUD: + out << "Device::SOPHGOTPUD"; + break; case Device::TIMVX: out << "Device::TIMVX"; break; @@ -205,8 +211,10 @@ std::string Str(const ModelFormat& f) { return "ModelFormat::PADDLE"; } else if (f == ModelFormat::ONNX) { return "ModelFormat::ONNX"; - }else if (f == ModelFormat::RKNN) { + } else if (f == ModelFormat::RKNN) { return "ModelFormat::RKNN"; + } else if (f == ModelFormat::SOPHGO) { + return "ModelFormat::SOPHGO"; } else if (f == ModelFormat::TORCHSCRIPT) { return "ModelFormat::TORCHSCRIPT"; } @@ -220,6 +228,8 @@ std::ostream& operator<<(std::ostream& out, const ModelFormat& format) { out << "ModelFormat::ONNX"; } else if (format == ModelFormat::RKNN) { out << "ModelFormat::RKNN"; + } else if (format == ModelFormat::SOPHGO) { + out << "ModelFormat::SOPHGO"; } else if (format == ModelFormat::TORCHSCRIPT) { out << "ModelFormat::TORCHSCRIPT"; } diff --git a/fastdeploy/core/fd_type.h b/fastdeploy/core/fd_type.h index fda26c1c8..5b49f1e86 100755 --- a/fastdeploy/core/fd_type.h +++ b/fastdeploy/core/fd_type.h @@ -22,7 +22,8 @@ namespace fastdeploy { -enum FASTDEPLOY_DECL Device { CPU, GPU, RKNPU, IPU, TIMVX, KUNLUNXIN, ASCEND}; +enum FASTDEPLOY_DECL Device {CPU, GPU, RKNPU, IPU, TIMVX, KUNLUNXIN, ASCEND, + SOPHGOTPUD}; FASTDEPLOY_DECL std::string Str(const Device& d); @@ -72,6 +73,7 @@ enum ModelFormat { ONNX, ///< Model with ONNX format RKNN, ///< Model with RKNN format TORCHSCRIPT, ///< Model with TorchScript format + SOPHGO, ///< Model with SOPHGO format }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, diff --git a/fastdeploy/fastdeploy_model.cc b/fastdeploy/fastdeploy_model.cc index 77c1539c3..d0f06a1ca 100755 --- a/fastdeploy/fastdeploy_model.cc +++ b/fastdeploy/fastdeploy_model.cc @@ -50,6 +50,7 @@ bool FastDeployModel::InitRuntimeWithSpecifiedBackend() { bool use_gpu = (runtime_option.device == Device::GPU); bool use_ipu = (runtime_option.device == Device::IPU); bool use_rknpu = (runtime_option.device == Device::RKNPU); + bool use_sophgotpu = (runtime_option.device == Device::SOPHGOTPUD); bool use_timvx = (runtime_option.device == Device::TIMVX); bool use_ascend = (runtime_option.device == Device::ASCEND); bool use_kunlunxin = (runtime_option.device == Device::KUNLUNXIN); @@ -64,6 +65,11 @@ bool FastDeployModel::InitRuntimeWithSpecifiedBackend() { FDERROR << "The valid rknpu backends of model " << ModelName() << " are " << Str(valid_rknpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; return false; } + } else if (use_sophgotpu) { + if (!IsSupported(valid_sophgonpu_backends, runtime_option.backend)) { + FDERROR << "The valid rknpu backends of model " << ModelName() << " are " << Str(valid_rknpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; + } } else if (use_timvx) { if (!IsSupported(valid_timvx_backends, runtime_option.backend)) { FDERROR << "The valid timvx backends of model " << ModelName() << " are " << Str(valid_timvx_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; @@ -118,6 +124,8 @@ bool FastDeployModel::InitRuntimeWithSpecifiedDevice() { return CreateASCENDBackend(); } else if (runtime_option.device == Device::KUNLUNXIN) { return CreateKunlunXinBackend(); + } else if (runtime_option.device == Device::SOPHGOTPUD) { + return CreateSophgoNPUBackend(); } else if (runtime_option.device == Device::IPU) { #ifdef WITH_IPU return CreateIpuBackend(); @@ -218,6 +226,30 @@ bool FastDeployModel::CreateRKNPUBackend() { return false; } +bool FastDeployModel::CreateSophgoNPUBackend() { + if (valid_sophgonpu_backends.empty()) { + FDERROR << "There's no valid npu backends for model: " << ModelName() + << std::endl; + return false; + } + + for (size_t i = 0; i < valid_sophgonpu_backends.size(); ++i) { + if (!IsBackendAvailable(valid_sophgonpu_backends[i])) { + continue; + } + runtime_option.backend = valid_sophgonpu_backends[i]; + runtime_ = std::unique_ptr(new Runtime()); + if (!runtime_->Init(runtime_option)) { + return false; + } + runtime_initialized_ = true; + return true; + } + FDERROR << "Cannot find an available npu backend to load this model." + << std::endl; + return false; +} + bool FastDeployModel::CreateTimVXBackend() { if (valid_timvx_backends.size() == 0) { FDERROR << "There's no valid timvx backends for model: " << ModelName() diff --git a/fastdeploy/fastdeploy_model.h b/fastdeploy/fastdeploy_model.h index 6d2f7e5b7..9b78c3d3f 100755 --- a/fastdeploy/fastdeploy_model.h +++ b/fastdeploy/fastdeploy_model.h @@ -54,6 +54,9 @@ class FASTDEPLOY_DECL FastDeployModel { /** Model's valid hardware backends. This member defined all the gpu backends have successfully tested for the model */ std::vector valid_rknpu_backends = {}; + /** Model's valid hardware backends. This member defined all the sophgo npu backends have successfully tested for the model + */ + std::vector valid_sophgonpu_backends = {}; /// Get number of inputs for this model virtual int NumInputsOfRuntime() { return runtime_->NumInputs(); } @@ -148,6 +151,7 @@ class FASTDEPLOY_DECL FastDeployModel { bool CreateGpuBackend(); bool CreateIpuBackend(); bool CreateRKNPUBackend(); + bool CreateSophgoNPUBackend(); bool CreateTimVXBackend(); bool CreateKunlunXinBackend(); bool CreateASCENDBackend(); diff --git a/fastdeploy/pybind/runtime.cc b/fastdeploy/pybind/runtime.cc index cbbb7f2e8..3402dd896 100755 --- a/fastdeploy/pybind/runtime.cc +++ b/fastdeploy/pybind/runtime.cc @@ -24,6 +24,7 @@ void BindRuntime(pybind11::module& m) { .def("use_gpu", &RuntimeOption::UseGpu) .def("use_cpu", &RuntimeOption::UseCpu) .def("use_rknpu2", &RuntimeOption::UseRKNPU2) + .def("use_sophgo", &RuntimeOption::UseSophgo) .def("use_ascend", &RuntimeOption::UseAscend) .def("use_kunlunxin", &RuntimeOption::UseKunlunXin) .def("set_external_stream", &RuntimeOption::SetExternalStream) @@ -241,19 +242,22 @@ void BindRuntime(pybind11::module& m) { .value("POROS", Backend::POROS) .value("PDINFER", Backend::PDINFER) .value("RKNPU2", Backend::RKNPU2) + .value("SOPHGOTPU", Backend::SOPHGOTPU) .value("LITE", Backend::LITE); pybind11::enum_(m, "ModelFormat", pybind11::arithmetic(), "ModelFormat for inference.") .value("PADDLE", ModelFormat::PADDLE) .value("TORCHSCRIPT", ModelFormat::TORCHSCRIPT) .value("RKNN", ModelFormat::RKNN) + .value("SOPHGO", ModelFormat::SOPHGO) .value("ONNX", ModelFormat::ONNX); pybind11::enum_(m, "Device", pybind11::arithmetic(), "Device for inference.") .value("CPU", Device::CPU) .value("GPU", Device::GPU) .value("IPU", Device::IPU) - .value("RKNPU", Device::RKNPU); + .value("RKNPU", Device::RKNPU) + .value("SOPHGOTPU", Device::SOPHGOTPUD); pybind11::enum_(m, "FDDataType", pybind11::arithmetic(), "Data type of FastDeploy.") diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 0110b4313..6c1949ed3 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -45,6 +45,10 @@ #include "fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h" #endif +#ifdef ENABLE_SOPHGO_BACKEND +#include "fastdeploy/backends/sophgo/sophgo_backend.h" +#endif + namespace fastdeploy { std::vector GetAvailableBackends() { @@ -69,6 +73,9 @@ std::vector GetAvailableBackends() { #endif #ifdef ENABLE_RKNPU2_BACKEND backends.push_back(Backend::RKNPU2); +#endif +#ifdef ENABLE_SOPHGO_BACKEND + backends.push_back(Backend::SOPHGOTPU); #endif return backends; } @@ -94,6 +101,8 @@ std::string Str(const Backend& b) { return "Backend::POROS"; } else if (b == Backend::RKNPU2) { return "Backend::RKNPU2"; + } else if (b == Backend::SOPHGOTPU) { + return "Backend::SOPHGOTPU"; } else if (b == Backend::OPENVINO) { return "Backend::OPENVINO"; } else if (b == Backend::LITE) { @@ -113,6 +122,8 @@ std::ostream& operator<<(std::ostream& out, const Backend& backend) { out << "Backend::OPENVINO"; } else if (backend == Backend::RKNPU2) { out << "Backend::RKNPU2"; + } else if (backend == Backend::SOPHGOTPU) { + out << "Backend::SOPHGOTPU"; } else if (backend == Backend::POROS) { out << "Backend::POROS"; } else if (backend == Backend::LITE) { @@ -158,6 +169,15 @@ bool CheckModelFormat(const std::string& model_file, << model_file << std::endl; return false; } + } else if (model_format == ModelFormat::SOPHGO) { + if (model_file.size() < 7 || + model_file.substr(model_file.size() -7, 7) != ".bmodel") { + FDERROR + << "With model format of ModelFormat::SOPHGO, the model file " + "should ends with `.bmodel`, but now it's " + << model_file << std::endl; + return false; + } } else { FDERROR << "Only support model format with frontend ModelFormat::PADDLE / " @@ -185,6 +205,10 @@ ModelFormat GuessModelFormat(const std::string& model_file) { model_file.substr(model_file.size() - 5, 5) == ".rknn") { FDINFO << "Model Format: RKNN." << std::endl; return ModelFormat::RKNN; + } else if (model_file.size() > 7 && + model_file.substr(model_file.size() - 7, 7) == ".bmodel") { + FDINFO << "Model Format: SOPHGO." << std::endl; + return ModelFormat::SOPHGO; } FDERROR << "Cannot guess which model format you are using, please set " @@ -288,6 +312,11 @@ void RuntimeOption::UseAscend(){ device = Device::ASCEND; } +void RuntimeOption::UseSophgo() { + device = Device::SOPHGOTPUD; + UseSophgoBackend(); +} + void RuntimeOption::SetExternalStream(void* external_stream) { external_stream_ = external_stream; } @@ -323,6 +352,15 @@ void RuntimeOption::UseOrtBackend() { #endif } +// use sophgoruntime backend +void RuntimeOption::UseSophgoBackend() { +#ifdef ENABLE_SOPHGO_BACKEND + backend = Backend::SOPHGOTPU; +#else + FDASSERT(false, "The FastDeploy didn't compile with SophgoBackend."); +#endif +} + // use poros backend void RuntimeOption::UsePorosBackend() { #ifdef ENABLE_POROS_BACKEND @@ -564,6 +602,8 @@ bool Runtime::Init(const RuntimeOption& _option) { option.backend = Backend::OPENVINO; } else if (IsBackendAvailable(Backend::RKNPU2)) { option.backend = Backend::RKNPU2; + } else if (IsBackendAvailable(Backend::SOPHGOTPU)) { + option.backend = Backend::SOPHGOTPU; } else { FDERROR << "Please define backend in RuntimeOption, current it's " "Backend::UNKNOWN." @@ -623,7 +663,15 @@ bool Runtime::Init(const RuntimeOption& _option) { FDINFO << "Runtime initialized with Backend::RKNPU2 in " << Str(option.device) << "." << std::endl; - } else { + } else if (option.backend == Backend::SOPHGOTPU) { + FDASSERT(option.device == Device::SOPHGOTPUD, + "Backend::SOPHGO only supports Device::SOPHGO"); + CreateSophgoNPUBackend(); + + FDINFO << "Runtime initialized with Backend::SOPHGO in " + << Str(option.device) << "." << std::endl; + } + else { FDERROR << "Runtime only support " "Backend::ORT/Backend::TRT/Backend::PDINFER/Backend::POROS as " "backend now." @@ -926,6 +974,21 @@ void Runtime::CreateRKNPU2Backend() { #endif } +void Runtime::CreateSophgoNPUBackend() { +#ifdef ENABLE_SOPHGO_BACKEND + auto sophgo_option = SophgoBackendOption(); + FDASSERT(option.model_format == ModelFormat::SOPHGO, + "SophgoBackend only support model format of ModelFormat::SOPHGO"); + backend_ = utils::make_unique(); + auto casted_backend = dynamic_cast(backend_.get()); + FDASSERT(casted_backend->InitFromSophgo(option.model_file, sophgo_option), + "Load model from nb file failed while initializing LiteBackend."); +#else + FDASSERT(false, "SophgoBackend is not available, please compiled with " + "ENABLE_SOPHGO_BACKEND=ON."); +#endif +} + Runtime* Runtime::Clone(void* stream, int device_id) { Runtime* runtime = new Runtime(); if (option.backend != Backend::OPENVINO && diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index c889d4337..46532b16b 100755 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -43,6 +43,7 @@ enum Backend { OPENVINO, ///< Intel OpenVINO, support Paddle/ONNX format, CPU only LITE, ///< Paddle Lite, support Paddle format model, ARM CPU only RKNPU2, ///< RKNPU2, support RKNN format model, Rockchip NPU only + SOPHGOTPU, ///< SOPHGOTPU, support SOPHGO format model, Sophgo TPU only }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, @@ -151,6 +152,9 @@ struct FASTDEPLOY_DECL RuntimeOption { bool adaptive_seqlen = false, bool enable_multi_stream = false); + /// Use Sophgo to inference + void UseSophgo(); + void SetExternalStream(void* external_stream); /* @@ -170,6 +174,9 @@ struct FASTDEPLOY_DECL RuntimeOption { /// Set ONNX Runtime as inference backend, support CPU/GPU void UseOrtBackend(); + /// Set SOPHGO Runtime as inference backend, support CPU/GPU + void UseSophgoBackend(); + /// Set TensorRT as inference backend, only support GPU void UseTrtBackend(); @@ -576,6 +583,7 @@ struct FASTDEPLOY_DECL Runtime { void CreateOpenVINOBackend(); void CreateLiteBackend(); void CreateRKNPU2Backend(); + void CreateSophgoNPUBackend(); std::unique_ptr backend_; std::vector input_tensors_; std::vector output_tensors_; diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index a9b5b46f0..6868c9c62 100755 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -32,7 +32,10 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, valid_ascend_backends = {Backend::LITE}; valid_kunlunxin_backends = {Backend::LITE}; valid_ipu_backends = {Backend::PDINFER}; - } else { + }else if (model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; + } + else { valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; valid_rknpu_backends = {Backend::RKNPU2}; diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc index 44e020ffd..107b1ab90 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc @@ -24,6 +24,8 @@ YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file, if (model_format == ModelFormat::ONNX) { valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else if (model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; diff --git a/fastdeploy/vision/detection/ppdet/model.h b/fastdeploy/vision/detection/ppdet/model.h index 2f4078f95..da69774fa 100755 --- a/fastdeploy/vision/detection/ppdet/model.h +++ b/fastdeploy/vision/detection/ppdet/model.h @@ -41,6 +41,7 @@ class FASTDEPLOY_DECL PicoDet : public PPDetBase { valid_rknpu_backends = {Backend::RKNPU2}; valid_kunlunxin_backends = {Backend::LITE}; valid_ascend_backends = {Backend::LITE}; + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; initialized = Initialize(); } diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 7baa7ac7f..54f978828 100755 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -25,12 +25,18 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) : preprocessor_(config_file), postprocessor_(config_file) { - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + if(model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; + } + else{ + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } valid_rknpu_backends = {Backend::RKNPU2}; valid_timvx_backends = {Backend::LITE}; valid_kunlunxin_backends = {Backend::LITE}; valid_ascend_backends = {Backend::LITE}; + runtime_option = custom_option; runtime_option.model_format = model_format; runtime_option.model_file = model_file; diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index 23cf697bf..d7035bd1b 100755 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -301,6 +301,11 @@ class RuntimeOption: rknpu2_core=rknpu2.CoreMask.RKNN_NPU_CORE_0): return self._option.use_rknpu2(rknpu2_name, rknpu2_core) + def use_sophgo(self): + """Inference with SOPHGO TPU + """ + return self._option.use_sophgo() + def use_ascend(self): """Inference with Huawei Ascend NPU """ diff --git a/python/setup.py b/python/setup.py index 80c6b80e9..d1b02254e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -56,6 +56,8 @@ if os.getenv("BUILD_ON_CPU", "OFF") == "ON": setup_configs = dict() setup_configs["ENABLE_RKNPU2_BACKEND"] = os.getenv("ENABLE_RKNPU2_BACKEND", "OFF") +setup_configs["ENABLE_SOPHGO_BACKEND"] = os.getenv("ENABLE_SOPHGO_BACKEND", + "OFF") setup_configs["WITH_ASCEND"] = os.getenv("WITH_ASCEND", "OFF") setup_configs["ENABLE_ORT_BACKEND"] = os.getenv("ENABLE_ORT_BACKEND", "OFF") setup_configs["ENABLE_OPENVINO_BACKEND"] = os.getenv("ENABLE_OPENVINO_BACKEND", From 07ad7216f6a85f8f6706665f0f4bb49a66a31a84 Mon Sep 17 00:00:00 2001 From: yunyaoXYY <109218879+yunyaoXYY@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:54:03 +0800 Subject: [PATCH 2/4] [Other] Add accuracy evaluation scripts (#1034) * add accuracy scripts * add accuracy scripts * Add FlyCV doc * fix conflict * fix conflict * fix conflict --- tests/acc_eval/README.md | 29 ++++ tests/acc_eval/classification/eval.py | 66 +++++++ tests/acc_eval/classification/prepare.sh | 28 +++ tests/acc_eval/classification/run.sh | 8 + tests/acc_eval/detection/eval_faster_rcnn.py | 69 ++++++++ tests/acc_eval/detection/eval_mask_rcnn.py | 76 ++++++++ tests/acc_eval/detection/eval_picodet.py | 67 +++++++ tests/acc_eval/detection/eval_ppyolo.py | 69 ++++++++ tests/acc_eval/detection/eval_ppyoloe.py | 68 ++++++++ tests/acc_eval/detection/eval_ssd.py | 56 ++++++ tests/acc_eval/detection/eval_yolov3.py | 67 +++++++ tests/acc_eval/detection/eval_yolov5.py | 60 +++++++ tests/acc_eval/detection/eval_yolov6.py | 60 +++++++ tests/acc_eval/detection/eval_yolov7.py | 60 +++++++ tests/acc_eval/detection/eval_yolox.py | 67 +++++++ tests/acc_eval/detection/prepare.sh | 25 +++ tests/acc_eval/detection/run.sh | 17 ++ tests/acc_eval/ppocr/eval_ppocrv2.py | 170 ++++++++++++++++++ tests/acc_eval/ppocr/eval_ppocrv3.py | 174 +++++++++++++++++++ tests/acc_eval/ppocr/prepare.sh | 22 +++ tests/acc_eval/ppocr/run.sh | 7 + tests/acc_eval/segmentation/eval.py | 58 +++++++ tests/acc_eval/segmentation/prepare.sh | 30 ++++ tests/acc_eval/segmentation/run.sh | 8 + 24 files changed, 1361 insertions(+) create mode 100644 tests/acc_eval/README.md create mode 100755 tests/acc_eval/classification/eval.py create mode 100644 tests/acc_eval/classification/prepare.sh create mode 100644 tests/acc_eval/classification/run.sh create mode 100644 tests/acc_eval/detection/eval_faster_rcnn.py create mode 100644 tests/acc_eval/detection/eval_mask_rcnn.py create mode 100644 tests/acc_eval/detection/eval_picodet.py create mode 100644 tests/acc_eval/detection/eval_ppyolo.py create mode 100644 tests/acc_eval/detection/eval_ppyoloe.py create mode 100644 tests/acc_eval/detection/eval_ssd.py create mode 100644 tests/acc_eval/detection/eval_yolov3.py create mode 100755 tests/acc_eval/detection/eval_yolov5.py create mode 100755 tests/acc_eval/detection/eval_yolov6.py create mode 100755 tests/acc_eval/detection/eval_yolov7.py create mode 100644 tests/acc_eval/detection/eval_yolox.py create mode 100644 tests/acc_eval/detection/prepare.sh create mode 100644 tests/acc_eval/detection/run.sh create mode 100644 tests/acc_eval/ppocr/eval_ppocrv2.py create mode 100644 tests/acc_eval/ppocr/eval_ppocrv3.py create mode 100644 tests/acc_eval/ppocr/prepare.sh create mode 100644 tests/acc_eval/ppocr/run.sh create mode 100644 tests/acc_eval/segmentation/eval.py create mode 100644 tests/acc_eval/segmentation/prepare.sh create mode 100644 tests/acc_eval/segmentation/run.sh diff --git a/tests/acc_eval/README.md b/tests/acc_eval/README.md new file mode 100644 index 000000000..5b62577f7 --- /dev/null +++ b/tests/acc_eval/README.md @@ -0,0 +1,29 @@ +# 模型精度批量验证脚本 + +本目录下的Python脚本可以在CPU/GPU/昆仑芯/昇腾,以及后续的新增硬件上, 完成对高优模型的精度批量验证. +各模型的精度测试代码是基于Python部署demo修改而成, 当后续有新增硬件或者新增模型时，用户可以通过同样的方式(新增option和模型),添加新的Python代码来完成精度验证. + + +## 用法 + +### 1.准备数据集 +- 分类模型需要ImageNet验证集以及标签 +- 检测模型需要COCO2017验证集以及标签 +- 分割模型需要Cityscape验证集以及标签 +- PP-OCRv2/v3的数据集在准备脚本中会自行下载. + +请将准备好的数据集解压至dataset目录中使用 + +### 2.精度验证 +分类/检测/分割/OCR四个场景的精度验证启用方式是一样的. +其中分类, 检测和分割模型会返回预测精度, OCR模型会返回与GPU预测结果的差异. + +```bash +# 进入分类模型目录下 +cd classification +# 执行prepare.sh脚本,自动下载并解压模型至models文件夹下 +bash prepare.sh +# 首先修改run.sh中的TARGET_DEVICE为想测试的硬件,之后执行run.sh脚本 +bash run.sh +# 验证完毕的输出以及精度数据,会保存至log文件夹下,用户自行查看 +``` diff --git a/tests/acc_eval/classification/eval.py b/tests/acc_eval/classification/eval.py new file mode 100755 index 000000000..b6a452855 --- /dev/null +++ b/tests/acc_eval/classification/eval.py @@ -0,0 +1,66 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of PaddleClas model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--topk", type=int, default=1, help="Return topk results.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu' or 'ipu' or 'kunlunxin' or 'ascend' ." + ) + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "ipu": + option.use_ipu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) + +model_file = os.path.join(args.model, "inference.pdmodel") +params_file = os.path.join(args.model, "inference.pdiparams") +config_file = os.path.join(args.model, "inference_cls.yaml") +model = fd.vision.classification.PaddleClasModel( + model_file, params_file, config_file, runtime_option=runtime_option) + +res = fd.vision.evaluation.eval_classify( + model=model, + image_file_path="../dataset/imagenet/", + label_file_path="../dataset/imagenet/val_list.txt", + topk=1) +print(res) diff --git a/tests/acc_eval/classification/prepare.sh b/tests/acc_eval/classification/prepare.sh new file mode 100644 index 000000000..b6d3ba344 --- /dev/null +++ b/tests/acc_eval/classification/prepare.sh @@ -0,0 +1,28 @@ +mkdir models +cd models + +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPLCNet_x1_0_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPLCNetV2_base_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/EfficientNetB7_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/EfficientNetB0_small_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/GhostNet_x1_3_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/GhostNet_x0_5_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV1_x0_25_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV1_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV2_x0_25_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV2_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV3_small_x0_35_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV3_large_x1_0_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ShuffleNetV2_x0_25_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ShuffleNetV2_x2_0_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/SqueezeNet1_1_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/InceptionV3_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPHGNet_tiny_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPHGNet_base_ssld_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz + +ls *.tgz | xargs -n1 tar xzvf + +rm -rf *.tgz + +cd .. diff --git a/tests/acc_eval/classification/run.sh b/tests/acc_eval/classification/run.sh new file mode 100644 index 000000000..16c1b2bb9 --- /dev/null +++ b/tests/acc_eval/classification/run.sh @@ -0,0 +1,8 @@ +TARGET_DEVICE=ascend + +model_dir=`ls ./models/` + +for MODEL_NAME in $model_dir +do + python infer.py --model ./models/$MODEL_NAME --image None --device $TARGET_DEVICE 2>&1 | tee ./log/${MODEL_NAME}_acc.log +done diff --git a/tests/acc_eval/detection/eval_faster_rcnn.py b/tests/acc_eval/detection/eval_faster_rcnn.py new file mode 100644 index 000000000..354ca6c0f --- /dev/null +++ b/tests/acc_eval/detection/eval_faster_rcnn.py @@ -0,0 +1,69 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("image", [1, 3, 640, 640]) + option.set_trt_input_shape("scale_factor", [1, 2]) + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='faster_rcnn_r50_vd_fpn_2x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.FasterRCNN( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_mask_rcnn.py b/tests/acc_eval/detection/eval_mask_rcnn.py new file mode 100644 index 000000000..5ee53097c --- /dev/null +++ b/tests/acc_eval/detection/eval_mask_rcnn.py @@ -0,0 +1,76 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + # option.use_gpu() + print( + """GPU inference with Backend::Paddle in python has not been supported yet. \ + \nWill ignore this option.""") + + if args.use_trt: + # TODO(qiuyanjun): may remove TRT option + # Backend::TRT has not been supported yet. + print( + """Backend::TRT has not been supported yet, will ignore this option.\ + \nPaddleDetection/MaskRCNN has only support Backend::Paddle now.""" + ) + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='mask_rcnn_r50_1x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.MaskRCNN( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_picodet.py b/tests/acc_eval/detection/eval_picodet.py new file mode 100644 index 000000000..c2120282a --- /dev/null +++ b/tests/acc_eval/detection/eval_picodet.py @@ -0,0 +1,67 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='picodet_l_320_coco_lcnet') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.PicoDet( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_ppyolo.py b/tests/acc_eval/detection/eval_ppyolo.py new file mode 100644 index 000000000..f5aa98d11 --- /dev/null +++ b/tests/acc_eval/detection/eval_ppyolo.py @@ -0,0 +1,69 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("image", [1, 3, 640, 640]) + option.set_trt_input_shape("scale_factor", [1, 2]) + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='ppyolo_r50vd_dcn_1x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.PPYOLO( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_ppyoloe.py b/tests/acc_eval/detection/eval_ppyoloe.py new file mode 100644 index 000000000..1fea5d425 --- /dev/null +++ b/tests/acc_eval/detection/eval_ppyoloe.py @@ -0,0 +1,68 @@ +import cv2 +import os + +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='ppyoloe_crn_l_300e_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.PPYOLOE( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_ssd.py b/tests/acc_eval/detection/eval_ssd.py new file mode 100644 index 000000000..1744a8f2f --- /dev/null +++ b/tests/acc_eval/detection/eval_ssd.py @@ -0,0 +1,56 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + required=True, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + return option + + +args = parse_arguments() + +model_file = os.path.join(args.model_dir, "model.pdmodel") +params_file = os.path.join(args.model_dir, "model.pdiparams") +config_file = os.path.join(args.model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.SSD(model_file, + params_file, + config_file, + runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_yolov3.py b/tests/acc_eval/detection/eval_yolov3.py new file mode 100644 index 000000000..7023676fb --- /dev/null +++ b/tests/acc_eval/detection/eval_yolov3.py @@ -0,0 +1,67 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='yolov3_darknet53_270e_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.YOLOv3( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/eval_yolov5.py b/tests/acc_eval/detection/eval_yolov5.py new file mode 100755 index 000000000..3d950b26a --- /dev/null +++ b/tests/acc_eval/detection/eval_yolov5.py @@ -0,0 +1,60 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", default=None, help="Path of yolov5 model.") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu' or 'kunlunxin'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("images", [1, 3, 640, 640]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.detection.YOLOv5( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.PADDLE) + +image_file_path = "/xieyunyao/Project/coco/val2017" +annotation_file_path = "/xieyunyao/Project/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path, 0.001, 0.65) +print(res) diff --git a/tests/acc_eval/detection/eval_yolov6.py b/tests/acc_eval/detection/eval_yolov6.py new file mode 100755 index 000000000..3641194ca --- /dev/null +++ b/tests/acc_eval/detection/eval_yolov6.py @@ -0,0 +1,60 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", default=None, help="Path of yolov5 model.") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu' or 'kunlunxin'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("images", [1, 3, 640, 640]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.detection.YOLOv6( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.PADDLE) + +image_file_path = "/xieyunyao/Project/coco/val2017" +annotation_file_path = "/xieyunyao/Project/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path, 0.001, 0.65) +print(res) diff --git a/tests/acc_eval/detection/eval_yolov7.py b/tests/acc_eval/detection/eval_yolov7.py new file mode 100755 index 000000000..3641194ca --- /dev/null +++ b/tests/acc_eval/detection/eval_yolov7.py @@ -0,0 +1,60 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", default=None, help="Path of yolov5 model.") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu' or 'kunlunxin'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("images", [1, 3, 640, 640]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.detection.YOLOv6( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.PADDLE) + +image_file_path = "/xieyunyao/Project/coco/val2017" +annotation_file_path = "/xieyunyao/Project/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path, 0.001, 0.65) +print(res) diff --git a/tests/acc_eval/detection/eval_yolox.py b/tests/acc_eval/detection/eval_yolox.py new file mode 100644 index 000000000..2885cc772 --- /dev/null +++ b/tests/acc_eval/detection/eval_yolox.py @@ -0,0 +1,67 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + default=None, + help="Path of PaddleDetection model directory") + parser.add_argument( + "--image", default=None, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + return option + + +args = parse_arguments() + +if args.model_dir is None: + model_dir = fd.download_model(name='yolox_s_300e_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.detection.PaddleYOLOX( + model_file, params_file, config_file, runtime_option=runtime_option) + +image_file_path = "../dataset/coco/val2017" +annotation_file_path = "../dataset/coco/annotations/instances_val2017.json" + +res = fd.vision.evaluation.eval_detection(model, image_file_path, + annotation_file_path) +print(res) diff --git a/tests/acc_eval/detection/prepare.sh b/tests/acc_eval/detection/prepare.sh new file mode 100644 index 000000000..2e36440a2 --- /dev/null +++ b/tests/acc_eval/detection/prepare.sh @@ -0,0 +1,25 @@ +mkdir models +cd models + +wget https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz +wget https://bj.bcebos.com/fastdeploy/models/ppyoloe_plus_crn_m_80e_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyolo_r50vd_dcn_1x_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ppyolov2_r101vd_dcn_365e_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov3_darknet53_270e_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolox_s_300e_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ssd_mobilenet_v1_300_120e_voc.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ssd_vgg16_300_240e_voc.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/ssdlite_mobilenet_v1_300_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/faster_rcnn_r50_vd_fpn_2x_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/mask_rcnn_r50_1x_coco.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s_infer.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov6s_infer.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov7_infer.tar + +ls *.tgz | xargs -n1 tar xzvf +ls *.tar | xargs -n1 tar xzvf +rm -rf *.tgz +rm -rf *.tar + +cd .. diff --git a/tests/acc_eval/detection/run.sh b/tests/acc_eval/detection/run.sh new file mode 100644 index 000000000..59dff2e9b --- /dev/null +++ b/tests/acc_eval/detection/run.sh @@ -0,0 +1,17 @@ +TARGET_DEVICE=ascend + +python eval_picodet.py --model_dir ./models/picodet_l_320_coco_lcnet --image None --device $TARGET_DEVICE 2>&1 | tee ./log/picodet_l_320_coco_lcnet.log +python eval_ppyolo.py --model_dir ./models/ppyolov2_r101vd_dcn_365e_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ppyolov2_r101vd_dcn_365e_coco.log +python eval_ppyolo.py --model_dir ./models/ppyolo_r50vd_dcn_1x_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ppyolo_r50vd_dcn_1x_coco.log +python eval_ppyoloe.py --model_dir ./models/ppyoloe_crn_l_300e_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ppyoloe_crn_l_300e_coco.log +python eval_ppyoloe.py --model_dir ./models/ppyoloe_plus_crn_m_80e_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ppyoloe_plus_crn_m_80e_coco.log +python eval_ssd.py --model_dir ./models/ssd_vgg16_300_240e_voc --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ssd_vgg16_300_240e_voc.log +python eval_ssd.py --model_dir ./models/ssdlite_mobilenet_v1_300_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ssdlite_mobilenet_v1_300_coco.log +python eval_ssd.py --model_dir ./models/ssd_mobilenet_v1_300_120e_voc --image None --device $TARGET_DEVICE 2>&1 | tee ./log/ssd_mobilenet_v1_300_120e_voc.log +python eval_yolov3.py --model_dir ./models/yolov3_darknet53_270e_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/yolov3_darknet53_270e_coco.log +python eval_yolox.py --model_dir ./models/yolox_s_300e_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/yolox_s_300e_coco.log +python eval_faster_rcnn.py --model_dir ./models/faster_rcnn_r50_vd_fpn_2x_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/faster_rcnn_r50_vd_fpn_2x_coco.log +python eval_mask_rcnn.py --model_dir ./models/mask_rcnn_r50_1x_coco --image None --device $TARGET_DEVICE 2>&1 | tee ./log/mask_rcnn_r50_1x_coco.log +python eval_yolov5.py --model_dir ./models/yolov5s_infer --image None --device $TARGET_DEVICE 2>&1 | tee ./log/yolov5s_infer.log +python eval_yolov6.py --model_dir ./models/yolov6s_infer --image None --device $TARGET_DEVICE 2>&1 | tee ./log/yolov6s_infer.log +python eval_yolov5.py --model_dir ./models/yolov7_infer --image None --device $TARGET_DEVICE 2>&1 | tee ./log/yolov7_infer.log diff --git a/tests/acc_eval/ppocr/eval_ppocrv2.py b/tests/acc_eval/ppocr/eval_ppocrv2.py new file mode 100644 index 000000000..f4742df66 --- /dev/null +++ b/tests/acc_eval/ppocr/eval_ppocrv2.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--det_model", required=True, help="Path of Detection model of PPOCR.") + parser.add_argument( + "--cls_model", + required=True, + help="Path of Classification model of PPOCR.") + parser.add_argument( + "--rec_model", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--rec_label_file", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--backend", + type=str, + default="default", + help="Type of inference backend, support ort/trt/paddle/openvino, default 'openvino' for cpu, 'tensorrt' for gpu" + ) + parser.add_argument( + "--device_id", + type=int, + default=0, + help="Define which GPU card used to run model.") + parser.add_argument( + "--cpu_thread_num", + type=int, + default=9, + help="Number of threads while inference on CPU.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.device.lower() == "gpu": + option.use_gpu() + + return option + + +args = parse_arguments() + +# Detection模型, 检测文字框 +det_model_file = os.path.join(args.det_model, "inference.pdmodel") +det_params_file = os.path.join(args.det_model, "inference.pdiparams") +# Classification模型，方向分类，可选 +cls_model_file = os.path.join(args.cls_model, "inference.pdmodel") +cls_params_file = os.path.join(args.cls_model, "inference.pdiparams") +# Recognition模型，文字识别模型 +rec_model_file = os.path.join(args.rec_model, "inference.pdmodel") +rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") +rec_label_file = args.rec_label_file + +# 对于三个模型，均采用同样的部署配置 +# 用户也可根据自行需求分别配置 +runtime_option = build_option(args) + +det_model = fd.vision.ocr.DBDetector( + det_model_file, det_params_file, runtime_option=runtime_option) +cls_model = fd.vision.ocr.Classifier( + cls_model_file, cls_params_file, runtime_option=runtime_option) +rec_model = fd.vision.ocr.Recognizer( + rec_model_file, + rec_params_file, + rec_label_file, + runtime_option=runtime_option) + +# PPOCR的Rec模型开启静态推理, 其他硬件不需要的话请注释掉. +rec_model.preprocessor.static_shape = True + +# 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None +ppocr_v2 = fd.vision.ocr.PPOCRv2( + det_model=det_model, cls_model=cls_model, rec_model=rec_model) + +##### +#准备输入图片数据 +img_dir = args.image +imgs_file_lists = [] +if os.path.isdir(img_dir): + for single_file in os.listdir(img_dir): + if 'jpg' in single_file: + file_path = os.path.join(img_dir, single_file) + if os.path.isfile(file_path): + imgs_file_lists.append(file_path) + +imgs_file_lists.sort() + +fd_result = [] +for idx, image in enumerate(imgs_file_lists): + img = cv2.imread(image) + result = ppocr_v2.predict(img) + for i in range(len(result.boxes)): + one_res = result.boxes[i] + [ + result.rec_scores[i] + ] + [result.cls_labels[i]] + [result.cls_scores[i]] + fd_result.append(one_res) + +local_result = [] +with open('PPOCRv2_ICDAR10_BS116_1221.txt', 'r') as f: + for line in f: + local_result.append(list(map(float, line.split(',')))) + +print("==== Begin to check OCR diff ====") +for list_local, list_fd in zip(local_result, fd_result): + + for i in range(len(list_local)): + + if (i < 8): + #Det + diff = list_local[i] - list_fd[i] + assert ( + abs(diff) < 1 + ), "Diff exist in Det box result, where is {} - {} .".format( + list_local, list_fd) + elif (i == 8): + #rec + diff = round(list_local[i], 6) - round(list_fd[i], 6) + assert ( + abs(diff) < 0.001 + ), "Diff exist in rec scores result, where is {} - {} .".format( + list_local, list_fd) + elif (i == 9): + diff = list_local[i] - list_fd[i] + assert ( + abs(diff) != 1 + ), "Diff exist in cls label result, where is {} - {} .".format( + list_local, list_fd) + else: + diff = round(list_local[i], 6) - round(list_fd[i], 6) + assert ( + abs(diff) < 0.001 + ), "Diff exist in cls score result, where is {} - {} .".format( + list_local, list_fd) diff --git a/tests/acc_eval/ppocr/eval_ppocrv3.py b/tests/acc_eval/ppocr/eval_ppocrv3.py new file mode 100644 index 000000000..b6f4dcced --- /dev/null +++ b/tests/acc_eval/ppocr/eval_ppocrv3.py @@ -0,0 +1,174 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--det_model", required=True, help="Path of Detection model of PPOCR.") + parser.add_argument( + "--cls_model", + required=True, + help="Path of Classification model of PPOCR.") + parser.add_argument( + "--rec_model", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--rec_label_file", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--backend", + type=str, + default="default", + help="Type of inference backend, support ort/trt/paddle/openvino, default 'openvino' for cpu, 'tensorrt' for gpu" + ) + parser.add_argument( + "--device_id", + type=int, + default=0, + help="Define which GPU card used to run model.") + parser.add_argument( + "--cpu_thread_num", + type=int, + default=9, + help="Number of threads while inference on CPU.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.device.lower() == "gpu": + option.use_gpu() + + return option + + +args = parse_arguments() + +# Detection模型, 检测文字框 +det_model_file = os.path.join(args.det_model, "inference.pdmodel") +det_params_file = os.path.join(args.det_model, "inference.pdiparams") +# Classification模型，方向分类，可选 +cls_model_file = os.path.join(args.cls_model, "inference.pdmodel") +cls_params_file = os.path.join(args.cls_model, "inference.pdiparams") +# Recognition模型，文字识别模型 +rec_model_file = os.path.join(args.rec_model, "inference.pdmodel") +rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") +rec_label_file = args.rec_label_file + +# 对于三个模型，均采用同样的部署配置 +# 用户也可根据自行需求分别配置 +runtime_option = build_option(args) + +det_model = fd.vision.ocr.DBDetector( + det_model_file, det_params_file, runtime_option=runtime_option) +cls_model = fd.vision.ocr.Classifier( + cls_model_file, cls_params_file, runtime_option=runtime_option) +rec_model = fd.vision.ocr.Recognizer( + rec_model_file, + rec_params_file, + rec_label_file, + runtime_option=runtime_option) + +# PPOCR的Rec模型开启静态推理, 其他硬件不需要的话请注释掉. +rec_model.preprocessor.static_shape = True + +# 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None +ppocr_v3 = fd.vision.ocr.PPOCRv3( + det_model=det_model, cls_model=cls_model, rec_model=rec_model) + +##### +#准备输入图片数据 +img_dir = args.image +imgs_file_lists = [] +if os.path.isdir(img_dir): + for single_file in os.listdir(img_dir): + if 'jpg' in single_file: + file_path = os.path.join(img_dir, single_file) + if os.path.isfile(file_path): + imgs_file_lists.append(file_path) + +imgs_file_lists.sort() + +fd_result = [] +for idx, image in enumerate(imgs_file_lists): + img = cv2.imread(image) + result = ppocr_v3.predict(img) + for i in range(len(result.boxes)): + one_res = result.boxes[i] + [ + result.rec_scores[i] + ] + [result.cls_labels[i]] + [result.cls_scores[i]] + fd_result.append(one_res) + +local_result = [] +with open('PPOCRv3_ICDAR10_BS116_1221.txt', 'r') as f: + for line in f: + local_result.append(list(map(float, line.split(',')))) + +# Begin to Diff Compare +total_num_res = len(local_result) * 11 +total_diff_num = 0 + +print("==== Begin to check OCR diff ====") +for list_local, list_fd in zip(local_result, fd_result): + + for i in range(len(list_local)): + + if (i < 8): + #Det + diff = list_local[i] - list_fd[i] + assert ( + abs(diff) < 1 + ), "Diff exist in Det box result, where is {} - {} .".format( + list_local, list_fd) + elif (i == 8): + #rec + diff = round(list_local[i], 6) - round(list_fd[i], 6) + assert ( + abs(diff) < 0.001 + ), "Diff exist in rec scores result, where is {} - {} .".format( + list_local, list_fd) + elif (i == 9): + diff = list_local[i] - list_fd[i] + assert ( + abs(diff) != 1 + ), "Diff exist in cls label result, where is {} - {} .".format( + list_local, list_fd) + else: + diff = round(list_local[i], 6) - round(list_fd[i], 6) + assert ( + abs(diff) < 0.001 + ), "Diff exist in cls score result, where is {} - {} .".format( + list_local, list_fd) diff --git a/tests/acc_eval/ppocr/prepare.sh b/tests/acc_eval/ppocr/prepare.sh new file mode 100644 index 000000000..8417ce879 --- /dev/null +++ b/tests/acc_eval/ppocr/prepare.sh @@ -0,0 +1,22 @@ +mkdir models +cd models + +# 下载模型 +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar +wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar +wget https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar +wget https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar +wget https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_infer.tar +wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_keys_v1.txt + +# 下载GPU预测结果 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPOCRv3_ICDAR10_BS116_1221.txt +wget https://bj.bcebos.com/paddlehub/fastdeploy/PPOCRv2_ICDAR10_BS116_1221.txt + +# 下载ICDAR2017数据集前10张图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/ICDAR2017_10.tar + +ls *.tar | xargs -n1 tar xzvf +rm -rf *.tar + +cd .. diff --git a/tests/acc_eval/ppocr/run.sh b/tests/acc_eval/ppocr/run.sh new file mode 100644 index 000000000..59deabd19 --- /dev/null +++ b/tests/acc_eval/ppocr/run.sh @@ -0,0 +1,7 @@ +TARGET_DEVICE=ascend + +python eval_ppocrv3.py.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt \ + --image ../ICDAR2017_10 --device $TARGET_DEVICE 2>&1 | tee ./log/ppocrv3_diff.log + +python eval_ppocrv2.py.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv2_rec_infer --rec_label_file ppocr_keys_v1.txt \ + --image ../ICDAR2017_10 --device $TARGET_DEVICE 2>&1 | tee ./log/ppocrv2_diff.log diff --git a/tests/acc_eval/segmentation/eval.py b/tests/acc_eval/segmentation/eval.py new file mode 100644 index 000000000..b77a69519 --- /dev/null +++ b/tests/acc_eval/segmentation/eval.py @@ -0,0 +1,58 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of PaddleSeg model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.device.lower() == "kunlunxin": + option.use_kunlunxin() + + if args.device.lower() == "ascend": + option.use_ascend() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("x", [1, 3, 256, 256], [1, 3, 1024, 1024], + [1, 3, 2048, 2048]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +config_file = os.path.join(args.model, "deploy.yaml") +model = fd.vision.segmentation.PaddleSegModel( + model_file, params_file, config_file, runtime_option=runtime_option) + +res = fd.vision.evaluation.eval_segmentation( + model=model, data_dir="../dataset/FD_dataset/data/cityscapes") +print(res) diff --git a/tests/acc_eval/segmentation/prepare.sh b/tests/acc_eval/segmentation/prepare.sh new file mode 100644 index 000000000..465eed00a --- /dev/null +++ b/tests/acc_eval/segmentation/prepare.sh @@ -0,0 +1,30 @@ +mkdir models +cd models + +wget https://bj.bcebos.com/paddlehub/fastdeploy/Unet_cityscapes_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/Unet_cityscapes_without_argmax_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/Portrait_PP_HumanSegV1_Lite_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_HumanSegV1_Lite_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_HumanSegV2_Lite_192x192_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_HumanSegV2_Mobile_192x192_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_HumanSegV1_Server_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP_HumanSegV1_Server_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/Portrait_PP_HumanSegV2_Lite_256x144_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/Portrait_PP_HumanSegV2_Lite_256x144_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/FCN_HRNet_W18_cityscapes_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/FCN_HRNet_W18_cityscapes_without_argmax_infer.tgz + +wget https://bj.bcebos.com/paddlehub/fastdeploy/Deeplabv3_ResNet101_OS8_cityscapes_with_argmax_infer.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/Deeplabv3_ResNet101_OS8_cityscapes_without_argmax_infer.tgz + + +ls *.tgz | xargs -n1 tar xzvf +rm -rf *.tgz diff --git a/tests/acc_eval/segmentation/run.sh b/tests/acc_eval/segmentation/run.sh new file mode 100644 index 000000000..73fe957d4 --- /dev/null +++ b/tests/acc_eval/segmentation/run.sh @@ -0,0 +1,8 @@ +TARGET_DEVICE=ascend + +model_dir=`ls ./models/` + +for MODEL_NAME in $model_dir +do + python eval.py --model ./models/$MODEL_NAME --image None --device $TARGET_DEVICE 2>&1 | tee ./log/${MODEL_NAME}_acc.log +done From d49160252bafee728052625e09c93574a6507c1f Mon Sep 17 00:00:00 2001 From: yunyaoXYY <109218879+yunyaoXYY@users.noreply.github.com> Date: Wed, 4 Jan 2023 16:18:38 +0800 Subject: [PATCH 3/4] [Other] Improve examples and readme for Ascend deployment (#1052) * Add Huawei Ascend NPU deploy through PaddleLite CANN * Add NNAdapter interface for paddlelite * Modify Huawei Ascend Cmake * Update way for compiling Huawei Ascend NPU deployment * remove UseLiteBackend in UseCANN * Support compile python whlee * Change names of nnadapter API * Add nnadapter pybind and remove useless API * Support Python deployment on Huawei Ascend NPU * Add models suppor for ascend * Add PPOCR rec reszie for ascend * fix conflict for ascend * Rename CANN to Ascend * Rename CANN to Ascend * Improve ascend * fix ascend bug * improve ascend docs * improve ascend docs * improve ascend docs * Improve Ascend * Improve Ascend * Move ascend python demo * Imporve ascend * Improve ascend * Improve ascend * Improve ascend * Improve ascend * Imporve ascend * Imporve ascend * Improve ascend * acc eval script * acc eval * remove acc_eval from branch huawei * Add detection and segmentation examples for Ascend deployment * Add detection and segmentation examples for Ascend deployment * Add PPOCR example for ascend deploy * Imporve paddle lite compiliation * Add FlyCV doc * Add FlyCV doc * Add FlyCV doc * Imporve Ascend docs * Imporve Ascend docs * Improve PPOCR example --- .../detection/paddledetection/cpp/README.md | 3 + .../vision/detection/yolov5/cpp/README.md | 3 + .../vision/detection/yolov6/cpp/README.md | 4 + .../vision/detection/yolov7/cpp/README.md | 4 + .../vision/ocr/PP-OCRv2/cpp/CMakeLists.txt | 4 + examples/vision/ocr/PP-OCRv2/cpp/README.md | 7 +- examples/vision/ocr/PP-OCRv2/cpp/infer.cc | 9 -- .../ocr/PP-OCRv2/cpp/infer_static_shape.cc | 107 +++++++++++++ examples/vision/ocr/PP-OCRv2/python/README.md | 4 +- examples/vision/ocr/PP-OCRv2/python/infer.py | 143 +++++++++++------- .../ocr/PP-OCRv2/python/infer_static_shape.py | 114 ++++++++++++++ .../vision/ocr/PP-OCRv3/cpp/CMakeLists.txt | 4 + examples/vision/ocr/PP-OCRv3/cpp/README.md | 7 +- examples/vision/ocr/PP-OCRv3/cpp/infer.cc | 9 -- .../ocr/PP-OCRv3/cpp/infer_static_shape.cc | 107 +++++++++++++ examples/vision/ocr/PP-OCRv3/python/README.md | 4 +- examples/vision/ocr/PP-OCRv3/python/infer.py | 143 +++++++++++------- .../ocr/PP-OCRv3/python/infer_static_shape.py | 114 ++++++++++++++ .../segmentation/paddleseg/cpp/README.md | 3 + 19 files changed, 663 insertions(+), 130 deletions(-) create mode 100755 examples/vision/ocr/PP-OCRv2/cpp/infer_static_shape.cc create mode 100755 examples/vision/ocr/PP-OCRv2/python/infer_static_shape.py create mode 100755 examples/vision/ocr/PP-OCRv3/cpp/infer_static_shape.cc create mode 100755 examples/vision/ocr/PP-OCRv3/python/infer_static_shape.py diff --git a/examples/vision/detection/paddledetection/cpp/README.md b/examples/vision/detection/paddledetection/cpp/README.md index d10be1525..0e944a465 100755 --- a/examples/vision/detection/paddledetection/cpp/README.md +++ b/examples/vision/detection/paddledetection/cpp/README.md @@ -41,6 +41,9 @@ tar xvf ppyoloe_crn_l_300e_coco.tgz 以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + ## PaddleDetection C++接口 ### 模型类 diff --git a/examples/vision/detection/yolov5/cpp/README.md b/examples/vision/detection/yolov5/cpp/README.md index 61abe3275..c70d0d118 100755 --- a/examples/vision/detection/yolov5/cpp/README.md +++ b/examples/vision/detection/yolov5/cpp/README.md @@ -55,6 +55,9 @@ wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/0000000 以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + ## YOLOv5 C++接口 ### YOLOv5类 diff --git a/examples/vision/detection/yolov6/cpp/README.md b/examples/vision/detection/yolov6/cpp/README.md index 765dde84c..eceb5bc46 100755 --- a/examples/vision/detection/yolov6/cpp/README.md +++ b/examples/vision/detection/yolov6/cpp/README.md @@ -33,6 +33,10 @@ wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/0000000 ./infer_paddle_demo yolov6s_infer 000000014439.jpg 3 ``` +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + + 如果想要验证ONNX模型的推理，可以参考如下命令： ```bash #下载官方转换好的YOLOv6 ONNX模型文件和测试图片 diff --git a/examples/vision/detection/yolov7/cpp/README.md b/examples/vision/detection/yolov7/cpp/README.md index 5cab3cc95..5308f7ddb 100755 --- a/examples/vision/detection/yolov7/cpp/README.md +++ b/examples/vision/detection/yolov7/cpp/README.md @@ -31,6 +31,10 @@ wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/0000000 # 华为昇腾推理 ./infer_paddle_model_demo yolov7_infer 000000014439.jpg 3 ``` + +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + 如果想要验证ONNX模型的推理，可以参考如下命令： ```bash #下载官方转换好的yolov7 ONNX模型文件和测试图片 diff --git a/examples/vision/ocr/PP-OCRv2/cpp/CMakeLists.txt b/examples/vision/ocr/PP-OCRv2/cpp/CMakeLists.txt index 93540a7e8..8b2f7aa61 100644 --- a/examples/vision/ocr/PP-OCRv2/cpp/CMakeLists.txt +++ b/examples/vision/ocr/PP-OCRv2/cpp/CMakeLists.txt @@ -12,3 +12,7 @@ include_directories(${FASTDEPLOY_INCS}) add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) # 添加FastDeploy库依赖 target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) + +add_executable(infer_static_shape_demo ${PROJECT_SOURCE_DIR}/infer_static_shape.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_static_shape_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/ocr/PP-OCRv2/cpp/README.md b/examples/vision/ocr/PP-OCRv2/cpp/README.md index e30d886d1..9052dd80e 100755 --- a/examples/vision/ocr/PP-OCRv2/cpp/README.md +++ b/examples/vision/ocr/PP-OCRv2/cpp/README.md @@ -43,13 +43,16 @@ wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_ ./infer_demo ./ch_PP-OCRv2_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv2_rec_infer ./ppocr_keys_v1.txt ./12.jpg 3 # 昆仑芯XPU推理 ./infer_demo ./ch_PP-OCRv2_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv2_rec_infer ./ppocr_keys_v1.txt ./12.jpg 4 -# 华为昇腾推理 -./infer_demo ./ch_PP-OCRv2_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv2_rec_infer ./ppocr_keys_v1.txt ./12.jpg 5 +# 华为昇腾推理, 需要使用静态shape的demo, 若用户需要连续地预测图片, 输入图片尺寸需要准备为统一尺寸 +./infer_static_shape_demo ./ch_PP-OCRv2_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv2_rec_infer ./ppocr_keys_v1.txt ./12.jpg 1 ``` 以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + 运行完成可视化结果如下图所示 diff --git a/examples/vision/ocr/PP-OCRv2/cpp/infer.cc b/examples/vision/ocr/PP-OCRv2/cpp/infer.cc index 0248367cc..72a7fcf7e 100755 --- a/examples/vision/ocr/PP-OCRv2/cpp/infer.cc +++ b/examples/vision/ocr/PP-OCRv2/cpp/infer.cc @@ -55,10 +55,6 @@ void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model auto cls_model = fastdeploy::vision::ocr::Classifier(cls_model_file, cls_params_file, cls_option); auto rec_model = fastdeploy::vision::ocr::Recognizer(rec_model_file, rec_params_file, rec_label_file, rec_option); - // Users could enable static shape infer for rec model when deploy PP-OCR on hardware - // which can not support dynamic shape infer well, like Huawei Ascend series. - // rec_model.GetPreprocessor().SetStaticShapeInfer(true); - assert(det_model.Initialized()); assert(cls_model.Initialized()); assert(rec_model.Initialized()); @@ -70,9 +66,6 @@ void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model // Set inference batch size for cls model and rec model, the value could be -1 and 1 to positive infinity. // When inference batch size is set to -1, it means that the inference batch size // of the cls and rec models will be the same as the number of boxes detected by the det model. - // When users enable static shape infer for rec model, the batch size of cls and rec model needs to be set to 1. - // ppocr_v2.SetClsBatchSize(1); - // ppocr_v2.SetRecBatchSize(1); ppocr_v2.SetClsBatchSize(cls_batch_size); ppocr_v2.SetRecBatchSize(rec_batch_size); @@ -129,8 +122,6 @@ int main(int argc, char* argv[]) { option.EnablePaddleToTrt(); } else if (flag == 4) { option.UseKunlunXin(); - } else if (flag == 5) { - option.UseAscend(); } std::string det_model_dir = argv[1]; diff --git a/examples/vision/ocr/PP-OCRv2/cpp/infer_static_shape.cc b/examples/vision/ocr/PP-OCRv2/cpp/infer_static_shape.cc new file mode 100755 index 000000000..ba5527a2e --- /dev/null +++ b/examples/vision/ocr/PP-OCRv2/cpp/infer_static_shape.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model_dir, const std::string& rec_model_dir, const std::string& rec_label_file, const std::string& image_file, const fastdeploy::RuntimeOption& option) { + auto det_model_file = det_model_dir + sep + "inference.pdmodel"; + auto det_params_file = det_model_dir + sep + "inference.pdiparams"; + + auto cls_model_file = cls_model_dir + sep + "inference.pdmodel"; + auto cls_params_file = cls_model_dir + sep + "inference.pdiparams"; + + auto rec_model_file = rec_model_dir + sep + "inference.pdmodel"; + auto rec_params_file = rec_model_dir + sep + "inference.pdiparams"; + + auto det_option = option; + auto cls_option = option; + auto rec_option = option; + + auto det_model = fastdeploy::vision::ocr::DBDetector(det_model_file, det_params_file, det_option); + auto cls_model = fastdeploy::vision::ocr::Classifier(cls_model_file, cls_params_file, cls_option); + auto rec_model = fastdeploy::vision::ocr::Recognizer(rec_model_file, rec_params_file, rec_label_file, rec_option); + + // Users could enable static shape infer for rec model when deploy PP-OCR on hardware + // which can not support dynamic shape infer well, like Huawei Ascend series. + rec_model.GetPreprocessor().SetStaticShapeInfer(true); + + assert(det_model.Initialized()); + assert(cls_model.Initialized()); + assert(rec_model.Initialized()); + + // The classification model is optional, so the PP-OCR can also be connected in series as follows + // auto ppocr_v2 = fastdeploy::pipeline::PPOCRv2(&det_model, &rec_model); + auto ppocr_v2 = fastdeploy::pipeline::PPOCRv2(&det_model, &cls_model, &rec_model); + + // When users enable static shape infer for rec model, the batch size of cls and rec model must to be set to 1. + ppocr_v2.SetClsBatchSize(1); + ppocr_v2.SetRecBatchSize(1); + + if(!ppocr_v2.Initialized()){ + std::cerr << "Failed to initialize PP-OCR." << std::endl; + return; + } + + auto im = cv::imread(image_file); + + fastdeploy::vision::OCRResult result; + if (!ppocr_v2.Predict(im, &result)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << result.Str() << std::endl; + + auto vis_im = fastdeploy::vision::VisOcr(im, result); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 7) { + std::cout << "Usage: infer_demo path/to/det_model path/to/cls_model " + "path/to/rec_model path/to/rec_label_file path/to/image " + "run_option, " + "e.g ./infer_demo ./ch_PP-OCRv2_det_infer " + "./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv2_rec_infer " + "./ppocr_keys_v1.txt ./12.jpg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with ascend." + << std::endl; + return -1; + } + + fastdeploy::RuntimeOption option; + int flag = std::atoi(argv[6]); + + if (flag == 0) { + option.UseCpu(); + } else if (flag == 1) { + option.UseAscend(); + } + + std::string det_model_dir = argv[1]; + std::string cls_model_dir = argv[2]; + std::string rec_model_dir = argv[3]; + std::string rec_label_file = argv[4]; + std::string test_image = argv[5]; + InitAndInfer(det_model_dir, cls_model_dir, rec_model_dir, rec_label_file, test_image, option); + return 0; +} diff --git a/examples/vision/ocr/PP-OCRv2/python/README.md b/examples/vision/ocr/PP-OCRv2/python/README.md index 270225ab7..1ea95695f 100755 --- a/examples/vision/ocr/PP-OCRv2/python/README.md +++ b/examples/vision/ocr/PP-OCRv2/python/README.md @@ -36,8 +36,8 @@ python infer.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2 python infer.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv2_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device gpu --backend trt # 昆仑芯XPU推理 python infer.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv2_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device kunlunxin -# 华为昇腾推理 -python infer.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv2_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device ascend +# 华为昇腾推理,需要使用静态shape脚本, 若用户需要连续地预测图片, 输入图片尺寸需要准备为统一尺寸 +python infer_static_shape.py --det_model ch_PP-OCRv2_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv2_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device ascend ``` 运行完成可视化结果如下图所示 diff --git a/examples/vision/ocr/PP-OCRv2/python/infer.py b/examples/vision/ocr/PP-OCRv2/python/infer.py index f7373b4c2..6e8fe62b1 100755 --- a/examples/vision/ocr/PP-OCRv2/python/infer.py +++ b/examples/vision/ocr/PP-OCRv2/python/infer.py @@ -58,43 +58,113 @@ def parse_arguments(): type=int, default=9, help="Number of threads while inference on CPU.") + parser.add_argument( + "--cls_bs", + type=int, + default=1, + help="Classification model inference batch size.") + parser.add_argument( + "--rec_bs", + type=int, + default=6, + help="Recognition model inference batch size") return parser.parse_args() def build_option(args): - option = fd.RuntimeOption() - if args.device.lower() == "gpu": - option.use_gpu(0) - option.set_cpu_thread_num(args.cpu_thread_num) + det_option = fd.RuntimeOption() + cls_option = fd.RuntimeOption() + rec_option = fd.RuntimeOption() + + det_option.set_cpu_thread_num(args.cpu_thread_num) + cls_option.set_cpu_thread_num(args.cpu_thread_num) + rec_option.set_cpu_thread_num(args.cpu_thread_num) + + if args.device.lower() == "gpu": + det_option.use_gpu(args.device_id) + cls_option.use_gpu(args.device_id) + rec_option.use_gpu(args.device_id) if args.device.lower() == "kunlunxin": - option.use_kunlunxin() - return option + det_option.use_kunlunxin() + cls_option.use_kunlunxin() + rec_option.use_kunlunxin() - if args.device.lower() == "ascend": - option.use_ascend() - return option + return det_option, cls_option, rec_option if args.backend.lower() == "trt": assert args.device.lower( ) == "gpu", "TensorRT backend require inference on device GPU." - option.use_trt_backend() + det_option.use_trt_backend() + cls_option.use_trt_backend() + rec_option.use_trt_backend() + + # 设置trt input shape + # 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. + det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], + [1, 3, 960, 960]) + cls_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.cls_bs, 3, 48, 320], + [args.cls_bs, 3, 48, 1024]) + rec_option.set_trt_input_shape("x", [1, 3, 32, 10], + [args.rec_bs, 3, 32, 320], + [args.rec_bs, 3, 32, 2304]) + + # 用户可以把TRT引擎文件保存至本地 + det_option.set_trt_cache_file(args.det_model + "/det_trt_cache.trt") + cls_option.set_trt_cache_file(args.cls_model + "/cls_trt_cache.trt") + rec_option.set_trt_cache_file(args.rec_model + "/rec_trt_cache.trt") + elif args.backend.lower() == "pptrt": assert args.device.lower( ) == "gpu", "Paddle-TensorRT backend require inference on device GPU." - option.use_trt_backend() - option.enable_paddle_trt_collect_shape() - option.enable_paddle_to_trt() + det_option.use_trt_backend() + det_option.enable_paddle_trt_collect_shape() + det_option.enable_paddle_to_trt() + + cls_option.use_trt_backend() + cls_option.enable_paddle_trt_collect_shape() + cls_option.enable_paddle_to_trt() + + rec_option.use_trt_backend() + rec_option.enable_paddle_trt_collect_shape() + rec_option.enable_paddle_to_trt() + + # 设置trt input shape + # 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. + det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], + [1, 3, 960, 960]) + cls_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.cls_bs, 3, 48, 320], + [args.cls_bs, 3, 48, 1024]) + rec_option.set_trt_input_shape("x", [1, 3, 32, 10], + [args.rec_bs, 3, 32, 320], + [args.rec_bs, 3, 32, 2304]) + + # 用户可以把TRT引擎文件保存至本地 + det_option.set_trt_cache_file(args.det_model) + cls_option.set_trt_cache_file(args.cls_model) + rec_option.set_trt_cache_file(args.rec_model) + elif args.backend.lower() == "ort": - option.use_ort_backend() + det_option.use_ort_backend() + cls_option.use_ort_backend() + rec_option.use_ort_backend() + elif args.backend.lower() == "paddle": - option.use_paddle_infer_backend() + det_option.use_paddle_infer_backend() + cls_option.use_paddle_infer_backend() + rec_option.use_paddle_infer_backend() + elif args.backend.lower() == "openvino": assert args.device.lower( ) == "cpu", "OpenVINO backend require inference on device CPU." - option.use_openvino_backend() - return option + det_option.use_openvino_backend() + cls_option.use_openvino_backend() + rec_option.use_openvino_backend() + + return det_option, cls_option, rec_option args = parse_arguments() @@ -111,49 +181,18 @@ rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") rec_label_file = args.rec_label_file # 对于三个模型，均采用同样的部署配置 -# 用户也可根据自行需求分别配置 -runtime_option = build_option(args) +# 用户也可根据自己的需求,个性化配置 +det_option, cls_option, rec_option = build_option(args) -# PPOCR的cls和rec模型现在已经支持推理一个Batch的数据 -# 定义下面两个变量后, 可用于设置trt输入shape, 并在PPOCR模型初始化后, 完成Batch推理设置 -# 当用户要把PP-OCR部署在对动态shape推理支持有限的设备上时,(例如华为昇腾) -# 需要把cls_batch_size和rec_batch_size都设置为1. -cls_batch_size = 1 -rec_batch_size = 6 - -# 当使用TRT时，分别给三个模型的runtime设置动态shape,并完成模型的创建. -# 注意: 需要在检测模型创建完成后，再设置分类模型的动态输入并创建分类模型, 识别模型同理. -# 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. -det_option = runtime_option -det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], - [1, 3, 960, 960]) -# 用户可以把TRT引擎文件保存至本地 -# det_option.set_trt_cache_file(args.det_model + "/det_trt_cache.trt") det_model = fd.vision.ocr.DBDetector( det_model_file, det_params_file, runtime_option=det_option) -cls_option = runtime_option -cls_option.set_trt_input_shape("x", [1, 3, 48, 10], - [cls_batch_size, 3, 48, 320], - [cls_batch_size, 3, 48, 1024]) -# 用户可以把TRT引擎文件保存至本地 -# cls_option.set_trt_cache_file(args.cls_model + "/cls_trt_cache.trt") cls_model = fd.vision.ocr.Classifier( cls_model_file, cls_params_file, runtime_option=cls_option) -rec_option = runtime_option -rec_option.set_trt_input_shape("x", [1, 3, 32, 10], - [rec_batch_size, 3, 32, 320], - [rec_batch_size, 3, 32, 2304]) -# 用户可以把TRT引擎文件保存至本地 -# rec_option.set_trt_cache_file(args.rec_model + "/rec_trt_cache.trt") rec_model = fd.vision.ocr.Recognizer( rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option) -# 当用户要把PP-OCR部署在对动态shape推理支持有限的设备上时,(例如华为昇腾) -# 需要使用下行代码, 来启用rec模型的静态shape推理. -# rec_model.preprocessor.static_shape_infer = True - # 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None ppocr_v2 = fd.vision.ocr.PPOCRv2( det_model=det_model, cls_model=cls_model, rec_model=rec_model) @@ -161,8 +200,8 @@ ppocr_v2 = fd.vision.ocr.PPOCRv2( # 给cls和rec模型设置推理时的batch size # 此值能为-1, 和1到正无穷 # 当此值为-1时, cls和rec模型的batch size将默认和det模型检测出的框的数量相同 -ppocr_v2.cls_batch_size = cls_batch_size -ppocr_v2.rec_batch_size = rec_batch_size +ppocr_v2.cls_batch_size = args.cls_bs +ppocr_v2.rec_batch_size = args.rec_bs # 预测图片准备 im = cv2.imread(args.image) diff --git a/examples/vision/ocr/PP-OCRv2/python/infer_static_shape.py b/examples/vision/ocr/PP-OCRv2/python/infer_static_shape.py new file mode 100755 index 000000000..29055fdaa --- /dev/null +++ b/examples/vision/ocr/PP-OCRv2/python/infer_static_shape.py @@ -0,0 +1,114 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--det_model", required=True, help="Path of Detection model of PPOCR.") + parser.add_argument( + "--cls_model", + required=True, + help="Path of Classification model of PPOCR.") + parser.add_argument( + "--rec_model", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--rec_label_file", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu', 'kunlunxin' or 'gpu'.") + parser.add_argument( + "--cpu_thread_num", + type=int, + default=9, + help="Number of threads while inference on CPU.") + return parser.parse_args() + + +def build_option(args): + + det_option = fd.RuntimeOption() + cls_option = fd.RuntimeOption() + rec_option = fd.RuntimeOption() + + # 当前需要对PP-OCR启用静态shape推理的硬件只有昇腾. + if args.device.lower() == "ascend": + det_option.use_ascend() + cls_option.use_ascend() + rec_option.use_ascend() + + return det_option, cls_option, rec_option + + +args = parse_arguments() + +# Detection模型, 检测文字框 +det_model_file = os.path.join(args.det_model, "inference.pdmodel") +det_params_file = os.path.join(args.det_model, "inference.pdiparams") +# Classification模型，方向分类，可选 +cls_model_file = os.path.join(args.cls_model, "inference.pdmodel") +cls_params_file = os.path.join(args.cls_model, "inference.pdiparams") +# Recognition模型，文字识别模型 +rec_model_file = os.path.join(args.rec_model, "inference.pdmodel") +rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") +rec_label_file = args.rec_label_file + +det_option, cls_option, rec_option = build_option(args) + +det_model = fd.vision.ocr.DBDetector( + det_model_file, det_params_file, runtime_option=det_option) + +cls_model = fd.vision.ocr.Classifier( + cls_model_file, cls_params_file, runtime_option=cls_option) + +rec_model = fd.vision.ocr.Recognizer( + rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option) + +# Rec模型启用静态shape推理 +rec_model.preprocessor.static_shape_infer = True + +# 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None +ppocr_v2 = fd.vision.ocr.PPOCRv2( + det_model=det_model, cls_model=cls_model, rec_model=rec_model) + +# Cls模型和Rec模型的batch size 必须设置为1, 开启静态shape推理 +ppocr_v2.cls_batch_size = 1 +ppocr_v2.rec_batch_size = 1 + +# 预测图片准备 +im = cv2.imread(args.image) + +#预测并打印结果 +result = ppocr_v2.predict(im) + +print(result) + +# 可视化结果 +vis_im = fd.vision.vis_ppocr(im, result) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/examples/vision/ocr/PP-OCRv3/cpp/CMakeLists.txt b/examples/vision/ocr/PP-OCRv3/cpp/CMakeLists.txt index 93540a7e8..8b2f7aa61 100644 --- a/examples/vision/ocr/PP-OCRv3/cpp/CMakeLists.txt +++ b/examples/vision/ocr/PP-OCRv3/cpp/CMakeLists.txt @@ -12,3 +12,7 @@ include_directories(${FASTDEPLOY_INCS}) add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) # 添加FastDeploy库依赖 target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) + +add_executable(infer_static_shape_demo ${PROJECT_SOURCE_DIR}/infer_static_shape.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_static_shape_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/ocr/PP-OCRv3/cpp/README.md b/examples/vision/ocr/PP-OCRv3/cpp/README.md index 6f48a69ac..7f557a213 100755 --- a/examples/vision/ocr/PP-OCRv3/cpp/README.md +++ b/examples/vision/ocr/PP-OCRv3/cpp/README.md @@ -43,13 +43,16 @@ wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/ppocr_ ./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 3 # 昆仑芯XPU推理 ./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 4 -# 华为昇腾推理, 请用户在代码里正确开启Rec模型的静态shape推理，并设置分类模型和识别模型的推理batch size为1. -./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 5 +# 华为昇腾推理,需要使用静态shape的demo, 若用户需要连续地预测图片, 输入图片尺寸需要准备为统一尺寸 +./infer_static_shape_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 1 ``` 以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + 运行完成可视化结果如下图所示 diff --git a/examples/vision/ocr/PP-OCRv3/cpp/infer.cc b/examples/vision/ocr/PP-OCRv3/cpp/infer.cc index 7fbcf835e..3b35c1d44 100755 --- a/examples/vision/ocr/PP-OCRv3/cpp/infer.cc +++ b/examples/vision/ocr/PP-OCRv3/cpp/infer.cc @@ -56,10 +56,6 @@ void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model auto cls_model = fastdeploy::vision::ocr::Classifier(cls_model_file, cls_params_file, cls_option); auto rec_model = fastdeploy::vision::ocr::Recognizer(rec_model_file, rec_params_file, rec_label_file, rec_option); - // Users could enable static shape infer for rec model when deploy PP-OCR on hardware - // which can not support dynamic shape infer well, like Huawei Ascend series. - // rec_model.GetPreprocessor().SetStaticShapeInfer(true); - assert(det_model.Initialized()); assert(cls_model.Initialized()); assert(rec_model.Initialized()); @@ -71,9 +67,6 @@ void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model // Set inference batch size for cls model and rec model, the value could be -1 and 1 to positive infinity. // When inference batch size is set to -1, it means that the inference batch size // of the cls and rec models will be the same as the number of boxes detected by the det model. - // When users enable static shape infer for rec model, the batch size of cls and rec model needs to be set to 1. - // ppocr_v3.SetClsBatchSize(1); - // ppocr_v3.SetRecBatchSize(1); ppocr_v3.SetClsBatchSize(cls_batch_size); ppocr_v3.SetRecBatchSize(rec_batch_size); @@ -130,8 +123,6 @@ int main(int argc, char* argv[]) { option.EnablePaddleToTrt(); } else if (flag == 4) { option.UseKunlunXin(); - } else if (flag == 5) { - option.UseAscend(); } std::string det_model_dir = argv[1]; diff --git a/examples/vision/ocr/PP-OCRv3/cpp/infer_static_shape.cc b/examples/vision/ocr/PP-OCRv3/cpp/infer_static_shape.cc new file mode 100755 index 000000000..aea3f5699 --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/cpp/infer_static_shape.cc @@ -0,0 +1,107 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& det_model_dir, const std::string& cls_model_dir, const std::string& rec_model_dir, const std::string& rec_label_file, const std::string& image_file, const fastdeploy::RuntimeOption& option) { + auto det_model_file = det_model_dir + sep + "inference.pdmodel"; + auto det_params_file = det_model_dir + sep + "inference.pdiparams"; + + auto cls_model_file = cls_model_dir + sep + "inference.pdmodel"; + auto cls_params_file = cls_model_dir + sep + "inference.pdiparams"; + + auto rec_model_file = rec_model_dir + sep + "inference.pdmodel"; + auto rec_params_file = rec_model_dir + sep + "inference.pdiparams"; + + auto det_option = option; + auto cls_option = option; + auto rec_option = option; + + auto det_model = fastdeploy::vision::ocr::DBDetector(det_model_file, det_params_file, det_option); + auto cls_model = fastdeploy::vision::ocr::Classifier(cls_model_file, cls_params_file, cls_option); + auto rec_model = fastdeploy::vision::ocr::Recognizer(rec_model_file, rec_params_file, rec_label_file, rec_option); + + // Users could enable static shape infer for rec model when deploy PP-OCR on hardware + // which can not support dynamic shape infer well, like Huawei Ascend series. + rec_model.GetPreprocessor().SetStaticShapeInfer(true); + + assert(det_model.Initialized()); + assert(cls_model.Initialized()); + assert(rec_model.Initialized()); + + // The classification model is optional, so the PP-OCR can also be connected in series as follows + // auto ppocr_v3 = fastdeploy::pipeline::PPOCRv3(&det_model, &rec_model); + auto ppocr_v3 = fastdeploy::pipeline::PPOCRv3(&det_model, &cls_model, &rec_model); + + // When users enable static shape infer for rec model, the batch size of cls and rec model must to be set to 1. + ppocr_v3.SetClsBatchSize(1); + ppocr_v3.SetRecBatchSize(1); + + if(!ppocr_v3.Initialized()){ + std::cerr << "Failed to initialize PP-OCR." << std::endl; + return; + } + + auto im = cv::imread(image_file); + + fastdeploy::vision::OCRResult result; + if (!ppocr_v3.Predict(im, &result)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << result.Str() << std::endl; + + auto vis_im = fastdeploy::vision::VisOcr(im, result); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 7) { + std::cout << "Usage: infer_demo path/to/det_model path/to/cls_model " + "path/to/rec_model path/to/rec_label_file path/to/image " + "run_option, " + "e.g ./infer_demo ./ch_PP-OCRv3_det_infer " + "./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer " + "./ppocr_keys_v1.txt ./12.jpg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with ascend." + << std::endl; + return -1; + } + + fastdeploy::RuntimeOption option; + int flag = std::atoi(argv[6]); + + if (flag == 0) { + option.UseCpu(); + } else if (flag == 1) { + option.UseAscend(); + } + + std::string det_model_dir = argv[1]; + std::string cls_model_dir = argv[2]; + std::string rec_model_dir = argv[3]; + std::string rec_label_file = argv[4]; + std::string test_image = argv[5]; + InitAndInfer(det_model_dir, cls_model_dir, rec_model_dir, rec_label_file, test_image, option); + return 0; +} diff --git a/examples/vision/ocr/PP-OCRv3/python/README.md b/examples/vision/ocr/PP-OCRv3/python/README.md index dd5965d33..3fcf372e0 100755 --- a/examples/vision/ocr/PP-OCRv3/python/README.md +++ b/examples/vision/ocr/PP-OCRv3/python/README.md @@ -35,8 +35,8 @@ python infer.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2 python infer.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device gpu --backend trt # 昆仑芯XPU推理 python infer.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device kunlunxin -# 华为昇腾推理 -python infer.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device ascend +# 华为昇腾推理,需要使用静态shape脚本, 若用户需要连续地预测图片, 输入图片尺寸需要准备为统一尺寸 +python infer_static_shape.py --det_model ch_PP-OCRv3_det_infer --cls_model ch_ppocr_mobile_v2.0_cls_infer --rec_model ch_PP-OCRv3_rec_infer --rec_label_file ppocr_keys_v1.txt --image 12.jpg --device ascend ``` 运行完成可视化结果如下图所示 diff --git a/examples/vision/ocr/PP-OCRv3/python/infer.py b/examples/vision/ocr/PP-OCRv3/python/infer.py index f6da98bdb..6dabce80e 100755 --- a/examples/vision/ocr/PP-OCRv3/python/infer.py +++ b/examples/vision/ocr/PP-OCRv3/python/infer.py @@ -58,43 +58,113 @@ def parse_arguments(): type=int, default=9, help="Number of threads while inference on CPU.") + parser.add_argument( + "--cls_bs", + type=int, + default=1, + help="Classification model inference batch size.") + parser.add_argument( + "--rec_bs", + type=int, + default=6, + help="Recognition model inference batch size") return parser.parse_args() def build_option(args): - option = fd.RuntimeOption() - if args.device.lower() == "gpu": - option.use_gpu(0) - option.set_cpu_thread_num(args.cpu_thread_num) + det_option = fd.RuntimeOption() + cls_option = fd.RuntimeOption() + rec_option = fd.RuntimeOption() + + det_option.set_cpu_thread_num(args.cpu_thread_num) + cls_option.set_cpu_thread_num(args.cpu_thread_num) + rec_option.set_cpu_thread_num(args.cpu_thread_num) + + if args.device.lower() == "gpu": + det_option.use_gpu(args.device_id) + cls_option.use_gpu(args.device_id) + rec_option.use_gpu(args.device_id) if args.device.lower() == "kunlunxin": - option.use_kunlunxin() - return option + det_option.use_kunlunxin() + cls_option.use_kunlunxin() + rec_option.use_kunlunxin() - if args.device.lower() == "ascend": - option.use_ascend() - return option + return det_option, cls_option, rec_option if args.backend.lower() == "trt": assert args.device.lower( ) == "gpu", "TensorRT backend require inference on device GPU." - option.use_trt_backend() + det_option.use_trt_backend() + cls_option.use_trt_backend() + rec_option.use_trt_backend() + + # 设置trt input shape + # 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. + det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], + [1, 3, 960, 960]) + cls_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.cls_bs, 3, 48, 320], + [args.cls_bs, 3, 48, 1024]) + rec_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.rec_bs, 3, 48, 320], + [args.rec_bs, 3, 48, 2304]) + + # 用户可以把TRT引擎文件保存至本地 + det_option.set_trt_cache_file(args.det_model + "/det_trt_cache.trt") + cls_option.set_trt_cache_file(args.cls_model + "/cls_trt_cache.trt") + rec_option.set_trt_cache_file(args.rec_model + "/rec_trt_cache.trt") + elif args.backend.lower() == "pptrt": assert args.device.lower( ) == "gpu", "Paddle-TensorRT backend require inference on device GPU." - option.use_trt_backend() - option.enable_paddle_trt_collect_shape() - option.enable_paddle_to_trt() + det_option.use_trt_backend() + det_option.enable_paddle_trt_collect_shape() + det_option.enable_paddle_to_trt() + + cls_option.use_trt_backend() + cls_option.enable_paddle_trt_collect_shape() + cls_option.enable_paddle_to_trt() + + rec_option.use_trt_backend() + rec_option.enable_paddle_trt_collect_shape() + rec_option.enable_paddle_to_trt() + + # 设置trt input shape + # 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. + det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], + [1, 3, 960, 960]) + cls_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.cls_bs, 3, 48, 320], + [args.cls_bs, 3, 48, 1024]) + rec_option.set_trt_input_shape("x", [1, 3, 48, 10], + [args.rec_bs, 3, 48, 320], + [args.rec_bs, 3, 48, 2304]) + + # 用户可以把TRT引擎文件保存至本地 + det_option.set_trt_cache_file(args.det_model) + cls_option.set_trt_cache_file(args.cls_model) + rec_option.set_trt_cache_file(args.rec_model) + elif args.backend.lower() == "ort": - option.use_ort_backend() + det_option.use_ort_backend() + cls_option.use_ort_backend() + rec_option.use_ort_backend() + elif args.backend.lower() == "paddle": - option.use_paddle_infer_backend() + det_option.use_paddle_infer_backend() + cls_option.use_paddle_infer_backend() + rec_option.use_paddle_infer_backend() + elif args.backend.lower() == "openvino": assert args.device.lower( ) == "cpu", "OpenVINO backend require inference on device CPU." - option.use_openvino_backend() - return option + det_option.use_openvino_backend() + cls_option.use_openvino_backend() + rec_option.use_openvino_backend() + + return det_option, cls_option, rec_option args = parse_arguments() @@ -111,49 +181,18 @@ rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") rec_label_file = args.rec_label_file # 对于三个模型，均采用同样的部署配置 -# 用户也可根据自行需求分别配置 -runtime_option = build_option(args) +# 用户也可根据自己的需求,个性化配置 +det_option, cls_option, rec_option = build_option(args) -# PPOCR的cls和rec模型现在已经支持推理一个Batch的数据 -# 定义下面两个变量后, 可用于设置trt输入shape, 并在PPOCR模型初始化后, 完成Batch推理设置 -# 当用户要把PP-OCR部署在对动态shape推理支持有限的设备上时,(例如华为昇腾) -# 需要把cls_batch_size和rec_batch_size都设置为1. -cls_batch_size = 1 -rec_batch_size = 6 - -# 当使用TRT时，分别给三个模型的runtime设置动态shape,并完成模型的创建. -# 注意: 需要在检测模型创建完成后，再设置分类模型的动态输入并创建分类模型, 识别模型同理. -# 如果用户想要自己改动检测模型的输入shape, 我们建议用户把检测模型的长和高设置为32的倍数. -det_option = runtime_option -det_option.set_trt_input_shape("x", [1, 3, 64, 64], [1, 3, 640, 640], - [1, 3, 960, 960]) -# 用户可以把TRT引擎文件保存至本地 -# det_option.set_trt_cache_file(args.det_model + "/det_trt_cache.trt") det_model = fd.vision.ocr.DBDetector( det_model_file, det_params_file, runtime_option=det_option) -cls_option = runtime_option -cls_option.set_trt_input_shape("x", [1, 3, 48, 10], - [cls_batch_size, 3, 48, 320], - [cls_batch_size, 3, 48, 1024]) -# 用户可以把TRT引擎文件保存至本地 -# cls_option.set_trt_cache_file(args.cls_model + "/cls_trt_cache.trt") cls_model = fd.vision.ocr.Classifier( cls_model_file, cls_params_file, runtime_option=cls_option) -rec_option = runtime_option -rec_option.set_trt_input_shape("x", [1, 3, 48, 10], - [rec_batch_size, 3, 48, 320], - [rec_batch_size, 3, 48, 2304]) -# 用户可以把TRT引擎文件保存至本地 -# rec_option.set_trt_cache_file(args.rec_model + "/rec_trt_cache.trt") rec_model = fd.vision.ocr.Recognizer( rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option) -# 当用户要把PP-OCR部署在对动态shape推理支持有限的设备上时,(例如华为昇腾) -# 需要使用下行代码, 来启用rec模型的静态shape推理. -# rec_model.preprocessor.static_shape_infer = True - # 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None ppocr_v3 = fd.vision.ocr.PPOCRv3( det_model=det_model, cls_model=cls_model, rec_model=rec_model) @@ -161,8 +200,8 @@ ppocr_v3 = fd.vision.ocr.PPOCRv3( # 给cls和rec模型设置推理时的batch size # 此值能为-1, 和1到正无穷 # 当此值为-1时, cls和rec模型的batch size将默认和det模型检测出的框的数量相同 -ppocr_v3.cls_batch_size = cls_batch_size -ppocr_v3.rec_batch_size = rec_batch_size +ppocr_v3.cls_batch_size = args.cls_bs +ppocr_v3.rec_batch_size = args.rec_bs # 预测图片准备 im = cv2.imread(args.image) diff --git a/examples/vision/ocr/PP-OCRv3/python/infer_static_shape.py b/examples/vision/ocr/PP-OCRv3/python/infer_static_shape.py new file mode 100755 index 000000000..e707d378c --- /dev/null +++ b/examples/vision/ocr/PP-OCRv3/python/infer_static_shape.py @@ -0,0 +1,114 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--det_model", required=True, help="Path of Detection model of PPOCR.") + parser.add_argument( + "--cls_model", + required=True, + help="Path of Classification model of PPOCR.") + parser.add_argument( + "--rec_model", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--rec_label_file", + required=True, + help="Path of Recognization model of PPOCR.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu', 'kunlunxin' or 'gpu'.") + parser.add_argument( + "--cpu_thread_num", + type=int, + default=9, + help="Number of threads while inference on CPU.") + return parser.parse_args() + + +def build_option(args): + + det_option = fd.RuntimeOption() + cls_option = fd.RuntimeOption() + rec_option = fd.RuntimeOption() + + # 当前需要对PP-OCR启用静态shape推理的硬件只有昇腾. + if args.device.lower() == "ascend": + det_option.use_ascend() + cls_option.use_ascend() + rec_option.use_ascend() + + return det_option, cls_option, rec_option + + +args = parse_arguments() + +# Detection模型, 检测文字框 +det_model_file = os.path.join(args.det_model, "inference.pdmodel") +det_params_file = os.path.join(args.det_model, "inference.pdiparams") +# Classification模型，方向分类，可选 +cls_model_file = os.path.join(args.cls_model, "inference.pdmodel") +cls_params_file = os.path.join(args.cls_model, "inference.pdiparams") +# Recognition模型，文字识别模型 +rec_model_file = os.path.join(args.rec_model, "inference.pdmodel") +rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") +rec_label_file = args.rec_label_file + +det_option, cls_option, rec_option = build_option(args) + +det_model = fd.vision.ocr.DBDetector( + det_model_file, det_params_file, runtime_option=det_option) + +cls_model = fd.vision.ocr.Classifier( + cls_model_file, cls_params_file, runtime_option=cls_option) + +rec_model = fd.vision.ocr.Recognizer( + rec_model_file, rec_params_file, rec_label_file, runtime_option=rec_option) + +# Rec模型启用静态shape推理 +rec_model.preprocessor.static_shape_infer = True + +# 创建PP-OCR，串联3个模型，其中cls_model可选，如无需求，可设置为None +ppocr_v3 = fd.vision.ocr.PPOCRv3( + det_model=det_model, cls_model=cls_model, rec_model=rec_model) + +# Cls模型和Rec模型的batch size 必须设置为1, 开启静态shape推理 +ppocr_v3.cls_batch_size = 1 +ppocr_v3.rec_batch_size = 1 + +# 预测图片准备 +im = cv2.imread(args.image) + +#预测并打印结果 +result = ppocr_v3.predict(im) + +print(result) + +# 可视化结果 +vis_im = fd.vision.vis_ppocr(im, result) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/examples/vision/segmentation/paddleseg/cpp/README.md b/examples/vision/segmentation/paddleseg/cpp/README.md index 6b1be6e5b..07f9f4c62 100755 --- a/examples/vision/segmentation/paddleseg/cpp/README.md +++ b/examples/vision/segmentation/paddleseg/cpp/README.md @@ -46,6 +46,9 @@ wget https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_demo.png 以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +如果用户使用华为昇腾NPU部署, 请参考以下方式在部署前初始化部署环境: +- [如何使用华为昇腾NPU部署](../../../../../docs/cn/faq/use_sdk_on_ascend.md) + ## PaddleSeg C++接口 ### PaddleSeg类 From 78a8c9afdae13515197ad71266f5ef5583aca9ec Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 4 Jan 2023 16:18:54 +0800 Subject: [PATCH 4/4] [Other] Finetune sophgo module (#1054) refine sophgo code --- fastdeploy/backends/sophgo/option.h | 25 ++++++ fastdeploy/backends/sophgo/sophgo_backend.cc | 86 ++++++++++---------- fastdeploy/backends/sophgo/sophgo_backend.h | 22 +++-- fastdeploy/backends/sophgo/sophgo_config.h | 0 4 files changed, 78 insertions(+), 55 deletions(-) create mode 100644 fastdeploy/backends/sophgo/option.h delete mode 100644 fastdeploy/backends/sophgo/sophgo_config.h diff --git a/fastdeploy/backends/sophgo/option.h b/fastdeploy/backends/sophgo/option.h new file mode 100644 index 000000000..320cb7ae2 --- /dev/null +++ b/fastdeploy/backends/sophgo/option.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include + +namespace fastdeploy { +struct SophgoBackendOption{ +}; +} // namespace fastdeploy diff --git a/fastdeploy/backends/sophgo/sophgo_backend.cc b/fastdeploy/backends/sophgo/sophgo_backend.cc index c4a75ce2d..2e52e4e5d 100644 --- a/fastdeploy/backends/sophgo/sophgo_backend.cc +++ b/fastdeploy/backends/sophgo/sophgo_backend.cc @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "fastdeploy/backends/sophgo/sophgo_backend.h" -#include -namespace fastdeploy { - SophgoBackend::~SophgoBackend() { - bm_dev_free(handle_); - -} +#include + +namespace fastdeploy { +SophgoBackend::~SophgoBackend() { bm_dev_free(handle_); } /*************************************************************** * @name GetSDKAndDeviceVersion * @brief get Sophgo sdk and device version @@ -26,10 +24,7 @@ namespace fastdeploy { * @return bool * @note None ***************************************************************/ -bool SophgoBackend::GetSDKAndDeviceVersion() { - - return true; -} +bool SophgoBackend::GetSDKAndDeviceVersion() { return true; } /*************************************************************** * @name BuildOption @@ -38,9 +33,9 @@ bool SophgoBackend::GetSDKAndDeviceVersion() { * @note None ***************************************************************/ void SophgoBackend::BuildOption(const SophgoBackendOption& option) { -// this->option_ = option; + // this->option_ = option; // save cpu_name -// this->option_.cpu_name = option.cpu_name; + // this->option_.cpu_name = option.cpu_name; } /*************************************************************** @@ -53,8 +48,7 @@ void SophgoBackend::BuildOption(const SophgoBackendOption& option) { * @note None ***************************************************************/ bool SophgoBackend::InitFromSophgo(const std::string& model_file, - const SophgoBackendOption& option) { - + const SophgoBackendOption& option) { // LoadModel if (!this->LoadModel((char*)model_file.data())) { FDERROR << "load model failed" << std::endl; @@ -98,7 +92,7 @@ bool SophgoBackend::LoadModel(void* model) { int network_num = bmrt_get_network_number(p_bmrt_); - const char **net_names = NULL; + const char** net_names = NULL; bmrt_get_network_names(p_bmrt_, &net_names); net_name_ = net_names[0]; free(net_names); @@ -119,15 +113,15 @@ bool SophgoBackend::LoadModel(void* model) { bool SophgoBackend::GetModelInputOutputInfos() { inputs_desc_.resize(net_info_->input_num); bm_shape_t* input_shapes = net_info_->stages->input_shapes; - for(int idx=0; idxinput_num; idx++){ + for (int idx = 0; idx < net_info_->input_num; idx++) { std::string temp_name = (net_info_->input_names)[idx]; std::vector temp_shape{}; temp_shape.resize(input_shapes[idx].num_dims); - for(int i=0; iinput_dtypes; - //SophgoType to FDDataType + // SophgoType to FDDataType FDDataType temp_dtype = SophgoTensorTypeToFDDataType(*input_dtypes); TensorInfo temp_input_info = {temp_name, temp_shape, temp_dtype}; inputs_desc_[idx] = temp_input_info; @@ -135,15 +129,15 @@ bool SophgoBackend::GetModelInputOutputInfos() { outputs_desc_.resize(net_info_->output_num); bm_shape_t* output_shapes = net_info_->stages->output_shapes; - for(int idx=0; idxoutput_num; idx++){ + for (int idx = 0; idx < net_info_->output_num; idx++) { std::string temp_name1 = (net_info_->output_names)[idx]; std::vector temp_shape1{}; temp_shape1.resize(output_shapes[idx].num_dims); - for(int i=0; ioutput_dtypes; - //SophgoType to FDDataType + // SophgoType to FDDataType FDDataType temp_dtype1 = SophgoTensorTypeToFDDataType(*output_dtypes); TensorInfo temp_output_info = {temp_name1, temp_shape1, temp_dtype1}; outputs_desc_[idx] = temp_output_info; @@ -167,11 +161,12 @@ TensorInfo SophgoBackend::GetOutputInfo(int index) { return outputs_desc_[index]; } -std::vector SophgoBackend::GetOutputInfos() { return outputs_desc_; } +std::vector SophgoBackend::GetOutputInfos() { + return outputs_desc_; +} bool SophgoBackend::Infer(std::vector& inputs, - std::vector* outputs, - bool copy_to_fd) { + std::vector* outputs, bool copy_to_fd) { int input_size = inputs.size(); assert(input_size != 0); assert(input_size == NumInputs()); @@ -179,48 +174,52 @@ bool SophgoBackend::Infer(std::vector& inputs, bm_status_t status = BM_SUCCESS; bm_data_type_t* input_dtypes = net_info_->input_dtypes; - for(int i=0;imax_input_bytes[i]); + for (int i = 0; i < input_size; i++) { + status = bm_malloc_device_byte(handle_, &input_tensors[i].device_mem, + net_info_->max_input_bytes[i]); assert(BM_SUCCESS == status); input_tensors[i].dtype = input_dtypes[i]; input_tensors[i].st_mode = BM_STORE_1N; input_tensors[i].shape = *(net_info_->stages[i].input_shapes); unsigned int input_byte = bmrt_tensor_bytesize(&input_tensors[i]); - bm_memcpy_s2d_partial(handle_, input_tensors[i].device_mem, (void *)inputs[i].Data(), - bmrt_tensor_bytesize(&input_tensors[i])); + bm_memcpy_s2d_partial(handle_, input_tensors[i].device_mem, + (void*)inputs[i].Data(), + bmrt_tensor_bytesize(&input_tensors[i])); } int output_size = NumOutputs(); bm_tensor_t output_tensors[output_size]; - for(int i=0;imax_output_bytes[i]); + net_info_->max_output_bytes[i]); assert(BM_SUCCESS == status); } - bool launch_status = bmrt_launch_tensor_ex(p_bmrt_, net_name_.c_str(), input_tensors, net_info_->input_num, - output_tensors, net_info_->output_num, true, false); + bool launch_status = bmrt_launch_tensor_ex( + p_bmrt_, net_name_.c_str(), input_tensors, net_info_->input_num, + output_tensors, net_info_->output_num, true, false); assert(launch_status); status = bm_thread_sync(handle_); assert(status == BM_SUCCESS); outputs->resize(outputs_desc_.size()); bm_data_type_t* output_dtypes = net_info_->output_dtypes; - for(int i=0;i temp_shape; temp_shape.resize(outputs_desc_[i].shape.size()); for (int j = 0; j < outputs_desc_[i].shape.size(); ++j) { - temp_shape[j] = outputs_desc_[i].shape[j]; + temp_shape[j] = outputs_desc_[i].shape[j]; } - (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, outputs_desc_[i].name); + (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, + outputs_desc_[i].name); - memcpy((*outputs)[i].MutableData(), temp_out, (*outputs)[i].Nbytes()); - free(temp_out); + memcpy((*outputs)[i].MutableData(), temp_out, (*outputs)[i].Nbytes()); + free(temp_out); } return true; @@ -264,7 +263,8 @@ FDDataType SophgoBackend::SophgoTensorTypeToFDDataType(bm_data_type_t type) { * @note None ***************************************************************/ // Sophgo_tensor_type -bm_data_type_t SophgoBackend::FDDataTypeToSophgoTensorType(fastdeploy::FDDataType type) { +bm_data_type_t SophgoBackend::FDDataTypeToSophgoTensorType( + fastdeploy::FDDataType type) { if (type == FDDataType::FP16) { return BM_FLOAT16; } @@ -287,4 +287,4 @@ bm_data_type_t SophgoBackend::FDDataTypeToSophgoTensorType(fastdeploy::FDDataTyp return BM_FLOAT32; } -} +} // namespace fastdeploy diff --git a/fastdeploy/backends/sophgo/sophgo_backend.h b/fastdeploy/backends/sophgo/sophgo_backend.h index 8007bfd13..348d844af 100644 --- a/fastdeploy/backends/sophgo/sophgo_backend.h +++ b/fastdeploy/backends/sophgo/sophgo_backend.h @@ -17,7 +17,7 @@ #include "fastdeploy/core/fd_tensor.h" #include "bmruntime_interface.h" // NOLINT #include "bmlib_runtime.h" // NOLINT -#include "fastdeploy/backends/sophgo/sophgo_config.h" +#include "fastdeploy/backends/sophgo/option.h" #include #include #include @@ -25,8 +25,6 @@ #include namespace fastdeploy { -struct SophgoBackendOption{ -}; class SophgoBackend : public BaseBackend { public: @@ -38,15 +36,15 @@ class SophgoBackend : public BaseBackend { void BuildOption(const SophgoBackendOption& option); bool InitFromSophgo(const std::string& model_file, const SophgoBackendOption& option = SophgoBackendOption()); - + int NumInputs() const override { return static_cast(inputs_desc_.size()); } - + int NumOutputs() const override { return static_cast(outputs_desc_.size()); } - + TensorInfo GetInputInfo(int index) override; TensorInfo GetOutputInfo(int index) override; std::vector GetInputInfos() override; @@ -59,17 +57,17 @@ class SophgoBackend : public BaseBackend { std::vector inputs_desc_; std::vector outputs_desc_; std::string net_name_; - + bm_handle_t handle_; void * p_bmrt_ = nullptr; - + bool infer_init = false; - + const bm_net_info_t* net_info_ = nullptr; - + // SophgoTPU2BackendOption option_; - + static FDDataType SophgoTensorTypeToFDDataType(bm_data_type_t type); static bm_data_type_t FDDataTypeToSophgoTensorType(FDDataType type); }; -} // namespace fastdeploy \ No newline at end of file +} // namespace fastdeploy diff --git a/fastdeploy/backends/sophgo/sophgo_config.h b/fastdeploy/backends/sophgo/sophgo_config.h deleted file mode 100644 index e69de29bb..000000000