diff --git a/CMakeLists.txt b/CMakeLists.txt index d4fe36fa3..42bc600bb 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ PROJECT(fastdeploy C CXX) CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + option(CSRCS_DIR_NAME "Name of source code directory") option(LIBRARY_NAME "Name of build library name") option(PY_LIBRARY_NAME "Name of build python library name") @@ -60,6 +61,7 @@ option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF) option(ENABLE_POROS_BACKEND "Whether to enable poros backend." OFF) option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF) option(ENABLE_RKNPU2_BACKEND "Whether to enable RKNPU2 backend." OFF) +option(ENABLE_SOPHGO_BACKEND "Whether to enable SOPHON backend." OFF) option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF) option(ENABLE_VISION "Whether to enable vision models usage." OFF) option(ENABLE_TEXT "Whether to enable text models usage." OFF) @@ -194,6 +196,7 @@ file(GLOB_RECURSE DEPLOY_POROS_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fast file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cpp) file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) file(GLOB_RECURSE DEPLOY_RKNPU2_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/rknpu/rknpu2/*.cc) +file(GLOB_RECURSE DEPLOY_SOPHGO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/sophgo/*.cc) file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) file(GLOB_RECURSE DEPLOY_ENCRYPTION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/encryption/*.cc) @@ -201,8 +204,7 @@ file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/f file(GLOB_RECURSE DEPLOY_VISION_CUDA_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cu) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) - -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS} ${DEPLOY_ENCRYPTION_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_POROS_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS} ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS} ${DEPLOY_SOPHGO_SRCS} ${DEPLOY_ENCRYPTION_SRCS}) set(DEPEND_LIBS "") @@ -266,6 +268,13 @@ if(ENABLE_RKNPU2_BACKEND) list(APPEND DEPEND_LIBS ${RKNN_RT_LIB}) endif() +if(ENABLE_SOPHGO_BACKEND) + add_definitions(-DENABLE_SOPHGO_BACKEND) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_SOPHGO_SRCS}) + include(${PROJECT_SOURCE_DIR}/cmake/sophgo.cmake) + list(APPEND DEPEND_LIBS ${SOPHGO_RT_LIB}) +endif() + if(ENABLE_POROS_BACKEND) set(CMAKE_CXX_STANDARD 14) add_definitions(-DENABLE_POROS_BACKEND) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 812caae54..6ba0b4307 100755 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -3,6 +3,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.8) set(WITH_GPU @WITH_GPU@) set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@) set(ENABLE_RKNPU2_BACKEND @ENABLE_RKNPU2_BACKEND@) +set(ENABLE_SOPHGO_BACKEND @ENABLE_SOPHGO_BACKEND@) set(ENABLE_LITE_BACKEND @ENABLE_LITE_BACKEND@) set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@) set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@) @@ -271,6 +272,7 @@ message(STATUS " CXX flags : ${CMAKE_CXX_FLAGS}") message(STATUS " WITH_GPU : ${WITH_GPU}") message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_RKNPU2_BACKEND : ${ENABLE_RKNPU2_BACKEND}") +message(STATUS " ENABLE_SOPHGO_BACKEND : ${ENABLE_SOPHGO_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") message(STATUS " ENABLE_POROS_BACKEND : ${ENABLE_POROS_BACKEND}") message(STATUS " ENABLE_OPENVINO_BACKEND : ${ENABLE_OPENVINO_BACKEND}") diff --git a/cmake/sophgo.cmake b/cmake/sophgo.cmake new file mode 100644 index 000000000..d51d61368 --- /dev/null +++ b/cmake/sophgo.cmake @@ -0,0 +1,7 @@ +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +find_package(libsophon REQUIRED) +message(${LIBSOPHON_LIB_DIRS}) +include_directories(${LIBSOPHON_INCLUDE_DIRS}) +message(${LIBSOPHON_LIB_DIRS}) +set(SOPHGO_RT_LIB ${LIBSOPHON_LIB_DIRS}/libbmrt.so) \ No newline at end of file diff --git a/cmake/summary.cmake b/cmake/summary.cmake index 38fb35530..9c3b1981c 100755 --- a/cmake/summary.cmake +++ b/cmake/summary.cmake @@ -32,6 +32,7 @@ function(fastdeploy_summary) message(STATUS " Paddle2ONNX version : ${PADDLE2ONNX_VERSION}") message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_RKNPU2_BACKEND : ${ENABLE_RKNPU2_BACKEND}") + message(STATUS " ENABLE_SOPHGO_BACKEND : ${ENABLE_SOPHGO_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") message(STATUS " ENABLE_LITE_BACKEND : ${ENABLE_LITE_BACKEND}") message(STATUS " ENABLE_POROS_BACKEND : ${ENABLE_POROS_BACKEND}") diff --git a/docs/cn/build_and_install/README.md b/docs/cn/build_and_install/README.md index 838a9f8ec..5175f50d9 100755 --- a/docs/cn/build_and_install/README.md +++ b/docs/cn/build_and_install/README.md @@ -26,6 +26,7 @@ | ENABLE_PADDLE_BACKEND | 默认OFF，是否编译集成Paddle Inference后端(CPU/GPU上推荐打开) | | ENABLE_LITE_BACKEND | 默认OFF，是否编译集成Paddle Lite后端(编译Android库时需要设置为ON) | | ENABLE_RKNPU2_BACKEND | 默认OFF，是否编译集成RKNPU2后端(RK3588/RK3568/RK3566上推荐打开) | +| ENABLE_SOPHGO_BACKEND | 默认OFF，是否编译集成SOPHGO后端, 当在SOPHGO TPU上部署时，需要设置为ON | | WITH_ASCEND | 默认OFF，当在华为昇腾NPU上部署时, 需要设置为ON | | WITH_KUNLUNXIN | 默认OFF，当在昆仑芯XPU上部署时，需设置为ON | | WITH_TIMVX | 默认OFF，需要在RV1126/RV1109/A311D上部署时，需设置为ON | diff --git a/docs/cn/build_and_install/sophgo.md b/docs/cn/build_and_install/sophgo.md new file mode 100644 index 000000000..f27432e71 --- /dev/null +++ b/docs/cn/build_and_install/sophgo.md @@ -0,0 +1,78 @@ +# SOPHGO 部署库编译 + +## SOPHGO 环境准备 +SOPHGO支持linux下进行编译,系统为Debian/Ubuntu +安装包由三个文件构成 +- [sophon-driver\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/KWqbmEcKp) +- [sophon-libsophon\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/PlvlBXhWY) +- [sophon-libsophon-dev\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/zTErLlpS7) + +其中“$arch”为当前机器的硬件架构，使用以下命令可以获取当前的服务器arch： +```shell +uname -m +``` +通常x86_64 机器对应的硬件架构为amd64,arm64 机器对应的硬件架构为 arm64: +```text +- sophon-driver_0.4.2_$arch.deb +- sophon-libsophon_0.4.2_$arch.deb +- sophon-libsophon-dev_0.4.2_$arch.deb +``` + +其中:sophon-driver 包含了 PCIe 加速卡驱动;sophon-libsophon 包含了运行时环境(库文 +件、工具等);sophon-libsophon-dev 包含了开发环境(头文件等)。如果只是在部署环境上安 +装,则不需要安装 sophon-libsophon-dev。 +可以通过如下步骤安装: +```shell +#安装依赖库,只需要执行一次: +sudo apt install dkms libncurses5 +#安装 libsophon: +sudo dpkg -i sophon-*.deb +#在终端执行如下命令,或者登出再登入当前用户后即可使用 bm-smi 等命令: +source /etc/profile +``` +安装位置为： +```text +/opt/sophon/ +├── driver-0.4.2 +├── libsophon-0.4.2 +| ├──bin +| ├──data +| ├──include +| └──lib +└── libsophon-current->/opt/sophon/libsophon-0.4.2 +``` + +## C++ SDK编译安装 +搭建好编译环境之后，编译命令如下： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy +mkdir build && cd build + +# CMake configuration with Ascend +cmake -DENABLE_SOPHGO_BACKEND=ON \ + -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-sophgo \ + -DENABLE_VISION=ON \ + .. + +# Build FastDeploy Ascend C++ SDK +make -j8 +make install +``` +编译完成之后，会在当前的build目录下生成 fastdeploy-sophgo 目录，编译完成。 + +## Python FastDeploy 库编译 +搭建好编译环境之后，编译命令如下： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/python +export ENABLE_SOPHGO_BACKEND=ON +export ENABLE_VISION=ON + +python setup.py build +python setup.py bdist_wheel + +#编译完成后,请用户自行安装当前目录的dist文件夹内的whl包. +``` diff --git a/docs/en/build_and_install/sophgo.md b/docs/en/build_and_install/sophgo.md new file mode 100644 index 000000000..08d18122c --- /dev/null +++ b/docs/en/build_and_install/sophgo.md @@ -0,0 +1,77 @@ + +# How to Build SOPHGO Deployment Environment + +## SOPHGO Environment Preparation +SOPHGO supports compilation on linux, using Debian/Ubuntu as an example +The installation package consists of three files +- [sophon-driver\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/KWqbmEcKp) +- [sophon-libsophon\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/PlvlBXhWY) +- [sophon-libsophon-dev\_0.4.2\_$arch.deb](http://219.142.246.77:65000/sharing/zTErLlpS7) + +$arch indicates the hardware architecture of the current machine. Run the following command to obtain the current server arch: +```shell +uname -m +``` +Generally, the hardware architecture of x86_64 machines is amd64, so the hardware architecture is arm64: +```text +- sophon-driver_0.4.2_$arch.deb +- sophon-libsophon_0.4.2_$arch.deb +- sophon-libsophon-dev_0.4.2_$arch.deb +``` + +sophon-driver contains PCIe acceleration card drivers; sophon-libsophon contains the runtime environment (librarys, tools, etc); sophon-libsophon-dev contains the development environment (header files, etc.). If you install packages only on a deployment environment, you do not need to install sophon-libsophon-dev. +You can perform the following steps to install: +```shell +#To install a dependency library, you only need to do this once: +sudo apt install dkms libncurses5 +#install libsophon: +sudo dpkg -i sophon-*.deb +#Run the following command on the terminal, log out and then log in the current user to use commands such as bm-smi: +source /etc/profile +``` +The position of installation:： +```text +/opt/sophon/ +├── driver-0.4.2 +├── libsophon-0.4.2 +| ├──bin +| ├──data +| ├──include +| └──lib +└── libsophon-current->/opt/sophon/libsophon-0.4.2 +``` + +## How to Build and Install C++ SDK +After setting up the compilation environment, the compilation command is as follows： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy +mkdir build && cd build + +# CMake configuration with Ascend +cmake -DENABLE_SOPHGO_BACKEND=ON \ + -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-sophgo \ + -DENABLE_VISION=ON \ + .. + +# Build FastDeploy Ascend C++ SDK +make -j8 +make install +``` +When the compilation is complete, the fastdeploy-sophgo directory is created in the current build directory, indicating that the FastDeploy library has been compiled. + +## Compiling Python FastDeploy Libraries +After setting up the compilation environment, the compilation command is as follows： +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/python +export ENABLE_SOPHGO_BACKEND=ON +export ENABLE_VISION=ON + +python setup.py build +python setup.py bdist_wheel + +#After the compilation is complete, please install the whl package in the dist folder of the current directory. +``` diff --git a/examples/vision/classification/paddleclas/sophgo/README.md b/examples/vision/classification/paddleclas/sophgo/README.md new file mode 100644 index 000000000..32bb3bfbf --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/README.md @@ -0,0 +1,84 @@ +# PaddleDetection SOPHGO部署示例 + +## 支持模型列表 + +目前FastDeploy支持的如下模型的部署[ResNet系列模型](https://github.com/PaddlePaddle/PaddleClas/blob/release/2.4/docs/zh_CN/models/ResNet_and_vd.md) + +## 准备ResNet部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- Paddle动态图模型转换为ONNX模型，请参考[Paddle2ONNX模型转换](https://github.com/PaddlePaddle/Paddle2ONNX/tree/develop/model_zoo/classification) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir)。 + +## 模型转换example + +下面以[ResNet50_vd](https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz)为例子,教大家如何转换Paddle模型到SOPHGO-TPU模型。 + +## 导出ONNX模型 + +### 下载Paddle ResNet50_vd静态图模型并解压 +```shell +wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz +tar xvf ResNet50_vd_infer.tgz +``` + +### 静态图转ONNX模型，注意，这里的save_file请和压缩包名对齐 +```shell +paddle2onnx --model_dir ResNet50_vd_infer \ + --model_filename inference.pdmodel \ + --params_filename inference.pdiparams \ + --save_file ResNet50_vd_infer.onnx \ + --enable_dev_version True +``` +### 导出bmodel模型 + +以转化BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir ResNet50_vd_infer && cd ResNet50_vd_infer + +# 在该文件中放入测试图片，同时将上一步转换好的ResNet50_vd_infer.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +# 放入onnx模型文件ResNet50_vd_infer.onnx + +mkdir workspace && cd workspace + +# 将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name ResNet50_vd_infer \ + --model_def ../ResNet50_vd_infer.onnx \ + --input_shapes [[1,3,224,224]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names save_infer_model/scale_0.tmp_1 \ + --test_input ../image/dog.jpg \ + --test_result ResNet50_vd_infer_top_outputs.npz \ + --mlir ResNet50_vd_infer.mlir + +# 将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir ResNet50_vd_infer.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input ResNet50_vd_infer_in_f32.npz \ + --test_reference ResNet50_vd_infer_top_outputs.npz \ + --model ResNet50_vd_infer_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型ResNet50_vd_infer_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt b/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/README.md b/examples/vision/classification/paddleclas/sophgo/cpp/README.md new file mode 100644 index 000000000..7edfd2c94 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/README.md @@ -0,0 +1,61 @@ +# PaddleClas C++部署示例 + +本目录下提供`infer.cc`快速完成ResNet50_vd模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +├── preprocess_config.yaml #示例前处理配置文件 +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 +将前处理配置文件也拷贝到model中 +```bash +cp preprocess_config.yaml ./model +``` + +### 准备测试图片至image文件夹 +```bash +wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg +cp ILSVRC2012_val_00000010.jpeg ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/ILSVRC2012_val_00000010.jpeg +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc b/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc new file mode 100644 index 000000000..b9281ffb0 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/cpp/infer.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "resnet50_1684x_f32.bmodel"; + auto params_file = model_dir + sep + ""; + auto config_file = model_dir + sep + "preprocess_config.yaml"; + + fastdeploy::RuntimeOption option; + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + auto model = fastdeploy::vision::classification::PaddleClasModel( + model_file, params_file, config_file, option, model_format); + + assert(model.Initialized()); + + auto im = cv::imread(image_file); + + fastdeploy::vision::ClassifyResult res; + if (!model.Predict(im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout << "Usage: infer_demo path/to/model " + "path/to/image " + "run_option, " + "e.g ./infer_demo ./bmodel ./test.jpeg" + << std::endl; + return -1; + } + + std::string model_dir = argv[1]; + std::string test_image = argv[2]; + InitAndInfer(model_dir, test_image); + return 0; +} diff --git a/examples/vision/classification/paddleclas/sophgo/python/README.md b/examples/vision/classification/paddleclas/sophgo/python/README.md new file mode 100644 index 000000000..f495e5830 --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/python/README.md @@ -0,0 +1,29 @@ +# PaddleClas Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 ResNet50_vd 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/classification/paddleclas/sophgo/python + +# 下载图片 +wget https://gitee.com/paddlepaddle/PaddleClas/raw/release/2.4/deploy/images/ImageNet/ILSVRC2012_val_00000010.jpeg + +# 推理 +python3 infer.py --model_file ./bmodel/resnet50_1684x_f32.bmodel --config_file ResNet50_vd_infer/inference_cls.yaml --image ILSVRC2012_val_00000010.jpeg + +# 运行完成后返回结果如下所示 +ClassifyResult( +label_ids: 153, +scores: 0.684570, +) +``` + +## 其它文档 +- [ResNet50_vd C++部署](../cpp) +- [转换ResNet50_vd SOPHGO模型文档](../README.md) diff --git a/examples/vision/classification/paddleclas/sophgo/python/infer.py b/examples/vision/classification/paddleclas/sophgo/python/infer.py new file mode 100644 index 000000000..5bc84789e --- /dev/null +++ b/examples/vision/classification/paddleclas/sophgo/python/infer.py @@ -0,0 +1,41 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--config_file", required=True, help="Path of config file.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--topk", type=int, default=1, help="Return topk results.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" +config_file = args.config_file + +model = fd.vision.classification.PaddleClasModel( + model_file, + params_file, + config_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im = cv2.imread(args.image) +result = model.predict(im, args.topk) +print(result) diff --git a/examples/vision/detection/yolov5/sophgo/README.md b/examples/vision/detection/yolov5/sophgo/README.md new file mode 100644 index 000000000..d4fa4f7a8 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/README.md @@ -0,0 +1,75 @@ +# YOLOv5 SOPHGO部署示例 + +## 支持模型列表 + +YOLOv5 v6.0部署模型实现来自[YOLOv5](https://github.com/ultralytics/yolov5/tree/v6.0),和[基于COCO的预训练模型](https://github.com/ultralytics/yolov5/releases/tag/v6.0) + +## 准备YOLOv5部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- 下载预训练ONNX模型，请参考[YOLOv5准备部署模型](https://github.com/PaddlePaddle/FastDeploy/tree/develop/examples/vision/detection/yolov5) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir) + +## 模型转换example + +下面以YOLOv5s为例子,教大家如何转换ONNX模型到SOPHGO-TPU模型 + +## 下载YOLOv5s模型 + +### 下载ONNX YOLOv5s静态图模型 +```shell +wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx + +``` +### 导出bmodel模型 + +以转化BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir YOLOv5s && cd YOLOv5s + +# 在该文件中放入测试图片，同时将上一步下载的yolov5s.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +# 放入onnx模型文件yolov5s.onnx + +mkdir workspace && cd workspace + +# 将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name yolov5s \ + --model_def ../yolov5s.onnx \ + --input_shapes [[1,3,640,640]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names output,350,498,646 \ + --test_input ../image/dog.jpg \ + --test_result yolov5s_top_outputs.npz \ + --mlir yolov5s.mlir + +# 将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir yolov5s.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input yolov5s_in_f32.npz \ + --test_reference yolov5s_top_outputs.npz \ + --model yolov5s_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型yolov5s_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt b/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/README.md b/examples/vision/detection/yolov5/sophgo/cpp/README.md new file mode 100644 index 000000000..e313da855 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/README.md @@ -0,0 +1,56 @@ +# YOLOv5 C++部署示例 + +本目录下提供`infer.cc`快速完成yolov5s模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 + +### 准备测试图片至image文件夹 +```bash +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg +cp 000000014439.jpg ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/000000014439.jpg +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/detection/yolov5/sophgo/cpp/infer.cc b/examples/vision/detection/yolov5/sophgo/cpp/infer.cc new file mode 100644 index 000000000..f1f63bcdc --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/cpp/infer.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "yolov5s_1684x_f32.bmodel"; + auto params_file = model_dir + sep + ""; + + fastdeploy::RuntimeOption option; + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + + auto model = fastdeploy::vision::detection::YOLOv5( + model_file, params_file, option, model_format); + + assert(model.Initialized()); + + auto im = cv::imread(image_file); + + fastdeploy::vision::DetectionResult res; + if (!model.Predict(im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout << "Usage: infer_demo path/to/model " + "path/to/image " + "run_option, " + "e.g ./infer_demo ./model ./test.jpeg" + << std::endl; + return -1; + } + + std::string model_dir = argv[1]; + std::string test_image = argv[2]; + InitAndInfer(model_dir, test_image); + return 0; +} diff --git a/examples/vision/detection/yolov5/sophgo/python/README.md b/examples/vision/detection/yolov5/sophgo/python/README.md new file mode 100644 index 000000000..ccf8ed7e8 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/python/README.md @@ -0,0 +1,46 @@ +# YOLOv5 Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 YOLOv5 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/detection/yolov5/sophgo/python + +# 下载图片 +wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg + +# 推理 +python3 infer.py --model_file ./bmodel/yolov5s_1684x_f32.bmodel --image 000000014439.jpg + +# 运行完成后返回结果如下所示 +DetectionResult: [xmin, ymin, xmax, ymax, score, label_id] +268.480255,81.053055, 298.694794, 169.439026, 0.896569, 0 +104.731163,45.661972, 127.583824, 93.449387, 0.869531, 0 +378.909363,39.750137, 395.608643, 84.243454, 0.868430, 0 +158.552979,80.361511, 199.185760, 168.181915, 0.842988, 0 +414.375305,90.948090, 506.321899, 280.405182, 0.835842, 0 +364.003448,56.608932, 381.978607, 115.968216, 0.815136, 0 +351.725128,42.635330, 366.910309, 98.048386, 0.808936, 0 +505.888306,114.366791, 593.124878, 275.995270, 0.801361, 0 +327.708618,38.363693, 346.849915, 80.893021, 0.794725, 0 +583.493408,114.532883, 612.354614, 175.873535, 0.760649, 0 +186.470657,44.941360, 199.664505, 61.037643, 0.632591, 0 +169.615891,48.014603, 178.141556, 60.888596, 0.613938, 0 +25.810200,117.199692, 59.888783, 152.850128, 0.590614, 0 +352.145294,46.712723, 381.946075, 106.752151, 0.505329, 0 +1.875000,150.734375, 37.968750, 173.781250, 0.404573, 24 +464.657288,15.901413, 472.512939, 34.116409, 0.346033, 0 +64.625000,135.171875, 84.500000, 154.406250, 0.332831, 24 +57.812500,151.234375, 103.000000, 174.156250, 0.332566, 24 +165.906250,88.609375, 527.906250, 339.953125, 0.259424, 33 +101.406250,152.562500, 118.890625, 169.140625, 0.253891, 24 +``` + +## 其它文档 +- [YOLOv5 C++部署](../cpp) +- [转换YOLOv5 SOPHGO模型文档](../README.md) diff --git a/examples/vision/detection/yolov5/sophgo/python/infer.py b/examples/vision/detection/yolov5/sophgo/python/infer.py new file mode 100644 index 000000000..d1ea190c5 --- /dev/null +++ b/examples/vision/detection/yolov5/sophgo/python/infer.py @@ -0,0 +1,40 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" + +model = fd.vision.detection.YOLOv5( + model_file, + params_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im = cv2.imread(args.image) +result = model.predict(im) +print(result) + +# 预测结果可视化 +vis_im = fd.vision.vis_detection(im, result) +cv2.imwrite("sophgo_result.jpg", vis_im) +print("Visualized result save in ./sophgo_result.jpg") diff --git a/examples/vision/segmentation/paddleseg/sophgo/README.md b/examples/vision/segmentation/paddleseg/sophgo/README.md new file mode 100644 index 000000000..afebe3451 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/README.md @@ -0,0 +1,89 @@ +# PaddleSeg C++部署示例 + +## 支持模型列表 + +- PP-LiteSeg部署模型实现来自[PaddleSeg PP-LiteSeg系列模型](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.6/configs/pp_liteseg/README.md) + +## 准备PP-LiteSeg部署模型以及转换模型 + +SOPHGO-TPU部署模型前需要将Paddle模型转换成bmodel模型，具体步骤如下: +- 下载Paddle模型[PP-LiteSeg-B(STDC2)-cityscapes-without-argmax](https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz) +- Pddle模型转换为ONNX模型，请参考[Paddle2ONNX](https://github.com/PaddlePaddle/Paddle2ONNX) +- ONNX模型转换bmodel模型的过程，请参考[TPU-MLIR](https://github.com/sophgo/tpu-mlir) + +## 模型转换example + +下面以[PP-LiteSeg-B(STDC2)-cityscapes-without-argmax](https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz)为例子,教大家如何转换Paddle模型到SOPHGO-TPU模型 + +### 下载PP-LiteSeg-B(STDC2)-cityscapes-without-argmax模型,并转换为ONNX模型 +```shell +https://bj.bcebos.com/paddlehub/fastdeploy/PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz +tar xvf PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer.tgz + +# 修改PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer模型的输入shape，由动态输入变成固定输入 +python paddle_infer_shape.py --model_dir PP_LiteSeg_B_STDC2_cityscapes_without_argmax_infer \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --save_dir pp_liteseg_fix \ + --input_shape_dict="{'x':[1,3,512,512]}" + +#将固定输入的Paddle模型转换成ONNX模型 +paddle2onnx --model_dir pp_liteseg_fix \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --save_file pp_liteseg.onnx \ + --enable_dev_version True +``` + +### 导出bmodel模型 + +以转换BM1684x的bmodel模型为例子，我们需要下载[TPU-MLIR](https://github.com/sophgo/tpu-mlir)工程，安装过程具体参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 +### 1. 安装 +``` shell +docker pull sophgo/tpuc_dev:latest + +# myname1234是一个示例，也可以设置其他名字 +docker run --privileged --name myname1234 -v $PWD:/workspace -it sophgo/tpuc_dev:latest + +source ./envsetup.sh +./build.sh +``` + +### 2. ONNX模型转换为bmodel模型 +``` shell +mkdir pp_liteseg && cd pp_liteseg + +#在该文件中放入测试图片，同时将上一步转换的pp_liteseg.onnx放入该文件夹中 +cp -rf ${REGRESSION_PATH}/dataset/COCO2017 . +cp -rf ${REGRESSION_PATH}/image . +#放入onnx模型文件pp_liteseg.onnx + +mkdir workspace && cd workspace + +#将ONNX模型转换为mlir模型，其中参数--output_names可以通过NETRON查看 +model_transform.py \ + --model_name pp_liteseg \ + --model_def ../pp_liteseg.onnx \ + --input_shapes [[1,3,512,512]] \ + --mean 0.0,0.0,0.0 \ + --scale 0.0039216,0.0039216,0.0039216 \ + --keep_aspect_ratio \ + --pixel_format rgb \ + --output_names bilinear_interp_v2_6.tmp_0 \ + --test_input ../image/dog.jpg \ + --test_result pp_liteseg_top_outputs.npz \ + --mlir pp_liteseg.mlir + +#将mlir模型转换为BM1684x的F32 bmodel模型 +model_deploy.py \ + --mlir pp_liteseg.mlir \ + --quantize F32 \ + --chip bm1684x \ + --test_input pp_liteseg_in_f32.npz \ + --test_reference pp_liteseg_top_outputs.npz \ + --model pp_liteseg_1684x_f32.bmodel +``` +最终获得可以在BM1684x上能够运行的bmodel模型pp_liteseg_1684x_f32.bmodel。如果需要进一步对模型进行加速，可以将ONNX模型转换为INT8 bmodel，具体步骤参见[TPU-MLIR文档](https://github.com/sophgo/tpu-mlir/blob/master/README.md)。 + +## 其他链接 +- [Cpp部署](./cpp) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt b/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt new file mode 100644 index 000000000..538370589 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +set(ENABLE_LITE_BACKEND OFF) +#set(FDLIB ${FASTDEPLOY_INSTALL_DIR}) + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md b/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md new file mode 100644 index 000000000..dac3ed565 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/README.md @@ -0,0 +1,56 @@ +# PaddleSeg C++部署示例 + +本目录下提供`infer.cc`快速完成pp_liteseg模型在SOPHGO BM1684x板子上加速部署的示例。 + +在部署前，需确认以下两个步骤: + +1. 软硬件环境满足要求 +2. 根据开发环境，从头编译FastDeploy仓库 + +以上步骤请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)实现 + +## 生成基本目录文件 + +该例程由以下几个部分组成 +```text +. +├── CMakeLists.txt +├── build # 编译文件夹 +├── image # 存放图片的文件夹 +├── infer.cc +└── model # 存放模型文件的文件夹 +``` + +## 编译 + +### 编译并拷贝SDK到thirdpartys文件夹 + +请参考[SOPHGO部署库编译](../../../../../../docs/cn/build_and_install/sophgo.md)仓库编译SDK，编译完成后，将在build目录下生成fastdeploy-0.0.3目录. + +### 拷贝模型文件，以及配置文件至model文件夹 +将Paddle模型转换为SOPHGO bmodel模型，转换步骤参考[文档](../README.md) +将转换后的SOPHGO bmodel模型文件拷贝至model中 + +### 准备测试图片至image文件夹 +```bash +wget https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_demo.png +cp cityscapes_demo.png ./images +``` + +### 编译example + +```bash +cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-0.0.3 +make +``` + +## 运行例程 + +```bash +./infer_demo model images/cityscapes_demo.png +``` + + +- [模型介绍](../../) +- [模型转换](../) diff --git a/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc b/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc new file mode 100644 index 000000000..934ab648c --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/cpp/infer.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include "fastdeploy/vision.h" + +void SophgoInfer(const std::string& model_dir, const std::string& image_file) { + std::string model_file = model_dir + "/pp_liteseg_1684x_f32.bmodel"; + std::string params_file; + std::string config_file = model_dir + "/deploy.yaml"; + auto option = fastdeploy::RuntimeOption(); + option.UseSophgo(); + auto model_format = fastdeploy::ModelFormat::SOPHGO; + + auto model = fastdeploy::vision::segmentation::PaddleSegModel( + model_file, params_file, config_file, option, model_format); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + //model.GetPreprocessor().DisableNormalizeAndPermute(); + + fastdeploy::TimeCounter tc; + tc.Start(); + auto im_org = cv::imread(image_file); + + //the input of bmodel should be fixed + int new_width = 512; + int new_height = 512; + cv::Mat im; + cv::resize(im_org, im, cv::Size(new_width, new_height), cv::INTER_LINEAR); + + fastdeploy::vision::SegmentationResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + tc.End(); + tc.PrintInfo("PPSeg in Sophgo"); + + cv::imwrite("infer_sophgo.jpg", vis_im); + std::cout + << "Visualized result saved in ./infer_sophgo.jpg" + << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./bmodel ./test.jpeg" + << std::endl; + return -1; + } + + SophgoInfer(argv[1], argv[2]); + return 0; +} + diff --git a/examples/vision/segmentation/paddleseg/sophgo/python/README.md b/examples/vision/segmentation/paddleseg/sophgo/python/README.md new file mode 100644 index 000000000..e04ad28c4 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/python/README.md @@ -0,0 +1,26 @@ +# PaddleSeg Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/sophgo.md) + +本目录下提供`infer.py`快速完成 pp_liteseg 在SOPHGO TPU上部署的示例。执行如下脚本即可完成 + +```bash +# 下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/segmentation/paddleseg/sophgo/python + +# 下载图片 +wget https://paddleseg.bj.bcebos.com/dygraph/demo/cityscapes_demo.png + +# 推理 +python3 infer.py --model_file ./bmodel/pp_liteseg_1684x_f32.bmodel --config_file ./bmodel/deploy.yaml --image cityscapes_demo.png + +# 运行完成后返回结果如下所示 +运行结果保存在sophgo_img.png中 +``` + +## 其它文档 +- [pp_liteseg C++部署](../cpp) +- [转换 pp_liteseg SOPHGO模型文档](../README.md) diff --git a/examples/vision/segmentation/paddleseg/sophgo/python/infer.py b/examples/vision/segmentation/paddleseg/sophgo/python/infer.py new file mode 100644 index 000000000..1b294da60 --- /dev/null +++ b/examples/vision/segmentation/paddleseg/sophgo/python/infer.py @@ -0,0 +1,45 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--config_file", required=True, help="Path of config file.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + + return parser.parse_args() + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = fd.RuntimeOption() +runtime_option.use_sophgo() + +model_file = args.model +params_file = "" +config_file = args.config_file + +model = fd.vision.segmentation.PaddleSegModel( + model_file, + params_file, + config_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.SOPHGO) + +# 预测图片分类结果 +im_org = cv2.imread(args.image) +#bmodel 是静态模型，模型输入固定，这里设置为[512, 512] +im = cv2.resize(im_org, [512, 512], interpolation=cv2.INTER_LINEAR) +result = model.predict(im) +print(result) + +# 预测结果可视化 +vis_im = fd.vision.vis_segmentation(im, result, weight=0.5) +cv2.imwrite("sophgo_img.png", vis_im) diff --git a/fastdeploy/backends/sophgo/sophgo_backend.cc b/fastdeploy/backends/sophgo/sophgo_backend.cc new file mode 100644 index 000000000..c4a75ce2d --- /dev/null +++ b/fastdeploy/backends/sophgo/sophgo_backend.cc @@ -0,0 +1,290 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/backends/sophgo/sophgo_backend.h" +#include +namespace fastdeploy { + SophgoBackend::~SophgoBackend() { + + bm_dev_free(handle_); + +} +/*************************************************************** + * @name GetSDKAndDeviceVersion + * @brief get Sophgo sdk and device version + * @param None + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::GetSDKAndDeviceVersion() { + + return true; +} + +/*************************************************************** + * @name BuildOption + * @brief save option + * @param SOPHGOTPU2BackendOption + * @note None + ***************************************************************/ +void SophgoBackend::BuildOption(const SophgoBackendOption& option) { +// this->option_ = option; + // save cpu_name +// this->option_.cpu_name = option.cpu_name; +} + +/*************************************************************** + * @name InitFromSophgo + * @brief Initialize Sophgo model + * @param model_file: Binary data for the Sophgo model. + * params_file: None + * option: config + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::InitFromSophgo(const std::string& model_file, + const SophgoBackendOption& option) { + + // LoadModel + if (!this->LoadModel((char*)model_file.data())) { + FDERROR << "load model failed" << std::endl; + return false; + } + + // GetSDKAndDeviceVersion + if (!this->GetSDKAndDeviceVersion()) { + FDERROR << "get SDK and device version failed" << std::endl; + return false; + } + + // BuildOption + this->BuildOption(option); + + // GetModelInputOutputInfos + if (!this->GetModelInputOutputInfos()) { + FDERROR << "get model input output infos failed" << std::endl; + return false; + } + + return true; +} + +/*************************************************************** + * @name LoadModel + * @brief read Sophgo bmodel + * @param model: Binary data for the Sophgo model. + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::LoadModel(void* model) { + unsigned int card_num = 0; + bm_status_t status = bm_get_card_num(&card_num); + status = bm_dev_request(&handle_, 0); + p_bmrt_ = bmrt_create(handle_); + assert(NULL != p_bmrt_); + + bool load_status = bmrt_load_bmodel(p_bmrt_, (char*)model); + assert(load_status); + + int network_num = bmrt_get_network_number(p_bmrt_); + + const char **net_names = NULL; + bmrt_get_network_names(p_bmrt_, &net_names); + net_name_ = net_names[0]; + free(net_names); + + net_info_ = bmrt_get_network_info(p_bmrt_, net_name_.c_str()); + assert(NULL != net_info_); + + return true; +} + +/*************************************************************** + * @name GetModelInputOutputInfos + * @brief Get the detailed input and output infos of Model + * @param None + * @return bool + * @note None + ***************************************************************/ +bool SophgoBackend::GetModelInputOutputInfos() { + inputs_desc_.resize(net_info_->input_num); + bm_shape_t* input_shapes = net_info_->stages->input_shapes; + for(int idx=0; idxinput_num; idx++){ + std::string temp_name = (net_info_->input_names)[idx]; + std::vector temp_shape{}; + temp_shape.resize(input_shapes[idx].num_dims); + for(int i=0; iinput_dtypes; + //SophgoType to FDDataType + FDDataType temp_dtype = SophgoTensorTypeToFDDataType(*input_dtypes); + TensorInfo temp_input_info = {temp_name, temp_shape, temp_dtype}; + inputs_desc_[idx] = temp_input_info; + } + + outputs_desc_.resize(net_info_->output_num); + bm_shape_t* output_shapes = net_info_->stages->output_shapes; + for(int idx=0; idxoutput_num; idx++){ + std::string temp_name1 = (net_info_->output_names)[idx]; + std::vector temp_shape1{}; + temp_shape1.resize(output_shapes[idx].num_dims); + for(int i=0; ioutput_dtypes; + //SophgoType to FDDataType + FDDataType temp_dtype1 = SophgoTensorTypeToFDDataType(*output_dtypes); + TensorInfo temp_output_info = {temp_name1, temp_shape1, temp_dtype1}; + outputs_desc_[idx] = temp_output_info; + } + return true; +} + +TensorInfo SophgoBackend::GetInputInfo(int index) { + FDASSERT(index < NumInputs(), + "The index: %d should less than the number of inputs: %d.", index, + NumInputs()) + return inputs_desc_[index]; +} + +std::vector SophgoBackend::GetInputInfos() { return inputs_desc_; } + +TensorInfo SophgoBackend::GetOutputInfo(int index) { + FDASSERT(index < NumOutputs(), + "The index: %d should less than the number of outputs %d.", index, + NumOutputs()) + return outputs_desc_[index]; +} + +std::vector SophgoBackend::GetOutputInfos() { return outputs_desc_; } + +bool SophgoBackend::Infer(std::vector& inputs, + std::vector* outputs, + bool copy_to_fd) { + int input_size = inputs.size(); + assert(input_size != 0); + assert(input_size == NumInputs()); + bm_tensor_t input_tensors[input_size]; + bm_status_t status = BM_SUCCESS; + + bm_data_type_t* input_dtypes = net_info_->input_dtypes; + for(int i=0;imax_input_bytes[i]); + assert(BM_SUCCESS == status); + input_tensors[i].dtype = input_dtypes[i]; + input_tensors[i].st_mode = BM_STORE_1N; + input_tensors[i].shape = *(net_info_->stages[i].input_shapes); + unsigned int input_byte = bmrt_tensor_bytesize(&input_tensors[i]); + bm_memcpy_s2d_partial(handle_, input_tensors[i].device_mem, (void *)inputs[i].Data(), + bmrt_tensor_bytesize(&input_tensors[i])); + } + + int output_size = NumOutputs(); + bm_tensor_t output_tensors[output_size]; + for(int i=0;imax_output_bytes[i]); + assert(BM_SUCCESS == status); + } + + bool launch_status = bmrt_launch_tensor_ex(p_bmrt_, net_name_.c_str(), input_tensors, net_info_->input_num, + output_tensors, net_info_->output_num, true, false); + assert(launch_status); + status = bm_thread_sync(handle_); + assert(status == BM_SUCCESS); + + outputs->resize(outputs_desc_.size()); + bm_data_type_t* output_dtypes = net_info_->output_dtypes; + for(int i=0;i temp_shape; + temp_shape.resize(outputs_desc_[i].shape.size()); + for (int j = 0; j < outputs_desc_[i].shape.size(); ++j) { + temp_shape[j] = outputs_desc_[i].shape[j]; + } + (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, outputs_desc_[i].name); + + memcpy((*outputs)[i].MutableData(), temp_out, (*outputs)[i].Nbytes()); + free(temp_out); + } + + return true; +} + +/*************************************************************** + * @name SophgoTensorTypeToFDDataType + * @brief Change SophgoTensorType To FDDataType + * @param bm_data_type_t + * @return None + * @note None + ***************************************************************/ +FDDataType SophgoBackend::SophgoTensorTypeToFDDataType(bm_data_type_t type) { + if (type == BM_FLOAT16) { + return FDDataType::FP32; + } + if (type == BM_FLOAT32) { + return FDDataType::FP32; + } + if (type == BM_INT8) { + return FDDataType::INT8; + } + if (type == BM_INT16) { + return FDDataType::INT16; + } + if (type == BM_INT32) { + return FDDataType::INT32; + } + if (type == BM_UINT8) { + return FDDataType::UINT8; + } + FDERROR << "FDDataType don't support this type" << std::endl; + return FDDataType::UNKNOWN1; +} + +/*************************************************************** + * @name FDDataTypeToSophgoTensorType + * @brief Change FDDataType To SophgoTensorType + * @param FDDataType + * @return None + * @note None + ***************************************************************/ +// Sophgo_tensor_type +bm_data_type_t SophgoBackend::FDDataTypeToSophgoTensorType(fastdeploy::FDDataType type) { + if (type == FDDataType::FP16) { + return BM_FLOAT16; + } + if (type == FDDataType::FP32) { + return BM_FLOAT32; + } + if (type == FDDataType::INT8) { + return BM_INT8; + } + if (type == FDDataType::INT16) { + return BM_INT16; + } + if (type == FDDataType::INT32) { + return BM_INT32; + } + if (type == FDDataType::UINT8) { + return BM_UINT8; + } + FDERROR << "Sophgo_tensor_type don't support this type" << std::endl; + return BM_FLOAT32; +} + +} diff --git a/fastdeploy/backends/sophgo/sophgo_backend.h b/fastdeploy/backends/sophgo/sophgo_backend.h new file mode 100644 index 000000000..8007bfd13 --- /dev/null +++ b/fastdeploy/backends/sophgo/sophgo_backend.h @@ -0,0 +1,75 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/backends/backend.h" +#include "fastdeploy/core/fd_tensor.h" +#include "bmruntime_interface.h" // NOLINT +#include "bmlib_runtime.h" // NOLINT +#include "fastdeploy/backends/sophgo/sophgo_config.h" +#include +#include +#include +#include +#include + +namespace fastdeploy { +struct SophgoBackendOption{ +}; + +class SophgoBackend : public BaseBackend { + public: + SophgoBackend() = default; + virtual ~SophgoBackend(); + bool LoadModel(void* model); + bool GetSDKAndDeviceVersion(); + bool GetModelInputOutputInfos(); + void BuildOption(const SophgoBackendOption& option); + bool InitFromSophgo(const std::string& model_file, + const SophgoBackendOption& option = SophgoBackendOption()); + + int NumInputs() const override { + return static_cast(inputs_desc_.size()); + } + + int NumOutputs() const override { + return static_cast(outputs_desc_.size()); + } + + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + bool Infer(std::vector& inputs, + std::vector* outputs, + bool copy_to_fd = true) override; + + private: + std::vector inputs_desc_; + std::vector outputs_desc_; + std::string net_name_; + + bm_handle_t handle_; + void * p_bmrt_ = nullptr; + + bool infer_init = false; + + const bm_net_info_t* net_info_ = nullptr; + + // SophgoTPU2BackendOption option_; + + static FDDataType SophgoTensorTypeToFDDataType(bm_data_type_t type); + static bm_data_type_t FDDataTypeToSophgoTensorType(FDDataType type); +}; +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/backends/sophgo/sophgo_config.h b/fastdeploy/backends/sophgo/sophgo_config.h new file mode 100644 index 000000000..e69de29bb diff --git a/fastdeploy/core/fd_type.cc b/fastdeploy/core/fd_type.cc index 21e3d30c3..420e03ff7 100755 --- a/fastdeploy/core/fd_type.cc +++ b/fastdeploy/core/fd_type.cc @@ -56,6 +56,9 @@ std::string Str(const Device& d) { case Device::RKNPU: out = "Device::RKNPU"; break; + case Device::SOPHGOTPUD: + out = "Device::SOPHGOTPUD"; + break; case Device::IPU: out = "Device::IPU"; break; @@ -85,6 +88,9 @@ std::ostream& operator<<(std::ostream& out,const Device& d){ case Device::RKNPU: out << "Device::RKNPU"; break; + case Device::SOPHGOTPUD: + out << "Device::SOPHGOTPUD"; + break; case Device::TIMVX: out << "Device::TIMVX"; break; @@ -205,8 +211,10 @@ std::string Str(const ModelFormat& f) { return "ModelFormat::PADDLE"; } else if (f == ModelFormat::ONNX) { return "ModelFormat::ONNX"; - }else if (f == ModelFormat::RKNN) { + } else if (f == ModelFormat::RKNN) { return "ModelFormat::RKNN"; + } else if (f == ModelFormat::SOPHGO) { + return "ModelFormat::SOPHGO"; } else if (f == ModelFormat::TORCHSCRIPT) { return "ModelFormat::TORCHSCRIPT"; } @@ -220,6 +228,8 @@ std::ostream& operator<<(std::ostream& out, const ModelFormat& format) { out << "ModelFormat::ONNX"; } else if (format == ModelFormat::RKNN) { out << "ModelFormat::RKNN"; + } else if (format == ModelFormat::SOPHGO) { + out << "ModelFormat::SOPHGO"; } else if (format == ModelFormat::TORCHSCRIPT) { out << "ModelFormat::TORCHSCRIPT"; } diff --git a/fastdeploy/core/fd_type.h b/fastdeploy/core/fd_type.h index fda26c1c8..5b49f1e86 100755 --- a/fastdeploy/core/fd_type.h +++ b/fastdeploy/core/fd_type.h @@ -22,7 +22,8 @@ namespace fastdeploy { -enum FASTDEPLOY_DECL Device { CPU, GPU, RKNPU, IPU, TIMVX, KUNLUNXIN, ASCEND}; +enum FASTDEPLOY_DECL Device {CPU, GPU, RKNPU, IPU, TIMVX, KUNLUNXIN, ASCEND, + SOPHGOTPUD}; FASTDEPLOY_DECL std::string Str(const Device& d); @@ -72,6 +73,7 @@ enum ModelFormat { ONNX, ///< Model with ONNX format RKNN, ///< Model with RKNN format TORCHSCRIPT, ///< Model with TorchScript format + SOPHGO, ///< Model with SOPHGO format }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, diff --git a/fastdeploy/fastdeploy_model.cc b/fastdeploy/fastdeploy_model.cc index 77c1539c3..d0f06a1ca 100755 --- a/fastdeploy/fastdeploy_model.cc +++ b/fastdeploy/fastdeploy_model.cc @@ -50,6 +50,7 @@ bool FastDeployModel::InitRuntimeWithSpecifiedBackend() { bool use_gpu = (runtime_option.device == Device::GPU); bool use_ipu = (runtime_option.device == Device::IPU); bool use_rknpu = (runtime_option.device == Device::RKNPU); + bool use_sophgotpu = (runtime_option.device == Device::SOPHGOTPUD); bool use_timvx = (runtime_option.device == Device::TIMVX); bool use_ascend = (runtime_option.device == Device::ASCEND); bool use_kunlunxin = (runtime_option.device == Device::KUNLUNXIN); @@ -64,6 +65,11 @@ bool FastDeployModel::InitRuntimeWithSpecifiedBackend() { FDERROR << "The valid rknpu backends of model " << ModelName() << " are " << Str(valid_rknpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; return false; } + } else if (use_sophgotpu) { + if (!IsSupported(valid_sophgonpu_backends, runtime_option.backend)) { + FDERROR << "The valid rknpu backends of model " << ModelName() << " are " << Str(valid_rknpu_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; + return false; + } } else if (use_timvx) { if (!IsSupported(valid_timvx_backends, runtime_option.backend)) { FDERROR << "The valid timvx backends of model " << ModelName() << " are " << Str(valid_timvx_backends) << ", " << runtime_option.backend << " is not supported." << std::endl; @@ -118,6 +124,8 @@ bool FastDeployModel::InitRuntimeWithSpecifiedDevice() { return CreateASCENDBackend(); } else if (runtime_option.device == Device::KUNLUNXIN) { return CreateKunlunXinBackend(); + } else if (runtime_option.device == Device::SOPHGOTPUD) { + return CreateSophgoNPUBackend(); } else if (runtime_option.device == Device::IPU) { #ifdef WITH_IPU return CreateIpuBackend(); @@ -218,6 +226,30 @@ bool FastDeployModel::CreateRKNPUBackend() { return false; } +bool FastDeployModel::CreateSophgoNPUBackend() { + if (valid_sophgonpu_backends.empty()) { + FDERROR << "There's no valid npu backends for model: " << ModelName() + << std::endl; + return false; + } + + for (size_t i = 0; i < valid_sophgonpu_backends.size(); ++i) { + if (!IsBackendAvailable(valid_sophgonpu_backends[i])) { + continue; + } + runtime_option.backend = valid_sophgonpu_backends[i]; + runtime_ = std::unique_ptr(new Runtime()); + if (!runtime_->Init(runtime_option)) { + return false; + } + runtime_initialized_ = true; + return true; + } + FDERROR << "Cannot find an available npu backend to load this model." + << std::endl; + return false; +} + bool FastDeployModel::CreateTimVXBackend() { if (valid_timvx_backends.size() == 0) { FDERROR << "There's no valid timvx backends for model: " << ModelName() diff --git a/fastdeploy/fastdeploy_model.h b/fastdeploy/fastdeploy_model.h index 6d2f7e5b7..9b78c3d3f 100755 --- a/fastdeploy/fastdeploy_model.h +++ b/fastdeploy/fastdeploy_model.h @@ -54,6 +54,9 @@ class FASTDEPLOY_DECL FastDeployModel { /** Model's valid hardware backends. This member defined all the gpu backends have successfully tested for the model */ std::vector valid_rknpu_backends = {}; + /** Model's valid hardware backends. This member defined all the sophgo npu backends have successfully tested for the model + */ + std::vector valid_sophgonpu_backends = {}; /// Get number of inputs for this model virtual int NumInputsOfRuntime() { return runtime_->NumInputs(); } @@ -148,6 +151,7 @@ class FASTDEPLOY_DECL FastDeployModel { bool CreateGpuBackend(); bool CreateIpuBackend(); bool CreateRKNPUBackend(); + bool CreateSophgoNPUBackend(); bool CreateTimVXBackend(); bool CreateKunlunXinBackend(); bool CreateASCENDBackend(); diff --git a/fastdeploy/pybind/runtime.cc b/fastdeploy/pybind/runtime.cc index cbbb7f2e8..3402dd896 100755 --- a/fastdeploy/pybind/runtime.cc +++ b/fastdeploy/pybind/runtime.cc @@ -24,6 +24,7 @@ void BindRuntime(pybind11::module& m) { .def("use_gpu", &RuntimeOption::UseGpu) .def("use_cpu", &RuntimeOption::UseCpu) .def("use_rknpu2", &RuntimeOption::UseRKNPU2) + .def("use_sophgo", &RuntimeOption::UseSophgo) .def("use_ascend", &RuntimeOption::UseAscend) .def("use_kunlunxin", &RuntimeOption::UseKunlunXin) .def("set_external_stream", &RuntimeOption::SetExternalStream) @@ -241,19 +242,22 @@ void BindRuntime(pybind11::module& m) { .value("POROS", Backend::POROS) .value("PDINFER", Backend::PDINFER) .value("RKNPU2", Backend::RKNPU2) + .value("SOPHGOTPU", Backend::SOPHGOTPU) .value("LITE", Backend::LITE); pybind11::enum_(m, "ModelFormat", pybind11::arithmetic(), "ModelFormat for inference.") .value("PADDLE", ModelFormat::PADDLE) .value("TORCHSCRIPT", ModelFormat::TORCHSCRIPT) .value("RKNN", ModelFormat::RKNN) + .value("SOPHGO", ModelFormat::SOPHGO) .value("ONNX", ModelFormat::ONNX); pybind11::enum_(m, "Device", pybind11::arithmetic(), "Device for inference.") .value("CPU", Device::CPU) .value("GPU", Device::GPU) .value("IPU", Device::IPU) - .value("RKNPU", Device::RKNPU); + .value("RKNPU", Device::RKNPU) + .value("SOPHGOTPU", Device::SOPHGOTPUD); pybind11::enum_(m, "FDDataType", pybind11::arithmetic(), "Data type of FastDeploy.") diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 0110b4313..6c1949ed3 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -45,6 +45,10 @@ #include "fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h" #endif +#ifdef ENABLE_SOPHGO_BACKEND +#include "fastdeploy/backends/sophgo/sophgo_backend.h" +#endif + namespace fastdeploy { std::vector GetAvailableBackends() { @@ -69,6 +73,9 @@ std::vector GetAvailableBackends() { #endif #ifdef ENABLE_RKNPU2_BACKEND backends.push_back(Backend::RKNPU2); +#endif +#ifdef ENABLE_SOPHGO_BACKEND + backends.push_back(Backend::SOPHGOTPU); #endif return backends; } @@ -94,6 +101,8 @@ std::string Str(const Backend& b) { return "Backend::POROS"; } else if (b == Backend::RKNPU2) { return "Backend::RKNPU2"; + } else if (b == Backend::SOPHGOTPU) { + return "Backend::SOPHGOTPU"; } else if (b == Backend::OPENVINO) { return "Backend::OPENVINO"; } else if (b == Backend::LITE) { @@ -113,6 +122,8 @@ std::ostream& operator<<(std::ostream& out, const Backend& backend) { out << "Backend::OPENVINO"; } else if (backend == Backend::RKNPU2) { out << "Backend::RKNPU2"; + } else if (backend == Backend::SOPHGOTPU) { + out << "Backend::SOPHGOTPU"; } else if (backend == Backend::POROS) { out << "Backend::POROS"; } else if (backend == Backend::LITE) { @@ -158,6 +169,15 @@ bool CheckModelFormat(const std::string& model_file, << model_file << std::endl; return false; } + } else if (model_format == ModelFormat::SOPHGO) { + if (model_file.size() < 7 || + model_file.substr(model_file.size() -7, 7) != ".bmodel") { + FDERROR + << "With model format of ModelFormat::SOPHGO, the model file " + "should ends with `.bmodel`, but now it's " + << model_file << std::endl; + return false; + } } else { FDERROR << "Only support model format with frontend ModelFormat::PADDLE / " @@ -185,6 +205,10 @@ ModelFormat GuessModelFormat(const std::string& model_file) { model_file.substr(model_file.size() - 5, 5) == ".rknn") { FDINFO << "Model Format: RKNN." << std::endl; return ModelFormat::RKNN; + } else if (model_file.size() > 7 && + model_file.substr(model_file.size() - 7, 7) == ".bmodel") { + FDINFO << "Model Format: SOPHGO." << std::endl; + return ModelFormat::SOPHGO; } FDERROR << "Cannot guess which model format you are using, please set " @@ -288,6 +312,11 @@ void RuntimeOption::UseAscend(){ device = Device::ASCEND; } +void RuntimeOption::UseSophgo() { + device = Device::SOPHGOTPUD; + UseSophgoBackend(); +} + void RuntimeOption::SetExternalStream(void* external_stream) { external_stream_ = external_stream; } @@ -323,6 +352,15 @@ void RuntimeOption::UseOrtBackend() { #endif } +// use sophgoruntime backend +void RuntimeOption::UseSophgoBackend() { +#ifdef ENABLE_SOPHGO_BACKEND + backend = Backend::SOPHGOTPU; +#else + FDASSERT(false, "The FastDeploy didn't compile with SophgoBackend."); +#endif +} + // use poros backend void RuntimeOption::UsePorosBackend() { #ifdef ENABLE_POROS_BACKEND @@ -564,6 +602,8 @@ bool Runtime::Init(const RuntimeOption& _option) { option.backend = Backend::OPENVINO; } else if (IsBackendAvailable(Backend::RKNPU2)) { option.backend = Backend::RKNPU2; + } else if (IsBackendAvailable(Backend::SOPHGOTPU)) { + option.backend = Backend::SOPHGOTPU; } else { FDERROR << "Please define backend in RuntimeOption, current it's " "Backend::UNKNOWN." @@ -623,7 +663,15 @@ bool Runtime::Init(const RuntimeOption& _option) { FDINFO << "Runtime initialized with Backend::RKNPU2 in " << Str(option.device) << "." << std::endl; - } else { + } else if (option.backend == Backend::SOPHGOTPU) { + FDASSERT(option.device == Device::SOPHGOTPUD, + "Backend::SOPHGO only supports Device::SOPHGO"); + CreateSophgoNPUBackend(); + + FDINFO << "Runtime initialized with Backend::SOPHGO in " + << Str(option.device) << "." << std::endl; + } + else { FDERROR << "Runtime only support " "Backend::ORT/Backend::TRT/Backend::PDINFER/Backend::POROS as " "backend now." @@ -926,6 +974,21 @@ void Runtime::CreateRKNPU2Backend() { #endif } +void Runtime::CreateSophgoNPUBackend() { +#ifdef ENABLE_SOPHGO_BACKEND + auto sophgo_option = SophgoBackendOption(); + FDASSERT(option.model_format == ModelFormat::SOPHGO, + "SophgoBackend only support model format of ModelFormat::SOPHGO"); + backend_ = utils::make_unique(); + auto casted_backend = dynamic_cast(backend_.get()); + FDASSERT(casted_backend->InitFromSophgo(option.model_file, sophgo_option), + "Load model from nb file failed while initializing LiteBackend."); +#else + FDASSERT(false, "SophgoBackend is not available, please compiled with " + "ENABLE_SOPHGO_BACKEND=ON."); +#endif +} + Runtime* Runtime::Clone(void* stream, int device_id) { Runtime* runtime = new Runtime(); if (option.backend != Backend::OPENVINO && diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index c889d4337..46532b16b 100755 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -43,6 +43,7 @@ enum Backend { OPENVINO, ///< Intel OpenVINO, support Paddle/ONNX format, CPU only LITE, ///< Paddle Lite, support Paddle format model, ARM CPU only RKNPU2, ///< RKNPU2, support RKNN format model, Rockchip NPU only + SOPHGOTPU, ///< SOPHGOTPU, support SOPHGO format model, Sophgo TPU only }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, @@ -151,6 +152,9 @@ struct FASTDEPLOY_DECL RuntimeOption { bool adaptive_seqlen = false, bool enable_multi_stream = false); + /// Use Sophgo to inference + void UseSophgo(); + void SetExternalStream(void* external_stream); /* @@ -170,6 +174,9 @@ struct FASTDEPLOY_DECL RuntimeOption { /// Set ONNX Runtime as inference backend, support CPU/GPU void UseOrtBackend(); + /// Set SOPHGO Runtime as inference backend, support CPU/GPU + void UseSophgoBackend(); + /// Set TensorRT as inference backend, only support GPU void UseTrtBackend(); @@ -576,6 +583,7 @@ struct FASTDEPLOY_DECL Runtime { void CreateOpenVINOBackend(); void CreateLiteBackend(); void CreateRKNPU2Backend(); + void CreateSophgoNPUBackend(); std::unique_ptr backend_; std::vector input_tensors_; std::vector output_tensors_; diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index a9b5b46f0..6868c9c62 100755 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -32,7 +32,10 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, valid_ascend_backends = {Backend::LITE}; valid_kunlunxin_backends = {Backend::LITE}; valid_ipu_backends = {Backend::PDINFER}; - } else { + }else if (model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; + } + else { valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; valid_rknpu_backends = {Backend::RKNPU2}; diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc index 44e020ffd..107b1ab90 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc @@ -24,6 +24,8 @@ YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file, if (model_format == ModelFormat::ONNX) { valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else if (model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; diff --git a/fastdeploy/vision/detection/ppdet/model.h b/fastdeploy/vision/detection/ppdet/model.h index 2f4078f95..da69774fa 100755 --- a/fastdeploy/vision/detection/ppdet/model.h +++ b/fastdeploy/vision/detection/ppdet/model.h @@ -41,6 +41,7 @@ class FASTDEPLOY_DECL PicoDet : public PPDetBase { valid_rknpu_backends = {Backend::RKNPU2}; valid_kunlunxin_backends = {Backend::LITE}; valid_ascend_backends = {Backend::LITE}; + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; initialized = Initialize(); } diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 7baa7ac7f..54f978828 100755 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -25,12 +25,18 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) : preprocessor_(config_file), postprocessor_(config_file) { - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + if(model_format == ModelFormat::SOPHGO) { + valid_sophgonpu_backends = {Backend::SOPHGOTPU}; + } + else{ + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } valid_rknpu_backends = {Backend::RKNPU2}; valid_timvx_backends = {Backend::LITE}; valid_kunlunxin_backends = {Backend::LITE}; valid_ascend_backends = {Backend::LITE}; + runtime_option = custom_option; runtime_option.model_format = model_format; runtime_option.model_file = model_file; diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index 23cf697bf..d7035bd1b 100755 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -301,6 +301,11 @@ class RuntimeOption: rknpu2_core=rknpu2.CoreMask.RKNN_NPU_CORE_0): return self._option.use_rknpu2(rknpu2_name, rknpu2_core) + def use_sophgo(self): + """Inference with SOPHGO TPU + """ + return self._option.use_sophgo() + def use_ascend(self): """Inference with Huawei Ascend NPU """ diff --git a/python/setup.py b/python/setup.py index 80c6b80e9..d1b02254e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -56,6 +56,8 @@ if os.getenv("BUILD_ON_CPU", "OFF") == "ON": setup_configs = dict() setup_configs["ENABLE_RKNPU2_BACKEND"] = os.getenv("ENABLE_RKNPU2_BACKEND", "OFF") +setup_configs["ENABLE_SOPHGO_BACKEND"] = os.getenv("ENABLE_SOPHGO_BACKEND", + "OFF") setup_configs["WITH_ASCEND"] = os.getenv("WITH_ASCEND", "OFF") setup_configs["ENABLE_ORT_BACKEND"] = os.getenv("ENABLE_ORT_BACKEND", "OFF") setup_configs["ENABLE_OPENVINO_BACKEND"] = os.getenv("ENABLE_OPENVINO_BACKEND",