From 0bb40ab1008cfd45b9a90ed9230a8ed1c51713ee Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Mon, 7 Nov 2022 19:49:12 +0800 Subject: [PATCH] [Other] Add runtime cpp demos (#515) * add paddle_trt in benchmark * update benchmark in device * update benchmark * update result doc * fixed for CI * update python api_docs * update index.rst * add runtime cpp examples * deal with comments * Update infer_paddle_tensorrt.py Co-authored-by: Jason <928090362@qq.com> --- docs/api_docs/python/index.rst | 4 ++ docs/api_docs/python/runtime.md | 9 +++ docs/api_docs/python/runtime_option.md | 9 +++ examples/runtime/cpp/CMakeLists.txt | 14 ++++ examples/runtime/cpp/infer_onnx_openvino.cc | 59 +++++++++++++++++ examples/runtime/cpp/infer_onnx_tensorrt.cc | 60 +++++++++++++++++ .../runtime/cpp/infer_paddle_onnxruntime.cc | 60 +++++++++++++++++ examples/runtime/cpp/infer_paddle_openvino.cc | 60 +++++++++++++++++ .../cpp/infer_paddle_paddle_inference.cc | 65 +++++++++++++++++++ examples/runtime/cpp/infer_paddle_tensorrt.cc | 61 +++++++++++++++++ .../runtime/python/infer_paddle_tensorrt.py | 2 + 11 files changed, 403 insertions(+) create mode 100644 docs/api_docs/python/runtime.md create mode 100644 docs/api_docs/python/runtime_option.md create mode 100644 examples/runtime/cpp/CMakeLists.txt create mode 100644 examples/runtime/cpp/infer_onnx_openvino.cc create mode 100644 examples/runtime/cpp/infer_onnx_tensorrt.cc create mode 100644 examples/runtime/cpp/infer_paddle_onnxruntime.cc create mode 100644 examples/runtime/cpp/infer_paddle_openvino.cc create mode 100644 examples/runtime/cpp/infer_paddle_paddle_inference.cc create mode 100644 examples/runtime/cpp/infer_paddle_tensorrt.cc diff --git a/docs/api_docs/python/index.rst b/docs/api_docs/python/index.rst index 06d4a95cb..69b65b3b1 100644 --- a/docs/api_docs/python/index.rst +++ b/docs/api_docs/python/index.rst @@ -20,4 +20,8 @@ FastDeploy matting.md face_recognition.md face_detection.md + face_alignment.md + headpose.md vision_results_en.md + runtime.md + runtime_option.md diff --git a/docs/api_docs/python/runtime.md b/docs/api_docs/python/runtime.md new file mode 100644 index 000000000..4a519ee7e --- /dev/null +++ b/docs/api_docs/python/runtime.md @@ -0,0 +1,9 @@ +# Runtime API + +## fastdeploy.Runtime + +```{eval-rst} +.. autoclass:: fastdeploy.Runtime + :members: + :inherited-members: +``` diff --git a/docs/api_docs/python/runtime_option.md b/docs/api_docs/python/runtime_option.md new file mode 100644 index 000000000..96eff8672 --- /dev/null +++ b/docs/api_docs/python/runtime_option.md @@ -0,0 +1,9 @@ +# Runtime Option API + +## fastdeploy.RuntimeOption + +```{eval-rst} +.. autoclass:: fastdeploy.RuntimeOption + :members: + :inherited-members: +``` diff --git a/examples/runtime/cpp/CMakeLists.txt b/examples/runtime/cpp/CMakeLists.txt new file mode 100644 index 000000000..09ea45c3b --- /dev/null +++ b/examples/runtime/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(runtime_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.12) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(runtime_demo ${PROJECT_SOURCE_DIR}/infer_onnx_openvino.cc) +# 添加FastDeploy库依赖 +target_link_libraries(runtime_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/runtime/cpp/infer_onnx_openvino.cc b/examples/runtime/cpp/infer_onnx_openvino.cc new file mode 100644 index 000000000..c2f270be9 --- /dev/null +++ b/examples/runtime/cpp/infer_onnx_openvino.cc @@ -0,0 +1,59 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2.onnx"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, "", fd::ModelFormat::ONNX); + runtime_option.UseOpenVINOBackend(); + runtime_option.SetCpuThreadNum(12); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/cpp/infer_onnx_tensorrt.cc b/examples/runtime/cpp/infer_onnx_tensorrt.cc new file mode 100644 index 000000000..084c1dfae --- /dev/null +++ b/examples/runtime/cpp/infer_onnx_tensorrt.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2.onnx"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, "", fd::ModelFormat::ONNX); + runtime_option.UseGpu(0); + runtime_option.UseTrtBackend(); + runtime_option.SetTrtInputShape("inputs", {1, 3, 224, 224}); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/cpp/infer_paddle_onnxruntime.cc b/examples/runtime/cpp/infer_paddle_onnxruntime.cc new file mode 100644 index 000000000..d8d036a03 --- /dev/null +++ b/examples/runtime/cpp/infer_paddle_onnxruntime.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2/inference.pdmodel"; + std::string params_file = "mobilenetv2/inference.pdiparams"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, params_file, fd::ModelFormat::PADDLE); + runtime_option.UseOrtBackend(); + runtime_option.SetCpuThreadNum(12); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/cpp/infer_paddle_openvino.cc b/examples/runtime/cpp/infer_paddle_openvino.cc new file mode 100644 index 000000000..3958cdcf0 --- /dev/null +++ b/examples/runtime/cpp/infer_paddle_openvino.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2/inference.pdmodel"; + std::string params_file = "mobilenetv2/inference.pdiparams"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, params_file, fd::ModelFormat::PADDLE); + runtime_option.UseOpenVINOBackend(); + runtime_option.SetCpuThreadNum(12); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/cpp/infer_paddle_paddle_inference.cc b/examples/runtime/cpp/infer_paddle_paddle_inference.cc new file mode 100644 index 000000000..1d0bd82ad --- /dev/null +++ b/examples/runtime/cpp/infer_paddle_paddle_inference.cc @@ -0,0 +1,65 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2/inference.pdmodel"; + std::string params_file = "mobilenetv2/inference.pdiparams"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, params_file, fd::ModelFormat::PADDLE); + // CPU + runtime_option.UsePaddleBackend(); + runtime_option.SetCpuThreadNum(12); + // GPU + // runtime_option.UseGpu(0); + // IPU + // runtime_option.UseIpu(); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/cpp/infer_paddle_tensorrt.cc b/examples/runtime/cpp/infer_paddle_tensorrt.cc new file mode 100644 index 000000000..04fe311b2 --- /dev/null +++ b/examples/runtime/cpp/infer_paddle_tensorrt.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/runtime.h" + +namespace fd = fastdeploy; + +int main(int argc, char* argv[]) { + std::string model_file = "mobilenetv2/inference.pdmodel"; + std::string params_file = "mobilenetv2/inference.pdiparams"; + + // setup option + fd::RuntimeOption runtime_option; + runtime_option.SetModelPath(model_file, params_file, fd::ModelFormat::PADDLE); + runtime_option.UseGpu(0); + runtime_option.UseTrtBackend(); + runtime_option.EnablePaddleToTrt(); + // init runtime + std::unique_ptr runtime = + std::unique_ptr(new fd::Runtime()); + if (!runtime->Init(runtime_option)) { + std::cerr << "--- Init FastDeploy Runitme Failed! " + << "\n--- Model: " << model_file << std::endl; + return -1; + } else { + std::cout << "--- Init FastDeploy Runitme Done! " + << "\n--- Model: " << model_file << std::endl; + } + // init input tensor shape + fd::TensorInfo info = runtime->GetInputInfo(0); + info.shape = {1, 3, 224, 224}; + + std::vector input_tensors(1); + std::vector output_tensors(1); + + std::vector inputs_data; + inputs_data.resize(1 * 3 * 224 * 224); + for (size_t i = 0; i < inputs_data.size(); ++i) { + inputs_data[i] = std::rand() % 1000 / 1000.0f; + } + input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data()); + + //get input name + input_tensors[0].name = info.name; + + runtime->Infer(input_tensors, &output_tensors); + + output_tensors[0].PrintInfo(); + return 0; +} \ No newline at end of file diff --git a/examples/runtime/python/infer_paddle_tensorrt.py b/examples/runtime/python/infer_paddle_tensorrt.py index ad2b8e197..94c95cb87 100644 --- a/examples/runtime/python/infer_paddle_tensorrt.py +++ b/examples/runtime/python/infer_paddle_tensorrt.py @@ -27,6 +27,8 @@ option.set_model_path("mobilenetv2/inference.pdmodel", # **** GPU 配置 *** option.use_gpu(0) option.use_trt_backend() +# using TensorRT integrated in Paddle Inference +# option.enable_paddle_to_trt() # 初始化构造runtime runtime = fd.Runtime(option)