From addce837bcab7a8111e362d33e17233a543fad6f Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 21 Sep 2022 13:22:34 +0800 Subject: [PATCH] Add paddlelite backend support (#260) * Add paddlelite backend support * Update CMakeLists.txt * Update __init__.py --- CMakeLists.txt | 12 +- cmake/paddlelite.cmake | 72 +++++++++ fastdeploy/backends/lite/lite_backend.cc | 141 ++++++++++++++++++ fastdeploy/backends/lite/lite_backend.h | 61 ++++++++ fastdeploy/core/allocate.cc | 1 - fastdeploy/fastdeploy_model.cc | 2 +- fastdeploy/fastdeploy_model.h | 2 +- fastdeploy/fastdeploy_runtime.cc | 47 +++++- fastdeploy/fastdeploy_runtime.h | 7 +- fastdeploy/pybind/fastdeploy_runtime.cc | 4 +- fastdeploy/pybind/main.cc.in | 4 - fastdeploy/utils/perf.h | 4 +- fastdeploy/utils/utils.cc | 9 +- fastdeploy/utils/utils.h | 32 ++-- .../vision/classification/ppcls/model.cc | 2 +- fastdeploy/vision/detection/ppdet/picodet.cc | 2 +- fastdeploy/vision/detection/ppdet/ppyoloe.cc | 2 + fastdeploy/vision/detection/ppdet/ppyoloe.h | 4 +- fastdeploy/vision/ocr/ppocr/classifier.cc | 5 +- fastdeploy/vision/ocr/ppocr/recognizer.cc | 8 +- python/fastdeploy/__init__.py | 2 +- python/fastdeploy/runtime.py | 3 + 22 files changed, 370 insertions(+), 56 deletions(-) create mode 100644 cmake/paddlelite.cmake create mode 100644 fastdeploy/backends/lite/lite_backend.cc create mode 100644 fastdeploy/backends/lite/lite_backend.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cd71e1430..93481dad8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,8 @@ option(WITH_GPU "Whether WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernc option(ENABLE_ORT_BACKEND "Whether to enable onnxruntime backend." OFF) option(ENABLE_TRT_BACKEND "Whether to enable tensorrt backend." OFF) option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF) -option(ENABLE_OPENVINO_BACKEND "Whether to enable paddle backend." OFF) +option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF) +option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF) option(CUDA_DIRECTORY "If build tensorrt backend, need to define path of cuda library.") option(TRT_DIRECTORY "If build tensorrt backend, need to define path of tensorrt library.") option(ENABLE_VISION "Whether to enable vision models usage." OFF) @@ -120,10 +121,11 @@ file(GLOB_RECURSE DEPLOY_ORT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastde file(GLOB_RECURSE DEPLOY_PADDLE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/paddle/*.cc) file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cpp) file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) +file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}) set(DEPEND_LIBS "") @@ -147,6 +149,12 @@ if(ENABLE_ORT_BACKEND) list(APPEND DEPEND_LIBS external_onnxruntime) endif() +if(ENABLE_LITE_BACKEND) + add_definitions(-DENABLE_LITE_BACKEND) + include(${PROJECT_SOURCE_DIR}/cmake/paddlelite.cmake) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_LITE_SRCS}) +endif() + if(ENABLE_PADDLE_BACKEND) add_definitions(-DENABLE_PADDLE_BACKEND) list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_SRCS}) diff --git a/cmake/paddlelite.cmake b/cmake/paddlelite.cmake new file mode 100644 index 000000000..8cd143157 --- /dev/null +++ b/cmake/paddlelite.cmake @@ -0,0 +1,72 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(ExternalProject) + +set(PADDLELITE_PROJECT "extern_paddlelite") +set(PADDLELITE_PREFIX_DIR ${THIRD_PARTY_PATH}/paddlelite) +set(PADDLELITE_SOURCE_DIR + ${THIRD_PARTY_PATH}/paddlelite/src/${PADDLELITE_PROJECT}) +set(PADDLELITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/paddlelite) +set(PADDLELITE_INC_DIR + "${PADDLELITE_INSTALL_DIR}/include" + CACHE PATH "paddlelite include directory." FORCE) +set(PADDLELITE_LIB_DIR + "${PADDLELITE_INSTALL_DIR}/lib" + CACHE PATH "paddlelite lib directory." FORCE) +set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}") + +#set(PADDLELITE_URL "https://github.com/PaddlePaddle/Paddle-Lite/releases/download/v2.11/inference_lite_lib.armlinux.armv8.gcc.with_extra.with_cv.tar.gz") +set(PADDLELITE_URL "https://bj.bcebos.com/fastdeploy/third_libs/lite-linux-arm64-20220920.tgz") + +if(WIN32) + message(FATAL_ERROR "Doesn't support windows platform with backend Paddle-Lite.") +elseif(APPLE) + message(FATAL_ERROR "Doesn't support Mac OSX platform with backend Paddle-Lite.") +else() + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + set(PADDLELITE_URL "https://bj.bcebos.com/fastdeploy/third_libs/lite-linux-arm64-20220920.tgz") + else() + message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle-Lite.") + endif() +endif() + +include_directories(${PADDLELITE_INC_DIR}) + +if(WIN32) +elseif(APPLE) +else() + set(PADDLELITE_LIB "${PADDLELITE_INSTALL_DIR}/lib/libpaddle_full_api_shared.so") +endif() + +ExternalProject_Add( + ${PADDLELITE_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${PADDLELITE_URL} + PREFIX ${PADDLELITE_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include + ${PADDLELITE_INC_DIR} + BUILD_BYPRODUCTS ${PADDLELITE_LIB}) + +add_library(external_paddle_lite STATIC IMPORTED GLOBAL) +set_property(TARGET external_paddle_lite PROPERTY IMPORTED_LOCATION ${PADDLELITE_LIB}) +add_dependencies(external_paddle_lite ${PADDLELITE_PROJECT}) diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc new file mode 100644 index 000000000..291132836 --- /dev/null +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -0,0 +1,141 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/backends/lite/lite_backend.h" +#include + +namespace fastdeploy { + +// Convert data type from paddle lite to fastdeploy +FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { + if (dtype == paddle::lite_api::PrecisionType::kFloat) { + return FDDataType::FP32; + } else if (dtype == paddle::lite_api::PrecisionType::kInt8) { + return FDDataType::INT8; + } else if (dtype == paddle::lite_api::PrecisionType::kInt32) { + return FDDataType::INT32; + } else if (dtype == paddle::lite_api::PrecisionType::kInt64) { + return FDDataType::INT64; + } else if (dtype == paddle::lite_api::PrecisionType::kInt16) { + return FDDataType::INT16; + } else if (dtype == paddle::lite_api::PrecisionType::kUInt8) { + return FDDataType::UINT8; + } else if (dtype == paddle::lite_api::PrecisionType::kFP64) { + return FDDataType::FP64; + } + FDASSERT(false, "Unexpected data type of %d.", dtype); + return FDDataType::FP32; +} + +void LiteBackend::BuildOption(const LiteBackendOption& option) { + std::vector valid_places; + valid_places.push_back(paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); + config_.set_valid_places(valid_places); +} + +bool LiteBackend::InitFromPaddle(const std::string& model_file, + const std::string& params_file, + const LiteBackendOption& option) { + if (initialized_) { + FDERROR << "LiteBackend is already initialized, cannot initialize again." + << std::endl; + return false; + } + + config_.set_model_file(model_file); + config_.set_param_file(params_file); + BuildOption(option); + predictor_ = paddle::lite_api::CreatePaddlePredictor(config_); + + inputs_desc_.clear(); + outputs_desc_.clear(); + inputs_order_.clear(); + std::vector input_names = predictor_->GetInputNames(); + std::vector output_names = predictor_->GetOutputNames(); + for (size_t i = 0; i < input_names.size(); ++i) { + inputs_order_[input_names[i]] = i; + TensorInfo info; + auto tensor = predictor_->GetInput(i); + auto shape = tensor->shape(); + info.shape.assign(shape.begin(), shape.end()); + info.name = input_names[i]; + info.dtype = LiteDataTypeToFD(tensor->precision()); + inputs_desc_.emplace_back(info); + } + for (size_t i = 0; i < output_names.size(); ++i) { + TensorInfo info; + auto tensor = predictor_->GetOutput(i); + auto shape = tensor->shape(); + info.shape.assign(shape.begin(), shape.end()); + info.name = output_names[i]; + info.dtype = LiteDataTypeToFD(tensor->precision()); + outputs_desc_.emplace_back(info); + } + + initialized_ = true; + return true; +} + +TensorInfo LiteBackend::GetInputInfo(int index) { + FDASSERT(index < NumInputs(), + "The index: %d should less than the number of inputs: %d.", index, + NumInputs()); + return inputs_desc_[index]; +} + +std::vector LiteBackend::GetInputInfos() { return inputs_desc_; } + +TensorInfo LiteBackend::GetOutputInfo(int index) { + FDASSERT(index < NumOutputs(), + "The index: %d should less than the number of outputs %d.", index, + NumOutputs()); + return outputs_desc_[index]; +} + +std::vector LiteBackend::GetOutputInfos() { + return outputs_desc_; +} + +bool LiteBackend::Infer(std::vector& inputs, + std::vector* outputs) { + if (inputs.size() != inputs_desc_.size()) { + FDERROR << "[LiteBackend] Size of inputs(" << inputs.size() + << ") should keep same with the inputs of this model(" + << inputs_desc_.size() << ")." << std::endl; + return false; + } + + for (size_t i = 0; i < inputs.size(); ++i) { + auto iter = inputs_order_.find(inputs[i].name); + if (iter == inputs_order_.end()) { + FDERROR << "Cannot find input with name:" << inputs[i].name << " in loaded model." << std::endl; + return false; + } + auto tensor = predictor_->GetInput(iter->second); + tensor->Resize(inputs[i].shape); + tensor->ShareExternalMemory(const_cast(inputs[i].CpuData()), inputs[i].Nbytes(), paddle::lite_api::TargetType::kARM); + } + + predictor_->Run(); + + outputs->resize(outputs_desc_.size()); + for (size_t i = 0; i < outputs_desc_.size(); ++i) { + auto tensor = predictor_->GetOutput(i); + (*outputs)[i].Resize(tensor->shape(), outputs_desc_[i].dtype, outputs_desc_[i].name); + memcpy((*outputs)[i].MutableData(), tensor->data(), (*outputs)[i].Nbytes()); + } + return true; +} + +} // namespace fastdeploy diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h new file mode 100644 index 000000000..0fdaf1920 --- /dev/null +++ b/fastdeploy/backends/lite/lite_backend.h @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "paddle_api.h" // NOLINT + +namespace fastdeploy { + +struct LiteBackendOption { +}; + +// Convert data type from paddle lite to fastdeploy +FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype); + +class LiteBackend : public BaseBackend { + public: + LiteBackend() {} + virtual ~LiteBackend() = default; + void BuildOption(const LiteBackendOption& option); + + bool InitFromPaddle(const std::string& model_file, + const std::string& params_file, + const LiteBackendOption& option = LiteBackendOption()); + + bool Infer(std::vector& inputs, std::vector* outputs); + + int NumInputs() const { return inputs_desc_.size(); } + + int NumOutputs() const { return outputs_desc_.size(); } + + TensorInfo GetInputInfo(int index); + TensorInfo GetOutputInfo(int index); + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + + private: + paddle::lite_api::CxxConfig config_; + std::shared_ptr predictor_; + std::vector inputs_desc_; + std::vector outputs_desc_; + std::map inputs_order_; +}; +} // namespace fastdeploy diff --git a/fastdeploy/core/allocate.cc b/fastdeploy/core/allocate.cc index 0615ee46d..44a9e7740 100644 --- a/fastdeploy/core/allocate.cc +++ b/fastdeploy/core/allocate.cc @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#pragma once #ifdef WITH_GPU #include diff --git a/fastdeploy/fastdeploy_model.cc b/fastdeploy/fastdeploy_model.cc index cd21720e5..bfe1c800a 100644 --- a/fastdeploy/fastdeploy_model.cc +++ b/fastdeploy/fastdeploy_model.cc @@ -166,7 +166,7 @@ std::map FastDeployModel::PrintStatisInfoOfRuntime() { std::map statis_info_of_runtime_dict; if (time_of_runtime_.size() < 10) { - std::cout << "[FastDeploy] [WARNING] PrintStatisInfoOfRuntime require the runtime ran 10 times at " + FDWARNING << "PrintStatisInfoOfRuntime require the runtime ran 10 times at " "least, but now you only ran " << time_of_runtime_.size() << " times." << std::endl; } diff --git a/fastdeploy/fastdeploy_model.h b/fastdeploy/fastdeploy_model.h index 0783546dd..57d3a754e 100644 --- a/fastdeploy/fastdeploy_model.h +++ b/fastdeploy/fastdeploy_model.h @@ -75,7 +75,7 @@ class FASTDEPLOY_DECL FastDeployModel { #define TIMERECORD_END(id, prefix) \ if (DebugEnabled()) { \ tc_##id.End(); \ - FDINFO << __FILE__ << "(" << __LINE__ << "):" << __FUNCTION__ << " " \ + FDLogger() << __FILE__ << "(" << __LINE__ << "):" << __FUNCTION__ << " " \ << prefix << " duration = " << tc_##id.Duration() << "s." \ << std::endl; \ } diff --git a/fastdeploy/fastdeploy_runtime.cc b/fastdeploy/fastdeploy_runtime.cc index c74fdb0b2..92ecc2424 100644 --- a/fastdeploy/fastdeploy_runtime.cc +++ b/fastdeploy/fastdeploy_runtime.cc @@ -33,6 +33,10 @@ #include "fastdeploy/backends/openvino/ov_backend.h" #endif +#ifdef ENABLE_LITE_BACKEND +#include "fastdeploy/backends/lite/lite_backend.h" +#endif + namespace fastdeploy { std::vector GetAvailableBackends() { @@ -48,6 +52,9 @@ std::vector GetAvailableBackends() { #endif #ifdef ENABLE_OPENVINO_BACKEND backends.push_back(Backend::OPENVINO); +#endif +#ifdef ENABLE_LITE_BACKEND + backends.push_back(Backend::LITE); #endif return backends; } @@ -71,6 +78,8 @@ std::string Str(const Backend& b) { return "Backend::PDINFER"; } else if (b == Backend::OPENVINO) { return "Backend::OPENVINO"; + } else if (b == Backend::LITE) { + return "Backend::LITE"; } return "UNKNOWN-Backend"; } @@ -194,6 +203,15 @@ void RuntimeOption::UseOpenVINOBackend() { FDASSERT(false, "The FastDeploy didn't compile with OpenVINO."); #endif } + +void RuntimeOption::UseLiteBackend() { +#ifdef ENABLE_LITE_BACKEND + backend = Backend::LITE; +#else + FDASSERT(false, "The FastDeploy didn't compile with Paddle Lite."); +#endif +} + void RuntimeOption::EnablePaddleMKLDNN() { pd_enable_mkldnn = true; } void RuntimeOption::DisablePaddleMKLDNN() { pd_enable_mkldnn = false; } @@ -262,12 +280,12 @@ bool Runtime::Init(const RuntimeOption& _option) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::ORT only supports Device::CPU/Device::GPU."); CreateOrtBackend(); - FDINFO << "Runtime initialized with Backend::ORT in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::ORT in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::TRT) { FDASSERT(option.device == Device::GPU, "Backend::TRT only supports Device::GPU."); CreateTrtBackend(); - FDINFO << "Runtime initialized with Backend::TRT in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::TRT in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::PDINFER) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::TRT only supports Device::CPU/Device::GPU."); @@ -275,12 +293,16 @@ bool Runtime::Init(const RuntimeOption& _option) { option.model_format == Frontend::PADDLE, "Backend::PDINFER only supports model format of Frontend::PADDLE."); CreatePaddleBackend(); - FDINFO << "Runtime initialized with Backend::PDINFER in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::PDINFER in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::OPENVINO) { FDASSERT(option.device == Device::CPU, "Backend::OPENVINO only supports Device::CPU"); CreateOpenVINOBackend(); - FDINFO << "Runtime initialized with Backend::OPENVINO in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::OPENVINO in " << Str(option.device) << "." << std::endl; + } else if (option.backend == Backend::LITE) { + FDASSERT(option.device == Device::CPU, "Backend::LITE only supports Device::CPU"); + CreateLiteBackend(); + FDINFO << "Runtime initialized with Backend::LITE in " << Str(option.device) << "." << std::endl; } else { FDERROR << "Runtime only support " "Backend::ORT/Backend::TRT/Backend::PDINFER as backend now." @@ -433,4 +455,21 @@ void Runtime::CreateTrtBackend() { "ENABLE_TRT_BACKEND=ON."); #endif } + +void Runtime::CreateLiteBackend() { +#ifdef ENABLE_LITE_BACKEND + auto lite_option = LiteBackendOption(); + FDASSERT(option.model_format == Frontend::PADDLE, + "LiteBackend only support model format of Frontend::PADDLE"); + backend_ = utils::make_unique(); + auto casted_backend = dynamic_cast(backend_.get()); + FDASSERT(casted_backend->InitFromPaddle(option.model_file, option.params_file, lite_option), + "Load model from nb file failed while initializing LiteBackend."); +#else + FDASSERT(false, + "LiteBackend is not available, please compiled with " + "ENABLE_LITE_BACKEND=ON."); +#endif +} + } // namespace fastdeploy diff --git a/fastdeploy/fastdeploy_runtime.h b/fastdeploy/fastdeploy_runtime.h index 58981687d..c2a5e664b 100644 --- a/fastdeploy/fastdeploy_runtime.h +++ b/fastdeploy/fastdeploy_runtime.h @@ -21,7 +21,7 @@ namespace fastdeploy { -enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO }; +enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO, LITE }; // AUTOREC will according to the name of model file // to decide which Frontend is enum FASTDEPLOY_DECL Frontend { AUTOREC, PADDLE, ONNX }; @@ -66,6 +66,9 @@ struct FASTDEPLOY_DECL RuntimeOption { // use openvino backend void UseOpenVINOBackend(); + // use paddle lite backend + void UseLiteBackend(); + // enable mkldnn while use paddle inference in CPU void EnablePaddleMKLDNN(); // disable mkldnn while use paddle inference in CPU @@ -161,6 +164,8 @@ struct FASTDEPLOY_DECL Runtime { void CreateOpenVINOBackend(); + void CreateLiteBackend(); + int NumInputs() { return backend_->NumInputs(); } int NumOutputs() { return backend_->NumOutputs(); } TensorInfo GetInputInfo(int index); diff --git a/fastdeploy/pybind/fastdeploy_runtime.cc b/fastdeploy/pybind/fastdeploy_runtime.cc index a5924b0af..ad4715a0f 100644 --- a/fastdeploy/pybind/fastdeploy_runtime.cc +++ b/fastdeploy/pybind/fastdeploy_runtime.cc @@ -27,6 +27,7 @@ void BindRuntime(pybind11::module& m) { .def("use_ort_backend", &RuntimeOption::UseOrtBackend) .def("use_trt_backend", &RuntimeOption::UseTrtBackend) .def("use_openvino_backend", &RuntimeOption::UseOpenVINOBackend) + .def("use_lite_backend", &RuntimeOption::UseLiteBackend) .def("enable_paddle_mkldnn", &RuntimeOption::EnablePaddleMKLDNN) .def("disable_paddle_mkldnn", &RuntimeOption::DisablePaddleMKLDNN) .def("enable_paddle_log_info", &RuntimeOption::EnablePaddleLogInfo) @@ -109,7 +110,8 @@ void BindRuntime(pybind11::module& m) { .value("UNKOWN", Backend::UNKNOWN) .value("ORT", Backend::ORT) .value("TRT", Backend::TRT) - .value("PDINFER", Backend::PDINFER); + .value("PDINFER", Backend::PDINFER) + .value("LITE", Backend::LITE); pybind11::enum_(m, "Frontend", pybind11::arithmetic(), "Frontend for inference.") .value("PADDLE", Frontend::PADDLE) diff --git a/fastdeploy/pybind/main.cc.in b/fastdeploy/pybind/main.cc.in index 417a59761..5aaac049c 100644 --- a/fastdeploy/pybind/main.cc.in +++ b/fastdeploy/pybind/main.cc.in @@ -137,10 +137,6 @@ PYBIND11_MODULE(@PY_LIBRARY_NAME@, m) { "Make programer easier to deploy deeplearning model, save time to save " "the world!"; - pybind11::class_(m, "FDLogger") - .def_readwrite_static("disable_info", &FDLogger::disable_info) - .def_readwrite_static("disable_warning", &FDLogger::disable_warning); - BindRuntime(m); BindFDModel(m); #ifdef ENABLE_VISION diff --git a/fastdeploy/utils/perf.h b/fastdeploy/utils/perf.h index 84c7ba8f8..783459064 100644 --- a/fastdeploy/utils/perf.h +++ b/fastdeploy/utils/perf.h @@ -38,7 +38,7 @@ class FASTDEPLOY_DECL TimeCounter { if (!print_out) { return; } - std::cout << prefix << " duration = " << Duration() << "s." << std::endl; + FDLogger() << prefix << " duration = " << Duration() << "s." << std::endl; } private: @@ -46,4 +46,4 @@ class FASTDEPLOY_DECL TimeCounter { std::chrono::time_point end_; }; -} // namespace fastdeploy +} // namespace fastdeploy diff --git a/fastdeploy/utils/utils.cc b/fastdeploy/utils/utils.cc index 7ea536e20..6e76c7888 100644 --- a/fastdeploy/utils/utils.cc +++ b/fastdeploy/utils/utils.cc @@ -16,17 +16,14 @@ namespace fastdeploy { -bool FDLogger::disable_info = false; -bool FDLogger::disable_warning = false; - -FDLogger::FDLogger(int level, const std::string& prefix) { +FDLogger::FDLogger(bool verbose, const std::string& prefix) { + verbose_ = verbose; line_ = ""; - level_ = level; prefix_ = prefix; } FDLogger& FDLogger::operator<<(std::ostream& (*os)(std::ostream&)) { - if (!verbose()) { + if (!verbose_) { return *this; } std::cout << prefix_ << " " << line_ << std::endl; diff --git a/fastdeploy/utils/utils.h b/fastdeploy/utils/utils.h index 556e2d768..2c7269763 100644 --- a/fastdeploy/utils/utils.h +++ b/fastdeploy/utils/utils.h @@ -37,26 +37,16 @@ namespace fastdeploy { class FASTDEPLOY_DECL FDLogger { public: - static bool disable_info; - static bool disable_warning; - // 0: INFO - // 1: WARNING - // 2: ERROR - explicit FDLogger(int level = 0, const std::string& prefix = "[FastDeploy]"); - - bool verbose() { - if (disable_info && level_ == 0) { - return false; - } - if (disable_warning && level_ == 1) { - return false; - } - return true; + FDLogger() { + line_ = ""; + prefix_ = "[FastDeploy]"; + verbose_ = true; } + explicit FDLogger(bool verbose, const std::string& prefix = "[FastDeploy]"); template FDLogger& operator<<(const T& val) { - if (!verbose()) { + if (!verbose_) { return *this; } std::stringstream ss; @@ -66,7 +56,7 @@ class FASTDEPLOY_DECL FDLogger { } FDLogger& operator<<(std::ostream& (*os)(std::ostream&)); ~FDLogger() { - if (!verbose() && line_ != "") { + if (!verbose_ && line_ != "") { std::cout << line_ << std::endl; } } @@ -74,7 +64,7 @@ class FASTDEPLOY_DECL FDLogger { private: std::string line_; std::string prefix_; - int level_ = 0; + bool verbose_ = true; }; FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file, @@ -85,15 +75,15 @@ FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file, #endif #define FDERROR \ - FDLogger(2, "[ERROR]") << __REL_FILE__ << "(" << __LINE__ \ + FDLogger(true, "[ERROR]") << __REL_FILE__ << "(" << __LINE__ \ << ")::" << __FUNCTION__ << "\t" #define FDWARNING \ - FDLogger(1, "[WARNING]") << __REL_FILE__ << "(" << __LINE__ \ + FDLogger(true, "[WARNING]") << __REL_FILE__ << "(" << __LINE__ \ << ")::" << __FUNCTION__ << "\t" #define FDINFO \ - FDLogger(0, "[INFO]") << __REL_FILE__ << "(" << __LINE__ \ + FDLogger(true, "[INFO]") << __REL_FILE__ << "(" << __LINE__ \ << ")::" << __FUNCTION__ << "\t" #define FDASSERT(condition, format, ...) \ diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index a738ac294..fb02125fc 100644 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -26,7 +26,7 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER}; + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/picodet.cc b/fastdeploy/vision/detection/ppdet/picodet.cc index bae51b409..3b651ccf5 100644 --- a/fastdeploy/vision/detection/ppdet/picodet.cc +++ b/fastdeploy/vision/detection/ppdet/picodet.cc @@ -24,7 +24,7 @@ PicoDet::PicoDet(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc index 69aacb3f1..8f95a1acd 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -24,6 +24,7 @@ PPYOLOE::PPYOLOE(const std::string& model_file, const std::string& params_file, } void PPYOLOE::GetNmsInfo() { +#ifdef ENABLE_PADDLE_FRONTEND if (runtime_option.model_format == Frontend::PADDLE) { std::string contents; if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { @@ -41,6 +42,7 @@ void PPYOLOE::GetNmsInfo() { normalized = reader.nms_params.normalized; } } +#endif } bool PPYOLOE::Initialize() { diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.h b/fastdeploy/vision/detection/ppdet/ppyoloe.h index 2d8cca99f..ecbce2190 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.h +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.h @@ -56,13 +56,13 @@ class FASTDEPLOY_DECL PPYOLOE : public FastDeployModel { float score_threshold = 0.01; int64_t nms_top_k = 10000; bool normalized = true; - bool has_nms_ = false; + bool has_nms_ = true; // This function will used to check if this model contains multiclass_nms // and get parameters from the operator void GetNmsInfo(); }; -} // namespace detection +} // namespace detection } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/classifier.cc b/fastdeploy/vision/ocr/ppocr/classifier.cc index 2dcb35eb3..c8ce1ccc1 100644 --- a/fastdeploy/vision/ocr/ppocr/classifier.cc +++ b/fastdeploy/vision/ocr/ppocr/classifier.cc @@ -26,12 +26,11 @@ Classifier::Classifier(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { if (model_format == Frontend::ONNX) { - valid_cpu_backends = {Backend::ORT, - Backend::OPENVINO}; // 指定可用的CPU后端 + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; // 指定可用的CPU后端 valid_gpu_backends = {Backend::ORT, Backend::TRT}; // 指定可用的GPU后端 } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + valid_gpu_backends = {Backend::PDINFER, Backend::TRT, Backend::ORT}; } runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/ocr/ppocr/recognizer.cc b/fastdeploy/vision/ocr/ppocr/recognizer.cc index b91719009..3e74f5a8c 100644 --- a/fastdeploy/vision/ocr/ppocr/recognizer.cc +++ b/fastdeploy/vision/ocr/ppocr/recognizer.cc @@ -44,12 +44,12 @@ Recognizer::Recognizer(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { if (model_format == Frontend::ONNX) { - valid_cpu_backends = {Backend::ORT, - Backend::OPENVINO}; // 指定可用的CPU后端 + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; // 指定可用的CPU后端 valid_gpu_backends = {Backend::ORT, Backend::TRT}; // 指定可用的GPU后端 } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + // NOTE:此模型暂不支持paddle-inference-Gpu推理 + valid_cpu_backends = {Backend::ORT, Backend::PDINFER, Backend::OPENVINO}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; } runtime_option = custom_option; diff --git a/python/fastdeploy/__init__.py b/python/fastdeploy/__init__.py index baa742e44..b27c6e5c8 100644 --- a/python/fastdeploy/__init__.py +++ b/python/fastdeploy/__init__.py @@ -17,7 +17,7 @@ import os import sys from .c_lib_wrap import (Frontend, Backend, FDDataType, TensorInfo, Device, - FDLogger, is_built_with_gpu, is_built_with_ort, + is_built_with_gpu, is_built_with_ort, is_built_with_paddle, is_built_with_trt, get_default_cuda_directory) from .runtime import Runtime, RuntimeOption diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index c3bac06a3..f19306a2c 100644 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -79,6 +79,9 @@ class RuntimeOption: def use_openvino_backend(self): return self._option.use_openvino_backend() + def use_lite_backend(self): + return self._option.use_lite_backend() + def enable_paddle_mkldnn(self): return self._option.enable_paddle_mkldnn()