From c5f85de35643f57f124aed37337d542fa07db4d8 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Wed, 28 Sep 2022 18:09:35 +0800 Subject: [PATCH] [lite] Add threads and power_mode option support (#298) * [cmake] support Android arm64-v8a & armeabi-v7a native c++ sdk * [cmake] fixed patchelf download on mac and android * [lite] Add threads and power_mode option support * [pybind] update runtime pybind for lite power mode * [python] Add set_lite_power_mode api to runtime --- fastdeploy/backends/lite/lite_backend.cc | 42 ++++++++++++++++-------- fastdeploy/backends/lite/lite_backend.h | 12 +++++++ fastdeploy/pybind/runtime.cc | 1 + fastdeploy/runtime.cc | 11 ++++++- fastdeploy/runtime.h | 9 +++++ python/fastdeploy/runtime.py | 7 +++- 6 files changed, 66 insertions(+), 16 deletions(-) diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc index 291132836..2d96ec4f0 100644 --- a/fastdeploy/backends/lite/lite_backend.cc +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/backends/lite/lite_backend.h" + #include namespace fastdeploy { @@ -40,13 +41,21 @@ FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { void LiteBackend::BuildOption(const LiteBackendOption& option) { std::vector valid_places; - valid_places.push_back(paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); + valid_places.push_back( + paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); config_.set_valid_places(valid_places); + if (option.threads > 0) { + config_.set_threads(option.threads); + } + if (option.power_mode > 0) { + config_.set_power_mode( + static_cast(option.power_mode)); + } } bool LiteBackend::InitFromPaddle(const std::string& model_file, - const std::string& params_file, - const LiteBackendOption& option) { + const std::string& params_file, + const LiteBackendOption& option) { if (initialized_) { FDERROR << "LiteBackend is already initialized, cannot initialize again." << std::endl; @@ -56,8 +65,10 @@ bool LiteBackend::InitFromPaddle(const std::string& model_file, config_.set_model_file(model_file); config_.set_param_file(params_file); BuildOption(option); - predictor_ = paddle::lite_api::CreatePaddlePredictor(config_); - + predictor_ = + paddle::lite_api::CreatePaddlePredictor( + config_); + inputs_desc_.clear(); outputs_desc_.clear(); inputs_order_.clear(); @@ -82,7 +93,7 @@ bool LiteBackend::InitFromPaddle(const std::string& model_file, info.dtype = LiteDataTypeToFD(tensor->precision()); outputs_desc_.emplace_back(info); } - + initialized_ = true; return true; } @@ -103,12 +114,10 @@ TensorInfo LiteBackend::GetOutputInfo(int index) { return outputs_desc_[index]; } -std::vector LiteBackend::GetOutputInfos() { - return outputs_desc_; -} +std::vector LiteBackend::GetOutputInfos() { return outputs_desc_; } bool LiteBackend::Infer(std::vector& inputs, - std::vector* outputs) { + std::vector* outputs) { if (inputs.size() != inputs_desc_.size()) { FDERROR << "[LiteBackend] Size of inputs(" << inputs.size() << ") should keep same with the inputs of this model(" @@ -119,12 +128,15 @@ bool LiteBackend::Infer(std::vector& inputs, for (size_t i = 0; i < inputs.size(); ++i) { auto iter = inputs_order_.find(inputs[i].name); if (iter == inputs_order_.end()) { - FDERROR << "Cannot find input with name:" << inputs[i].name << " in loaded model." << std::endl; + FDERROR << "Cannot find input with name:" << inputs[i].name + << " in loaded model." << std::endl; return false; } auto tensor = predictor_->GetInput(iter->second); tensor->Resize(inputs[i].shape); - tensor->ShareExternalMemory(const_cast(inputs[i].CpuData()), inputs[i].Nbytes(), paddle::lite_api::TargetType::kARM); + tensor->ShareExternalMemory(const_cast(inputs[i].CpuData()), + inputs[i].Nbytes(), + paddle::lite_api::TargetType::kARM); } predictor_->Run(); @@ -132,8 +144,10 @@ bool LiteBackend::Infer(std::vector& inputs, outputs->resize(outputs_desc_.size()); for (size_t i = 0; i < outputs_desc_.size(); ++i) { auto tensor = predictor_->GetOutput(i); - (*outputs)[i].Resize(tensor->shape(), outputs_desc_[i].dtype, outputs_desc_[i].name); - memcpy((*outputs)[i].MutableData(), tensor->data(), (*outputs)[i].Nbytes()); + (*outputs)[i].Resize(tensor->shape(), outputs_desc_[i].dtype, + outputs_desc_[i].name); + memcpy((*outputs)[i].MutableData(), tensor->data(), + (*outputs)[i].Nbytes()); } return true; } diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h index 0fdaf1920..5984c4d34 100644 --- a/fastdeploy/backends/lite/lite_backend.h +++ b/fastdeploy/backends/lite/lite_backend.h @@ -25,6 +25,18 @@ namespace fastdeploy { struct LiteBackendOption { + // cpu num threads + int threads = 1; + // lite power mode + // 0: LITE_POWER_HIGH + // 1: LITE_POWER_LOW + // 2: LITE_POWER_FULL + // 3: LITE_POWER_NO_BIND + // 4: LITE_POWER_RAND_HIGH + // 5: LITE_POWER_RAND_LOW + int power_mode = 0; + // TODO(qiuyanjun): support more options for lite backend. + // Such as fp16, different device target (kARM/kXPU/kNPU/...) }; // Convert data type from paddle lite to fastdeploy diff --git a/fastdeploy/pybind/runtime.cc b/fastdeploy/pybind/runtime.cc index f5cbdce6c..ddf9a9585 100644 --- a/fastdeploy/pybind/runtime.cc +++ b/fastdeploy/pybind/runtime.cc @@ -34,6 +34,7 @@ void BindRuntime(pybind11::module& m) { .def("disable_paddle_log_info", &RuntimeOption::DisablePaddleLogInfo) .def("set_paddle_mkldnn_cache_size", &RuntimeOption::SetPaddleMKLDNNCacheSize) + .def("set_lite_power_mode", &RuntimeOption::SetLitePowerMode) .def("set_trt_input_shape", &RuntimeOption::SetTrtInputShape) .def("enable_trt_fp16", &RuntimeOption::EnableTrtFP16) .def("disable_trt_fp16", &RuntimeOption::DisableTrtFP16) diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index dc600c46d..67974fefe 100644 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -148,7 +148,9 @@ void RuntimeOption::SetModelPath(const std::string& model_path, model_file = model_path; model_format = ModelFormat::ONNX; } else { - FDASSERT(false, "The model format only can be ModelFormat::PADDLE/ModelFormat::ONNX."); + FDASSERT( + false, + "The model format only can be ModelFormat::PADDLE/ModelFormat::ONNX."); } } @@ -228,6 +230,11 @@ void RuntimeOption::SetPaddleMKLDNNCacheSize(int size) { pd_mkldnn_cache_size = size; } +void RuntimeOption::SetLitePowerMode(int mode) { + FDASSERT(mode > -1, "Parameter mode must greater than -1."); + lite_power_mode = mode; +} + void RuntimeOption::SetTrtInputShape(const std::string& input_name, const std::vector& min_shape, const std::vector& opt_shape, @@ -465,6 +472,8 @@ void Runtime::CreateTrtBackend() { void Runtime::CreateLiteBackend() { #ifdef ENABLE_LITE_BACKEND auto lite_option = LiteBackendOption(); + lite_option.threads = option.cpu_thread_num; + lite_option.power_mode = option.lite_power_mode; FDASSERT(option.model_format == ModelFormat::PADDLE, "LiteBackend only support model format of ModelFormat::PADDLE"); backend_ = utils::make_unique(); diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index 84efdc4c8..b6f0affd5 100644 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -84,6 +84,9 @@ struct FASTDEPLOY_DECL RuntimeOption { // set size of cached shape while enable mkldnn with paddle inference backend void SetPaddleMKLDNNCacheSize(int size); + // set the power mode of paddle lite backend. + void SetLitePowerMode(int mode); + // set tensorrt shape while the inputs of model contain dynamic shape // min_shape: the minimum shape // opt_shape: the most common shape while inference, default be empty @@ -126,6 +129,12 @@ struct FASTDEPLOY_DECL RuntimeOption { int pd_mkldnn_cache_size = 1; std::vector pd_delete_pass_names; + // ======Only for Paddle-Lite Backend===== + // 0: LITE_POWER_HIGH 1: LITE_POWER_LOW 2: LITE_POWER_FULL + // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH + // 5: LITE_POWER_RAND_LOW + int lite_power_mode = 0; + // ======Only for Trt Backend======= std::map> trt_max_shape; std::map> trt_min_shape; diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index 2a6edbdf6..8fb5e8ec7 100644 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -54,7 +54,9 @@ class RuntimeOption: def __init__(self): self._option = C.RuntimeOption() - def set_model_path(self, model_path, params_path="", + def set_model_path(self, + model_path, + params_path="", model_format=C.ModelFormat.PADDLE): return self._option.set_model_path(model_path, params_path, model_format) @@ -98,6 +100,9 @@ class RuntimeOption: def set_paddle_mkldnn_cache_size(self, cache_size): return self._option.set_paddle_mkldnn_cache_size(cache_size) + def set_lite_power_mode(self, mode): + return self._option.set_lite_power_mode(mode) + def set_trt_input_shape(self, tensor_name, min_shape,