[lite] add more options for lite backend (#313)

* [cmake] support Android arm64-v8a & armeabi-v7a native c++ sdk * [cmake] fixed patchelf download on mac and android * [lite] Add threads and power_mode option support * [pybind] update runtime pybind for lite power mode * [python] Add set_lite_power_mode api to runtime * [Lite] add lite enable_fp16 option support * [lite] add more options for lite backend. * [cmake] fixed Paddle Lite typo * [runtime] format LitePowerMode enum comments * [runtime] format lite option comments
2025-10-05 00:33:03 +08:00 · 2022-10-04 17:31:48 +08:00
parent f2e984fdfa
commit c4288a92e1
10 changed files with 180 additions and 40 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -63,7 +63,9 @@ option(ENABLE_TEXT "Whether to enable text models usage." OFF)
 option(WITH_TESTING "Whether to compile with unittest." OFF)
 ######################### Options for Android cross compiling ####################
 option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF)
-option(WITH_LITE_STATIC "Use Paddle-Lite static lib for Android." OFF)
+option(WITH_LITE_STATIC "Use Paddle Lite static lib for Android." OFF)
+option(WITH_LITE_FULL_API "Use Paddle Lite full API lib for Android." ON)
+option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." OFF)

 # Please don't open this flag now, some bugs exists.
 # Only support Linux Now
@@ -413,6 +415,11 @@ if(WIN32)
    ARCHIVE DESTINATION lib
    RUNTIME DESTINATION lib
  )
+elseif(ANDROID)
+  install(
+    TARGETS ${LIBRARY_NAME}
+    LIBRARY DESTINATION lib/${ANDROID_ABI}
+  )
 else()
  install(
    TARGETS ${LIBRARY_NAME}
--- a/FastDeploy.cmake.in
+++ b/FastDeploy.cmake.in
@@ -14,6 +14,8 @@ set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@)
 set(OPENVINO_VERSION @OPENVINO_VERSION@)
 set(WITH_LITE_STATIC @WITH_LITE_STATIC@)
 set(WITH_OPENCV_STATIC @WITH_OPENCV_STATIC@)
+set(WITH_LITE_FULL_API @WITH_LITE_FULL_API@)
+set(WITH_LITE_FP16 @WITH_LITE_FP16@)
 # set(ENABLE_OPENCV_CUDA @ENABLE_OPENCV_CUDA@)
 set(OPENCV_FILENAME @OPENCV_FILENAME@)
 set(OPENVINO_FILENAME @OPENVINO_FILENAME@)
@@ -36,7 +38,7 @@ endif()

 if(ANDROID)
  add_library(fastdeploy STATIC IMPORTED GLOBAL)
-  set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/lib/lib${LIBRARY_NAME}.so)
+  set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/lib/${ANDROID_ABI}/lib${LIBRARY_NAME}.so)
  list(APPEND FASTDEPLOY_LIBS fastdeploy)
 else()
  find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib NO_DEFAULT_PATH)
@@ -85,10 +87,19 @@ endif()
 if(ENABLE_LITE_BACKEND)
  set(LITE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${PADDLELITE_FILENAME})
  if(ANDROID)
+    if(NOT WITH_LITE_STATIC)
+      if(WITH_LITE_FULL_API)
        add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL)
-    set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/libpaddle_full_api_shared.so)
+        set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_full_api_shared.so)
        list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared)
      else()
+        add_library(paddle_light_api_shared STATIC IMPORTED GLOBAL)
+        set_property(TARGET paddle_light_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_light_api_shared.so)
+        list(APPEND FASTDEPLOY_LIBS paddle_light_api_shared)
+      endif()
+    endif()
+  else()
+    # Linux/Mac/Win/...
    find_library(LITE_LIB paddle_full_api_shared ${LITE_DIR}/lib NO_DEFAULT_PATH)
    list(APPEND FASTDEPLOY_LIBS ${LITE_LIB})
  endif()
@@ -231,6 +242,8 @@ if(ANDROID)
  message(STATUS "  WITH_OPENCV_STATIC:       : ${WITH_OPENCV_STATIC}")
  if(ENABLE_LITE_BACKEND)
    message(STATUS "  WITH_LITE_STATIC          : ${WITH_LITE_STATIC}")
+    message(STATUS "  WITH_LITE_FULL_API        : ${WITH_LITE_FULL_API}")
+    message(STATUS "  WITH_LITE_FP16            : ${WITH_LITE_FP16}")
  endif()
 endif()
 message(STATUS "  DEPENDENCY_LIBS           : ${FASTDEPLOY_LIBS}")
--- a/cmake/paddlelite.cmake
+++ b/cmake/paddlelite.cmake
@@ -24,7 +24,7 @@ set(PADDLELITE_INC_DIR
    "${PADDLELITE_INSTALL_DIR}/include"
    CACHE PATH "paddlelite include directory." FORCE)
 set(PADDLELITE_LIB_DIR
-    "${PADDLELITE_INSTALL_DIR}/lib"
+    "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}"
    CACHE PATH "paddlelite lib directory." FORCE)
 set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}")

@@ -32,39 +32,80 @@ set(PADDLELITE_URL_PREFIX "https://bj.bcebos.com/fastdeploy/third_libs")

 if(ANDROID)
  if(WITH_LITE_STATIC)
-      message(FATAL_ERROR "Doesn't support WTIH_LITE_STATIC=ON for Paddle-Lite now.")
+      message(FATAL_ERROR "Doesn't support WTIH_LITE_STATIC=ON for Paddle Lite now.")
+  endif()  
+  if(NOT WITH_LITE_FULL_API)
+      message(FATAL_ERROR "Doesn't support WITH_LITE_FULL_API=OFF for Paddle Lite now.")
  endif()
  # check ABI, toolchain
  if((NOT ANDROID_ABI MATCHES "armeabi-v7a") AND (NOT ANDROID_ABI MATCHES "arm64-v8a"))
-    message(FATAL_ERROR "FastDeploy with Paddle-Lite only support armeabi-v7a, arm64-v8a now.")
+    message(FATAL_ERROR "FastDeploy with Paddle Lite only support armeabi-v7a, arm64-v8a now.")
  endif()
  if(NOT ANDROID_TOOLCHAIN MATCHES "clang")
-     message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with Paddle-Lite backend, but found ${ANDROID_TOOLCHAIN}.")
+     message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with Paddle Lite backend, but found ${ANDROID_TOOLCHAIN}.")
  endif()  
 endif()

 if(WIN32 OR APPLE OR IOS)
-  message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle-Lite now.")
+  message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.")
 elseif(ANDROID)
-  set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-2.11.tgz")
+  set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-latest.tgz")
+  if(WITH_LITE_FP16) 
+    if(ANDROID_ABI MATCHES "arm64-v8a") 
+      set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-fp16-latest.tgz")
+    else()
+      message(FATAL_ERROR "Doesn't support fp16 for ${ANDROID_ABI} now !")
+    endif()
+  endif()
 else() # Linux
  if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
    set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-linux-arm64-20220920.tgz")
  else()
-    message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle-Lite.")
+    message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle Lite.")
  endif()
 endif()

 if(WIN32 OR APPLE OR IOS)
-  message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle-Lite now.")
+  message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.")
 elseif(ANDROID AND WITH_LITE_STATIC)
-  set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.a")
+  if(WITH_LITE_FULL_API)
+    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled..a")
+    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a")
  else()
+    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a")
+    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled.a")
+  endif()  
+else()
+  if(WITH_LITE_FULL_API)
    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so")
+    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so")
+  else()
+    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so")
+    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so")
+  endif()  
 endif()

 include_directories(${PADDLELITE_INC_DIR})

+if(ANDROID)
+  ExternalProject_Add(
+    ${PADDLELITE_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    URL ${PADDLELITE_URL}
+    PREFIX ${PADDLELITE_PREFIX_DIR}
+    DOWNLOAD_NO_PROGRESS 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} &&
+      ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} &&  
+      ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR}/lib &&
+      ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI} &&
+      ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include ${PADDLELITE_INC_DIR} && 
+      ${CMAKE_COMMAND} -E remove ${PADDLELITE_REMOVE_LIB}
+    BUILD_BYPRODUCTS ${PADDLELITE_LIB})
+else()  
  ExternalProject_Add(
    ${PADDLELITE_PROJECT}
    ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -78,9 +119,9 @@ ExternalProject_Add(
      ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} &&
      ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} &&
      ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib &&
-    ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include
-    ${PADDLELITE_INC_DIR}
+      ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include ${PADDLELITE_INC_DIR}
    BUILD_BYPRODUCTS ${PADDLELITE_LIB})
+endif()  

 add_library(external_paddle_lite STATIC IMPORTED GLOBAL)
 set_property(TARGET external_paddle_lite PROPERTY IMPORTED_LOCATION ${PADDLELITE_LIB})
--- a/cmake/summary.cmake
+++ b/cmake/summary.cmake
@@ -56,9 +56,12 @@ function(fastdeploy_summary)
    message(STATUS "  ANDROID_ABI               : ${ANDROID_ABI}")
    message(STATUS "  ANDROID_PLATFORM          : ${ANDROID_PLATFORM}")
    message(STATUS "  ANDROID_NDK               : ${ANDROID_NDK}")
+    message(STATUS "  ANDROID_NDK_VERSION       : ${CMAKE_ANDROID_NDK_VERSION}")
    message(STATUS "  WITH_OPENCV_STATIC:       : ${WITH_OPENCV_STATIC}")
    if(ENABLE_LITE_BACKEND)
      message(STATUS "  WITH_LITE_STATIC          : ${WITH_LITE_STATIC}")
+      message(STATUS "  WITH_LITE_FULL_API        : ${WITH_LITE_FULL_API}")
+      message(STATUS "  WITH_LITE_FP16            : ${WITH_LITE_FP16}")
    endif()
  endif() 
  if (${BUILD_FASTDEPLOY_PYTHON})
--- a/fastdeploy/backends/lite/lite_backend.cc
+++ b/fastdeploy/backends/lite/lite_backend.cc
@@ -40,7 +40,21 @@ FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) {
 }

 void LiteBackend::BuildOption(const LiteBackendOption& option) {
+  option_ = option;
  std::vector<paddle::lite_api::Place> valid_places;
+  if (option.enable_fp16) {
+    paddle::lite_api::MobileConfig check_fp16_config;
+    // Determine whether the device supports the FP16
+    // instruction set (or whether it is an arm device
+    // of the armv8.2 architecture)
+    supported_fp16_ = check_fp16_config.check_fp16_valid();
+    if (supported_fp16_) {
+      valid_places.push_back(
+          paddle::lite_api::Place{TARGET(kARM), PRECISION(kFP16)});
+    } else {
+      FDWARNING << "This device is not supported fp16, will skip fp16 option.";
+    }
+  }
  valid_places.push_back(
      paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)});
  config_.set_valid_places(valid_places);
@@ -68,6 +82,11 @@ bool LiteBackend::InitFromPaddle(const std::string& model_file,
  predictor_ =
      paddle::lite_api::CreatePaddlePredictor<paddle::lite_api::CxxConfig>(
          config_);
+  if (option_.optimized_model_dir != "") {
+    FDINFO << "Optimzed model dir is not empty, will save optimized model to: "
+           << option_.optimized_model_dir << std::endl;
+    predictor_->SaveOptimizedModel(option_.optimized_model_dir);
+  }

  inputs_desc_.clear();
  outputs_desc_.clear();
--- a/fastdeploy/backends/lite/lite_backend.h
+++ b/fastdeploy/backends/lite/lite_backend.h
@@ -34,7 +34,11 @@ struct LiteBackendOption {
  // 3: LITE_POWER_NO_BIND
  // 4: LITE_POWER_RAND_HIGH
  // 5: LITE_POWER_RAND_LOW
-  int power_mode = 0;
+  int power_mode = 3;
+  // enable fp16
+  bool enable_fp16 = false;
+  // optimized model dir for CxxConfig
+  std::string optimized_model_dir = "";
  // TODO(qiuyanjun): support more options for lite backend.
  // Such as fp16, different device target (kARM/kXPU/kNPU/...)
 };
@@ -69,5 +73,7 @@ class LiteBackend : public BaseBackend {
  std::vector<TensorInfo> inputs_desc_;
  std::vector<TensorInfo> outputs_desc_;
  std::map<std::string, int> inputs_order_;
+  LiteBackendOption option_;
+  bool supported_fp16_ = false;
 };
 }  // namespace fastdeploy
--- a/fastdeploy/pybind/runtime.cc
+++ b/fastdeploy/pybind/runtime.cc
@@ -34,6 +34,8 @@ void BindRuntime(pybind11::module& m) {
      .def("disable_paddle_log_info", &RuntimeOption::DisablePaddleLogInfo)
      .def("set_paddle_mkldnn_cache_size",
           &RuntimeOption::SetPaddleMKLDNNCacheSize)
+      .def("enable_lite_fp16", &RuntimeOption::EnableLiteFP16)
+      .def("disable_lite_fp16", &RuntimeOption::DisableLiteFP16)
      .def("set_lite_power_mode", &RuntimeOption::SetLitePowerMode)
      .def("set_trt_input_shape", &RuntimeOption::SetTrtInputShape)
      .def("set_trt_max_workspace_size", &RuntimeOption::SetTrtMaxWorkspaceSize)
--- a/fastdeploy/runtime.cc
+++ b/fastdeploy/runtime.cc
@@ -256,11 +256,24 @@ void RuntimeOption::SetPaddleMKLDNNCacheSize(int size) {
  pd_mkldnn_cache_size = size;
 }

-void RuntimeOption::SetLitePowerMode(int mode) {
-  FDASSERT(mode > -1, "Parameter mode must greater than -1.");
+void RuntimeOption::EnableLiteFP16() {
+  FDASSERT(false,
+           "FP16 with LiteBackend for FastDeploy is not fully supported, "
+           "please do not use it now!");
+  lite_enable_fp16 = true;
+}
+
+void RuntimeOption::DisableLiteFP16() { lite_enable_fp16 = false; }
+
+void RuntimeOption::SetLitePowerMode(LitePowerMode mode) {
  lite_power_mode = mode;
 }

+void RuntimeOption::SetLiteOptimizedModelDir(
+    const std::string& optimized_model_dir) {
+  lite_optimized_model_dir = optimized_model_dir;
+}
+
 void RuntimeOption::SetTrtInputShape(const std::string& input_name,
                                     const std::vector<int32_t>& min_shape,
                                     const std::vector<int32_t>& opt_shape,
@@ -503,7 +516,9 @@ void Runtime::CreateLiteBackend() {
 #ifdef ENABLE_LITE_BACKEND
  auto lite_option = LiteBackendOption();
  lite_option.threads = option.cpu_thread_num;
-  lite_option.power_mode = option.lite_power_mode;
+  lite_option.enable_fp16 = option.lite_enable_fp16;
+  lite_option.power_mode = static_cast<int>(option.lite_power_mode);
+  lite_option.optimized_model_dir = option.lite_optimized_model_dir;
  FDASSERT(option.model_format == ModelFormat::PADDLE,
           "LiteBackend only support model format of ModelFormat::PADDLE");
  backend_ = utils::make_unique<LiteBackend>();
--- a/fastdeploy/runtime.h
+++ b/fastdeploy/runtime.h
@@ -53,6 +53,16 @@ FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,
 FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,
                                         const ModelFormat& format);

+/*! Paddle Lite power mode for mobile device. */
+enum FASTDEPLOY_DECL LitePowerMode {
+  LITE_POWER_HIGH = 0,  ///< Use Lite Backend with high power mode
+  LITE_POWER_LOW = 1,  ///< Use Lite Backend with low power mode
+  LITE_POWER_FULL = 2,  ///< Use Lite Backend with full power mode
+  LITE_POWER_NO_BIND = 3,  ///< Use Lite Backend with no bind power mode
+  LITE_POWER_RAND_HIGH = 4,  ///< Use Lite Backend with rand high mode
+  LITE_POWER_RAND_LOW = 5  ///< Use Lite Backend with rand low power mode
+};
+
 FASTDEPLOY_DECL std::string Str(const Backend& b);
 FASTDEPLOY_DECL std::string Str(const ModelFormat& f);

@@ -135,11 +145,25 @@ struct FASTDEPLOY_DECL RuntimeOption {
   */
  void SetPaddleMKLDNNCacheSize(int size);

+  /**
+   * @brief Set optimzed model dir for Paddle Lite backend.
+   */
+  void SetLiteOptimizedModelDir(const std::string& optimized_model_dir);
+
+  /**
+   * @brief enable half precision while use paddle lite backend
+   */
+  void EnableLiteFP16();
+
+  /**
+   * @brief disable half precision, change to full precision(float32)
+   */
+  void DisableLiteFP16();
+
  /**
   * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details)
   */
-  void SetLitePowerMode(int mode);
-
+  void SetLitePowerMode(LitePowerMode mode);

  /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend
   *
@@ -196,7 +220,11 @@ struct FASTDEPLOY_DECL RuntimeOption {
  // 0: LITE_POWER_HIGH 1: LITE_POWER_LOW 2: LITE_POWER_FULL
  // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH
  // 5: LITE_POWER_RAND_LOW
-  int lite_power_mode = 0;
+  LitePowerMode lite_power_mode = LitePowerMode::LITE_POWER_NO_BIND;
+  // enable fp16 or not
+  bool lite_enable_fp16 = false;
+  // optimized model dir for CxxConfig
+  std::string lite_optimized_model_dir = "";

  // ======Only for Trt Backend=======
  std::map<std::string, std::vector<int32_t>> trt_max_shape;
--- a/python/fastdeploy/runtime.py
+++ b/python/fastdeploy/runtime.py
@@ -100,6 +100,12 @@ class RuntimeOption:
    def set_paddle_mkldnn_cache_size(self, cache_size):
        return self._option.set_paddle_mkldnn_cache_size(cache_size)

+    def enable_lite_fp16(self):
+        return self._option.enable_lite_fp16()
+
+    def disable_lite_fp16(self):
+        return self._option.disable_lite_fp16()
+
    def set_lite_power_mode(self, mode):
        return self._option.set_lite_power_mode(mode)