diff --git a/CMakeLists.txt b/CMakeLists.txt index 49598cd6c..9e0bfbda5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,9 @@ option(ENABLE_TEXT "Whether to enable text models usage." OFF) option(WITH_TESTING "Whether to compile with unittest." OFF) ######################### Options for Android cross compiling #################### option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF) -option(WITH_LITE_STATIC "Use Paddle-Lite static lib for Android." OFF) +option(WITH_LITE_STATIC "Use Paddle Lite static lib for Android." OFF) +option(WITH_LITE_FULL_API "Use Paddle Lite full API lib for Android." ON) +option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." OFF) # Please don't open this flag now, some bugs exists. # Only support Linux Now @@ -413,6 +415,11 @@ if(WIN32) ARCHIVE DESTINATION lib RUNTIME DESTINATION lib ) +elseif(ANDROID) + install( + TARGETS ${LIBRARY_NAME} + LIBRARY DESTINATION lib/${ANDROID_ABI} + ) else() install( TARGETS ${LIBRARY_NAME} diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index cd05d4acc..c59191198 100644 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -14,6 +14,8 @@ set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@) set(OPENVINO_VERSION @OPENVINO_VERSION@) set(WITH_LITE_STATIC @WITH_LITE_STATIC@) set(WITH_OPENCV_STATIC @WITH_OPENCV_STATIC@) +set(WITH_LITE_FULL_API @WITH_LITE_FULL_API@) +set(WITH_LITE_FP16 @WITH_LITE_FP16@) # set(ENABLE_OPENCV_CUDA @ENABLE_OPENCV_CUDA@) set(OPENCV_FILENAME @OPENCV_FILENAME@) set(OPENVINO_FILENAME @OPENVINO_FILENAME@) @@ -36,7 +38,7 @@ endif() if(ANDROID) add_library(fastdeploy STATIC IMPORTED GLOBAL) - set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/lib/lib${LIBRARY_NAME}.so) + set_property(TARGET fastdeploy PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/lib/${ANDROID_ABI}/lib${LIBRARY_NAME}.so) list(APPEND FASTDEPLOY_LIBS fastdeploy) else() find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib NO_DEFAULT_PATH) @@ -85,10 +87,19 @@ endif() if(ENABLE_LITE_BACKEND) set(LITE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${PADDLELITE_FILENAME}) if(ANDROID) - add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL) - set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/libpaddle_full_api_shared.so) - list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared) + if(NOT WITH_LITE_STATIC) + if(WITH_LITE_FULL_API) + add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL) + set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_full_api_shared.so) + list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared) + else() + add_library(paddle_light_api_shared STATIC IMPORTED GLOBAL) + set_property(TARGET paddle_light_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_light_api_shared.so) + list(APPEND FASTDEPLOY_LIBS paddle_light_api_shared) + endif() + endif() else() + # Linux/Mac/Win/... find_library(LITE_LIB paddle_full_api_shared ${LITE_DIR}/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${LITE_LIB}) endif() @@ -123,10 +134,10 @@ if(WITH_GPU) endif() if(ENABLE_VISION) - if (OPENCV_DIRECTORY) + if(OPENCV_DIRECTORY) set(OpenCV_DIR ${OPENCV_DIRECTORY}) else() - if (ANDROID) + if(ANDROID) set(OpenCV_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENCV_FILENAME}/sdk/native/jni) else() set(OpenCV_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${OPENCV_FILENAME}) @@ -231,6 +242,8 @@ if(ANDROID) message(STATUS " WITH_OPENCV_STATIC: : ${WITH_OPENCV_STATIC}") if(ENABLE_LITE_BACKEND) message(STATUS " WITH_LITE_STATIC : ${WITH_LITE_STATIC}") + message(STATUS " WITH_LITE_FULL_API : ${WITH_LITE_FULL_API}") + message(STATUS " WITH_LITE_FP16 : ${WITH_LITE_FP16}") endif() endif() message(STATUS " DEPENDENCY_LIBS : ${FASTDEPLOY_LIBS}") diff --git a/cmake/paddlelite.cmake b/cmake/paddlelite.cmake index c7bbb091a..a2489c34b 100644 --- a/cmake/paddlelite.cmake +++ b/cmake/paddlelite.cmake @@ -24,7 +24,7 @@ set(PADDLELITE_INC_DIR "${PADDLELITE_INSTALL_DIR}/include" CACHE PATH "paddlelite include directory." FORCE) set(PADDLELITE_LIB_DIR - "${PADDLELITE_INSTALL_DIR}/lib" + "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}" CACHE PATH "paddlelite lib directory." FORCE) set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}") @@ -32,55 +32,96 @@ set(PADDLELITE_URL_PREFIX "https://bj.bcebos.com/fastdeploy/third_libs") if(ANDROID) if(WITH_LITE_STATIC) - message(FATAL_ERROR "Doesn't support WTIH_LITE_STATIC=ON for Paddle-Lite now.") + message(FATAL_ERROR "Doesn't support WTIH_LITE_STATIC=ON for Paddle Lite now.") + endif() + if(NOT WITH_LITE_FULL_API) + message(FATAL_ERROR "Doesn't support WITH_LITE_FULL_API=OFF for Paddle Lite now.") endif() # check ABI, toolchain if((NOT ANDROID_ABI MATCHES "armeabi-v7a") AND (NOT ANDROID_ABI MATCHES "arm64-v8a")) - message(FATAL_ERROR "FastDeploy with Paddle-Lite only support armeabi-v7a, arm64-v8a now.") + message(FATAL_ERROR "FastDeploy with Paddle Lite only support armeabi-v7a, arm64-v8a now.") endif() if(NOT ANDROID_TOOLCHAIN MATCHES "clang") - message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with Paddle-Lite backend, but found ${ANDROID_TOOLCHAIN}.") + message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with Paddle Lite backend, but found ${ANDROID_TOOLCHAIN}.") endif() endif() if(WIN32 OR APPLE OR IOS) - message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle-Lite now.") + message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.") elseif(ANDROID) - set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-2.11.tgz") + set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-latest.tgz") + if(WITH_LITE_FP16) + if(ANDROID_ABI MATCHES "arm64-v8a") + set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-fp16-latest.tgz") + else() + message(FATAL_ERROR "Doesn't support fp16 for ${ANDROID_ABI} now !") + endif() + endif() else() # Linux if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-linux-arm64-20220920.tgz") else() - message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle-Lite.") + message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle Lite.") endif() endif() if(WIN32 OR APPLE OR IOS) - message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle-Lite now.") + message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.") elseif(ANDROID AND WITH_LITE_STATIC) - set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.a") + if(WITH_LITE_FULL_API) + set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled..a") + set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a") + else() + set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a") + set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled.a") + endif() else() - set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so") + if(WITH_LITE_FULL_API) + set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so") + set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so") + else() + set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so") + set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so") + endif() endif() include_directories(${PADDLELITE_INC_DIR}) -ExternalProject_Add( - ${PADDLELITE_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${PADDLELITE_URL} - PREFIX ${PADDLELITE_PREFIX_DIR} - DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - UPDATE_COMMAND "" - INSTALL_COMMAND - ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} && - ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} && - ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib && - ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include - ${PADDLELITE_INC_DIR} - BUILD_BYPRODUCTS ${PADDLELITE_LIB}) +if(ANDROID) + ExternalProject_Add( + ${PADDLELITE_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${PADDLELITE_URL} + PREFIX ${PADDLELITE_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI} && + ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include ${PADDLELITE_INC_DIR} && + ${CMAKE_COMMAND} -E remove ${PADDLELITE_REMOVE_LIB} + BUILD_BYPRODUCTS ${PADDLELITE_LIB}) +else() + ExternalProject_Add( + ${PADDLELITE_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${PADDLELITE_URL} + PREFIX ${PADDLELITE_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include ${PADDLELITE_INC_DIR} + BUILD_BYPRODUCTS ${PADDLELITE_LIB}) +endif() add_library(external_paddle_lite STATIC IMPORTED GLOBAL) set_property(TARGET external_paddle_lite PROPERTY IMPORTED_LOCATION ${PADDLELITE_LIB}) diff --git a/cmake/summary.cmake b/cmake/summary.cmake index cf2836384..e522a130a 100644 --- a/cmake/summary.cmake +++ b/cmake/summary.cmake @@ -56,9 +56,12 @@ function(fastdeploy_summary) message(STATUS " ANDROID_ABI : ${ANDROID_ABI}") message(STATUS " ANDROID_PLATFORM : ${ANDROID_PLATFORM}") message(STATUS " ANDROID_NDK : ${ANDROID_NDK}") + message(STATUS " ANDROID_NDK_VERSION : ${CMAKE_ANDROID_NDK_VERSION}") message(STATUS " WITH_OPENCV_STATIC: : ${WITH_OPENCV_STATIC}") if(ENABLE_LITE_BACKEND) message(STATUS " WITH_LITE_STATIC : ${WITH_LITE_STATIC}") + message(STATUS " WITH_LITE_FULL_API : ${WITH_LITE_FULL_API}") + message(STATUS " WITH_LITE_FP16 : ${WITH_LITE_FP16}") endif() endif() if (${BUILD_FASTDEPLOY_PYTHON}) diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc index 2d96ec4f0..689595865 100644 --- a/fastdeploy/backends/lite/lite_backend.cc +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -40,7 +40,21 @@ FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { } void LiteBackend::BuildOption(const LiteBackendOption& option) { + option_ = option; std::vector valid_places; + if (option.enable_fp16) { + paddle::lite_api::MobileConfig check_fp16_config; + // Determine whether the device supports the FP16 + // instruction set (or whether it is an arm device + // of the armv8.2 architecture) + supported_fp16_ = check_fp16_config.check_fp16_valid(); + if (supported_fp16_) { + valid_places.push_back( + paddle::lite_api::Place{TARGET(kARM), PRECISION(kFP16)}); + } else { + FDWARNING << "This device is not supported fp16, will skip fp16 option."; + } + } valid_places.push_back( paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); config_.set_valid_places(valid_places); @@ -68,6 +82,11 @@ bool LiteBackend::InitFromPaddle(const std::string& model_file, predictor_ = paddle::lite_api::CreatePaddlePredictor( config_); + if (option_.optimized_model_dir != "") { + FDINFO << "Optimzed model dir is not empty, will save optimized model to: " + << option_.optimized_model_dir << std::endl; + predictor_->SaveOptimizedModel(option_.optimized_model_dir); + } inputs_desc_.clear(); outputs_desc_.clear(); diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h index 5984c4d34..fed33e285 100644 --- a/fastdeploy/backends/lite/lite_backend.h +++ b/fastdeploy/backends/lite/lite_backend.h @@ -34,7 +34,11 @@ struct LiteBackendOption { // 3: LITE_POWER_NO_BIND // 4: LITE_POWER_RAND_HIGH // 5: LITE_POWER_RAND_LOW - int power_mode = 0; + int power_mode = 3; + // enable fp16 + bool enable_fp16 = false; + // optimized model dir for CxxConfig + std::string optimized_model_dir = ""; // TODO(qiuyanjun): support more options for lite backend. // Such as fp16, different device target (kARM/kXPU/kNPU/...) }; @@ -69,5 +73,7 @@ class LiteBackend : public BaseBackend { std::vector inputs_desc_; std::vector outputs_desc_; std::map inputs_order_; + LiteBackendOption option_; + bool supported_fp16_ = false; }; } // namespace fastdeploy diff --git a/fastdeploy/pybind/runtime.cc b/fastdeploy/pybind/runtime.cc index 44647aa29..e6c0a8d58 100644 --- a/fastdeploy/pybind/runtime.cc +++ b/fastdeploy/pybind/runtime.cc @@ -34,6 +34,8 @@ void BindRuntime(pybind11::module& m) { .def("disable_paddle_log_info", &RuntimeOption::DisablePaddleLogInfo) .def("set_paddle_mkldnn_cache_size", &RuntimeOption::SetPaddleMKLDNNCacheSize) + .def("enable_lite_fp16", &RuntimeOption::EnableLiteFP16) + .def("disable_lite_fp16", &RuntimeOption::DisableLiteFP16) .def("set_lite_power_mode", &RuntimeOption::SetLitePowerMode) .def("set_trt_input_shape", &RuntimeOption::SetTrtInputShape) .def("set_trt_max_workspace_size", &RuntimeOption::SetTrtMaxWorkspaceSize) diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 01973ef38..b4c4cd858 100644 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -256,11 +256,24 @@ void RuntimeOption::SetPaddleMKLDNNCacheSize(int size) { pd_mkldnn_cache_size = size; } -void RuntimeOption::SetLitePowerMode(int mode) { - FDASSERT(mode > -1, "Parameter mode must greater than -1."); +void RuntimeOption::EnableLiteFP16() { + FDASSERT(false, + "FP16 with LiteBackend for FastDeploy is not fully supported, " + "please do not use it now!"); + lite_enable_fp16 = true; +} + +void RuntimeOption::DisableLiteFP16() { lite_enable_fp16 = false; } + +void RuntimeOption::SetLitePowerMode(LitePowerMode mode) { lite_power_mode = mode; } +void RuntimeOption::SetLiteOptimizedModelDir( + const std::string& optimized_model_dir) { + lite_optimized_model_dir = optimized_model_dir; +} + void RuntimeOption::SetTrtInputShape(const std::string& input_name, const std::vector& min_shape, const std::vector& opt_shape, @@ -503,7 +516,9 @@ void Runtime::CreateLiteBackend() { #ifdef ENABLE_LITE_BACKEND auto lite_option = LiteBackendOption(); lite_option.threads = option.cpu_thread_num; - lite_option.power_mode = option.lite_power_mode; + lite_option.enable_fp16 = option.lite_enable_fp16; + lite_option.power_mode = static_cast(option.lite_power_mode); + lite_option.optimized_model_dir = option.lite_optimized_model_dir; FDASSERT(option.model_format == ModelFormat::PADDLE, "LiteBackend only support model format of ModelFormat::PADDLE"); backend_ = utils::make_unique(); diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index 7cd8bf6fe..3a17e8fba 100644 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -53,6 +53,16 @@ FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ModelFormat& format); +/*! Paddle Lite power mode for mobile device. */ +enum FASTDEPLOY_DECL LitePowerMode { + LITE_POWER_HIGH = 0, ///< Use Lite Backend with high power mode + LITE_POWER_LOW = 1, ///< Use Lite Backend with low power mode + LITE_POWER_FULL = 2, ///< Use Lite Backend with full power mode + LITE_POWER_NO_BIND = 3, ///< Use Lite Backend with no bind power mode + LITE_POWER_RAND_HIGH = 4, ///< Use Lite Backend with rand high mode + LITE_POWER_RAND_LOW = 5 ///< Use Lite Backend with rand low power mode +}; + FASTDEPLOY_DECL std::string Str(const Backend& b); FASTDEPLOY_DECL std::string Str(const ModelFormat& f); @@ -135,11 +145,25 @@ struct FASTDEPLOY_DECL RuntimeOption { */ void SetPaddleMKLDNNCacheSize(int size); + /** + * @brief Set optimzed model dir for Paddle Lite backend. + */ + void SetLiteOptimizedModelDir(const std::string& optimized_model_dir); + + /** + * @brief enable half precision while use paddle lite backend + */ + void EnableLiteFP16(); + + /** + * @brief disable half precision, change to full precision(float32) + */ + void DisableLiteFP16(); + /** * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details) */ - void SetLitePowerMode(int mode); - + void SetLitePowerMode(LitePowerMode mode); /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend * @@ -196,7 +220,11 @@ struct FASTDEPLOY_DECL RuntimeOption { // 0: LITE_POWER_HIGH 1: LITE_POWER_LOW 2: LITE_POWER_FULL // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH // 5: LITE_POWER_RAND_LOW - int lite_power_mode = 0; + LitePowerMode lite_power_mode = LitePowerMode::LITE_POWER_NO_BIND; + // enable fp16 or not + bool lite_enable_fp16 = false; + // optimized model dir for CxxConfig + std::string lite_optimized_model_dir = ""; // ======Only for Trt Backend======= std::map> trt_max_shape; diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index abd1d4bac..7a16844a3 100644 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -100,6 +100,12 @@ class RuntimeOption: def set_paddle_mkldnn_cache_size(self, cache_size): return self._option.set_paddle_mkldnn_cache_size(cache_size) + def enable_lite_fp16(self): + return self._option.enable_lite_fp16() + + def disable_lite_fp16(self): + return self._option.disable_lite_fp16() + def set_lite_power_mode(self, mode): return self._option.set_lite_power_mode(mode)