diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e2621f6c..e79e253a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,9 @@ if(ENABLE_TRT_BACKEND) if(NOT WITH_GPU) message(FATAL_ERROR "While -DENABLE_TRT_BACKEND=ON, must set -DWITH_GPU=ON, but now it's OFF") endif() + if(NOT TRT_DIRECTORY) + message(FATAL_ERROR "While -DENABLE_TRT_BACKEND=ON, must define -DTRT_DIRECTORY, e.g -DTRT_DIRECTORY=/Downloads/TensorRT-8.4") + endif() add_definitions(-DENABLE_TRT_BACKEND) include_directories(${TRT_DIRECTORY}/include) include_directories(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/common) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 50910f073..8ad4fc6e6 100644 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED (VERSION 3.16) +CMAKE_MINIMUM_REQUIRED (VERSION 3.12) set(WITH_GPU @WITH_GPU@) set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@) @@ -19,16 +19,16 @@ if(NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 11) endif() -find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib) +find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${FDLIB}) if(ENABLE_ORT_BACKEND) - find_library(ORT_LIB onnxruntime ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib) + find_library(ORT_LIB onnxruntime ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${ORT_LIB}) endif() if(ENABLE_PADDLE_BACKEND) - find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib) + find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib NO_DEFAULT_PATH) if(WIN32) set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/mkldnn.lib") set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5.lib") @@ -57,10 +57,10 @@ if(WITH_GPU) list(APPEND FASTDEPLOY_LIBS ${CUDA_LIB}) if (ENABLE_TRT_BACKEND) - find_library(TRT_INFER_LIB nvinfer ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib) - find_library(TRT_ONNX_LIB nvonnxparser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib) - find_library(TRT_CAFFE_LIB nvcaffe_parser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib) - find_library(TRT_PLUGIN_LIB nvinfer_plugin ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib) + find_library(TRT_INFER_LIB nvinfer ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH) + find_library(TRT_ONNX_LIB nvonnxparser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH) + find_library(TRT_CAFFE_LIB nvcaffe_parser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH) + find_library(TRT_PLUGIN_LIB nvinfer_plugin ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB}) endif() endif() @@ -72,17 +72,17 @@ if(ENABLE_VISION) list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS}) list(APPEND FASTDEPLOY_LIBS ${OpenCV_LIBS}) else() - find_library(OPENCV_CORE_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) - find_library(OPENCV_HIGHGUI_LIB opencv_highgui ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) - find_library(OPENCV_IMGPROC_LIB opencv_imgproc ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) - find_library(OPENCV_IMGCODESC_LIB opencv_imgcodecs ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) + find_library(OPENCV_CORE_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) + find_library(OPENCV_HIGHGUI_LIB opencv_highgui ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) + find_library(OPENCV_IMGPROC_LIB opencv_imgproc ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) + find_library(OPENCV_IMGCODESC_LIB opencv_imgcodecs ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${OPENCV_CORE_LIB} ${OPENCV_HIGHGUI_LIB} ${OPENCV_IMGPROC_LIB} ${OPENCV_IMGCODESC_LIB}) list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/include) if(ENABLE_OPENCV_CUDA) - find_library(OPENCV_CUDAARITHM_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) - find_library(OPENCV_CUDAIMGPROC_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) - find_library(OPENCV_CUDAWARPING_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib) + find_library(OPENCV_CUDAARITHM_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) + find_library(OPENCV_CUDAIMGPROC_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) + find_library(OPENCV_CUDAWARPING_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${OPENCV_CUDAARITHM_LIB} ${OPENCV_CUDAIMGPROC_LIB} ${OPENCV_CUDAWARPING_LIB}) endif() endif() @@ -93,7 +93,7 @@ if (ENABLE_TEXT) endif() if(ENABLE_PADDLE_FRONTEND) - find_library(PADDLE2ONNX_LIB paddle2onnx ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib) + find_library(PADDLE2ONNX_LIB paddle2onnx ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib NO_DEFAULT_PATH) list(APPEND FASTDEPLOY_LIBS ${PADDLE2ONNX_LIB}) endif() diff --git a/csrc/fastdeploy/fastdeploy_model.cc b/csrc/fastdeploy/fastdeploy_model.cc index 31781ac3a..a60d7cae5 100644 --- a/csrc/fastdeploy/fastdeploy_model.cc +++ b/csrc/fastdeploy/fastdeploy_model.cc @@ -165,7 +165,52 @@ bool FastDeployModel::CreateGpuBackend() { bool FastDeployModel::Infer(std::vector& input_tensors, std::vector* output_tensors) { - return runtime_->Infer(input_tensors, output_tensors); + TimeCounter tc; + if (enable_record_time_of_runtime_) { + tc.Start(); + } + auto ret = runtime_->Infer(input_tensors, output_tensors); + if (enable_record_time_of_runtime_) { + tc.End(); + if (time_of_runtime_.size() > 50000) { + FDWARNING << "There are already 50000 records of runtime, will force to " + "disable record time of runtime now." + << std::endl; + enable_record_time_of_runtime_ = false; + } + time_of_runtime_.push_back(tc.Duration()); + } + return ret; +} + +void FastDeployModel::PrintStatisInfoOfRuntime() { + if (time_of_runtime_.size() < 10) { + FDWARNING << "PrintStatisInfoOfRuntime require the runtime ran 10 times at " + "least, but now you only ran " + << time_of_runtime_.size() << " times." << std::endl; + return; + } + double warmup_time = 0.0; + double remain_time = 0.0; + int warmup_iter = time_of_runtime_.size() / 5; + for (size_t i = 0; i < time_of_runtime_.size(); ++i) { + if (i < warmup_iter) { + warmup_time += time_of_runtime_[i]; + } else { + remain_time += time_of_runtime_[i]; + } + } + double avg_time = remain_time / (time_of_runtime_.size() - warmup_iter); + std::cout << "============= Runtime Statis Info(" << ModelName() + << ") =============" << std::endl; + std::cout << "Total iterations: " << time_of_runtime_.size() << std::endl; + std::cout << "Total time of runtime: " << warmup_time + remain_time << "s." + << std::endl; + std::cout << "Warmup iterations: " << warmup_iter << std::endl; + std::cout << "Total time of runtime in warmup step: " << warmup_time << "s." + << std::endl; + std::cout << "Average time of runtime exclude warmup step: " << avg_time + << "s." << std::endl; } void FastDeployModel::EnableDebug() { diff --git a/csrc/fastdeploy/fastdeploy_model.h b/csrc/fastdeploy/fastdeploy_model.h index df83ac525..3fc851f6c 100644 --- a/csrc/fastdeploy/fastdeploy_model.h +++ b/csrc/fastdeploy/fastdeploy_model.h @@ -43,13 +43,30 @@ class FASTDEPLOY_DECL FastDeployModel { return runtime_initialized_ && initialized; } + virtual void EnableRecordTimeOfRuntime() { + time_of_runtime_.clear(); + std::vector().swap(time_of_runtime_); + enable_record_time_of_runtime_ = true; + } + + virtual void DisableRecordTimeOfRuntime() { + enable_record_time_of_runtime_ = false; + } + + virtual void PrintStatisInfoOfRuntime(); + virtual void EnableDebug(); virtual bool DebugEnabled(); private: std::unique_ptr runtime_; bool runtime_initialized_ = false; + // whether to record inference time + bool enable_record_time_of_runtime_ = false; bool debug_ = false; + + // record inference time for backend + std::vector time_of_runtime_; }; #define TIMERECORD_START(id) \ diff --git a/csrc/fastdeploy/pybind/fastdeploy_model.cc b/csrc/fastdeploy/pybind/fastdeploy_model.cc index b59c0fd0f..0b138fa60 100644 --- a/csrc/fastdeploy/pybind/fastdeploy_model.cc +++ b/csrc/fastdeploy/pybind/fastdeploy_model.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "fastdeploy/pybind/main.h" #include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/pybind/main.h" namespace fastdeploy { @@ -25,6 +25,11 @@ void BindFDModel(pybind11::module& m) { .def("num_outputs_of_runtime", &FastDeployModel::NumOutputsOfRuntime) .def("input_info_of_runtime", &FastDeployModel::InputInfoOfRuntime) .def("output_info_of_runtime", &FastDeployModel::OutputInfoOfRuntime) + .def("enable_record_time_of_runtime", &FastDeployModel::EnableRecordTimeOfRuntime) + .def("disable_record_time_of_runtime", + &FastDeployModel::DisableRecordTimeOfRuntime) + .def("print_statis_info_of_runtime", + &FastDeployModel::PrintStatisInfoOfRuntime) .def("initialized", &FastDeployModel::Initialized) .def_readwrite("runtime_option", &FastDeployModel::runtime_option) .def_readwrite("valid_cpu_backends", &FastDeployModel::valid_cpu_backends) @@ -32,4 +37,4 @@ void BindFDModel(pybind11::module& m) { &FastDeployModel::valid_gpu_backends); } -} // namespace fastdeploy +} // namespace fastdeploy diff --git a/examples/vision/detection/README.md b/examples/vision/detection/README.md new file mode 100644 index 000000000..f6e73e659 --- /dev/null +++ b/examples/vision/detection/README.md @@ -0,0 +1,13 @@ +# 目标检测模型 + +FastDeploy目前支持如下目标检测模型部署 + +| 模型 | 说明 | 模型格式 | 版本 | +| :--- | :--- | :------- | :--- | +| [PaddleDetection/PPYOLOE](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PPYOLOE系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [PaddleDetection/PicoDet](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PicoDet系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [PaddleDetection/YOLOX](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | Paddle版本的YOLOX系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [PaddleDetection/YOLOv3](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | YOLOv3系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [PaddleDetection/PPYOLO](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PPYOLO系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [PaddleDetection/FasterRCNN](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | FasterRCNN系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) | +| [WongKinYiu/YOLOv7](https://github.com/WongKinYiu/yolov7) | YOLOv7、YOLOv7-X等系列模型 | ONNX | [v0.1](https://github.com/WongKinYiu/yolov7/tree/v0.1) | diff --git a/external/paddle_inference.cmake b/external/paddle_inference.cmake index 8894209f4..897823f98 100644 --- a/external/paddle_inference.cmake +++ b/external/paddle_inference.cmake @@ -49,7 +49,7 @@ else() endif(WIN32) set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/paddle2onnx/libs/") -set(PADDLEINFERENCE_VERSION "2.3.1") +set(PADDLEINFERENCE_VERSION "2.3.2") if(WIN32) message(FATAL_ERROR "Paddle Backend doesn't support Windows now.") set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-${PADDLEINFERENCE_VERSION}.zip") diff --git a/fastdeploy/model.py b/fastdeploy/model.py index dd48ae18f..a04b65d1a 100644 --- a/fastdeploy/model.py +++ b/fastdeploy/model.py @@ -45,6 +45,15 @@ class FastDeployModel: index, self.num_outputs()) return self._model.get_output_info(index) + def enable_record_time_of_runtime(self): + self._model.enable_record_time_of_runtime() + + def disable_record_time_of_runtime(self): + self._model.disable_record_time_of_runtime() + + def print_statis_info_of_runtime(self): + self._model.print_statis_info_of_runtime() + @property def runtime_option(self): return self._model.runtime_option if self._model is not None else None