Add time statis functions for fastdeploy model (#96)

* Add time statis functions for fastdeploy model

* Create README.md

* Update README.md

* Update fastdeploy_model.cc

* Update fastdeploy_model.cc

CMakeLists.txt

@@ -159,6 +159,9 @@ if(ENABLE_TRT_BACKEND)
   if(NOT WITH_GPU)
     message(FATAL_ERROR "While -DENABLE_TRT_BACKEND=ON, must set -DWITH_GPU=ON, but now it's OFF")
   endif()
+  if(NOT TRT_DIRECTORY)
+    message(FATAL_ERROR "While -DENABLE_TRT_BACKEND=ON, must define -DTRT_DIRECTORY, e.g -DTRT_DIRECTORY=/Downloads/TensorRT-8.4")
+  endif()
   add_definitions(-DENABLE_TRT_BACKEND)
   include_directories(${TRT_DIRECTORY}/include)
   include_directories(${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/common)

FastDeploy.cmake.in

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED (VERSION 3.16)
+CMAKE_MINIMUM_REQUIRED (VERSION 3.12)
 
 set(WITH_GPU @WITH_GPU@)
 set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@)
@@ -19,16 +19,16 @@ if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 11)
 endif()
 
-find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib)
+find_library(FDLIB ${LIBRARY_NAME} ${CMAKE_CURRENT_LIST_DIR}/lib NO_DEFAULT_PATH)
 list(APPEND FASTDEPLOY_LIBS ${FDLIB})
 
 if(ENABLE_ORT_BACKEND)
-  find_library(ORT_LIB onnxruntime ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib)
+  find_library(ORT_LIB onnxruntime ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib NO_DEFAULT_PATH)
   list(APPEND FASTDEPLOY_LIBS ${ORT_LIB})
 endif()
 
 if(ENABLE_PADDLE_BACKEND)
-  find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib)
+  find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib NO_DEFAULT_PATH)
   if(WIN32)
     set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/mkldnn.lib")
     set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5.lib")
@@ -57,10 +57,10 @@ if(WITH_GPU)
   list(APPEND FASTDEPLOY_LIBS ${CUDA_LIB})
 
   if (ENABLE_TRT_BACKEND)
-    find_library(TRT_INFER_LIB nvinfer ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib)
+    find_library(TRT_INFER_LIB nvinfer ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-    find_library(TRT_ONNX_LIB nvonnxparser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib)
+    find_library(TRT_ONNX_LIB nvonnxparser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-    find_library(TRT_CAFFE_LIB nvcaffe_parser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib)
+    find_library(TRT_CAFFE_LIB nvcaffe_parser ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
-    find_library(TRT_PLUGIN_LIB nvinfer_plugin ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib)
+    find_library(TRT_PLUGIN_LIB nvinfer_plugin ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/tensorrt/lib NO_DEFAULT_PATH)
     list(APPEND FASTDEPLOY_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB})
   endif()
 endif()
@@ -72,17 +72,17 @@ if(ENABLE_VISION)
     list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS})
     list(APPEND FASTDEPLOY_LIBS ${OpenCV_LIBS})
   else()
-    find_library(OPENCV_CORE_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+    find_library(OPENCV_CORE_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
-    find_library(OPENCV_HIGHGUI_LIB opencv_highgui ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+    find_library(OPENCV_HIGHGUI_LIB opencv_highgui ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
-    find_library(OPENCV_IMGPROC_LIB opencv_imgproc ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+    find_library(OPENCV_IMGPROC_LIB opencv_imgproc ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
-    find_library(OPENCV_IMGCODESC_LIB opencv_imgcodecs ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+    find_library(OPENCV_IMGCODESC_LIB opencv_imgcodecs ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
     list(APPEND FASTDEPLOY_LIBS ${OPENCV_CORE_LIB} ${OPENCV_HIGHGUI_LIB} ${OPENCV_IMGPROC_LIB} ${OPENCV_IMGCODESC_LIB})
     list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/include)
 
     if(ENABLE_OPENCV_CUDA)
-      find_library(OPENCV_CUDAARITHM_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+      find_library(OPENCV_CUDAARITHM_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
-      find_library(OPENCV_CUDAIMGPROC_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+      find_library(OPENCV_CUDAIMGPROC_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
-      find_library(OPENCV_CUDAWARPING_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib)
+      find_library(OPENCV_CUDAWARPING_LIB opencv_core ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/opencv/lib NO_DEFAULT_PATH)
       list(APPEND FASTDEPLOY_LIBS ${OPENCV_CUDAARITHM_LIB} ${OPENCV_CUDAIMGPROC_LIB} ${OPENCV_CUDAWARPING_LIB})
     endif()
   endif()
@@ -93,7 +93,7 @@ if (ENABLE_TEXT)
 endif()
 
 if(ENABLE_PADDLE_FRONTEND)
-  find_library(PADDLE2ONNX_LIB paddle2onnx  ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib)
+  find_library(PADDLE2ONNX_LIB paddle2onnx  ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib NO_DEFAULT_PATH)
   list(APPEND FASTDEPLOY_LIBS ${PADDLE2ONNX_LIB})
 endif()
 

csrc/fastdeploy/fastdeploy_model.cc

@@ -165,7 +165,52 @@ bool FastDeployModel::CreateGpuBackend() {
 
 bool FastDeployModel::Infer(std::vector<FDTensor>& input_tensors,
                             std::vector<FDTensor>* output_tensors) {
-  return runtime_->Infer(input_tensors, output_tensors);
+  TimeCounter tc;
+  if (enable_record_time_of_runtime_) {
+    tc.Start();
+  }
+  auto ret = runtime_->Infer(input_tensors, output_tensors);
+  if (enable_record_time_of_runtime_) {
+    tc.End();
+    if (time_of_runtime_.size() > 50000) {
+      FDWARNING << "There are already 50000 records of runtime, will force to "
+                   "disable record time of runtime now."
+                << std::endl;
+      enable_record_time_of_runtime_ = false;
+    }
+    time_of_runtime_.push_back(tc.Duration());
+  }
+  return ret;
+}
+
+void FastDeployModel::PrintStatisInfoOfRuntime() {
+  if (time_of_runtime_.size() < 10) {
+    FDWARNING << "PrintStatisInfoOfRuntime require the runtime ran 10 times at "
+                 "least, but now you only ran "
+              << time_of_runtime_.size() << " times." << std::endl;
+    return;
+  }
+  double warmup_time = 0.0;
+  double remain_time = 0.0;
+  int warmup_iter = time_of_runtime_.size() / 5;
+  for (size_t i = 0; i < time_of_runtime_.size(); ++i) {
+    if (i < warmup_iter) {
+      warmup_time += time_of_runtime_[i];
+    } else {
+      remain_time += time_of_runtime_[i];
+    }
+  }
+  double avg_time = remain_time / (time_of_runtime_.size() - warmup_iter);
+  std::cout << "============= Runtime Statis Info(" << ModelName()
+            << ") =============" << std::endl;
+  std::cout << "Total iterations: " << time_of_runtime_.size() << std::endl;
+  std::cout << "Total time of runtime: " << warmup_time + remain_time << "s."
+            << std::endl;
+  std::cout << "Warmup iterations: " << warmup_iter << std::endl;
+  std::cout << "Total time of runtime in warmup step: " << warmup_time << "s."
+            << std::endl;
+  std::cout << "Average time of runtime exclude warmup step: " << avg_time
+            << "s." << std::endl;
 }
 
 void FastDeployModel::EnableDebug() {

csrc/fastdeploy/fastdeploy_model.h

@@ -43,13 +43,30 @@ class FASTDEPLOY_DECL FastDeployModel {
     return runtime_initialized_ && initialized;
   }
 
+  virtual void EnableRecordTimeOfRuntime() {
+    time_of_runtime_.clear();
+    std::vector<double>().swap(time_of_runtime_);
+    enable_record_time_of_runtime_ = true;
+  }
+
+  virtual void DisableRecordTimeOfRuntime() {
+    enable_record_time_of_runtime_ = false;
+  }
+
+  virtual void PrintStatisInfoOfRuntime();
+
   virtual void EnableDebug();
   virtual bool DebugEnabled();
 
  private:
   std::unique_ptr<Runtime> runtime_;
   bool runtime_initialized_ = false;
+  // whether to record inference time
+  bool enable_record_time_of_runtime_ = false;
   bool debug_ = false;
+
+  // record inference time for backend
+  std::vector<double> time_of_runtime_;
 };
 
 #define TIMERECORD_START(id) \

csrc/fastdeploy/pybind/fastdeploy_model.cc

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "fastdeploy/pybind/main.h"
 #include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/pybind/main.h"
 
 namespace fastdeploy {
 
@@ -25,6 +25,11 @@ void BindFDModel(pybind11::module& m) {
       .def("num_outputs_of_runtime", &FastDeployModel::NumOutputsOfRuntime)
       .def("input_info_of_runtime", &FastDeployModel::InputInfoOfRuntime)
       .def("output_info_of_runtime", &FastDeployModel::OutputInfoOfRuntime)
+      .def("enable_record_time_of_runtime", &FastDeployModel::EnableRecordTimeOfRuntime)
+      .def("disable_record_time_of_runtime",
+           &FastDeployModel::DisableRecordTimeOfRuntime)
+      .def("print_statis_info_of_runtime",
+           &FastDeployModel::PrintStatisInfoOfRuntime)
       .def("initialized", &FastDeployModel::Initialized)
       .def_readwrite("runtime_option", &FastDeployModel::runtime_option)
       .def_readwrite("valid_cpu_backends", &FastDeployModel::valid_cpu_backends)

examples/vision/detection/README.md

@@ -0,0 +1,13 @@
+# 目标检测模型
+
+FastDeploy目前支持如下目标检测模型部署
+
+| 模型 | 说明 | 模型格式 | 版本 |
+| :--- | :--- | :------- | :--- |
+| [PaddleDetection/PPYOLOE](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PPYOLOE系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [PaddleDetection/PicoDet](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PicoDet系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [PaddleDetection/YOLOX](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | Paddle版本的YOLOX系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [PaddleDetection/YOLOv3](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | YOLOv3系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [PaddleDetection/PPYOLO](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | PPYOLO系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [PaddleDetection/FasterRCNN](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe) | FasterRCNN系列模型 | Paddle | [Release/2.4](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4) |
+| [WongKinYiu/YOLOv7](https://github.com/WongKinYiu/yolov7) | YOLOv7、YOLOv7-X等系列模型 | ONNX | [v0.1](https://github.com/WongKinYiu/yolov7/tree/v0.1) |

external/paddle_inference.cmake

@@ -49,7 +49,7 @@ else()
 endif(WIN32)
 
 set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/paddle2onnx/libs/")
-set(PADDLEINFERENCE_VERSION "2.3.1")
+set(PADDLEINFERENCE_VERSION "2.3.2")
 if(WIN32)
   message(FATAL_ERROR "Paddle Backend doesn't support Windows now.")
   set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-${PADDLEINFERENCE_VERSION}.zip")

fastdeploy/model.py

@@ -45,6 +45,15 @@ class FastDeployModel:
             index, self.num_outputs())
         return self._model.get_output_info(index)
 
+    def enable_record_time_of_runtime(self):
+        self._model.enable_record_time_of_runtime()
+
+    def disable_record_time_of_runtime(self):
+        self._model.disable_record_time_of_runtime()
+
+    def print_statis_info_of_runtime(self):
+        self._model.print_statis_info_of_runtime()
+
     @property
     def runtime_option(self):
         return self._model.runtime_option if self._model is not None else None