[Benchmark] Add ppocr && ppseg benchmark (#1344)

* add GPL lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* support yolov8

* add pybind for yolov8

* add yolov8 readme

* add cpp benchmark

* add cpu and gpu mem

* public part split

* add runtime mode

* fixed bugs

* add cpu_thread_nums

* deal with comments

* deal with comments

* deal with comments

* rm useless code

* add FASTDEPLOY_DECL

* add FASTDEPLOY_DECL

* fixed for windows

* mv rss to pss

* mv rss to pss

* Update utils.cc

* use thread to collect mem

* Add ResourceUsageMonitor

* rm useless code

* fixed bug

* fixed typo

* update ResourceUsageMonitor

* fixed bug

* fixed bug

* add note for ResourceUsageMonitor

* deal with comments

* add macros

* deal with comments

* deal with comments

* deal with comments

* re-lint

* rm pmap and use mem api

* rm pmap and use mem api

* add mem api

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* deal with comments

* fixed enable_paddle_to_trt

* add log for paddle_trt

* support ppcls benchmark

* use new trt option api

* update benchmark info

* simplify benchmark.cc

* simplify benchmark.cc

* deal with comments

* Add ppseg && ppocr benchmark

* add OCR rec img

* add ocr benchmark

* fixed trt shape

* add trt shape

* resolve conflict

* add ENABLE_BENCHMARK define

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

benchmark/cpp/CMakeLists.txt

@@ -12,15 +12,21 @@ add_executable(benchmark_yolov5 ${PROJECT_SOURCE_DIR}/benchmark_yolov5.cc)
 add_executable(benchmark_ppyolov8 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov8.cc)
 add_executable(benchmark_ppcls ${PROJECT_SOURCE_DIR}/benchmark_ppcls.cc)
 add_executable(benchmark_precision_ppyolov8 ${PROJECT_SOURCE_DIR}/benchmark_precision_ppyolov8.cc)
+add_executable(benchmark_ppseg ${PROJECT_SOURCE_DIR}/benchmark_ppseg.cc)
+add_executable(benchmark_ppocr ${PROJECT_SOURCE_DIR}/benchmark_ppocr.cc)
 
 if(UNIX AND (NOT APPLE) AND (NOT ANDROID))
   target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_precision_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppocr ${FASTDEPLOY_LIBS} gflags pthread)
 else()
   target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_precision_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppocr ${FASTDEPLOY_LIBS} gflags)
 endif()

benchmark/cpp/benchmark_ppcls.cc

@@ -17,6 +17,7 @@
 #include "option.h"
 
 int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option, argc, argv, true)) {
@@ -24,7 +25,9 @@ int main(int argc, char* argv[]) {
   }
   auto im = cv::imread(FLAGS_image);
   // Set max_batch_size 1 for best performance
-  option.trt_option.max_batch_size = 1;
+  if (FLAGS_backend == "paddle_trt") {
+    option.trt_option.max_batch_size = 1;
+  }
   auto model_file = FLAGS_model + sep + "inference.pdmodel";
   auto params_file = FLAGS_model + sep + "inference.pdiparams";
   auto config_file = FLAGS_model + sep + "inference_cls.yaml";
@@ -32,5 +35,6 @@ int main(int argc, char* argv[]) {
       model_file, params_file, config_file, option);
   fastdeploy::vision::ClassifyResult res;
   BENCHMARK_MODEL(model_ppcls, model_ppcls.Predict(im, &res))
+#endif
   return 0;
 }

benchmark/cpp/benchmark_ppocr.cc

@@ -0,0 +1,90 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flags.h"
+#include "macros.h"
+#include "option.h"
+
+int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
+  // Initialization
+  auto option = fastdeploy::RuntimeOption();
+  if (!CreateRuntimeOption(&option, argc, argv, true)) {
+    return -1;
+  }
+  auto im = cv::imread(FLAGS_image);
+  auto im_rec = cv::imread(FLAGS_image_rec);
+  // Detection Model
+  auto det_model_file =
+      FLAGS_model + sep + FLAGS_det_model + sep + "inference.pdmodel";
+  auto det_params_file =
+      FLAGS_model + sep + FLAGS_det_model + sep + "inference.pdiparams";
+  // Classification Model
+  auto cls_model_file =
+      FLAGS_model + sep + FLAGS_cls_model + sep + "inference.pdmodel";
+  auto cls_params_file =
+      FLAGS_model + sep + FLAGS_cls_model + sep + "inference.pdiparams";
+  // Recognition Model
+  auto rec_model_file =
+      FLAGS_model + sep + FLAGS_rec_model + sep + "inference.pdmodel";
+  auto rec_params_file =
+      FLAGS_model + sep + FLAGS_rec_model + sep + "inference.pdiparams";
+  auto rec_label_file = FLAGS_rec_label_file;
+  if (FLAGS_backend == "paddle_trt") {
+    option.paddle_infer_option.collect_trt_shape = true;
+  }
+  auto det_option = option;
+  auto cls_option = option;
+  auto rec_option = option;
+  if (FLAGS_backend == "paddle_trt" || FLAGS_backend == "trt") {
+    det_option.trt_option.SetShape("x", {1, 3, 64, 64}, {1, 3, 640, 640},
+                                   {1, 3, 960, 960});
+    cls_option.trt_option.SetShape("x", {1, 3, 48, 10}, {4, 3, 48, 320},
+                                   {8, 3, 48, 1024});
+    rec_option.trt_option.SetShape("x", {1, 3, 48, 10}, {4, 3, 48, 320},
+                                   {8, 3, 48, 2304});
+  }
+  auto det_model = fastdeploy::vision::ocr::DBDetector(
+      det_model_file, det_params_file, det_option);
+  auto cls_model = fastdeploy::vision::ocr::Classifier(
+      cls_model_file, cls_params_file, cls_option);
+  auto rec_model = fastdeploy::vision::ocr::Recognizer(
+      rec_model_file, rec_params_file, rec_label_file, rec_option);
+  // Only for runtime
+  if (FLAGS_profile_mode == "runtime") {
+    std::vector<std::array<int, 8>> boxes_result;
+    std::cout << "====Detection model====" << std::endl;
+    BENCHMARK_MODEL(det_model, det_model.Predict(im, &boxes_result));
+    int32_t cls_label;
+    float cls_score;
+    std::cout << "====Classification model====" << std::endl;
+    BENCHMARK_MODEL(cls_model,
+                    cls_model.Predict(im_rec, &cls_label, &cls_score));
+    std::string text;
+    float rec_score;
+    std::cout << "====Recognization model====" << std::endl;
+    BENCHMARK_MODEL(rec_model, rec_model.Predict(im_rec, &text, &rec_score));
+  }
+  auto model_ppocrv3 =
+      fastdeploy::pipeline::PPOCRv3(&det_model, &cls_model, &rec_model);
+  fastdeploy::vision::OCRResult res;
+  if (FLAGS_profile_mode == "end2end") {
+    BENCHMARK_MODEL(model_ppocrv3, model_ppocrv3.Predict(im, &res))
+  }
+  auto vis_im = fastdeploy::vision::VisOcr(im, res);
+  cv::imwrite("vis_result.jpg", vis_im);
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+#endif
+  return 0;
+}

benchmark/cpp/benchmark_ppseg.cc

@@ -0,0 +1,46 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flags.h"
+#include "macros.h"
+#include "option.h"
+
+int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
+  // Initialization
+  auto option = fastdeploy::RuntimeOption();
+  if (!CreateRuntimeOption(&option, argc, argv, true)) {
+    return -1;
+  }
+  auto im = cv::imread(FLAGS_image);
+  auto model_file = FLAGS_model + sep + "model.pdmodel";
+  auto params_file = FLAGS_model + sep + "model.pdiparams";
+  auto config_file = FLAGS_model + sep + "deploy.yaml";
+  if (FLAGS_backend == "paddle_trt") {
+    option.paddle_infer_option.collect_trt_shape = true;
+  }
+  if (FLAGS_backend == "paddle_trt" || FLAGS_backend == "trt") {
+    option.trt_option.SetShape("x", {1, 3, 192, 192}, {1, 3, 192, 192},
+                               {1, 3, 192, 192});
+  }
+  auto model_ppseg = fastdeploy::vision::segmentation::PaddleSegModel(
+      model_file, params_file, config_file, option);
+  fastdeploy::vision::SegmentationResult res;
+  BENCHMARK_MODEL(model_ppseg, model_ppseg.Predict(im, &res))
+  auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5);
+  cv::imwrite("vis_result.jpg", vis_im);
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+#endif
+  return 0;
+}

benchmark/cpp/benchmark_ppyolov8.cc

@@ -17,6 +17,7 @@
 #include "option.h"
 
 int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option, argc, argv, true)) {
@@ -33,5 +34,6 @@ int main(int argc, char* argv[]) {
   auto vis_im = fastdeploy::vision::VisDetection(im, res);
   cv::imwrite("vis_result.jpg", vis_im);
   std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+#endif
   return 0;
 }

benchmark/cpp/benchmark_yolov5.cc

@@ -17,6 +17,7 @@
 #include "option.h"
 
 int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option, argc, argv, true)) {
@@ -30,5 +31,6 @@ int main(int argc, char* argv[]) {
   auto vis_im = fastdeploy::vision::VisDetection(im, res);
   cv::imwrite("vis_result.jpg", vis_im);
   std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+#endif
   return 0;
 }

benchmark/cpp/flags.h

@@ -44,6 +44,12 @@ DEFINE_bool(
 DEFINE_bool(
     collect_memory_info, false, "Whether to collect memory info");
 DEFINE_int32(sampling_interval, 50, "How often to collect memory info(ms).");
+// Only for ppocr
+DEFINE_string(det_model, "", "Path of Detection model of PPOCR.");
+DEFINE_string(cls_model, "", "Path of Classification model of PPOCR.");
+DEFINE_string(rec_model, "", "Path of Recognization model of PPOCR.");
+DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR.");
+DEFINE_string(image_rec, "", "Path of Recognization img file of PPOCR.");
 
 static void PrintUsage() {
   std::cout << "Usage: infer_demo --model model_path --image img_path --device "
@@ -60,6 +66,10 @@ static void PrintBenchmarkInfo() {
   // Get model name
   std::vector<std::string> model_names;
   fastdeploy::benchmark::Split(FLAGS_model, model_names, sep);
+  if (model_names.empty()) {
+    std::cout << "Directory of the inference model is invalid!!!" << std::endl;
+    return;
+  }
   // Save benchmark info
   std::stringstream ss;
   ss.precision(3);

benchmark/python/benchmark_ppocr.py

@@ -38,7 +38,7 @@ def parse_arguments():
     parser.add_argument(
         "--rec_label_file",
         required=True,
-        help="Path of Recognization model of PPOCR.")
+        help="Path of Recognization label file of PPOCR.")
     parser.add_argument(
         "--image", type=str, required=False, help="Path of test image file.")
     parser.add_argument(
@@ -272,19 +272,19 @@ if __name__ == '__main__':
         if "OCRv2" in args.model_dir:
             det_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                det_option.set_trt_input_shape(
+                det_option.trt_option.set_shape(
                     "x", [1, 3, 64, 64], [1, 3, 640, 640], [1, 3, 960, 960])
             det_model = fd.vision.ocr.DBDetector(
                 det_model_file, det_params_file, runtime_option=det_option)
             cls_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                cls_option.set_trt_input_shape(
+                cls_option.trt_option.set_shape(
                     "x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 1024])
             cls_model = fd.vision.ocr.Classifier(
                 cls_model_file, cls_params_file, runtime_option=cls_option)
             rec_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                rec_option.set_trt_input_shape(
+                rec_option.trt_option.set_shape(
                     "x", [1, 3, 32, 10], [10, 3, 32, 320], [32, 3, 32, 2304])
             rec_model = fd.vision.ocr.Recognizer(
                 rec_model_file,
@@ -296,19 +296,19 @@ if __name__ == '__main__':
         elif "OCRv3" in args.model_dir:
             det_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                det_option.set_trt_input_shape(
+                det_option.trt_option.set_shape(
                     "x", [1, 3, 64, 64], [1, 3, 640, 640], [1, 3, 960, 960])
             det_model = fd.vision.ocr.DBDetector(
                 det_model_file, det_params_file, runtime_option=det_option)
             cls_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                cls_option.set_trt_input_shape(
+                cls_option.trt_option.set_shape(
                     "x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 1024])
             cls_model = fd.vision.ocr.Classifier(
                 cls_model_file, cls_params_file, runtime_option=cls_option)
             rec_option = option
             if args.backend in ["trt", "paddle_trt"]:
-                rec_option.set_trt_input_shape(
+                rec_option.trt_option.set_shape(
                     "x", [1, 3, 48, 10], [10, 3, 48, 320], [64, 3, 48, 2304])
             rec_model = fd.vision.ocr.Recognizer(
                 rec_model_file,

benchmark/python/benchmark_ppseg.py

@@ -83,18 +83,18 @@ def build_option(args):
         elif backend in ["trt", "paddle_trt"]:
             option.use_trt_backend()
             if "Deeplabv3_ResNet101" in args.model or "FCN_HRNet_W18" in args.model or "Unet_cityscapes" in args.model or "PP_LiteSeg_B_STDC2_cityscapes" in args.model:
-                option.set_trt_input_shape("x", [1, 3, 1024, 2048],
+                option.trt_option.set_shape("x", [1, 3, 1024, 2048],
-                                           [1, 3, 1024,
+                                            [1, 3, 1024, 2048],
-                                            2048], [1, 3, 1024, 2048])
+                                            [1, 3, 1024, 2048])
             elif "Portrait_PP_HumanSegV2_Lite_256x144" in args.model:
-                option.set_trt_input_shape("x", [1, 3, 144, 256],
+                option.trt_option.set_shape(
-                                           [1, 3, 144, 256], [1, 3, 144, 256])
+                    "x", [1, 3, 144, 256], [1, 3, 144, 256], [1, 3, 144, 256])
             elif "PP_HumanSegV1_Server" in args.model:
-                option.set_trt_input_shape("x", [1, 3, 512, 512],
+                option.trt_option.set_shape(
-                                           [1, 3, 512, 512], [1, 3, 512, 512])
+                    "x", [1, 3, 512, 512], [1, 3, 512, 512], [1, 3, 512, 512])
             else:
-                option.set_trt_input_shape("x", [1, 3, 192, 192],
+                option.trt_option.set_shape(
-                                           [1, 3, 192, 192], [1, 3, 192, 192])
+                    "x", [1, 3, 192, 192], [1, 3, 192, 192], [1, 3, 192, 192])
             if backend == "paddle_trt":
                 option.paddle_infer_option.collect_trt_shape = True
                 option.use_paddle_infer_backend()

benchmark/python/benchmark_uie.py

@@ -80,22 +80,22 @@ def build_option(args):
             option.use_paddle_infer_backend()
             option.paddle_infer_option.enable_trt = True
         trt_file = os.path.join(args.model_dir, "infer.trt")
-        option.set_trt_input_shape(
+        option.trt_option.set_shape(
             'input_ids',
             min_shape=[1, 1],
             opt_shape=[args.batch_size, args.max_length // 2],
             max_shape=[args.batch_size, args.max_length])
-        option.set_trt_input_shape(
+        option.trt_option.set_shape(
             'token_type_ids',
             min_shape=[1, 1],
             opt_shape=[args.batch_size, args.max_length // 2],
             max_shape=[args.batch_size, args.max_length])
-        option.set_trt_input_shape(
+        option.trt_option.set_shape(
             'pos_ids',
             min_shape=[1, 1],
             opt_shape=[args.batch_size, args.max_length // 2],
             max_shape=[args.batch_size, args.max_length])
-        option.set_trt_input_shape(
+        option.trt_option.set_shape(
             'att_mask',
             min_shape=[1, 1],
             opt_shape=[args.batch_size, args.max_length // 2],

fastdeploy/runtime/backends/paddle/option.h

@@ -60,7 +60,7 @@ struct PaddleBackendOption {
    */
   IpuOption ipu_option;
 
-  /// Collect shape for model while enabel_trt is true
+  /// Collect shape for model while enable_trt is true
   bool collect_trt_shape = false;
   /// Cache input shape for mkldnn while the input data will change dynamiclly
   int mkldnn_cache_size = -1;