[Benchmark]Add trt cache option (#1653)

* add GPL lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* support yolov8

* add pybind for yolov8

* add yolov8 readme

* add cpp benchmark

* add cpu and gpu mem

* public part split

* add runtime mode

* fixed bugs

* add cpu_thread_nums

* deal with comments

* deal with comments

* deal with comments

* rm useless code

* add FASTDEPLOY_DECL

* add FASTDEPLOY_DECL

* fixed for windows

* mv rss to pss

* mv rss to pss

* Update utils.cc

* use thread to collect mem

* Add ResourceUsageMonitor

* rm useless code

* fixed bug

* fixed typo

* update ResourceUsageMonitor

* fixed bug

* fixed bug

* add note for ResourceUsageMonitor

* deal with comments

* add macros

* deal with comments

* deal with comments

* deal with comments

* re-lint

* rm pmap and use mem api

* rm pmap and use mem api

* add mem api

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* deal with comments

* fixed enable_paddle_to_trt

* add log for paddle_trt

* support ppcls benchmark

* use new trt option api

* update benchmark info

* simplify benchmark.cc

* simplify benchmark.cc

* deal with comments

* Add ppseg && ppocr benchmark

* add OCR rec img

* add ocr benchmark

* fixed trt shape

* add trt shape

* resolve conflict

* add ENABLE_BENCHMARK define

* Add ClassifyDiff

* Add Resize for ClassifyResult

* deal with comments

* add convert info script

* resolve conflict

* Add SaveBenchmarkResult func

* fixed bug

* fixed bug

* fixed bug

* add config.txt for option

* fixed bug

* fixed bug

* fixed bug

* add benchmark.sh

* mv thread_nums from 8 to 1

* deal with comments

* deal with comments

* fixed readme

* deal with comments

* add all platform shell

* Update config.arm.txt

* Update config.gpu.txt

* Update config.x86.txt

* fixed printinfo bug

* rm proxy

* add more model support

* all backend config.txt

* deal with comments

* Add MattingDiff compare

* fixed predict bug

* adjust warmup/repeat times

* add e2e/mem configs

* fixed typo

* open collect_mem

* fixed typo

* add trt cache option

* fixed bug

* fixed repeat times

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

benchmark/cpp/benchmark_gpu_trt.sh

@@ -44,4 +44,4 @@ fi
 ./benchmark_ppocr_cls --model ch_ppocr_mobile_v2.0_cls_infer --image rec_img.jpg --config_path $CONFIG_PATH --trt_shape 1,3,48,10:4,3,48,320:8,3,48,1024
 ./benchmark_ppocr_rec --model ch_PP-OCRv3_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH --trt_shape 1,3,48,10:4,3,48,320:8,3,48,2304
 ./benchmark_ppocr_det --model ch_PP-OCRv2_det_infer --image 12.jpg --config_path $CONFIG_PATH --trt_shape 1,3,64,64:1,3,640,640:1,3,960,960
-./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH --trt_shape 1,3,48,10:4,3,48,320:8,3,48,2304
+./benchmark_ppocr_rec --model ch_PP-OCRv2_rec_infer --image rec_img.jpg --rec_label_file ppocr_keys_v1.txt --config_path $CONFIG_PATH --trt_shape 1,3,32,10:4,3,32,320:8,3,32,2304

benchmark/cpp/benchmark_ppocr_rec.cc

@@ -55,6 +55,11 @@ int main(int argc, char* argv[]) {
   }
   auto model_ppocr_rec = vision::ocr::Recognizer(
       model_file, params_file, FLAGS_rec_label_file, option, model_format);
+  std::vector<std::string> model_names;
+  fastdeploy::benchmark::Split(FLAGS_model, model_names, sep);
+  if (model_names[model_names.size() - 1] == "ch_PP-OCRv2_rec_infer") {
+    model_ppocr_rec.GetPreprocessor().SetRecImageShape({3, 32, 320});
+  }
   std::string text;
   float rec_score;
   if (config_info["precision_compare"] == "true") {

benchmark/cpp/config/config.gpu.ort.fp32.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: ort
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle.fp32.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: paddle
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp16.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: paddle_trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp16.e2e.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: paddle_trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp16.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: paddle_trt
 profile_mode: runtime
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp32.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: paddle_trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp32.e2e.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: paddle_trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.paddle_trt.fp32.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: paddle_trt
 profile_mode: runtime
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp16.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp16.e2e.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp16.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: trt
 profile_mode: runtime
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp32.e2e.mem.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 10
-repeat: 50
+warmup: 20
+repeat: 100
 backend: trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp32.e2e.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: trt
 profile_mode: end2end
 include_h2d_d2h: false

benchmark/cpp/config/config.gpu.trt.fp32.txt

@@ -1,8 +1,8 @@
 device: gpu
 device_id: 3
 cpu_thread_nums: 1
-warmup: 20
-repeat: 100
+warmup: 200
+repeat: 1000
 backend: trt
 profile_mode: runtime
 include_h2d_d2h: false

benchmark/cpp/option.h

@@ -43,6 +43,8 @@ static bool CreateRuntimeOption(fastdeploy::RuntimeOption* option,
       option->UsePaddleInferBackend();
     } else if (config_info["backend"] == "trt" ||
                config_info["backend"] == "paddle_trt") {
+      option->trt_option.serialize_file = FLAGS_model +
+                                          sep + "trt_serialized.trt";
       option->UseTrtBackend();
       if (config_info["backend"] == "paddle_trt") {
         option->UsePaddleInferBackend();