[Benchmark] Add ClassifyDiff to compare ClassifyResult diff (#1381)

* add GPL lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* support yolov8

* add pybind for yolov8

* add yolov8 readme

* add cpp benchmark

* add cpu and gpu mem

* public part split

* add runtime mode

* fixed bugs

* add cpu_thread_nums

* deal with comments

* deal with comments

* deal with comments

* rm useless code

* add FASTDEPLOY_DECL

* add FASTDEPLOY_DECL

* fixed for windows

* mv rss to pss

* mv rss to pss

* Update utils.cc

* use thread to collect mem

* Add ResourceUsageMonitor

* rm useless code

* fixed bug

* fixed typo

* update ResourceUsageMonitor

* fixed bug

* fixed bug

* add note for ResourceUsageMonitor

* deal with comments

* add macros

* deal with comments

* deal with comments

* deal with comments

* re-lint

* rm pmap and use mem api

* rm pmap and use mem api

* add mem api

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* deal with comments

* fixed enable_paddle_to_trt

* add log for paddle_trt

* support ppcls benchmark

* use new trt option api

* update benchmark info

* simplify benchmark.cc

* simplify benchmark.cc

* deal with comments

* Add ppseg && ppocr benchmark

* add OCR rec img

* add ocr benchmark

* fixed trt shape

* add trt shape

* resolve conflict

* add ENABLE_BENCHMARK define

* Add ClassifyDiff

* Add Resize for ClassifyResult

* deal with comments

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

benchmark/cpp/benchmark_ppcls.cc

@@ -16,6 +16,9 @@
 #include "macros.h"
 #include "option.h"
 
+namespace vision = fastdeploy::vision;
+namespace benchmark = fastdeploy::benchmark;
+
 int main(int argc, char* argv[]) {
 #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
@@ -31,9 +34,28 @@ int main(int argc, char* argv[]) {
   auto model_file = FLAGS_model + sep + "inference.pdmodel";
   auto params_file = FLAGS_model + sep + "inference.pdiparams";
   auto config_file = FLAGS_model + sep + "inference_cls.yaml";
-  auto model_ppcls = fastdeploy::vision::classification::PaddleClasModel(
+  auto model_ppcls = vision::classification::PaddleClasModel(
       model_file, params_file, config_file, option);
-  fastdeploy::vision::ClassifyResult res;
+  vision::ClassifyResult res;
+  // Run once at least
+  model_ppcls.Predict(im, &res);
+  // 1. Test result diff
+  std::cout << "=============== Test result diff =================\n";
+  // Save result to -> disk.
+  std::string cls_result_path = "ppcls_result.txt";
+  benchmark::ResultManager::SaveClassifyResult(res, cls_result_path);
+  // Load result from <- disk.
+  vision::ClassifyResult res_loaded;
+  benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path);
+  // Calculate diff between two results.
+  auto cls_diff =
+      benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded);
+  std::cout << "Labels diff: mean=" << cls_diff.labels.mean
+            << ", max=" << cls_diff.labels.max
+            << ", min=" << cls_diff.labels.min << std::endl;
+  std::cout << "Scores diff: mean=" << cls_diff.scores.mean
+            << ", max=" << cls_diff.scores.max
+            << ", min=" << cls_diff.scores.min << std::endl;
   BENCHMARK_MODEL(model_ppcls, model_ppcls.Predict(im, &res))
 #endif
   return 0;

benchmark/cpp/benchmark_ppocr.cc

@@ -16,6 +16,13 @@
 #include "macros.h"
 #include "option.h"
 
+// Only for ppocr
+DEFINE_string(det_model, "", "Path of Detection model of PPOCR.");
+DEFINE_string(cls_model, "", "Path of Classification model of PPOCR.");
+DEFINE_string(rec_model, "", "Path of Recognization model of PPOCR.");
+DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR.");
+DEFINE_string(image_rec, "", "Path of Recognization img file of PPOCR.");
+
 int main(int argc, char* argv[]) {
 #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization

benchmark/cpp/benchmark_precision_ppyolov8.cc

@@ -46,11 +46,15 @@ int main(int argc, char* argv[]) {
   // Calculate diff between two results.
   auto det_diff =
       benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded);
-  std::cout << "diff: mean=" << det_diff.mean << ",max=" << det_diff.max
+  std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
-            << ",min=" << det_diff.min << std::endl;
+            << ", max=" << det_diff.boxes.max << ", min=" << det_diff.boxes.min
+            << std::endl;
+  std::cout << "Label_ids diff: mean=" << det_diff.labels.mean
+            << ", max=" << det_diff.labels.max
+            << ", min=" << det_diff.labels.min << std::endl;
   // 2. Test tensor diff
   std::cout << "=============== Test tensor diff =================\n";
-  std::vector<vision::DetectionResult> bacth_res;
+  std::vector<vision::DetectionResult> batch_res;
   std::vector<fastdeploy::FDTensor> input_tensors, output_tensors;
   std::vector<cv::Mat> imgs;
   imgs.push_back(im);
@@ -62,7 +66,7 @@ int main(int argc, char* argv[]) {
   input_tensors[2].name = "im_shape";
   input_tensors.pop_back();
   model_ppyolov8.Infer(input_tensors, &output_tensors);
-  model_ppyolov8.GetPostprocessor().Run(output_tensors, &bacth_res);
+  model_ppyolov8.GetPostprocessor().Run(output_tensors, &batch_res);
   // Save tensor to -> disk.
   auto& tensor_dump = output_tensors[0];
   std::string det_tensor_path = "ppyolov8_tensor.txt";
@@ -73,9 +77,9 @@ int main(int argc, char* argv[]) {
   // Calculate diff between two tensors.
   auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis(
       &tensor_dump, &tensor_loaded);
-  std::cout << "diff: mean=" << det_tensor_diff.mean
+  std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean
-            << ",max=" << det_tensor_diff.max << ",min=" << det_tensor_diff.min
+            << ", max=" << det_tensor_diff.data.max
-            << std::endl;
+            << ", min=" << det_tensor_diff.data.min << std::endl;
   // 3. Run profiling
   BENCHMARK_MODEL(model_ppyolov8, model_ppyolov8.Predict(im, &res))
   auto vis_im = vision::VisDetection(im, res);

benchmark/cpp/flags.h

@@ -44,12 +44,6 @@ DEFINE_bool(
 DEFINE_bool(
     collect_memory_info, false, "Whether to collect memory info");
 DEFINE_int32(sampling_interval, 50, "How often to collect memory info(ms).");
-// Only for ppocr
-DEFINE_string(det_model, "", "Path of Detection model of PPOCR.");
-DEFINE_string(cls_model, "", "Path of Classification model of PPOCR.");
-DEFINE_string(rec_model, "", "Path of Recognization model of PPOCR.");
-DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR.");
-DEFINE_string(image_rec, "", "Path of Recognization img file of PPOCR.");
 
 static void PrintUsage() {
   std::cout << "Usage: infer_demo --model model_path --image img_path --device "

fastdeploy/benchmark/utils.cc

@@ -321,8 +321,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
     TensorDiff diff;
-    CalculateStatisInfo<int64_t>(tensor_diff.data(), numel, &(diff.mean),
+    CalculateStatisInfo<int64_t>(tensor_diff.data(), numel, &(diff.data.mean),
-                                 &(diff.max), &(diff.min));
+                                 &(diff.data.max), &(diff.data.min));
     return diff;
   } else if (dtype == FDDataType::INT32) {
     std::vector<int32_t> tensor_diff(numel);
@@ -332,8 +332,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
     TensorDiff diff;
-    CalculateStatisInfo<float>(tensor_diff.data(), numel, &(diff.mean),
+    CalculateStatisInfo<float>(tensor_diff.data(), numel, &(diff.data.mean),
-                               &(diff.max), &(diff.min));
+                               &(diff.data.max), &(diff.data.min));
     return diff;
   } else {  // FP32
     std::vector<float> tensor_diff(numel);
@@ -343,8 +343,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
     TensorDiff diff;
-    CalculateStatisInfo<float>(tensor_diff.data(), numel, &(diff.mean),
+    CalculateStatisInfo<float>(tensor_diff.data(), numel, &(diff.data.mean),
-                               &(diff.max), &(diff.min));
+                               &(diff.data.max), &(diff.data.min));
     return diff;
   }
 }
@@ -399,6 +399,42 @@ bool ResultManager::SaveDetectionResult(const vision::DetectionResult& res,
   return true;
 }
 
+bool ResultManager::SaveClassifyResult(const vision::ClassifyResult& res,
+                                       const std::string& path) {
+  if (res.label_ids.empty()) {
+    FDERROR << "ClassifyResult can not be empty!" << std::endl;
+    return false;
+  }
+  std::ofstream fs(path, std::ios::out);
+  if (!fs.is_open()) {
+    FDERROR << "Fail to open file:" << path << std::endl;
+    return false;
+  }
+  fs.precision(20);
+  // label_ids
+  fs << "label_ids" << KEY_VALUE_SEP;
+  for (int i = 0; i < res.label_ids.size(); ++i) {
+    if (i < res.label_ids.size() - 1) {
+      fs << res.label_ids[i] << VALUE_SEP;
+    } else {
+      fs << res.label_ids[i];
+    }
+  }
+  fs << "\n";
+  // scores
+  fs << "scores" << KEY_VALUE_SEP;
+  for (int i = 0; i < res.scores.size(); ++i) {
+    if (i < res.scores.size() - 1) {
+      fs << res.scores[i] << VALUE_SEP;
+    } else {
+      fs << res.scores[i];
+    }
+  }
+  fs << "\n";
+  fs.close();
+  return true;
+}
+
 bool ResultManager::LoadDetectionResult(vision::DetectionResult* res,
                                         const std::string& path) {
   if (!CheckFileExists(path)) {
@@ -432,6 +468,28 @@ bool ResultManager::LoadDetectionResult(vision::DetectionResult* res,
   return true;
 }
 
+bool ResultManager::LoadClassifyResult(vision::ClassifyResult* res,
+                                       const std::string& path) {
+  if (!CheckFileExists(path)) {
+    FDERROR << "Can't found file from" << path << std::endl;
+    return false;
+  }
+  auto lines = ReadLines(path);
+  std::map<std::string, std::vector<std::string>> data;
+  // label_ids
+  data = SplitDataLine(lines[0]);
+  res->Resize(data.begin()->second.size());
+  for (int i = 0; i < data.begin()->second.size(); ++i) {
+    res->label_ids[i] = std::stoi(data.begin()->second[i]);
+  }
+  // scores
+  data = SplitDataLine(lines[1]);
+  for (int i = 0; i < data.begin()->second.size(); ++i) {
+    res->scores[i] = std::stof(data.begin()->second[i]);
+  }
+  return true;
+}
+
 DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs,
                                                  vision::DetectionResult* rhs,
                                                  float score_threshold) {
@@ -469,11 +527,29 @@ DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs,
   CalculateStatisInfo<int32_t>(labels_diff.data(), labels_diff.size(),
                                &(diff.labels.mean), &(diff.labels.max),
                                &(diff.labels.min));
-  diff.mean = diff.boxes.mean;
-  diff.max = diff.boxes.max;
-  diff.min = diff.boxes.min;
   return diff;
 }
+
+ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs,
+                                                vision::ClassifyResult* rhs) {
+  const int class_nums = std::min(lhs->label_ids.size(), rhs->label_ids.size());
+  std::vector<float> scores_diff;
+  std::vector<int32_t> labels_diff;
+  for (int i = 0; i < class_nums; ++i) {
+    scores_diff.push_back(lhs->scores[i] - rhs->scores[i]);
+    labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]);
+  }
+
+  ClassifyDiff diff;
+  CalculateStatisInfo<float>(scores_diff.data(), scores_diff.size(),
+                             &(diff.scores.mean), &(diff.scores.max),
+                             &(diff.scores.min));
+  CalculateStatisInfo<int32_t>(labels_diff.data(), labels_diff.size(),
+                               &(diff.labels.mean), &(diff.labels.max),
+                               &(diff.labels.min));
+  return diff;
+}
+
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 

fastdeploy/benchmark/utils.h

@@ -101,14 +101,21 @@ struct FASTDEPLOY_DECL EvalStatis {
   double max = -1.0;
 };
 
-struct FASTDEPLOY_DECL TensorDiff: public BaseDiff, public EvalStatis {};
+struct FASTDEPLOY_DECL TensorDiff: public BaseDiff {
+  EvalStatis data;
+};
 
 #if defined(ENABLE_VISION)
-struct FASTDEPLOY_DECL DetectionDiff: public BaseDiff, public EvalStatis {
+struct FASTDEPLOY_DECL DetectionDiff: public BaseDiff {
   EvalStatis boxes;
   EvalStatis scores;
   EvalStatis labels;
 };
+
+struct FASTDEPLOY_DECL ClassifyDiff: public BaseDiff {
+  EvalStatis scores;
+  EvalStatis labels;
+};
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 
@@ -127,10 +134,16 @@ struct FASTDEPLOY_DECL ResultManager {
                                   const std::string& path);
   static bool LoadDetectionResult(vision::DetectionResult* res,
                                   const std::string& path);
+  static bool SaveClassifyResult(const vision::ClassifyResult& res,
+                                 const std::string& path);
+  static bool LoadClassifyResult(vision::ClassifyResult* res,
+                                 const std::string& path);
   /// Calculate diff value between two basic results.
   static DetectionDiff CalculateDiffStatis(vision::DetectionResult* lhs,
                                            vision::DetectionResult* rhs,
                                            float score_threshold = 0.3f);
+  static ClassifyDiff CalculateDiffStatis(vision::ClassifyResult* lhs,
+                                          vision::ClassifyResult* rhs);
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 };

fastdeploy/vision/common/result.cc

@@ -26,6 +26,11 @@ void ClassifyResult::Clear() {
   scores.clear();
 }
 
+void ClassifyResult::Resize(int size) {
+  label_ids.resize(size);
+  scores.resize(size);
+}
+
 std::string ClassifyResult::Str() {
   std::string out;
   out = "ClassifyResult(\nlabel_ids: ";

fastdeploy/vision/common/result.h

@@ -51,6 +51,9 @@ struct FASTDEPLOY_DECL ClassifyResult : public BaseResult {
   std::vector<float> scores;
   ResultType type = ResultType::CLASSIFY;
 
+  /// Resize ClassifyResult data buffer
+  void Resize(int size);
+
   /// Clear ClassifyResult
   void Clear();