[Benchmark]Add SegmentationDiff to compare SegmentationResult diff (#1404)

* avoid mem copy for cpp benchmark

* set CMAKE_BUILD_TYPE to Release

* Add SegmentationDiff

* change pointer to reference

* fixed bug

* cast uint8 to int32

benchmark/cpp/benchmark_ppcls.cc

@@ -49,7 +49,7 @@ int main(int argc, char* argv[]) {
   benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path);
   // Calculate diff between two results.
   auto cls_diff =
-      benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded);
+      benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
   std::cout << "Labels diff: mean=" << cls_diff.labels.mean
             << ", max=" << cls_diff.labels.max
             << ", min=" << cls_diff.labels.min << std::endl;

benchmark/cpp/benchmark_ppseg.cc

@@ -16,6 +16,9 @@
 #include "macros.h"
 #include "option.h"
 
+namespace vision = fastdeploy::vision;
+namespace benchmark = fastdeploy::benchmark;
+
 int main(int argc, char* argv[]) {
 #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
@@ -34,11 +37,33 @@ int main(int argc, char* argv[]) {
     option.trt_option.SetShape("x", {1, 3, 192, 192}, {1, 3, 192, 192},
                                {1, 3, 192, 192});
   }
-  auto model_ppseg = fastdeploy::vision::segmentation::PaddleSegModel(
+  auto model_ppseg = vision::segmentation::PaddleSegModel(
       model_file, params_file, config_file, option);
-  fastdeploy::vision::SegmentationResult res;
+  vision::SegmentationResult res;
+  // Run once at least
+  model_ppseg.Predict(im, &res);
+  // 1. Test result diff
+  std::cout << "=============== Test result diff =================\n";
+  // Save result to -> disk.
+  std::string seg_result_path = "ppseg_result.txt";
+  benchmark::ResultManager::SaveSegmentationResult(res, seg_result_path);
+  // Load result from <- disk.
+  vision::SegmentationResult res_loaded;
+  benchmark::ResultManager::LoadSegmentationResult(&res_loaded,
+                                                   seg_result_path);
+  // Calculate diff between two results.
+  auto seg_diff =
+      benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
+  std::cout << "Labels diff: mean=" << seg_diff.labels.mean
+            << ", max=" << seg_diff.labels.max
+            << ", min=" << seg_diff.labels.min << std::endl;
+  if (res_loaded.contain_score_map) {
+    std::cout << "Scores diff: mean=" << seg_diff.scores.mean
+              << ", max=" << seg_diff.scores.max
+              << ", min=" << seg_diff.scores.min << std::endl;
+  }
   BENCHMARK_MODEL(model_ppseg, model_ppseg.Predict(im, &res))
-  auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5);
+  auto vis_im = vision::VisSegmentation(im, res, 0.5);
   cv::imwrite("vis_result.jpg", vis_im);
   std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
 #endif

benchmark/cpp/benchmark_precision_ppyolov8.cc

@@ -45,7 +45,7 @@ int main(int argc, char* argv[]) {
   benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path);
   // Calculate diff between two results.
   auto det_diff =
-      benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded);
+      benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
   std::cout << "Boxes diff: mean=" << det_diff.boxes.mean
             << ", max=" << det_diff.boxes.max << ", min=" << det_diff.boxes.min
             << std::endl;
@@ -75,8 +75,8 @@ int main(int argc, char* argv[]) {
   fastdeploy::FDTensor tensor_loaded;
   benchmark::ResultManager::LoadFDTensor(&tensor_loaded, det_tensor_path);
   // Calculate diff between two tensors.
-  auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis(
+  auto det_tensor_diff =
-      &tensor_dump, &tensor_loaded);
+      benchmark::ResultManager::CalculateDiffStatis(tensor_dump, tensor_loaded);
   std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean
             << ", max=" << det_tensor_diff.data.max
             << ", min=" << det_tensor_diff.data.min << std::endl;

fastdeploy/benchmark/utils.cc

@@ -298,16 +298,17 @@ bool ResultManager::LoadFDTensor(FDTensor* tensor, const std::string& path) {
   return true;
 }
 
-TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
+TensorDiff ResultManager::CalculateDiffStatis(const FDTensor& lhs,
-  if (lhs->Numel() != rhs->Numel() || lhs->Dtype() != rhs->Dtype()) {
+                                              const FDTensor& rhs) {
+  if (lhs.Numel() != rhs.Numel() || lhs.Dtype() != rhs.Dtype()) {
     FDASSERT(false,
              "The size and dtype of input FDTensor must be equal!"
              " But got size %d, %d, dtype %s, %s",
-             lhs->Numel(), rhs->Numel(), Str(lhs->Dtype()).c_str(),
+             lhs.Numel(), rhs.Numel(), Str(lhs.Dtype()).c_str(),
-             Str(rhs->Dtype()).c_str())
+             Str(rhs.Dtype()).c_str())
   }
-  FDDataType dtype = lhs->Dtype();
+  FDDataType dtype = lhs.Dtype();
-  int numel = lhs->Numel();
+  int numel = lhs.Numel();
   if (dtype != FDDataType::FP32 && dtype != FDDataType::INT64 &&
       dtype != FDDataType::INT32) {
     FDASSERT(false, "Only support FP32/INT64/INT32 now, but got %s",
@@ -315,8 +316,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
   }
   if (dtype == FDDataType::INT64) {
     std::vector<int64_t> tensor_diff(numel);
-    const int64_t* lhs_data_ptr = static_cast<const int64_t*>(lhs->CpuData());
+    const int64_t* lhs_data_ptr = static_cast<const int64_t*>(lhs.CpuData());
-    const int64_t* rhs_data_ptr = static_cast<const int64_t*>(rhs->CpuData());
+    const int64_t* rhs_data_ptr = static_cast<const int64_t*>(rhs.CpuData());
     for (int i = 0; i < numel; ++i) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
@@ -326,8 +327,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
     return diff;
   } else if (dtype == FDDataType::INT32) {
     std::vector<int32_t> tensor_diff(numel);
-    const int32_t* lhs_data_ptr = static_cast<const int32_t*>(lhs->CpuData());
+    const int32_t* lhs_data_ptr = static_cast<const int32_t*>(lhs.CpuData());
-    const int32_t* rhs_data_ptr = static_cast<const int32_t*>(rhs->CpuData());
+    const int32_t* rhs_data_ptr = static_cast<const int32_t*>(rhs.CpuData());
     for (int i = 0; i < numel; ++i) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
@@ -337,8 +338,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) {
     return diff;
   } else {  // FP32
     std::vector<float> tensor_diff(numel);
-    const float* lhs_data_ptr = static_cast<const float*>(lhs->CpuData());
+    const float* lhs_data_ptr = static_cast<const float*>(lhs.CpuData());
-    const float* rhs_data_ptr = static_cast<const float*>(rhs->CpuData());
+    const float* rhs_data_ptr = static_cast<const float*>(rhs.CpuData());
     for (int i = 0; i < numel; ++i) {
       tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i];
     }
@@ -435,6 +436,44 @@ bool ResultManager::SaveClassifyResult(const vision::ClassifyResult& res,
   return true;
 }
 
+bool ResultManager::SaveSegmentationResult(
+    const vision::SegmentationResult& res, const std::string& path) {
+  if (res.label_map.empty()) {
+    FDERROR << "SegmentationResult can not be empty!" << std::endl;
+    return false;
+  }
+  std::ofstream fs(path, std::ios::out);
+  if (!fs.is_open()) {
+    FDERROR << "Fail to open file:" << path << std::endl;
+    return false;
+  }
+  fs.precision(20);
+  // label_map
+  fs << "label_map" << KEY_VALUE_SEP;
+  for (int i = 0; i < res.label_map.size(); ++i) {
+    if (i < res.label_map.size() - 1) {
+      fs << static_cast<int32_t>(res.label_map[i]) << VALUE_SEP;
+    } else {
+      fs << static_cast<int32_t>(res.label_map[i]);
+    }
+  }
+  fs << "\n";
+  // score_map
+  if (res.contain_score_map) {
+    fs << "score_map" << KEY_VALUE_SEP;
+    for (int i = 0; i < res.score_map.size(); ++i) {
+      if (i < res.score_map.size() - 1) {
+        fs << res.score_map[i] << VALUE_SEP;
+      } else {
+        fs << res.score_map[i];
+      }
+    }
+    fs << "\n";
+  }
+  fs.close();
+  return true;
+}
+
 bool ResultManager::LoadDetectionResult(vision::DetectionResult* res,
                                         const std::string& path) {
   if (!CheckFileExists(path)) {
@@ -490,32 +529,62 @@ bool ResultManager::LoadClassifyResult(vision::ClassifyResult* res,
   return true;
 }
 
-DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs,
+bool ResultManager::LoadSegmentationResult(vision::SegmentationResult* res,
-                                                 vision::DetectionResult* rhs,
+                                           const std::string& path) {
-                                                 float score_threshold) {
+  if (!CheckFileExists(path)) {
+    FDERROR << "Can't found file from" << path << std::endl;
+    return false;
+  }
+  auto lines = ReadLines(path);
+  if (lines.size() > 1) {
+    res->contain_score_map = true;
+  }
+  std::map<std::string, std::vector<std::string>> data;
+  // label_map
+  data = SplitDataLine(lines[0]);
+  res->Resize(data.begin()->second.size());
+  for (int i = 0; i < data.begin()->second.size(); ++i) {
+    res->label_map[i] = std::stoi(data.begin()->second[i]);
+  }
+  // score_map
+  if (lines.size() > 1) {
+    data = SplitDataLine(lines[1]);
+    for (int i = 0; i < data.begin()->second.size(); ++i) {
+      res->score_map[i] = std::stof(data.begin()->second[i]);
+    }
+  }
+  return true;
+}
+
+DetectionDiff ResultManager::CalculateDiffStatis(
+    const vision::DetectionResult& lhs, const vision::DetectionResult& rhs,
+    const float& score_threshold) {
+  vision::DetectionResult lhs_sort = lhs;
+  vision::DetectionResult rhs_sort = rhs;
   // lex sort by x(w) & y(h)
-  vision::utils::LexSortDetectionResultByXY(lhs);
+  vision::utils::LexSortDetectionResultByXY(&lhs_sort);
-  vision::utils::LexSortDetectionResultByXY(rhs);
+  vision::utils::LexSortDetectionResultByXY(&rhs_sort);
   // get value diff & trunc it by score_threshold
-  const int boxes_num = std::min(lhs->boxes.size(), rhs->boxes.size());
+  const int boxes_num = std::min(lhs_sort.boxes.size(), rhs_sort.boxes.size());
   std::vector<float> boxes_diff;
   std::vector<float> scores_diff;
   std::vector<int32_t> labels_diff;
   // TODO(qiuyanjun): process the diff of masks.
   for (int i = 0; i < boxes_num; ++i) {
-    if (lhs->scores[i] > score_threshold && rhs->scores[i] > score_threshold) {
+    if (lhs_sort.scores[i] > score_threshold &&
-      scores_diff.push_back(lhs->scores[i] - rhs->scores[i]);
+        rhs_sort.scores[i] > score_threshold) {
-      labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]);
+      scores_diff.push_back(lhs_sort.scores[i] - rhs_sort.scores[i]);
-      boxes_diff.push_back(lhs->boxes[i][0] - rhs->boxes[i][0]);
+      labels_diff.push_back(lhs_sort.label_ids[i] - rhs_sort.label_ids[i]);
-      boxes_diff.push_back(lhs->boxes[i][1] - rhs->boxes[i][1]);
+      boxes_diff.push_back(lhs_sort.boxes[i][0] - rhs_sort.boxes[i][0]);
-      boxes_diff.push_back(lhs->boxes[i][2] - rhs->boxes[i][2]);
+      boxes_diff.push_back(lhs_sort.boxes[i][1] - rhs_sort.boxes[i][1]);
-      boxes_diff.push_back(lhs->boxes[i][3] - rhs->boxes[i][3]);
+      boxes_diff.push_back(lhs_sort.boxes[i][2] - rhs_sort.boxes[i][2]);
+      boxes_diff.push_back(lhs_sort.boxes[i][3] - rhs_sort.boxes[i][3]);
     }
   }
   FDASSERT(boxes_diff.size() > 0,
            "Can't get any valid boxes while score_threshold is %f, "
            "The boxes.size of lhs is %d, the boxes.size of rhs is %d",
-           score_threshold, lhs->boxes.size(), rhs->boxes.size())
+           score_threshold, lhs_sort.boxes.size(), rhs_sort.boxes.size())
 
   DetectionDiff diff;
   CalculateStatisInfo<float>(boxes_diff.data(), boxes_diff.size(),
@@ -530,14 +599,14 @@ DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs,
   return diff;
 }
 
-ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs,
+ClassifyDiff ResultManager::CalculateDiffStatis(
-                                                vision::ClassifyResult* rhs) {
+    const vision::ClassifyResult& lhs, const vision::ClassifyResult& rhs) {
-  const int class_nums = std::min(lhs->label_ids.size(), rhs->label_ids.size());
+  const int class_nums = std::min(lhs.label_ids.size(), rhs.label_ids.size());
   std::vector<float> scores_diff;
   std::vector<int32_t> labels_diff;
   for (int i = 0; i < class_nums; ++i) {
-    scores_diff.push_back(lhs->scores[i] - rhs->scores[i]);
+    scores_diff.push_back(lhs.scores[i] - rhs.scores[i]);
-    labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]);
+    labels_diff.push_back(lhs.label_ids[i] - rhs.label_ids[i]);
   }
 
   ClassifyDiff diff;
@@ -550,6 +619,30 @@ ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs,
   return diff;
 }
 
+SegmentationDiff ResultManager::CalculateDiffStatis(
+    const vision::SegmentationResult& lhs,
+    const vision::SegmentationResult& rhs) {
+  const int pixel_nums = std::min(lhs.label_map.size(), rhs.label_map.size());
+  std::vector<int32_t> labels_diff;
+  std::vector<float> scores_diff;
+  for (int i = 0; i < pixel_nums; ++i) {
+    labels_diff.push_back(lhs.label_map[i] - rhs.label_map[i]);
+    if (lhs.contain_score_map && rhs.contain_score_map) {
+      scores_diff.push_back(lhs.score_map[i] - rhs.score_map[i]);
+    }
+  }
+  SegmentationDiff diff;
+  CalculateStatisInfo<int32_t>(labels_diff.data(), labels_diff.size(),
+                               &(diff.labels.mean), &(diff.labels.max),
+                               &(diff.labels.min));
+  if (lhs.contain_score_map && rhs.contain_score_map) {
+    CalculateStatisInfo<float>(scores_diff.data(), scores_diff.size(),
+                               &(diff.scores.mean), &(diff.scores.max),
+                               &(diff.scores.min));
+  }
+  return diff;
+}
+
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 

fastdeploy/benchmark/utils.h

@@ -116,6 +116,12 @@ struct FASTDEPLOY_DECL ClassifyDiff: public BaseDiff {
   EvalStatis scores;
   EvalStatis labels;
 };
+
+struct FASTDEPLOY_DECL SegmentationDiff: public BaseDiff {
+  EvalStatis scores;
+  EvalStatis labels;
+};
+
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 
@@ -126,8 +132,8 @@ struct FASTDEPLOY_DECL ResultManager {
   static bool SaveFDTensor(const FDTensor& tensor, const std::string& path);
   static bool LoadFDTensor(FDTensor* tensor, const std::string& path);
   /// Calculate diff value between two FDTensor results.
-  static TensorDiff CalculateDiffStatis(FDTensor* lhs,
+  static TensorDiff CalculateDiffStatis(const FDTensor& lhs,
-                                        FDTensor* rhs);
+                                        const FDTensor& rhs);
 #if defined(ENABLE_VISION)
   /// Save & Load functions for basic results.
   static bool SaveDetectionResult(const vision::DetectionResult& res,
@@ -138,12 +144,19 @@ struct FASTDEPLOY_DECL ResultManager {
                                  const std::string& path);
   static bool LoadClassifyResult(vision::ClassifyResult* res,
                                  const std::string& path);
+  static bool SaveSegmentationResult(const vision::SegmentationResult& res,
+                                     const std::string& path);
+  static bool LoadSegmentationResult(vision::SegmentationResult* res,
+                                     const std::string& path);
   /// Calculate diff value between two basic results.
-  static DetectionDiff CalculateDiffStatis(vision::DetectionResult* lhs,
+  static DetectionDiff CalculateDiffStatis(const vision::DetectionResult& lhs,
-                                           vision::DetectionResult* rhs,
+                                           const vision::DetectionResult& rhs,
-                                           float score_threshold = 0.3f);
+                                           const float& score_threshold = 0.3f);
-  static ClassifyDiff CalculateDiffStatis(vision::ClassifyResult* lhs,
+  static ClassifyDiff CalculateDiffStatis(const vision::ClassifyResult& lhs,
-                                          vision::ClassifyResult* rhs);
+                                          const vision::ClassifyResult& rhs);
+  static SegmentationDiff CalculateDiffStatis(
+      const vision::SegmentationResult& lhs,
+      const vision::SegmentationResult& rhs);
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 };

fastdeploy/runtime/backends/tensorrt/utils.h

@@ -241,6 +241,7 @@ class FDTrtLogger : public nvinfer1::ILogger {
       FDASSERT(false, "%s", msg);
     }
   }
+
  private:
   bool enable_info_ = false;
   bool enable_warning_ = false;