[Benchmark]Add MattingResult Compare (#1619)

* add GPL lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* add GPL-3.0 lisence

* support yolov8

* add pybind for yolov8

* add yolov8 readme

* add cpp benchmark

* add cpu and gpu mem

* public part split

* add runtime mode

* fixed bugs

* add cpu_thread_nums

* deal with comments

* deal with comments

* deal with comments

* rm useless code

* add FASTDEPLOY_DECL

* add FASTDEPLOY_DECL

* fixed for windows

* mv rss to pss

* mv rss to pss

* Update utils.cc

* use thread to collect mem

* Add ResourceUsageMonitor

* rm useless code

* fixed bug

* fixed typo

* update ResourceUsageMonitor

* fixed bug

* fixed bug

* add note for ResourceUsageMonitor

* deal with comments

* add macros

* deal with comments

* deal with comments

* deal with comments

* re-lint

* rm pmap and use mem api

* rm pmap and use mem api

* add mem api

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* Add PrintBenchmarkInfo func

* deal with comments

* fixed enable_paddle_to_trt

* add log for paddle_trt

* support ppcls benchmark

* use new trt option api

* update benchmark info

* simplify benchmark.cc

* simplify benchmark.cc

* deal with comments

* Add ppseg && ppocr benchmark

* add OCR rec img

* add ocr benchmark

* fixed trt shape

* add trt shape

* resolve conflict

* add ENABLE_BENCHMARK define

* Add ClassifyDiff

* Add Resize for ClassifyResult

* deal with comments

* add convert info script

* resolve conflict

* Add SaveBenchmarkResult func

* fixed bug

* fixed bug

* fixed bug

* add config.txt for option

* fixed bug

* fixed bug

* fixed bug

* add benchmark.sh

* mv thread_nums from 8 to 1

* deal with comments

* deal with comments

* fixed readme

* deal with comments

* add all platform shell

* Update config.arm.txt

* Update config.gpu.txt

* Update config.x86.txt

* fixed printinfo bug

* rm proxy

* add more model support

* all backend config.txt

* deal with comments

* Add MattingDiff compare

* fixed predict bug

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

benchmark/cpp/benchmark_ppmatting.cc

@@ -50,6 +50,30 @@ int main(int argc, char* argv[]) {
   auto model_ppmatting =
       vision::matting::PPMatting(model_file, params_file, config_file, option);
   vision::MattingResult res;
+  if (config_info["precision_compare"] == "true") {
+    // Run once at least
+    model_ppmatting.Predict(&im, &res);
+    // 1. Test result diff
+    std::cout << "=============== Test result diff =================\n";
+    // Save result to -> disk.
+    std::string matting_result_path = "ppmatting_result.txt";
+    benchmark::ResultManager::SaveMattingResult(res, matting_result_path);
+    // Load result from <- disk.
+    vision::MattingResult res_loaded;
+    benchmark::ResultManager::LoadMattingResult(&res_loaded,
+                                                matting_result_path);
+    // Calculate diff between two results.
+    auto matting_diff =
+        benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
+    std::cout << "Alpha diff: mean=" << matting_diff.alpha.mean
+              << ", max=" << matting_diff.alpha.max
+              << ", min=" << matting_diff.alpha.min << std::endl;
+    if (res_loaded.contain_foreground) {
+      std::cout << "Foreground diff: mean=" << matting_diff.foreground.mean
+                << ", max=" << matting_diff.foreground.max
+                << ", min=" << matting_diff.foreground.min << std::endl;
+    }
+  }
   BENCHMARK_MODEL(model_ppmatting, model_ppmatting.Predict(&im, &res))
   auto vis_im = vision::VisMatting(im, res);
   cv::imwrite("vis_result.jpg", vis_im);

fastdeploy/benchmark/utils.cc

@@ -556,6 +556,44 @@ bool ResultManager::SaveOCRDetResult(const std::vector<std::array<int, 8>>& res,
   return true;
 }
 
+bool ResultManager::SaveMattingResult(const vision::MattingResult& res,
+                                      const std::string& path) {
+  if (res.alpha.empty()) {
+    FDERROR << "MattingResult can not be empty!" << std::endl;
+    return false;
+  }
+  std::ofstream fs(path, std::ios::out);
+  if (!fs.is_open()) {
+    FDERROR << "Fail to open file:" << path << std::endl;
+    return false;
+  }
+  fs.precision(20);
+  // alpha
+  fs << "alpha" << KEY_VALUE_SEP;
+  for (int i = 0; i < res.alpha.size(); ++i) {
+    if (i < res.alpha.size() - 1) {
+      fs << res.alpha[i] << VALUE_SEP;
+    } else {
+      fs << res.alpha[i];
+    }
+  }
+  fs << "\n";
+  // foreground
+  if (res.contain_foreground) {
+    fs << "foreground" << KEY_VALUE_SEP;
+    for (int i = 0; i < res.foreground.size(); ++i) {
+      if (i < res.foreground.size() - 1) {
+        fs << res.foreground[i] << VALUE_SEP;
+      } else {
+        fs << res.foreground[i];
+      }
+    }
+    fs << "\n";
+  }
+  fs.close();
+  return true;
+}
+
 bool ResultManager::LoadDetectionResult(vision::DetectionResult* res,
                                         const std::string& path) {
   if (!CheckFileExists(path)) {
@@ -658,6 +696,33 @@ bool ResultManager::LoadOCRDetResult(std::vector<std::array<int, 8>>* res,
   return true;
 }
 
+bool ResultManager::LoadMattingResult(vision::MattingResult* res,
+                                      const std::string& path) {
+  if (!CheckFileExists(path)) {
+    FDERROR << "Can't found file from " << path << std::endl;
+    return false;
+  }
+  auto lines = ReadLines(path);
+  if (lines.size() > 1) {
+    res->contain_foreground = true;
+  }
+  std::map<std::string, std::vector<std::string>> data;
+  // alpha
+  data = SplitDataLine(lines[0]);
+  res->Resize(data.begin()->second.size());
+  for (int i = 0; i < data.begin()->second.size(); ++i) {
+    res->alpha[i] = std::stof(data.begin()->second[i]);
+  }
+  // foreground
+  if (lines.size() > 1) {
+    data = SplitDataLine(lines[1]);
+    for (int i = 0; i < data.begin()->second.size(); ++i) {
+      res->foreground[i] = std::stof(data.begin()->second[i]);
+    }
+  }
+  return true;
+}
+
 DetectionDiff ResultManager::CalculateDiffStatis(
     const vision::DetectionResult& lhs, const vision::DetectionResult& rhs,
     const float& score_threshold) {
@@ -770,6 +835,29 @@ OCRDetDiff ResultManager::CalculateDiffStatis(
   return diff;
 }
 
+MattingDiff ResultManager::CalculateDiffStatis(
+    const vision::MattingResult& lhs, const vision::MattingResult& rhs) {
+  const int pixel_nums = std::min(lhs.alpha.size(), rhs.alpha.size());
+  std::vector<float> alpha_diff;
+  std::vector<float> foreground_diff;
+  for (int i = 0; i < pixel_nums; ++i) {
+    alpha_diff.push_back(lhs.alpha[i] - rhs.alpha[i]);
+    if (lhs.contain_foreground && rhs.contain_foreground) {
+      foreground_diff.push_back(lhs.foreground[i] - rhs.foreground[i]);
+    }
+  }
+  MattingDiff diff;
+  CalculateStatisInfo<float>(alpha_diff.data(), alpha_diff.size(),
+                             &(diff.alpha.mean), &(diff.alpha.max),
+                             &(diff.alpha.min));
+  if (lhs.contain_foreground && rhs.contain_foreground) {
+    CalculateStatisInfo<float>(foreground_diff.data(), foreground_diff.size(),
+                               &(diff.foreground.mean), &(diff.foreground.max),
+                               &(diff.foreground.min));
+  }
+  return diff;
+}
+
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 

fastdeploy/benchmark/utils.h

@@ -127,6 +127,11 @@ struct FASTDEPLOY_DECL OCRDetDiff: public BaseDiff {
   EvalStatis boxes;
 };
 
+struct FASTDEPLOY_DECL MattingDiff: public BaseDiff {
+  EvalStatis alpha;
+  EvalStatis foreground;
+};
+
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 
@@ -166,6 +171,10 @@ struct FASTDEPLOY_DECL ResultManager {
                                const std::string& path);
   static bool LoadOCRDetResult(std::vector<std::array<int, 8>>* res,
                                const std::string& path);
+  static bool SaveMattingResult(const vision::MattingResult& res,
+                                const std::string& path);
+  static bool LoadMattingResult(vision::MattingResult* res,
+                                const std::string& path);
   /// Calculate diff value between two basic results.
   static DetectionDiff CalculateDiffStatis(const vision::DetectionResult& lhs,
                                            const vision::DetectionResult& rhs,
@@ -178,6 +187,9 @@ struct FASTDEPLOY_DECL ResultManager {
   static OCRDetDiff CalculateDiffStatis(
       const std::vector<std::array<int, 8>>& lhs,
       const std::vector<std::array<int, 8>>& rhs);
+  static MattingDiff CalculateDiffStatis(
+      const vision::MattingResult& lhs,
+      const vision::MattingResult& rhs);
 #endif  // ENABLE_VISION
 #endif  // ENABLE_BENCHMARK
 };