From a598e7adc4c37557b3dc08c9f91a3dff2e2ea93d Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Wed, 15 Mar 2023 17:41:19 +0800 Subject: [PATCH] [Benchmark]Add MattingResult Compare (#1619) * add GPL lisence * add GPL-3.0 lisence * add GPL-3.0 lisence * add GPL-3.0 lisence * support yolov8 * add pybind for yolov8 * add yolov8 readme * add cpp benchmark * add cpu and gpu mem * public part split * add runtime mode * fixed bugs * add cpu_thread_nums * deal with comments * deal with comments * deal with comments * rm useless code * add FASTDEPLOY_DECL * add FASTDEPLOY_DECL * fixed for windows * mv rss to pss * mv rss to pss * Update utils.cc * use thread to collect mem * Add ResourceUsageMonitor * rm useless code * fixed bug * fixed typo * update ResourceUsageMonitor * fixed bug * fixed bug * add note for ResourceUsageMonitor * deal with comments * add macros * deal with comments * deal with comments * deal with comments * re-lint * rm pmap and use mem api * rm pmap and use mem api * add mem api * Add PrintBenchmarkInfo func * Add PrintBenchmarkInfo func * Add PrintBenchmarkInfo func * deal with comments * fixed enable_paddle_to_trt * add log for paddle_trt * support ppcls benchmark * use new trt option api * update benchmark info * simplify benchmark.cc * simplify benchmark.cc * deal with comments * Add ppseg && ppocr benchmark * add OCR rec img * add ocr benchmark * fixed trt shape * add trt shape * resolve conflict * add ENABLE_BENCHMARK define * Add ClassifyDiff * Add Resize for ClassifyResult * deal with comments * add convert info script * resolve conflict * Add SaveBenchmarkResult func * fixed bug * fixed bug * fixed bug * add config.txt for option * fixed bug * fixed bug * fixed bug * add benchmark.sh * mv thread_nums from 8 to 1 * deal with comments * deal with comments * fixed readme * deal with comments * add all platform shell * Update config.arm.txt * Update config.gpu.txt * Update config.x86.txt * fixed printinfo bug * rm proxy * add more model support * all backend config.txt * deal with comments * Add MattingDiff compare * fixed predict bug --------- Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> --- benchmark/cpp/benchmark_ppmatting.cc | 24 ++++++++ fastdeploy/benchmark/utils.cc | 88 ++++++++++++++++++++++++++++ fastdeploy/benchmark/utils.h | 12 ++++ 3 files changed, 124 insertions(+) mode change 100644 => 100755 benchmark/cpp/benchmark_ppmatting.cc mode change 100755 => 100644 fastdeploy/benchmark/utils.cc diff --git a/benchmark/cpp/benchmark_ppmatting.cc b/benchmark/cpp/benchmark_ppmatting.cc old mode 100644 new mode 100755 index 020bfa5fa..18598c4bb --- a/benchmark/cpp/benchmark_ppmatting.cc +++ b/benchmark/cpp/benchmark_ppmatting.cc @@ -50,6 +50,30 @@ int main(int argc, char* argv[]) { auto model_ppmatting = vision::matting::PPMatting(model_file, params_file, config_file, option); vision::MattingResult res; + if (config_info["precision_compare"] == "true") { + // Run once at least + model_ppmatting.Predict(&im, &res); + // 1. Test result diff + std::cout << "=============== Test result diff =================\n"; + // Save result to -> disk. + std::string matting_result_path = "ppmatting_result.txt"; + benchmark::ResultManager::SaveMattingResult(res, matting_result_path); + // Load result from <- disk. + vision::MattingResult res_loaded; + benchmark::ResultManager::LoadMattingResult(&res_loaded, + matting_result_path); + // Calculate diff between two results. + auto matting_diff = + benchmark::ResultManager::CalculateDiffStatis(res, res_loaded); + std::cout << "Alpha diff: mean=" << matting_diff.alpha.mean + << ", max=" << matting_diff.alpha.max + << ", min=" << matting_diff.alpha.min << std::endl; + if (res_loaded.contain_foreground) { + std::cout << "Foreground diff: mean=" << matting_diff.foreground.mean + << ", max=" << matting_diff.foreground.max + << ", min=" << matting_diff.foreground.min << std::endl; + } + } BENCHMARK_MODEL(model_ppmatting, model_ppmatting.Predict(&im, &res)) auto vis_im = vision::VisMatting(im, res); cv::imwrite("vis_result.jpg", vis_im); diff --git a/fastdeploy/benchmark/utils.cc b/fastdeploy/benchmark/utils.cc old mode 100755 new mode 100644 index 894a3df46..bf978edac --- a/fastdeploy/benchmark/utils.cc +++ b/fastdeploy/benchmark/utils.cc @@ -556,6 +556,44 @@ bool ResultManager::SaveOCRDetResult(const std::vector>& res, return true; } +bool ResultManager::SaveMattingResult(const vision::MattingResult& res, + const std::string& path) { + if (res.alpha.empty()) { + FDERROR << "MattingResult can not be empty!" << std::endl; + return false; + } + std::ofstream fs(path, std::ios::out); + if (!fs.is_open()) { + FDERROR << "Fail to open file:" << path << std::endl; + return false; + } + fs.precision(20); + // alpha + fs << "alpha" << KEY_VALUE_SEP; + for (int i = 0; i < res.alpha.size(); ++i) { + if (i < res.alpha.size() - 1) { + fs << res.alpha[i] << VALUE_SEP; + } else { + fs << res.alpha[i]; + } + } + fs << "\n"; + // foreground + if (res.contain_foreground) { + fs << "foreground" << KEY_VALUE_SEP; + for (int i = 0; i < res.foreground.size(); ++i) { + if (i < res.foreground.size() - 1) { + fs << res.foreground[i] << VALUE_SEP; + } else { + fs << res.foreground[i]; + } + } + fs << "\n"; + } + fs.close(); + return true; +} + bool ResultManager::LoadDetectionResult(vision::DetectionResult* res, const std::string& path) { if (!CheckFileExists(path)) { @@ -658,6 +696,33 @@ bool ResultManager::LoadOCRDetResult(std::vector>* res, return true; } +bool ResultManager::LoadMattingResult(vision::MattingResult* res, + const std::string& path) { + if (!CheckFileExists(path)) { + FDERROR << "Can't found file from " << path << std::endl; + return false; + } + auto lines = ReadLines(path); + if (lines.size() > 1) { + res->contain_foreground = true; + } + std::map> data; + // alpha + data = SplitDataLine(lines[0]); + res->Resize(data.begin()->second.size()); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->alpha[i] = std::stof(data.begin()->second[i]); + } + // foreground + if (lines.size() > 1) { + data = SplitDataLine(lines[1]); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->foreground[i] = std::stof(data.begin()->second[i]); + } + } + return true; +} + DetectionDiff ResultManager::CalculateDiffStatis( const vision::DetectionResult& lhs, const vision::DetectionResult& rhs, const float& score_threshold) { @@ -770,6 +835,29 @@ OCRDetDiff ResultManager::CalculateDiffStatis( return diff; } +MattingDiff ResultManager::CalculateDiffStatis( + const vision::MattingResult& lhs, const vision::MattingResult& rhs) { + const int pixel_nums = std::min(lhs.alpha.size(), rhs.alpha.size()); + std::vector alpha_diff; + std::vector foreground_diff; + for (int i = 0; i < pixel_nums; ++i) { + alpha_diff.push_back(lhs.alpha[i] - rhs.alpha[i]); + if (lhs.contain_foreground && rhs.contain_foreground) { + foreground_diff.push_back(lhs.foreground[i] - rhs.foreground[i]); + } + } + MattingDiff diff; + CalculateStatisInfo(alpha_diff.data(), alpha_diff.size(), + &(diff.alpha.mean), &(diff.alpha.max), + &(diff.alpha.min)); + if (lhs.contain_foreground && rhs.contain_foreground) { + CalculateStatisInfo(foreground_diff.data(), foreground_diff.size(), + &(diff.foreground.mean), &(diff.foreground.max), + &(diff.foreground.min)); + } + return diff; +} + #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK diff --git a/fastdeploy/benchmark/utils.h b/fastdeploy/benchmark/utils.h index 669da9c57..e90838d79 100755 --- a/fastdeploy/benchmark/utils.h +++ b/fastdeploy/benchmark/utils.h @@ -127,6 +127,11 @@ struct FASTDEPLOY_DECL OCRDetDiff: public BaseDiff { EvalStatis boxes; }; +struct FASTDEPLOY_DECL MattingDiff: public BaseDiff { + EvalStatis alpha; + EvalStatis foreground; +}; + #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK @@ -166,6 +171,10 @@ struct FASTDEPLOY_DECL ResultManager { const std::string& path); static bool LoadOCRDetResult(std::vector>* res, const std::string& path); + static bool SaveMattingResult(const vision::MattingResult& res, + const std::string& path); + static bool LoadMattingResult(vision::MattingResult* res, + const std::string& path); /// Calculate diff value between two basic results. static DetectionDiff CalculateDiffStatis(const vision::DetectionResult& lhs, const vision::DetectionResult& rhs, @@ -178,6 +187,9 @@ struct FASTDEPLOY_DECL ResultManager { static OCRDetDiff CalculateDiffStatis( const std::vector>& lhs, const std::vector>& rhs); + static MattingDiff CalculateDiffStatis( + const vision::MattingResult& lhs, + const vision::MattingResult& rhs); #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK };