diff --git a/benchmark/cpp/benchmark_ppcls.cc b/benchmark/cpp/benchmark_ppcls.cc index f925d52c6..b8bfcc989 100755 --- a/benchmark/cpp/benchmark_ppcls.cc +++ b/benchmark/cpp/benchmark_ppcls.cc @@ -49,7 +49,7 @@ int main(int argc, char* argv[]) { benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path); // Calculate diff between two results. auto cls_diff = - benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded); + benchmark::ResultManager::CalculateDiffStatis(res, res_loaded); std::cout << "Labels diff: mean=" << cls_diff.labels.mean << ", max=" << cls_diff.labels.max << ", min=" << cls_diff.labels.min << std::endl; diff --git a/benchmark/cpp/benchmark_ppseg.cc b/benchmark/cpp/benchmark_ppseg.cc index 23b98b3f5..02968cf2a 100755 --- a/benchmark/cpp/benchmark_ppseg.cc +++ b/benchmark/cpp/benchmark_ppseg.cc @@ -16,6 +16,9 @@ #include "macros.h" #include "option.h" +namespace vision = fastdeploy::vision; +namespace benchmark = fastdeploy::benchmark; + int main(int argc, char* argv[]) { #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION) // Initialization @@ -34,11 +37,33 @@ int main(int argc, char* argv[]) { option.trt_option.SetShape("x", {1, 3, 192, 192}, {1, 3, 192, 192}, {1, 3, 192, 192}); } - auto model_ppseg = fastdeploy::vision::segmentation::PaddleSegModel( + auto model_ppseg = vision::segmentation::PaddleSegModel( model_file, params_file, config_file, option); - fastdeploy::vision::SegmentationResult res; + vision::SegmentationResult res; + // Run once at least + model_ppseg.Predict(im, &res); + // 1. Test result diff + std::cout << "=============== Test result diff =================\n"; + // Save result to -> disk. + std::string seg_result_path = "ppseg_result.txt"; + benchmark::ResultManager::SaveSegmentationResult(res, seg_result_path); + // Load result from <- disk. + vision::SegmentationResult res_loaded; + benchmark::ResultManager::LoadSegmentationResult(&res_loaded, + seg_result_path); + // Calculate diff between two results. + auto seg_diff = + benchmark::ResultManager::CalculateDiffStatis(res, res_loaded); + std::cout << "Labels diff: mean=" << seg_diff.labels.mean + << ", max=" << seg_diff.labels.max + << ", min=" << seg_diff.labels.min << std::endl; + if (res_loaded.contain_score_map) { + std::cout << "Scores diff: mean=" << seg_diff.scores.mean + << ", max=" << seg_diff.scores.max + << ", min=" << seg_diff.scores.min << std::endl; + } BENCHMARK_MODEL(model_ppseg, model_ppseg.Predict(im, &res)) - auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5); + auto vis_im = vision::VisSegmentation(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; #endif diff --git a/benchmark/cpp/benchmark_precision_ppyolov8.cc b/benchmark/cpp/benchmark_precision_ppyolov8.cc index e5caf8004..7792d98c6 100644 --- a/benchmark/cpp/benchmark_precision_ppyolov8.cc +++ b/benchmark/cpp/benchmark_precision_ppyolov8.cc @@ -45,7 +45,7 @@ int main(int argc, char* argv[]) { benchmark::ResultManager::LoadDetectionResult(&res_loaded, det_result_path); // Calculate diff between two results. auto det_diff = - benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded); + benchmark::ResultManager::CalculateDiffStatis(res, res_loaded); std::cout << "Boxes diff: mean=" << det_diff.boxes.mean << ", max=" << det_diff.boxes.max << ", min=" << det_diff.boxes.min << std::endl; @@ -75,8 +75,8 @@ int main(int argc, char* argv[]) { fastdeploy::FDTensor tensor_loaded; benchmark::ResultManager::LoadFDTensor(&tensor_loaded, det_tensor_path); // Calculate diff between two tensors. - auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis( - &tensor_dump, &tensor_loaded); + auto det_tensor_diff = + benchmark::ResultManager::CalculateDiffStatis(tensor_dump, tensor_loaded); std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean << ", max=" << det_tensor_diff.data.max << ", min=" << det_tensor_diff.data.min << std::endl; diff --git a/fastdeploy/benchmark/utils.cc b/fastdeploy/benchmark/utils.cc old mode 100755 new mode 100644 index e4b4e8a84..5af28e4b1 --- a/fastdeploy/benchmark/utils.cc +++ b/fastdeploy/benchmark/utils.cc @@ -298,16 +298,17 @@ bool ResultManager::LoadFDTensor(FDTensor* tensor, const std::string& path) { return true; } -TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { - if (lhs->Numel() != rhs->Numel() || lhs->Dtype() != rhs->Dtype()) { +TensorDiff ResultManager::CalculateDiffStatis(const FDTensor& lhs, + const FDTensor& rhs) { + if (lhs.Numel() != rhs.Numel() || lhs.Dtype() != rhs.Dtype()) { FDASSERT(false, "The size and dtype of input FDTensor must be equal!" " But got size %d, %d, dtype %s, %s", - lhs->Numel(), rhs->Numel(), Str(lhs->Dtype()).c_str(), - Str(rhs->Dtype()).c_str()) + lhs.Numel(), rhs.Numel(), Str(lhs.Dtype()).c_str(), + Str(rhs.Dtype()).c_str()) } - FDDataType dtype = lhs->Dtype(); - int numel = lhs->Numel(); + FDDataType dtype = lhs.Dtype(); + int numel = lhs.Numel(); if (dtype != FDDataType::FP32 && dtype != FDDataType::INT64 && dtype != FDDataType::INT32) { FDASSERT(false, "Only support FP32/INT64/INT32 now, but got %s", @@ -315,8 +316,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { } if (dtype == FDDataType::INT64) { std::vector tensor_diff(numel); - const int64_t* lhs_data_ptr = static_cast(lhs->CpuData()); - const int64_t* rhs_data_ptr = static_cast(rhs->CpuData()); + const int64_t* lhs_data_ptr = static_cast(lhs.CpuData()); + const int64_t* rhs_data_ptr = static_cast(rhs.CpuData()); for (int i = 0; i < numel; ++i) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } @@ -326,8 +327,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { return diff; } else if (dtype == FDDataType::INT32) { std::vector tensor_diff(numel); - const int32_t* lhs_data_ptr = static_cast(lhs->CpuData()); - const int32_t* rhs_data_ptr = static_cast(rhs->CpuData()); + const int32_t* lhs_data_ptr = static_cast(lhs.CpuData()); + const int32_t* rhs_data_ptr = static_cast(rhs.CpuData()); for (int i = 0; i < numel; ++i) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } @@ -337,8 +338,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { return diff; } else { // FP32 std::vector tensor_diff(numel); - const float* lhs_data_ptr = static_cast(lhs->CpuData()); - const float* rhs_data_ptr = static_cast(rhs->CpuData()); + const float* lhs_data_ptr = static_cast(lhs.CpuData()); + const float* rhs_data_ptr = static_cast(rhs.CpuData()); for (int i = 0; i < numel; ++i) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } @@ -435,6 +436,44 @@ bool ResultManager::SaveClassifyResult(const vision::ClassifyResult& res, return true; } +bool ResultManager::SaveSegmentationResult( + const vision::SegmentationResult& res, const std::string& path) { + if (res.label_map.empty()) { + FDERROR << "SegmentationResult can not be empty!" << std::endl; + return false; + } + std::ofstream fs(path, std::ios::out); + if (!fs.is_open()) { + FDERROR << "Fail to open file:" << path << std::endl; + return false; + } + fs.precision(20); + // label_map + fs << "label_map" << KEY_VALUE_SEP; + for (int i = 0; i < res.label_map.size(); ++i) { + if (i < res.label_map.size() - 1) { + fs << static_cast(res.label_map[i]) << VALUE_SEP; + } else { + fs << static_cast(res.label_map[i]); + } + } + fs << "\n"; + // score_map + if (res.contain_score_map) { + fs << "score_map" << KEY_VALUE_SEP; + for (int i = 0; i < res.score_map.size(); ++i) { + if (i < res.score_map.size() - 1) { + fs << res.score_map[i] << VALUE_SEP; + } else { + fs << res.score_map[i]; + } + } + fs << "\n"; + } + fs.close(); + return true; +} + bool ResultManager::LoadDetectionResult(vision::DetectionResult* res, const std::string& path) { if (!CheckFileExists(path)) { @@ -490,32 +529,62 @@ bool ResultManager::LoadClassifyResult(vision::ClassifyResult* res, return true; } -DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs, - vision::DetectionResult* rhs, - float score_threshold) { +bool ResultManager::LoadSegmentationResult(vision::SegmentationResult* res, + const std::string& path) { + if (!CheckFileExists(path)) { + FDERROR << "Can't found file from" << path << std::endl; + return false; + } + auto lines = ReadLines(path); + if (lines.size() > 1) { + res->contain_score_map = true; + } + std::map> data; + // label_map + data = SplitDataLine(lines[0]); + res->Resize(data.begin()->second.size()); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->label_map[i] = std::stoi(data.begin()->second[i]); + } + // score_map + if (lines.size() > 1) { + data = SplitDataLine(lines[1]); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->score_map[i] = std::stof(data.begin()->second[i]); + } + } + return true; +} + +DetectionDiff ResultManager::CalculateDiffStatis( + const vision::DetectionResult& lhs, const vision::DetectionResult& rhs, + const float& score_threshold) { + vision::DetectionResult lhs_sort = lhs; + vision::DetectionResult rhs_sort = rhs; // lex sort by x(w) & y(h) - vision::utils::LexSortDetectionResultByXY(lhs); - vision::utils::LexSortDetectionResultByXY(rhs); + vision::utils::LexSortDetectionResultByXY(&lhs_sort); + vision::utils::LexSortDetectionResultByXY(&rhs_sort); // get value diff & trunc it by score_threshold - const int boxes_num = std::min(lhs->boxes.size(), rhs->boxes.size()); + const int boxes_num = std::min(lhs_sort.boxes.size(), rhs_sort.boxes.size()); std::vector boxes_diff; std::vector scores_diff; std::vector labels_diff; // TODO(qiuyanjun): process the diff of masks. for (int i = 0; i < boxes_num; ++i) { - if (lhs->scores[i] > score_threshold && rhs->scores[i] > score_threshold) { - scores_diff.push_back(lhs->scores[i] - rhs->scores[i]); - labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]); - boxes_diff.push_back(lhs->boxes[i][0] - rhs->boxes[i][0]); - boxes_diff.push_back(lhs->boxes[i][1] - rhs->boxes[i][1]); - boxes_diff.push_back(lhs->boxes[i][2] - rhs->boxes[i][2]); - boxes_diff.push_back(lhs->boxes[i][3] - rhs->boxes[i][3]); + if (lhs_sort.scores[i] > score_threshold && + rhs_sort.scores[i] > score_threshold) { + scores_diff.push_back(lhs_sort.scores[i] - rhs_sort.scores[i]); + labels_diff.push_back(lhs_sort.label_ids[i] - rhs_sort.label_ids[i]); + boxes_diff.push_back(lhs_sort.boxes[i][0] - rhs_sort.boxes[i][0]); + boxes_diff.push_back(lhs_sort.boxes[i][1] - rhs_sort.boxes[i][1]); + boxes_diff.push_back(lhs_sort.boxes[i][2] - rhs_sort.boxes[i][2]); + boxes_diff.push_back(lhs_sort.boxes[i][3] - rhs_sort.boxes[i][3]); } } FDASSERT(boxes_diff.size() > 0, "Can't get any valid boxes while score_threshold is %f, " "The boxes.size of lhs is %d, the boxes.size of rhs is %d", - score_threshold, lhs->boxes.size(), rhs->boxes.size()) + score_threshold, lhs_sort.boxes.size(), rhs_sort.boxes.size()) DetectionDiff diff; CalculateStatisInfo(boxes_diff.data(), boxes_diff.size(), @@ -530,14 +599,14 @@ DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs, return diff; } -ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs, - vision::ClassifyResult* rhs) { - const int class_nums = std::min(lhs->label_ids.size(), rhs->label_ids.size()); +ClassifyDiff ResultManager::CalculateDiffStatis( + const vision::ClassifyResult& lhs, const vision::ClassifyResult& rhs) { + const int class_nums = std::min(lhs.label_ids.size(), rhs.label_ids.size()); std::vector scores_diff; std::vector labels_diff; for (int i = 0; i < class_nums; ++i) { - scores_diff.push_back(lhs->scores[i] - rhs->scores[i]); - labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]); + scores_diff.push_back(lhs.scores[i] - rhs.scores[i]); + labels_diff.push_back(lhs.label_ids[i] - rhs.label_ids[i]); } ClassifyDiff diff; @@ -550,6 +619,30 @@ ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs, return diff; } +SegmentationDiff ResultManager::CalculateDiffStatis( + const vision::SegmentationResult& lhs, + const vision::SegmentationResult& rhs) { + const int pixel_nums = std::min(lhs.label_map.size(), rhs.label_map.size()); + std::vector labels_diff; + std::vector scores_diff; + for (int i = 0; i < pixel_nums; ++i) { + labels_diff.push_back(lhs.label_map[i] - rhs.label_map[i]); + if (lhs.contain_score_map && rhs.contain_score_map) { + scores_diff.push_back(lhs.score_map[i] - rhs.score_map[i]); + } + } + SegmentationDiff diff; + CalculateStatisInfo(labels_diff.data(), labels_diff.size(), + &(diff.labels.mean), &(diff.labels.max), + &(diff.labels.min)); + if (lhs.contain_score_map && rhs.contain_score_map) { + CalculateStatisInfo(scores_diff.data(), scores_diff.size(), + &(diff.scores.mean), &(diff.scores.max), + &(diff.scores.min)); + } + return diff; +} + #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK diff --git a/fastdeploy/benchmark/utils.h b/fastdeploy/benchmark/utils.h index 918844b51..f4d608133 100755 --- a/fastdeploy/benchmark/utils.h +++ b/fastdeploy/benchmark/utils.h @@ -116,6 +116,12 @@ struct FASTDEPLOY_DECL ClassifyDiff: public BaseDiff { EvalStatis scores; EvalStatis labels; }; + +struct FASTDEPLOY_DECL SegmentationDiff: public BaseDiff { + EvalStatis scores; + EvalStatis labels; +}; + #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK @@ -126,8 +132,8 @@ struct FASTDEPLOY_DECL ResultManager { static bool SaveFDTensor(const FDTensor& tensor, const std::string& path); static bool LoadFDTensor(FDTensor* tensor, const std::string& path); /// Calculate diff value between two FDTensor results. - static TensorDiff CalculateDiffStatis(FDTensor* lhs, - FDTensor* rhs); + static TensorDiff CalculateDiffStatis(const FDTensor& lhs, + const FDTensor& rhs); #if defined(ENABLE_VISION) /// Save & Load functions for basic results. static bool SaveDetectionResult(const vision::DetectionResult& res, @@ -138,12 +144,19 @@ struct FASTDEPLOY_DECL ResultManager { const std::string& path); static bool LoadClassifyResult(vision::ClassifyResult* res, const std::string& path); + static bool SaveSegmentationResult(const vision::SegmentationResult& res, + const std::string& path); + static bool LoadSegmentationResult(vision::SegmentationResult* res, + const std::string& path); /// Calculate diff value between two basic results. - static DetectionDiff CalculateDiffStatis(vision::DetectionResult* lhs, - vision::DetectionResult* rhs, - float score_threshold = 0.3f); - static ClassifyDiff CalculateDiffStatis(vision::ClassifyResult* lhs, - vision::ClassifyResult* rhs); + static DetectionDiff CalculateDiffStatis(const vision::DetectionResult& lhs, + const vision::DetectionResult& rhs, + const float& score_threshold = 0.3f); + static ClassifyDiff CalculateDiffStatis(const vision::ClassifyResult& lhs, + const vision::ClassifyResult& rhs); + static SegmentationDiff CalculateDiffStatis( + const vision::SegmentationResult& lhs, + const vision::SegmentationResult& rhs); #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK }; diff --git a/fastdeploy/runtime/backends/tensorrt/utils.h b/fastdeploy/runtime/backends/tensorrt/utils.h old mode 100644 new mode 100755 index b2fe8ee99..91402b305 --- a/fastdeploy/runtime/backends/tensorrt/utils.h +++ b/fastdeploy/runtime/backends/tensorrt/utils.h @@ -241,6 +241,7 @@ class FDTrtLogger : public nvinfer1::ILogger { FDASSERT(false, "%s", msg); } } + private: bool enable_info_ = false; bool enable_warning_ = false;