diff --git a/benchmark/cpp/benchmark_ppcls.cc b/benchmark/cpp/benchmark_ppcls.cc index 734a09a48..f925d52c6 100755 --- a/benchmark/cpp/benchmark_ppcls.cc +++ b/benchmark/cpp/benchmark_ppcls.cc @@ -16,6 +16,9 @@ #include "macros.h" #include "option.h" +namespace vision = fastdeploy::vision; +namespace benchmark = fastdeploy::benchmark; + int main(int argc, char* argv[]) { #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION) // Initialization @@ -31,9 +34,28 @@ int main(int argc, char* argv[]) { auto model_file = FLAGS_model + sep + "inference.pdmodel"; auto params_file = FLAGS_model + sep + "inference.pdiparams"; auto config_file = FLAGS_model + sep + "inference_cls.yaml"; - auto model_ppcls = fastdeploy::vision::classification::PaddleClasModel( + auto model_ppcls = vision::classification::PaddleClasModel( model_file, params_file, config_file, option); - fastdeploy::vision::ClassifyResult res; + vision::ClassifyResult res; + // Run once at least + model_ppcls.Predict(im, &res); + // 1. Test result diff + std::cout << "=============== Test result diff =================\n"; + // Save result to -> disk. + std::string cls_result_path = "ppcls_result.txt"; + benchmark::ResultManager::SaveClassifyResult(res, cls_result_path); + // Load result from <- disk. + vision::ClassifyResult res_loaded; + benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path); + // Calculate diff between two results. + auto cls_diff = + benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded); + std::cout << "Labels diff: mean=" << cls_diff.labels.mean + << ", max=" << cls_diff.labels.max + << ", min=" << cls_diff.labels.min << std::endl; + std::cout << "Scores diff: mean=" << cls_diff.scores.mean + << ", max=" << cls_diff.scores.max + << ", min=" << cls_diff.scores.min << std::endl; BENCHMARK_MODEL(model_ppcls, model_ppcls.Predict(im, &res)) #endif return 0; diff --git a/benchmark/cpp/benchmark_ppocr.cc b/benchmark/cpp/benchmark_ppocr.cc index 398d0feb0..e81080c54 100755 --- a/benchmark/cpp/benchmark_ppocr.cc +++ b/benchmark/cpp/benchmark_ppocr.cc @@ -16,6 +16,13 @@ #include "macros.h" #include "option.h" +// Only for ppocr +DEFINE_string(det_model, "", "Path of Detection model of PPOCR."); +DEFINE_string(cls_model, "", "Path of Classification model of PPOCR."); +DEFINE_string(rec_model, "", "Path of Recognization model of PPOCR."); +DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR."); +DEFINE_string(image_rec, "", "Path of Recognization img file of PPOCR."); + int main(int argc, char* argv[]) { #if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION) // Initialization diff --git a/benchmark/cpp/benchmark_precision_ppyolov8.cc b/benchmark/cpp/benchmark_precision_ppyolov8.cc index caea3be19..e5caf8004 100644 --- a/benchmark/cpp/benchmark_precision_ppyolov8.cc +++ b/benchmark/cpp/benchmark_precision_ppyolov8.cc @@ -46,11 +46,15 @@ int main(int argc, char* argv[]) { // Calculate diff between two results. auto det_diff = benchmark::ResultManager::CalculateDiffStatis(&res, &res_loaded); - std::cout << "diff: mean=" << det_diff.mean << ",max=" << det_diff.max - << ",min=" << det_diff.min << std::endl; + std::cout << "Boxes diff: mean=" << det_diff.boxes.mean + << ", max=" << det_diff.boxes.max << ", min=" << det_diff.boxes.min + << std::endl; + std::cout << "Label_ids diff: mean=" << det_diff.labels.mean + << ", max=" << det_diff.labels.max + << ", min=" << det_diff.labels.min << std::endl; // 2. Test tensor diff std::cout << "=============== Test tensor diff =================\n"; - std::vector bacth_res; + std::vector batch_res; std::vector input_tensors, output_tensors; std::vector imgs; imgs.push_back(im); @@ -62,7 +66,7 @@ int main(int argc, char* argv[]) { input_tensors[2].name = "im_shape"; input_tensors.pop_back(); model_ppyolov8.Infer(input_tensors, &output_tensors); - model_ppyolov8.GetPostprocessor().Run(output_tensors, &bacth_res); + model_ppyolov8.GetPostprocessor().Run(output_tensors, &batch_res); // Save tensor to -> disk. auto& tensor_dump = output_tensors[0]; std::string det_tensor_path = "ppyolov8_tensor.txt"; @@ -73,9 +77,9 @@ int main(int argc, char* argv[]) { // Calculate diff between two tensors. auto det_tensor_diff = benchmark::ResultManager::CalculateDiffStatis( &tensor_dump, &tensor_loaded); - std::cout << "diff: mean=" << det_tensor_diff.mean - << ",max=" << det_tensor_diff.max << ",min=" << det_tensor_diff.min - << std::endl; + std::cout << "Tensor diff: mean=" << det_tensor_diff.data.mean + << ", max=" << det_tensor_diff.data.max + << ", min=" << det_tensor_diff.data.min << std::endl; // 3. Run profiling BENCHMARK_MODEL(model_ppyolov8, model_ppyolov8.Predict(im, &res)) auto vis_im = vision::VisDetection(im, res); diff --git a/benchmark/cpp/flags.h b/benchmark/cpp/flags.h index 59361162f..e32e39eab 100755 --- a/benchmark/cpp/flags.h +++ b/benchmark/cpp/flags.h @@ -44,12 +44,6 @@ DEFINE_bool( DEFINE_bool( collect_memory_info, false, "Whether to collect memory info"); DEFINE_int32(sampling_interval, 50, "How often to collect memory info(ms)."); -// Only for ppocr -DEFINE_string(det_model, "", "Path of Detection model of PPOCR."); -DEFINE_string(cls_model, "", "Path of Classification model of PPOCR."); -DEFINE_string(rec_model, "", "Path of Recognization model of PPOCR."); -DEFINE_string(rec_label_file, "", "Path of Recognization label file of PPOCR."); -DEFINE_string(image_rec, "", "Path of Recognization img file of PPOCR."); static void PrintUsage() { std::cout << "Usage: infer_demo --model model_path --image img_path --device " diff --git a/fastdeploy/benchmark/utils.cc b/fastdeploy/benchmark/utils.cc old mode 100644 new mode 100755 index 825cb5977..e4b4e8a84 --- a/fastdeploy/benchmark/utils.cc +++ b/fastdeploy/benchmark/utils.cc @@ -321,8 +321,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } TensorDiff diff; - CalculateStatisInfo(tensor_diff.data(), numel, &(diff.mean), - &(diff.max), &(diff.min)); + CalculateStatisInfo(tensor_diff.data(), numel, &(diff.data.mean), + &(diff.data.max), &(diff.data.min)); return diff; } else if (dtype == FDDataType::INT32) { std::vector tensor_diff(numel); @@ -332,8 +332,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } TensorDiff diff; - CalculateStatisInfo(tensor_diff.data(), numel, &(diff.mean), - &(diff.max), &(diff.min)); + CalculateStatisInfo(tensor_diff.data(), numel, &(diff.data.mean), + &(diff.data.max), &(diff.data.min)); return diff; } else { // FP32 std::vector tensor_diff(numel); @@ -343,8 +343,8 @@ TensorDiff ResultManager::CalculateDiffStatis(FDTensor* lhs, FDTensor* rhs) { tensor_diff[i] = lhs_data_ptr[i] - rhs_data_ptr[i]; } TensorDiff diff; - CalculateStatisInfo(tensor_diff.data(), numel, &(diff.mean), - &(diff.max), &(diff.min)); + CalculateStatisInfo(tensor_diff.data(), numel, &(diff.data.mean), + &(diff.data.max), &(diff.data.min)); return diff; } } @@ -399,6 +399,42 @@ bool ResultManager::SaveDetectionResult(const vision::DetectionResult& res, return true; } +bool ResultManager::SaveClassifyResult(const vision::ClassifyResult& res, + const std::string& path) { + if (res.label_ids.empty()) { + FDERROR << "ClassifyResult can not be empty!" << std::endl; + return false; + } + std::ofstream fs(path, std::ios::out); + if (!fs.is_open()) { + FDERROR << "Fail to open file:" << path << std::endl; + return false; + } + fs.precision(20); + // label_ids + fs << "label_ids" << KEY_VALUE_SEP; + for (int i = 0; i < res.label_ids.size(); ++i) { + if (i < res.label_ids.size() - 1) { + fs << res.label_ids[i] << VALUE_SEP; + } else { + fs << res.label_ids[i]; + } + } + fs << "\n"; + // scores + fs << "scores" << KEY_VALUE_SEP; + for (int i = 0; i < res.scores.size(); ++i) { + if (i < res.scores.size() - 1) { + fs << res.scores[i] << VALUE_SEP; + } else { + fs << res.scores[i]; + } + } + fs << "\n"; + fs.close(); + return true; +} + bool ResultManager::LoadDetectionResult(vision::DetectionResult* res, const std::string& path) { if (!CheckFileExists(path)) { @@ -432,6 +468,28 @@ bool ResultManager::LoadDetectionResult(vision::DetectionResult* res, return true; } +bool ResultManager::LoadClassifyResult(vision::ClassifyResult* res, + const std::string& path) { + if (!CheckFileExists(path)) { + FDERROR << "Can't found file from" << path << std::endl; + return false; + } + auto lines = ReadLines(path); + std::map> data; + // label_ids + data = SplitDataLine(lines[0]); + res->Resize(data.begin()->second.size()); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->label_ids[i] = std::stoi(data.begin()->second[i]); + } + // scores + data = SplitDataLine(lines[1]); + for (int i = 0; i < data.begin()->second.size(); ++i) { + res->scores[i] = std::stof(data.begin()->second[i]); + } + return true; +} + DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs, vision::DetectionResult* rhs, float score_threshold) { @@ -469,11 +527,29 @@ DetectionDiff ResultManager::CalculateDiffStatis(vision::DetectionResult* lhs, CalculateStatisInfo(labels_diff.data(), labels_diff.size(), &(diff.labels.mean), &(diff.labels.max), &(diff.labels.min)); - diff.mean = diff.boxes.mean; - diff.max = diff.boxes.max; - diff.min = diff.boxes.min; return diff; } + +ClassifyDiff ResultManager::CalculateDiffStatis(vision::ClassifyResult* lhs, + vision::ClassifyResult* rhs) { + const int class_nums = std::min(lhs->label_ids.size(), rhs->label_ids.size()); + std::vector scores_diff; + std::vector labels_diff; + for (int i = 0; i < class_nums; ++i) { + scores_diff.push_back(lhs->scores[i] - rhs->scores[i]); + labels_diff.push_back(lhs->label_ids[i] - rhs->label_ids[i]); + } + + ClassifyDiff diff; + CalculateStatisInfo(scores_diff.data(), scores_diff.size(), + &(diff.scores.mean), &(diff.scores.max), + &(diff.scores.min)); + CalculateStatisInfo(labels_diff.data(), labels_diff.size(), + &(diff.labels.mean), &(diff.labels.max), + &(diff.labels.min)); + return diff; +} + #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK diff --git a/fastdeploy/benchmark/utils.h b/fastdeploy/benchmark/utils.h index fc7835745..918844b51 100755 --- a/fastdeploy/benchmark/utils.h +++ b/fastdeploy/benchmark/utils.h @@ -101,14 +101,21 @@ struct FASTDEPLOY_DECL EvalStatis { double max = -1.0; }; -struct FASTDEPLOY_DECL TensorDiff: public BaseDiff, public EvalStatis {}; +struct FASTDEPLOY_DECL TensorDiff: public BaseDiff { + EvalStatis data; +}; #if defined(ENABLE_VISION) -struct FASTDEPLOY_DECL DetectionDiff: public BaseDiff, public EvalStatis { +struct FASTDEPLOY_DECL DetectionDiff: public BaseDiff { EvalStatis boxes; EvalStatis scores; EvalStatis labels; }; + +struct FASTDEPLOY_DECL ClassifyDiff: public BaseDiff { + EvalStatis scores; + EvalStatis labels; +}; #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK @@ -127,10 +134,16 @@ struct FASTDEPLOY_DECL ResultManager { const std::string& path); static bool LoadDetectionResult(vision::DetectionResult* res, const std::string& path); + static bool SaveClassifyResult(const vision::ClassifyResult& res, + const std::string& path); + static bool LoadClassifyResult(vision::ClassifyResult* res, + const std::string& path); /// Calculate diff value between two basic results. static DetectionDiff CalculateDiffStatis(vision::DetectionResult* lhs, vision::DetectionResult* rhs, float score_threshold = 0.3f); + static ClassifyDiff CalculateDiffStatis(vision::ClassifyResult* lhs, + vision::ClassifyResult* rhs); #endif // ENABLE_VISION #endif // ENABLE_BENCHMARK }; diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index d48d9ddc4..1c8d20c0f 100755 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -26,6 +26,11 @@ void ClassifyResult::Clear() { scores.clear(); } +void ClassifyResult::Resize(int size) { + label_ids.resize(size); + scores.resize(size); +} + std::string ClassifyResult::Str() { std::string out; out = "ClassifyResult(\nlabel_ids: "; diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index 7c4efde23..6b40bf314 100755 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -51,6 +51,9 @@ struct FASTDEPLOY_DECL ClassifyResult : public BaseResult { std::vector scores; ResultType type = ResultType::CLASSIFY; + /// Resize ClassifyResult data buffer + void Resize(int size); + /// Clear ClassifyResult void Clear();