apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-07 01:22:59 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Backend] Enable TensorRT BatchedNMSDynamic_TRT plugin (#449)

Browse Source 
* Enable TensorRT EfficientNMS plugin

* remove some temporary code

* Update trt_backend.cc

* Update utils.h
This commit is contained in:
Jason
2022-11-04 11:46:29 +08:00
committed by GitHub
parent 7150e6405c
commit 9fa612c24b
8 changed files with 36 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
CMakeLists.txt
View File

@@ -50,7 +50,6 @@ if(ANDROID)
endif()
############################# Basic Options for FastDeploy ################################
option(ENABLE_PADDLE_FRONTEND "Whether to enable PaddlePaddle frontend to support load paddle model in fastdeploy." ON)
option(WITH_GPU "Whether WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernce-gpu/poros-gpu" OFF)
option(WITH_IPU "Whether WITH_IPU=ON, will enable paddle-infernce-ipu" OFF)
option(ENABLE_ORT_BACKEND "Whether to enable onnxruntime backend." OFF)
@@ -190,13 +189,8 @@ if(WITH_SW)
add_definitions(-DEIGEN_AVOID_THREAD_LOCAL)
endif()
if(ENABLE_PADDLE_FRONTEND)
add_definitions(-DENABLE_PADDLE_FRONTEND)
include(${PROJECT_SOURCE_DIR}/cmake/paddle2onnx.cmake)
list(APPEND DEPEND_LIBS external_paddle2onnx)
endif(ENABLE_PADDLE_FRONTEND)
if(ENABLE_ORT_BACKEND)
set(ENABLE_PADDLE_FRONTEND ON)
add_definitions(-DENABLE_ORT_BACKEND)
list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS})
include(${PROJECT_SOURCE_DIR}/cmake/onnxruntime.cmake)
@@ -224,6 +218,7 @@ if(ENABLE_PADDLE_BACKEND)
endif()
if(ENABLE_OPENVINO_BACKEND)
set(ENABLE_PADDLE_FRONTEND ON)
add_definitions(-DENABLE_OPENVINO_BACKEND)
list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_OPENVINO_SRCS})
include(${PROJECT_SOURCE_DIR}/cmake/openvino.cmake)
@@ -329,6 +324,7 @@ if(WITH_IPU)
endif()
if(ENABLE_TRT_BACKEND)
set(ENABLE_PADDLE_FRONTEND ON)
if(APPLE OR ANDROID OR IOS)
message(FATAL_ERROR "Cannot enable tensorrt backend in mac/ios/android os, please set -DENABLE_TRT_BACKEND=OFF.")
endif()
@@ -382,7 +378,6 @@ endif()
if(ENABLE_VISION)
add_definitions(-DENABLE_VISION)
# set(ENABLE_VISION_VISUALIZE ON)
add_definitions(-DENABLE_VISION_VISUALIZE)
if(ENABLE_OPENCV_CUDA)
if(NOT WITH_GPU)
@@ -424,6 +419,13 @@ if(ENABLE_TEXT)
include(${PROJECT_SOURCE_DIR}/cmake/faster_tokenizer.cmake)
endif()
if(ENABLE_PADDLE_FRONTEND)
add_definitions(-DENABLE_PADDLE_FRONTEND)
include(${PROJECT_SOURCE_DIR}/cmake/paddle2onnx.cmake)
list(APPEND DEPEND_LIBS external_paddle2onnx)
endif(ENABLE_PADDLE_FRONTEND)
configure_file(${PROJECT_SOURCE_DIR}/FastDeploy.cmake.in ${PROJECT_SOURCE_DIR}/FastDeploy.cmake @ONLY)
configure_file(${PROJECT_SOURCE_DIR}/python/fastdeploy/c_lib_wrap.py.in ${PROJECT_SOURCE_DIR}/python/fastdeploy/c_lib_wrap.py)
configure_file(${PROJECT_SOURCE_DIR}/python/scripts/process_libraries.py.in ${PROJECT_SOURCE_DIR}/python/scripts/process_libraries.py)

3
cmake/paddle2onnx.cmake
View File

@@ -43,13 +43,14 @@ else()
endif(WIN32)
set(PADDLE2ONNX_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/")
set(PADDLE2ONNX_VERSION "1.0.1")
set(PADDLE2ONNX_VERSION "1.0.2rc")
if(WIN32)
set(PADDLE2ONNX_FILE "paddle2onnx-win-x64-${PADDLE2ONNX_VERSION}.zip")
if(NOT CMAKE_CL_64)
set(PADDLE2ONNX_FILE "paddle2onnx-win-x86-${PADDLE2ONNX_VERSION}.zip")
endif()
elseif(APPLE)
set(PADDLE2ONNX_VERSION "1.0.1")
if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
set(PADDLE2ONNX_FILE "paddle2onnx-osx-arm64-${PADDLE2ONNX_VERSION}.tgz")
else()

17
fastdeploy/backends/ort/ort_backend.cc
View File

@@ -80,21 +80,18 @@ bool OrtBackend::InitFromPaddle(const std::string& model_file,
<< std::endl;
return false;
}
#ifdef ENABLE_PADDLE_FRONTEND
char* model_content_ptr;
int model_content_size = 0;
std::vector<paddle2onnx::CustomOp> custom_ops;
for (auto& item : option.custom_op_info_) {
#ifdef ENABLE_PADDLE_FRONTEND
paddle2onnx::CustomOp op;
strcpy(op.op_name, item.first.c_str());
strcpy(op.export_op_name, item.second.c_str());
custom_ops.emplace_back(op);
}
strcpy(op.op_name, "multiclass_nms3");
strcpy(op.export_op_name, "MultiClassNMS");
if (!paddle2onnx::Export(model_file.c_str(), params_file.c_str(),
&model_content_ptr, &model_content_size, 11, true,
verbose, true, true, true, custom_ops.data(),
custom_ops.size())) {
verbose, true, true, true, &op,
1)) {
FDERROR << "Error occured while export PaddlePaddle to ONNX format."
<< std::endl;
return false;
@@ -106,7 +103,7 @@ bool OrtBackend::InitFromPaddle(const std::string& model_file,
model_content_ptr = nullptr;
return InitFromOnnx(onnx_model_proto, option, true);
#else
FDERROR << "Didn't compile with PaddlePaddle frontend, you can try to "
FDERROR << "Didn't compile with PaddlePaddle Frontend, you can try to "
"call `InitFromOnnx` instead."
<< std::endl;
#endif

32
fastdeploy/backends/tensorrt/trt_backend.cc
View File

@@ -124,48 +124,20 @@ bool TrtBackend::InitFromPaddle(const std::string& model_file,
option_ = option;
#ifdef ENABLE_PADDLE_FRONTEND
std::vector<paddle2onnx::CustomOp> custom_ops;
for (auto& item : option_.custom_op_info_) {
paddle2onnx::CustomOp op;
std::strcpy(op.op_name, item.first.c_str());
std::strcpy(op.export_op_name, item.second.c_str());
custom_ops.emplace_back(op);
}
char* model_content_ptr;
int model_content_size = 0;
char* calibration_cache_ptr;
int calibration_cache_size = 0;
if (!paddle2onnx::Export(model_file.c_str(), params_file.c_str(),
&model_content_ptr, &model_content_size, 11, true,
verbose, true, true, true, custom_ops.data(),
custom_ops.size(), "tensorrt",
verbose, true, true, true, nullptr,
0, "tensorrt",
&calibration_cache_ptr, &calibration_cache_size)) {
FDERROR << "Error occured while export PaddlePaddle to ONNX format."
<< std::endl;
return false;
}
if (option_.remove_multiclass_nms_) {
char* new_model = nullptr;
int new_model_size = 0;
if (!paddle2onnx::RemoveMultiClassNMS(model_content_ptr, model_content_size,
&new_model, &new_model_size)) {
FDERROR << "Try to remove MultiClassNMS failed." << std::endl;
return false;
}
delete[] model_content_ptr;
std::string onnx_model_proto(new_model, new_model + new_model_size);
delete[] new_model;
if (calibration_cache_size) {
std::string calibration_str(
calibration_cache_ptr,
calibration_cache_ptr + calibration_cache_size);
calibration_str_ = calibration_str;
delete[] calibration_cache_ptr;
}
return InitFromOnnx(onnx_model_proto, option, true);
}
std::string onnx_model_proto(model_content_ptr,
model_content_ptr + model_content_size);
delete[] model_content_ptr;

4
fastdeploy/backends/tensorrt/trt_backend.h
View File

@@ -73,10 +73,6 @@ struct TrtBackendOption {
std::string serialize_file = "";
bool enable_pinned_memory = false;
void* external_stream_ = nullptr;
// inside parameter, maybe remove next version
bool remove_multiclass_nms_ = false;
std::map<std::string, std::string> custom_op_info_;
};
std::vector<int> toVec(const nvinfer1::Dims& dim);

8
fastdeploy/runtime.cc
View File

@@ -675,10 +675,6 @@ void Runtime::CreateOrtBackend() {
ort_option.gpu_id = option.device_id;
ort_option.external_stream_ = option.external_stream_;
// TODO(jiangjiajun): inside usage, maybe remove this later
ort_option.remove_multiclass_nms_ = option.remove_multiclass_nms_;
ort_option.custom_op_info_ = option.custom_op_info_;
FDASSERT(option.model_format == ModelFormat::PADDLE ||
option.model_format == ModelFormat::ONNX,
"OrtBackend only support model format of ModelFormat::PADDLE / "
@@ -715,10 +711,6 @@ void Runtime::CreateTrtBackend() {
trt_option.enable_pinned_memory = option.enable_pinned_memory;
trt_option.external_stream_ = option.external_stream_;
// TODO(jiangjiajun): inside usage, maybe remove this later
trt_option.remove_multiclass_nms_ = option.remove_multiclass_nms_;
trt_option.custom_op_info_ = option.custom_op_info_;
FDASSERT(option.model_format == ModelFormat::PADDLE ||
option.model_format == ModelFormat::ONNX,
"TrtBackend only support model format of ModelFormat::PADDLE / "

6
fastdeploy/runtime.h Executable file → Normal file
View File

@@ -338,12 +338,6 @@ struct FASTDEPLOY_DECL RuntimeOption {
std::string model_file = ""; // Path of model file
std::string params_file = ""; // Path of parameters file, can be empty
ModelFormat model_format = ModelFormat::AUTOREC; // format of input model
// inside parameters, only for inside usage
// remove multiclass_nms in Paddle2ONNX
bool remove_multiclass_nms_ = false;
// for Paddle2ONNX to export custom operators
std::map<std::string, std::string> custom_op_info_;
};
/*! @brief Runtime object used to inference the loaded model on different devices

42
fastdeploy/vision/detection/ppdet/ppyoloe.cc
View File

@@ -46,13 +46,6 @@ void PPYOLOE::GetNmsInfo() {
}
bool PPYOLOE::Initialize() {
#ifdef ENABLE_PADDLE_FRONTEND
// remove multiclass_nms3 now
// this is a trick operation for ppyoloe while inference on trt
GetNmsInfo();
runtime_option.remove_multiclass_nms_ = true;
runtime_option.custom_op_info_["multiclass_nms3"] = "MultiClassNMS";
#endif
if (!BuildPreprocessPipelineFromConfig()) {
FDERROR << "Failed to build preprocess pipeline from configuration file."
<< std::endl;
@@ -63,16 +56,6 @@ bool PPYOLOE::Initialize() {
return false;
}
if (has_nms_ && runtime_option.backend == Backend::TRT) {
FDINFO << "Detected operator multiclass_nms3 in your model, will replace "
"it with fastdeploy::backend::MultiClassNMS(background_label="
<< background_label << ", keep_top_k=" << keep_top_k
<< ", nms_eta=" << nms_eta << ", nms_threshold=" << nms_threshold
<< ", score_threshold=" << score_threshold
<< ", nms_top_k=" << nms_top_k << ", normalized=" << normalized
<< ")." << std::endl;
has_nms_ = false;
}
return true;
}
@@ -198,6 +181,7 @@ bool PPYOLOE::Postprocess(std::vector<FDTensor>& infer_result,
FDASSERT(infer_result[1].shape[0] == 1,
"Only support batch = 1 in FastDeploy now.");
has_nms_ = true;
if (!has_nms_) {
int boxes_index = 0;
int scores_index = 1;
@@ -237,19 +221,23 @@ bool PPYOLOE::Postprocess(std::vector<FDTensor>& infer_result,
nms.out_box_data[i * 6 + 4], nms.out_box_data[i * 6 + 5]});
}
} else {
int box_num = 0;
std::vector<int> num_boxes(infer_result[1].shape[0]);
if (infer_result[1].dtype == FDDataType::INT32) {
box_num = *(static_cast<int32_t*>(infer_result[1].Data()));
} else if (infer_result[1].dtype == FDDataType::INT64) {
box_num = *(static_cast<int64_t*>(infer_result[1].Data()));
} else {
FDASSERT(
false,
"The output box_num of PPYOLOE model should be type of int32/int64.");
int32_t* data = static_cast<int32_t*>(infer_result[1].Data());
for (size_t i = 0; i < infer_result[1].shape[0]; ++i) {
num_boxes[i] = static_cast<int>(data[i]);
}
result->Reserve(box_num);
} else if (infer_result[1].dtype == FDDataType::INT64) {
int64_t* data = static_cast<int64_t*>(infer_result[1].Data());
for (size_t i = 0; i < infer_result[1].shape[0]; ++i) {
num_boxes[i] = static_cast<int>(data[i]);
}
}
// Only support batch = 1 now
result->Reserve(num_boxes[0]);
float* box_data = static_cast<float*>(infer_result[0].Data());
for (size_t i = 0; i < box_num; ++i) {
for (size_t i = 0; i < num_boxes[0]; ++i) {
result->label_ids.push_back(box_data[i * 6]);
result->scores.push_back(box_data[i * 6 + 1]);
result->boxes.emplace_back(