apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 09:07:10 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Serving][backend]serving support multi stream and backend support external stream (#431)

Browse Source 
* serving support multi stream

* pybind add external stream

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
heliqi
2022-10-26 14:46:13 +08:00
committed by GitHub
parent 718698a32a
commit b064ddf7ed
10 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fastdeploy/backends/tensorrt/trt_backend.cc
View File

@@ -258,8 +258,12 @@ bool TrtBackend::InitFromOnnx(const std::string& model_file,
ReaderDtypeToTrtDtype(onnx_reader.outputs[i].dtype);
}
FDASSERT(cudaStreamCreate(&stream_) == 0,
if (option_.external_stream_) {
stream_ = reinterpret_cast<cudaStream_t>(option_.external_stream_);
} else {
FDASSERT(cudaStreamCreate(&stream_) == 0,
"[ERROR] Error occurs while calling cudaStreamCreate().");
}
if (!CreateTrtEngineFromOnnx(onnx_content)) {
FDERROR << "Failed to create tensorrt engine." << std::endl;

1
fastdeploy/backends/tensorrt/trt_backend.h
View File

@@ -71,6 +71,7 @@ struct TrtBackendOption {
std::map<std::string, std::vector<int32_t>> opt_shape;
std::string serialize_file = "";
bool enable_pinned_memory = false;
void* external_stream_ = nullptr;
// inside parameter, maybe remove next version
bool remove_multiclass_nms_ = false;