apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 08:37:06 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Serving][backend]serving support multi stream and backend support external stream (#431)

Browse Source 
* serving support multi stream

* pybind add external stream

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
heliqi
2022-10-26 14:46:13 +08:00
committed by GitHub
parent 718698a32a
commit b064ddf7ed
10 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fastdeploy/runtime.cc
View File

@@ -223,6 +223,10 @@ void RuntimeOption::UseGpu(int gpu_id) {
void RuntimeOption::UseCpu() { device = Device::CPU; }
void RuntimeOption::SetExternalStream(void* external_stream) {
external_stream_ = external_stream;
}
void RuntimeOption::SetCpuThreadNum(int thread_num) {
FDASSERT(thread_num > 0, "The thread_num must be greater than 0.");
cpu_thread_num = thread_num;
@@ -508,6 +512,7 @@ void Runtime::CreatePaddleBackend() {
pd_option.delete_pass_names = option.pd_delete_pass_names;
pd_option.cpu_thread_num = option.cpu_thread_num;
pd_option.enable_pinned_memory = option.enable_pinned_memory;
pd_option.external_stream_ = option.external_stream_;
#ifdef ENABLE_TRT_BACKEND
if (pd_option.use_gpu && option.pd_enable_trt) {
pd_option.enable_trt = true;
@@ -574,6 +579,7 @@ void Runtime::CreateOrtBackend() {
ort_option.execution_mode = option.ort_execution_mode;
ort_option.use_gpu = (option.device == Device::GPU) ? true : false;
ort_option.gpu_id = option.device_id;
ort_option.external_stream_ = option.external_stream_;
// TODO(jiangjiajun): inside usage, maybe remove this later
ort_option.remove_multiclass_nms_ = option.remove_multiclass_nms_;
@@ -613,6 +619,7 @@ void Runtime::CreateTrtBackend() {
trt_option.opt_shape = option.trt_opt_shape;
trt_option.serialize_file = option.trt_serialize_file;
trt_option.enable_pinned_memory = option.enable_pinned_memory;
trt_option.external_stream_ = option.external_stream_;
// TODO(jiangjiajun): inside usage, maybe remove this later
trt_option.remove_multiclass_nms_ = option.remove_multiclass_nms_;