apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-14 04:44:00 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

[Serving][backend]serving support multi stream and backend support external stream (#431)

Browse Source 
* serving support multi stream

* pybind add external stream

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
heliqi
2022-10-26 14:46:13 +08:00
committed by GitHub
parent 718698a32a
commit b064ddf7ed
10 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
serving/src/fastdeploy_runtime.cc
View File

@@ -379,6 +379,7 @@ TRITONSERVER_Error* ModelState::LoadModel(
if ((instance_group_kind == TRITONSERVER_INSTANCEGROUPKIND_GPU) ||
(instance_group_kind == TRITONSERVER_INSTANCEGROUPKIND_AUTO)) {
runtime_options_->UseGpu(instance_group_device_id);
runtime_options_->SetExternalStream((void*)stream);
} else {
runtime_options_->UseCpu();
}
@@ -1001,9 +1002,7 @@ TRITONSERVER_Error* ModelInstanceState::Run(
runtime_->Infer(input_tensors_, &output_tensors_);
#ifdef TRITON_ENABLE_GPU
if (Kind() == TRITONSERVER_INSTANCEGROUPKIND_GPU) {
// TODO: stream controll
cudaDeviceSynchronize();
// cudaStreamSynchronize(CudaStream());
cudaStreamSynchronize(CudaStream());
}
#endif
return nullptr;