apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 17:17:14 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Serving][backend]serving support multi stream and backend support external stream (#431)

Browse Source 
* serving support multi stream

* pybind add external stream

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
heliqi
2022-10-26 14:46:13 +08:00
committed by GitHub
parent 718698a32a
commit b064ddf7ed
10 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fastdeploy/backends/ort/ort_backend.h
View File

@@ -44,6 +44,7 @@ struct OrtBackendOption {
int execution_mode = -1;
bool use_gpu = false;
int gpu_id = 0;
void* external_stream_ = nullptr;
// inside parameter, maybe remove next version
bool remove_multiclass_nms_ = false;
@@ -66,7 +67,8 @@ class OrtBackend : public BaseBackend {
const OrtBackendOption& option = OrtBackendOption(),
bool from_memory_buffer = false);
bool Infer(std::vector<FDTensor>& inputs, std::vector<FDTensor>* outputs) override;
bool Infer(std::vector<FDTensor>& inputs,
std::vector<FDTensor>* outputs) override;
int NumInputs() const override { return inputs_desc_.size(); }