apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-17 06:00:59 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

[Other] Optimize Poros backend (#1232)

Browse Source 
* Optimize Poros backend

* fix error

* Add more pybind

* fix conflicts

* add some deprecate notices
This commit is contained in:
Jason
2023-02-07 10:18:03 +08:00
committed by GitHub
parent f73a538f61
commit 7c9bf11c44
15 changed files with 397 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
fastdeploy/runtime/runtime.cc
View File

@@ -495,25 +495,24 @@ Runtime* Runtime::Clone(void* stream, int device_id) {
bool Runtime::Compile(std::vector<std::vector<FDTensor>>& prewarm_tensors,
const RuntimeOption& _option) {
#ifdef ENABLE_POROS_BACKEND
option = _option;
auto poros_option = PorosBackendOption();
poros_option.use_gpu = (option.device == Device::GPU) ? true : false;
poros_option.gpu_id = option.device_id;
poros_option.long_to_int = option.long_to_int;
poros_option.use_nvidia_tf32 = option.use_nvidia_tf32;
poros_option.unconst_ops_thres = option.unconst_ops_thres;
poros_option.poros_file = option.poros_file;
poros_option.is_dynamic = option.is_dynamic;
poros_option.enable_fp16 = option.trt_enable_fp16;
poros_option.max_batch_size = option.trt_max_batch_size;
poros_option.max_workspace_size = option.trt_max_workspace_size;
FDASSERT(
option.model_format == ModelFormat::TORCHSCRIPT,
"PorosBackend only support model format of ModelFormat::TORCHSCRIPT.");
if (option.device != Device::CPU && option.device != Device::GPU) {
FDERROR << "PorosBackend only supports CPU/GPU, but now its "
<< option.device << "." << std::endl;
return false;
}
option.poros_option.device = option.device;
option.poros_option.device_id = option.device_id;
option.poros_option.enable_fp16 = option.trt_enable_fp16;
option.poros_option.max_batch_size = option.trt_max_batch_size;
option.poros_option.max_workspace_size = option.trt_max_workspace_size;
backend_ = utils::make_unique<PorosBackend>();
auto casted_backend = dynamic_cast<PorosBackend*>(backend_.get());
FDASSERT(
casted_backend->Compile(option.model_file, prewarm_tensors, poros_option),
casted_backend->Compile(option.model_file, prewarm_tensors,
option.poros_option),
"Load model from Torchscript failed while initliazing PorosBackend.");
#else
FDASSERT(false,