apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 16:48:03 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Backend] Support Intel GPU with heterogeneous mode (#701)

Browse Source 
* Add some comments for python api

* support openvino gpu

* Add cpu operators

* add interface to specify hetero operators

* remove useless dir

* format code

* remove debug code

* Support GPU for ONNX
This commit is contained in:
Jason
2022-11-25 19:40:56 +08:00
committed by GitHub
parent ad5c9c08b2
commit 5b3fd9dd88
7 changed files with 156 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
python/fastdeploy/runtime.py
View File

@@ -35,7 +35,7 @@ class Runtime:
self.runtime_option._option), "Initialize Runtime Failed!"
def forward(self, *inputs):
"""Inference with input data for poros
"""[Only for Poros backend] Inference with input data for poros
:param data: (list[str : numpy.ndarray])The input data list
:return list of numpy.ndarray
@@ -60,7 +60,7 @@ class Runtime:
return self._runtime.infer(data)
def compile(self, warm_datas):
"""compile with prewarm data for poros
"""[Only for Poros backend] compile with prewarm data for poros
:param data: (list[str : numpy.ndarray])The prewarm data list
:return TorchScript Model
@@ -122,6 +122,9 @@ class RuntimeOption:
"""
def __init__(self):
"""Initialize a FastDeploy RuntimeOption object.
"""
self._option = C.RuntimeOption()
@property
@@ -210,8 +213,6 @@ class RuntimeOption:
def use_rknpu2(self,
rknpu2_name=rknpu2.CpuName.RK3588,
rknpu2_core=rknpu2.CoreMask.RKNN_NPU_CORE_0):
"""Inference with CPU
"""
return self._option.use_rknpu2(rknpu2_name, rknpu2_core)
def set_cpu_thread_num(self, thread_num=-1):
@@ -222,6 +223,10 @@ class RuntimeOption:
return self._option.set_cpu_thread_num(thread_num)
def set_ort_graph_opt_level(self, level=-1):
"""Set graph optimization level for ONNX Runtime backend
:param level: (int)Optimization level, -1 means the default setting
"""
return self._option.set_ort_graph_opt_level(level)
def use_paddle_backend(self):
@@ -274,6 +279,20 @@ class RuntimeOption:
"""
return self._option.set_openvino_device(name)
def set_openvino_shape_info(self, shape_info):
"""Set shape information of the models' inputs, used for GPU to fix the shape
:param shape_info: (dict{str, list of int})Shape information of model's inputs, e.g {"image": [1, 3, 640, 640], "scale_factor": [1, 2]}
"""
return self._option.set_openvino_shape_info(shape_info)
def set_openvino_cpu_operators(self, operators):
"""While using OpenVINO backend and intel GPU, this interface specifies unsupported operators to run on CPU
:param operators: (list of string)list of operators' name, e.g ["MulticlasNms"]
"""
return self._option.set_openvino_cpu_operators(operators)
def enable_paddle_log_info(self):
"""Enable print out the debug log information while using Paddle Inference backend, the log information is disabled by default.
"""
@@ -367,9 +386,13 @@ class RuntimeOption:
return self._option.set_trt_max_batch_size(trt_max_batch_size)
def enable_paddle_trt_collect_shape(self):
"""Enable collect subgraph shape information while using Paddle Inference with TensorRT
"""
return self._option.enable_paddle_trt_collect_shape()
def disable_paddle_trt_collect_shape(self):
"""Disable collect subgraph shape information while using Paddle Inference with TensorRT
"""
return self._option.disable_paddle_trt_collect_shape()
def use_ipu(self,