Support Poros Backend (#188)

* Add poros backend * Add torch lib * Add python3 lib * set c++ 14 for poros * fixed bugs * fixed grammer bugs * fixed grammer bugs * fixed code bugs * fixed code bugs * fixed CreatePorosValue bug * Add AtType2String for Log * fixed trt_option * fixed poros.cmake path * fixed grammer bug * fixed grammer bug * fixed ambiguous reference * fixed ambiguous reference * fixed reference error * fixed include files * rm ENABLE_TRT_BACKEND in poros * update CMakeLists.txt * fixed CMakeLists.txt * Add libtorch.so in CMakeLists.txt * Fixed CMakeLists.txt * Fixed CMakeLists.txt * Fixed copy bug * Fixed copy bug * Fixed copy bug * Fixed Cmake * Fixed Cmake * debug * debug * debug * debug * debug * debug * debug utils * debug utils * copy to cpu * rm log info * test share mem * test share mem * test share mem * test multi outputs * test multi outputs * test multi outputs * test multi outputs * test multi outputs * test multi outputs * test multi outputs * time cost * time cost * fixed bug * time collect * mem copy * mem copy * rm time log * rm share mem * fixed multi inputs bug * add set_input_dtypes func * add SetInputDtypes * fixed bug * fixed bug * fixed prewarm data order * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * fixed bug * Add compile func * Add compile func * Add compile func * Add is_dynamic option * Add is_dynamic option * Add is_dynamic option * Add is_dynamic option * rm infer log * add cuda11.6 poros lib * fixed bug * fixed bug * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * fixed multi outputs * rm logs * test * test * test * add test log * add test log * add test log * add test log * support cpu * support cpu * support cpu * support cpu * support member variable definition * rm useless log * fixed name * resolve conflict * resolve conflict * resolve conflict * fixed cmake * add GetInputInfos&GetOutputInfos * add GetInputInfos&GetOutputInfos * fixed bug * fixed runtime.py * add compile func * add np * deal with comments * rm to_inter func * add property
2025-10-05 16:48:03 +08:00 · 2022-10-17 15:28:12 +08:00
parent c8db2dd1ef
commit f5c94e5471
19 changed files with 1333 additions and 12 deletions
--- a/python/fastdeploy/runtime.py
+++ b/python/fastdeploy/runtime.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 import logging
+import numpy as np
 from . import ModelFormat
 from . import c_lib_wrap as C

@@ -28,8 +29,24 @@ class Runtime:
        """

        self._runtime = C.Runtime()
+        self.runtime_option = runtime_option
        assert self._runtime.init(
-            runtime_option._option), "Initialize Runtime Failed!"
+            self.runtime_option._option), "Initialize Runtime Failed!"
+
+    def forward(self, *inputs):
+        """Inference with input data for poros
+
+        :param data: (list[str : numpy.ndarray])The input data list
+        :return list of numpy.ndarray
+        """
+        if self.runtime_option._option.model_format != ModelFormat.TORCHSCRIPT:
+            raise Exception(
+                "The forward function is only used for Poros backend, please call infer function"
+            )
+        inputs_dict = dict()
+        for i in range(len(inputs)):
+            inputs_dict["x" + str(i)] = inputs[i]
+        return self.infer(inputs_dict)

    def infer(self, data):
        """Inference with input data.
@@ -41,6 +58,27 @@ class Runtime:
            data, list), "The input data should be type of dict or list."
        return self._runtime.infer(data)

+    def compile(self, warm_datas):
+        """compile with prewarm data for poros
+
+        :param data: (list[str : numpy.ndarray])The prewarm data list
+        :return TorchScript Model
+        """
+        if self.runtime_option._option.model_format != ModelFormat.TORCHSCRIPT:
+            raise Exception(
+                "The compile function is only used for Poros backend, please call infer function"
+            )
+        assert isinstance(warm_datas,
+                          list), "The prewarm data should be type of list."
+        for i in range(len(warm_datas)):
+            warm_data = warm_datas[i]
+            if isinstance(warm_data[0], np.ndarray):
+                warm_data = list(data for data in warm_data)
+            else:
+                warm_data = list(data.numpy() for data in warm_data)
+            warm_datas[i] = warm_data
+        return self._runtime.compile(warm_datas, self.runtime_option._option)
+
    def num_inputs(self):
        """Get number of inputs of the loaded model.
        """
@@ -85,6 +123,65 @@ class RuntimeOption:
    def __init__(self):
        self._option = C.RuntimeOption()

+    @property
+    def is_dynamic(self):
+        """Only for Poros backend
+
+        :param value: (bool)Whether to enable dynamic shape, default False
+        """
+        return self._option.is_dynamic
+
+    @property
+    def unconst_ops_thres(self):
+        """Only for Poros backend
+
+        :param value: (int)Minimum number of subgraph OPs, default 10
+        """
+        return self._option.unconst_ops_thres
+
+    @property
+    def long_to_int(self):
+        """Only for Poros backend
+
+        :param value: (bool)Whether to convert long dtype to int dtype, default True
+        """
+        return self._option.long_to_int
+
+    @property
+    def use_nvidia_tf32(self):
+        """Only for Poros backend
+
+        :param value: (bool)The calculation accuracy of tf32 mode exists on the A card, which can bring some performance improvements, default False
+        """
+        return self._option.use_nvidia_tf32
+
+    @is_dynamic.setter
+    def is_dynamic(self, value):
+        assert isinstance(
+            value, bool), "The value to set `is_dynamic` must be type of bool."
+        self._option.is_dynamic = value
+
+    @unconst_ops_thres.setter
+    def unconst_ops_thres(self, value):
+        assert isinstance(
+            value,
+            int), "The value to set `unconst_ops_thres` must be type of int."
+        self._option.unconst_ops_thres = value
+
+    @long_to_int.setter
+    def long_to_int(self, value):
+        assert isinstance(
+            value,
+            bool), "The value to set `long_to_int` must be type of bool."
+        self._option.long_to_int = value
+
+    @use_nvidia_tf32.setter
+    def use_nvidia_tf32(self, value):
+        assert isinstance(
+            value,
+            bool), "The value to set `use_nvidia_tf32` must be type of bool."
+        self._option.use_nvidia_tf32 = value
+
    def set_model_path(self,
                       model_path,
                       params_path="",
@@ -125,6 +222,11 @@ class RuntimeOption:
        """
        return self._option.use_paddle_backend()

+    def use_poros_backend(self):
+        """Use Poros backend, support inference TorchScript model on CPU/Nvidia GPU.
+        """
+        return self._option.use_poros_backend()
+
    def use_ort_backend(self):
        """Use ONNX Runtime backend, support inference Paddle/ONNX model on CPU/Nvidia GPU.
        """
@@ -235,7 +337,8 @@ class RuntimeOption:
                continue
            if hasattr(getattr(self._option, attr), "__call__"):
                continue
-            message += "  {} : {}\t\n".format(attr, getattr(self._option, attr))
+            message += "  {} : {}\t\n".format(attr,
+                                              getattr(self._option, attr))
        message.strip("\n")
        message += ")"
        return message