[Model] Add Paddle3D smoke model (#1766)

* add smoke model

* add 3d vis

* update code

* update doc

* mv paddle3d from detection to perception

* update result for velocity

* update code for CI

* add set input data for TRT backend

* add serving support for smoke model

* update code

* update code

* update code

---------

Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>

fastdeploy/runtime/backends/paddle/paddle_backend.cc

@@ -226,10 +226,17 @@ bool PaddleBackend::InitFromPaddle(const std::string& model,
       std::map<std::string, std::vector<int>> min_shape;
       std::map<std::string, std::vector<int>> opt_shape;
       GetDynamicShapeFromOption(option, &max_shape, &min_shape, &opt_shape);
+      std::map<std::string, std::vector<float>> max_input_data;
+      std::map<std::string, std::vector<float>> min_input_data;
+      std::map<std::string, std::vector<float>> opt_input_data;
+      if (!option.trt_option.min_input_data.empty()) {
+        GetInputDataFromOption(option, &max_input_data, &min_input_data,
+                               &opt_input_data);
+      }
       // Need to run once to get the shape range info file.
-      CollectShapeRun(predictor_tmp.get(), max_shape);
-      CollectShapeRun(predictor_tmp.get(), min_shape);
-      CollectShapeRun(predictor_tmp.get(), opt_shape);
+      CollectShapeRun(predictor_tmp.get(), max_shape, max_input_data);
+      CollectShapeRun(predictor_tmp.get(), min_shape, min_input_data);
+      CollectShapeRun(predictor_tmp.get(), opt_shape, min_input_data);
       FDINFO << "Finish generating shape range info file." << std::endl;
     }
     FDINFO << "Start loading shape range info file " << shape_range_info
@@ -400,9 +407,33 @@ void PaddleBackend::GetDynamicShapeFromOption(
   }
 }
 
+void PaddleBackend::GetInputDataFromOption(
+    const PaddleBackendOption& option,
+    std::map<std::string, std::vector<float>>* max_input_data,
+    std::map<std::string, std::vector<float>>* min_input_data,
+    std::map<std::string, std::vector<float>>* opt_input_data) const {
+  for (const auto& item : option.trt_option.min_input_data) {
+    auto max_iter = option.trt_option.max_input_data.find(item.first);
+    auto opt_iter = option.trt_option.opt_input_data.find(item.first);
+    FDASSERT(max_iter != option.trt_option.max_input_data.end(),
+             "Cannot find %s in TrtBackendOption::min_input_data.",
+             item.first.c_str());
+    FDASSERT(opt_iter != option.trt_option.opt_input_data.end(),
+             "Cannot find %s in TrtBackendOption::opt_input_data.",
+             item.first.c_str());
+    (*max_input_data)[item.first].assign(max_iter->second.begin(),
+                                         max_iter->second.end());
+    (*opt_input_data)[item.first].assign(opt_iter->second.begin(),
+                                         opt_iter->second.end());
+    (*min_input_data)[item.first].assign(item.second.begin(),
+                                         item.second.end());
+  }
+}
+
 void PaddleBackend::CollectShapeRun(
     paddle_infer::Predictor* predictor,
-    const std::map<std::string, std::vector<int>>& shape) const {
+    const std::map<std::string, std::vector<int>>& shape,
+    const std::map<std::string, std::vector<float>>& data) const {
   auto input_names = predictor->GetInputNames();
   auto input_type = predictor->GetInputTypes();
   for (const auto& name : input_names) {
@@ -418,21 +449,47 @@ void PaddleBackend::CollectShapeRun(
     int shape_num = std::accumulate(shape_value.begin(), shape_value.end(), 1,
                                     std::multiplies<int>());
     tensor->Reshape(shape_value);
+
+    if (data.find(name) != data.end()) {
+      FDASSERT(data.at(name).size() == shape_num,
+               "The data num and accumulate of shape must be equal for input: "
+               "[\"%s\"], "
+               " When Use the (C++)RuntimeOption.trt_option.SetInputData/ "
+               " (Python)RuntimeOption.trt_option.set_input_data/",
+               name.c_str());
+    }
+
     auto dtype = input_type[name];
     switch (dtype) {
       case paddle_infer::DataType::FLOAT32: {
-        std::vector<float> input_data(shape_num, 1.0);
-        tensor->CopyFromCpu(input_data.data());
+        if (data.find(name) != data.end()) {
+          tensor->CopyFromCpu(data.at(name).data());
+        } else {
+          std::vector<float> input_data(shape_num, 1.0);
+          tensor->CopyFromCpu(input_data.data());
+        }
         break;
       }
       case paddle_infer::DataType::INT32: {
-        std::vector<int> input_data(shape_num, 1);
-        tensor->CopyFromCpu(input_data.data());
+        if (data.find(name) != data.end()) {
+          std::vector<int> input_data(data.at(name).begin(),
+                                      data.at(name).end());
+          tensor->CopyFromCpu(input_data.data());
+        } else {
+          std::vector<int> input_data(shape_num, 1);
+          tensor->CopyFromCpu(input_data.data());
+        }
         break;
       }
       case paddle_infer::DataType::INT64: {
-        std::vector<int64_t> input_data(shape_num, 1);
-        tensor->CopyFromCpu(input_data.data());
+        if (data.find(name) != data.end()) {
+          std::vector<int64_t> input_data(data.at(name).begin(),
+                                          data.at(name).end());
+          tensor->CopyFromCpu(input_data.data());
+        } else {
+          std::vector<int64_t> input_data(shape_num, 1);
+          tensor->CopyFromCpu(input_data.data());
+        }
         break;
       }
       default: {

fastdeploy/runtime/backends/paddle/paddle_backend.h

@@ -81,12 +81,18 @@ class PaddleBackend : public BaseBackend {
 
   void
   CollectShapeRun(paddle_infer::Predictor* predictor,
-                  const std::map<std::string, std::vector<int>>& shape) const;
+                  const std::map<std::string, std::vector<int>>& shape,
+                  const std::map<std::string, std::vector<float>>& data) const;
   void GetDynamicShapeFromOption(
       const PaddleBackendOption& option,
       std::map<std::string, std::vector<int>>* max_shape,
       std::map<std::string, std::vector<int>>* min_shape,
       std::map<std::string, std::vector<int>>* opt_shape) const;
+  void GetInputDataFromOption(
+    const PaddleBackendOption& option,
+    std::map<std::string, std::vector<float>>* max_input_data,
+    std::map<std::string, std::vector<float>>* min_input_data,
+    std::map<std::string, std::vector<float>>* opt_input_data) const;
   void SetTRTDynamicShapeToConfig(const PaddleBackendOption& option);
   PaddleBackendOption option_;
   paddle_infer::Config config_;

fastdeploy/runtime/backends/tensorrt/option.h

@@ -33,8 +33,9 @@ struct TrtBackendOption {
   /// Enable log while converting onnx model to tensorrt
   bool enable_log_info = false;
 
-  
-  /// Enable half precison inference, on some device not support half precision, it will fallback to float32 mode
+
+  /// Enable half precison inference, on some device not support half precision,
+  /// it will fallback to float32 mode
   bool enable_fp16 = false;
 
   /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend
@@ -63,9 +64,44 @@ struct TrtBackendOption {
       max_shape[tensor_name].assign(max.begin(), max.end());
     }
   }
-  /// Set cache file path while use TensorRT backend. Loadding a Paddle/ONNX model and initialize TensorRT will take a long time, by this interface it will save the tensorrt engine to `cache_file_path`, and load it directly while execute the code again
+
+    /** \brief Set Input data for input tensor for the model  while using TensorRT backend
+   *
+   * \param[in] tensor_name The name of input for the model which is dynamic shape
+   * \param[in] min_data The input data for minimal shape for the input tensor
+   * \param[in] opt_data The input data for optimized shape for the input tensor
+   * \param[in] max_data The input data for maximum shape for the input tensor, if set as default value, it will keep same with min_data
+   */
+  void SetInputData(const std::string& tensor_name,
+                const std::vector<float> min_data,
+                const std::vector<float> opt_data = std::vector<float>(),
+                const std::vector<float> max_data = std::vector<float>()) {
+    max_input_data[tensor_name].clear();
+    min_input_data[tensor_name].clear();
+    opt_input_data[tensor_name].clear();
+    min_input_data[tensor_name].assign(min_data.begin(), min_data.end());
+    if (opt_data.empty()) {
+      opt_input_data[tensor_name].assign(min_data.begin(), min_data.end());
+    } else {
+      opt_input_data[tensor_name].assign(opt_data.begin(), opt_data.end());
+    }
+    if (max_data.empty()) {
+      max_input_data[tensor_name].assign(min_data.begin(), min_data.end());
+    } else {
+      max_input_data[tensor_name].assign(max_data.begin(), max_data.end());
+    }
+  }
+
+  /// Set cache file path while use TensorRT backend.
+  /// Loadding a Paddle/ONNX model and initialize TensorRT will
+  /// take a long time,
+  /// by this interface it will save the tensorrt engine to `cache_file_path`,
+  /// and load it directly while execute the code again
   std::string serialize_file = "";
 
+  std::map<std::string, std::vector<float>> max_input_data;
+  std::map<std::string, std::vector<float>> min_input_data;
+  std::map<std::string, std::vector<float>> opt_input_data;
   // The below parameters may be removed in next version, please do not
   // visit or use them directly
   std::map<std::string, std::vector<int32_t>> max_shape;

fastdeploy/runtime/backends/tensorrt/option_pybind.cc

@@ -26,7 +26,8 @@ void BindTrtOption(pybind11::module& m) {
       .def_readwrite("max_workspace_size",
                      &TrtBackendOption::max_workspace_size)
       .def_readwrite("serialize_file", &TrtBackendOption::serialize_file)
-      .def("set_shape", &TrtBackendOption::SetShape);
+      .def("set_shape", &TrtBackendOption::SetShape)
+      .def("set_input_data", &TrtBackendOption::SetInputData);
 }
 
 }  // namespace fastdeploy

fastdeploy/runtime/runtime_option.cc

@@ -379,6 +379,17 @@ void RuntimeOption::SetTrtInputShape(const std::string& input_name,
   trt_option.SetShape(input_name, min_shape, opt_shape, max_shape);
 }
 
+void RuntimeOption::SetTrtInputData(const std::string& input_name,
+                                    const std::vector<float>& min_shape_data,
+                                    const std::vector<float>& opt_shape_data,
+                                    const std::vector<float>& max_shape_data) {
+  FDWARNING << "`RuntimeOption::SetTrtInputData` will be removed in v1.2.0, "
+               "please use `RuntimeOption.trt_option.SetInputData()` instead."
+            << std::endl;
+  trt_option.SetInputData(input_name, min_shape_data, opt_shape_data,
+                          max_shape_data);
+}
+
 void RuntimeOption::SetTrtMaxWorkspaceSize(size_t max_workspace_size) {
   FDWARNING << "`RuntimeOption::SetTrtMaxWorkspaceSize` will be removed in "
                "v1.2.0, please modify its member variable directly, e.g "

fastdeploy/runtime/runtime_option.h

@@ -257,6 +257,12 @@ struct FASTDEPLOY_DECL RuntimeOption {
       const std::string& input_name, const std::vector<int32_t>& min_shape,
       const std::vector<int32_t>& opt_shape = std::vector<int32_t>(),
       const std::vector<int32_t>& max_shape = std::vector<int32_t>());
+
+  void SetTrtInputData(
+      const std::string& input_name, const std::vector<float>& min_shape_data,
+      const std::vector<float>& opt_shape_data = std::vector<float>(),
+      const std::vector<float>& max_shape_data = std::vector<float>());
+
   void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size);
   void SetTrtMaxBatchSize(size_t max_batch_size);
   void EnableTrtFP16();